def test(self, test):
try:
result = self.client.query(test.format()).result()
except Exception as E:
message = f"Bad test query: {test}"
logging.exception(message)
return message
if result.total_rows != 1:
message = f"'{test}' must return exactly one boolean value, but returned {result.total_rows} rows"
logging.exception(message)
else:
row = [row for row in result][0]
if len(row) != 1:
message = f"'{test}' must return exactly one boolean value, but returned {len(row)} columns"
logging.exception(message)
value = row[0]
if isinstance(value, bool):
if not value:
message = f"'{test}' did not pass"
logging.exception(message)
else:
message = value
else:
message = f"'{test}' must return exactly one boolean value, but returned type {type(value)}"
logging.exception(message)
return message
def get_previous_release_timestamp(commitsUrl, latest_tag):
try:
commit = call_github_api(f"{commitsUrl}/{latest_tag}")
return commit["commit"]["author"]["date"]
except KeyError:
logging.exception(f"Commit not found for {latest_tag}: {commit}")
exit(2)
def get_latest_tag(tagsUrl):
try:
tags = call_github_api(tagsUrl)
return tags[0]["name"]
except KeyError:
logging.exception(f"Malformed response from tags: {tags}")
exit(2)
def query(self, query):
query_job = self.client.query(query.format())
try:
query_job.result()
message = 'Query executed correctly'
except Exception as E:
message = f"Your query '{query}' is incorrect"
logging.exception(f"Your query '{query}' is incorrect")
return message
def get_pull_request_data(owner, repo, pr_nbr):
pullsUrl = f"{API_URL}/repos/{owner}/{repo}/pulls/{pr_nbr}"
pull_request = call_github_api(pullsUrl)
try:
target_branch = pull_request["base"]["ref"]
except KeyError:
logging.exception(f"Pull request not found: {pull_request}")
exit(2)
time_delta = datetime.now() - datetime.strptime(pull_request["created_at"],
DATE_FORMAT)
commits = call_github_api(f"{pullsUrl}/commits")
change_sets = get_change_sets(commits)
return {
"repo": repo,
"pullRequest": pr_nbr,
"branch": target_branch,
"changeSets": change_sets,
"leadTimeMinutes": math.floor(time_delta.total_seconds() / 60)
}
def merge(self, data_batch):
# TODO: read https://cloud.google.com/bigquery/streaming-data-into-bigquery#template-tables
tmp_id = _id_generator()
table_tmp_id = f"{self.table_id}_tmp_{tmp_id}"
table = self.client.create_table(table=bigquery.Table(table_tmp_id, schema=self.schema.schema_api))
print(f"Created table {table_tmp_id}")
# set table to expire expiration hours from now
table.expires = datetime.datetime.now(pytz.utc) + datetime.timedelta(hours=self.expiration)
table = self.client.update_table(table, ["expires"]) # API request
print(f"Updated expiration date of table {table_tmp_id}")
print(f"Going to insert {len(data_batch)} rows to {table.table_id}")
self.errors = self.client.insert_rows(table, data_batch)
self.handle_errors(data_batch)
# insert without duplicates
query = self.query_template.format(table_id=self.table_id, table_tmp_id=table_tmp_id)
query_job = self.client.query(query)
try:
query_job.result() # Waits for job to complete.
except google.api_core.exceptions.BadRequest as E:
# catch streaming buffer error because it will be fixed in at max 90 minutes
if 'affect rows in the streaming buffer, which is not supported' in repr(E):
logging.exception(E)
elif 'UPDATE/MERGE must match at most one source row for each target row' in repr(E):
backtick_fields = ', '.join(map(lambda s: f"`{s}`", self.how))
query = f"""SELECT T.* FROM `{self.table_id}` T
JOIN (SELECT {backtick_fields}, COUNT(*)
FROM `{self.table_id}`
GROUP BY {backtick_fields}
HAVING COUNT(*) > 1) USING ({backtick_fields})"""
query_job = self.client.query(query)
print(query_job.result().to_dataframe().to_dict(orient='records')[:3]) # top three rows
else:
raise E
def extra(request):
id = '0'
user = None
type = None
value = None
new_doc = None
if request.method == "POST":
try:
data = simplejson.loads(request.raw_post_data)
id = data.get('id').upper()
user = data.get('user')
type = data.get('type')
value = data.get('value')
logging.debug('user: %s, id: %s, type: %s, value: %s' %
(user, id, type, value))
except Exception as e:
logging.warning('Error %s' % (e))
try:
id = int(id)
except:
pass
channelurl = ChannelUrl.get_by_id(id)
if not channelurl:
retval = simplejson.dumps({'id': 0, 'extra': ''})
return HttpResponse(retval, mimetype="application/json")
else:
if type == 'comment':
Extra(user=user, comment=value, channelurl=channelurl.key).put()
if type == 'tag':
Extra(user=user, tag=value, channelurl=channelurl.key).put()
if type == 'related':
Extra(user=user, related=value, channelurl=channelurl.key).put()
# Update Document (FullTextSearch)
url = channelurl.url.get()
doc_id = str(url.key.id())
try:
doc = search.Index(name='url').get(doc_id)
if not doc:
logging.warning('Document not found.')
try:
taskqueue.add(name=str(doc_id) + '_extra',
queue_name='document',
url='/tasks/update_document',
params={'doc_id': doc_id})
except taskqueue.TombstonedTaskError:
logging.warning('TombstonedTaskError %s_extra' %
(str(doc_id)))
except taskqueue.TaskAlreadyExistsError:
logging.warning('TaskAlreadyExistsError %s_extra' %
(str(doc_id)))
else:
new_fields = []
for field in doc.fields:
if type == 'tag' and field.name == 'tag':
if field.value:
new_value = field.value + ' ' + value
else:
new_value = value
logging.debug('Updating tags: %s + %s = %s' %
(field.value, value, new_value))
new_fields.append(
search.TextField(name=field.name, value=new_value))
if type == 'comment' and field.name == 'comment':
if field.value:
new_value = field.value + ' ' + value
else:
new_value = value
logging.debug('Updating comments: %s + %s = %s' %
(field.value, value, new_value))
new_fields.append(
search.TextField(name=field.name, value=new_value))
elif field.name == 'rate':
new_fields.append(
search.NumberField(name=field.name,
value=field.value))
elif field.name == 'date':
new_fields.append(
search.DateField(name=field.name,
value=field.value))
else:
new_fields.append(
search.TextField(name=field.name,
value=field.value))
new_doc = search.Document(doc_id=doc_id,
fields=new_fields,
language='en')
except Exception as e:
logging.warning('Error %s' % (e))
try:
if new_doc:
search.Index(name='url').put(new_doc)
else:
logging.warning('New document (new_doc) missing?')
except search.Error:
logging.exception('Create/Update Document failed.')
retval = simplejson.dumps({'id': id, type: value})
return HttpResponse(retval, mimetype="application/json")
def rate(request):
id = 0
user = None
value = 0
if request.method == "POST":
try:
data = simplejson.loads(request.raw_post_data)
id = data.get('id').upper()
user = data.get('user')
value = int(data.get('value', 0))
logging.debug('user: %s, id: %s, value: %s' % (user, id, value))
except Exception as e:
logging.warning('Error %s' % (e))
try:
id = int(id)
except:
pass
channelurl = ChannelUrl.get_by_id(id)
if not channelurl:
retval = simplejson.dumps({'id': 0, 'rate': ''})
return HttpResponse(retval, mimetype="application/json")
else:
rate = Rate(user=user, value=value, channelurl=channelurl.key)
rate.put()
# Update Document (FullTextSearch)
url = channelurl.url.get()
doc_id = str(url.key.id())
try:
doc = search.Index(name='url').get(doc_id)
if not doc:
logging.warning('Document not found.')
try:
taskqueue.add(name=str(doc_id) + '_update',
queue_name='document',
url='/tasks/update_document',
params={'doc_id': doc_id})
except taskqueue.TombstonedTaskError:
logging.warning('TombstonedTaskError %s_update' %
(str(doc_id)))
except taskqueue.TaskAlreadyExistsError:
logging.warning('TaskAlreadyExistsError %s_update' %
(str(doc_id)))
else:
new_fields = []
for field in doc.fields:
if field.name == 'rate':
new_value = float(field.value) + float(value)
logging.debug('Updating rate: %s + %s = %s' %
(field.value, value, new_value))
new_fields.append(
search.NumberField(name='rate', value=new_value))
elif field.name == 'date':
new_fields.append(
search.DateField(name=field.name,
value=field.value))
else:
new_fields.append(
search.TextField(name=field.name,
value=field.value))
new_doc = search.Document(doc_id=doc_id,
fields=new_fields,
language='en')
except Exception as e:
logging.warning('Error %s' % (e))
try:
search.Index(name='url').put(new_doc)
except search.Error:
logging.exception('Create/Update Document failed.')
retval = simplejson.dumps({'id': id, 'rate': channelurl.rating()})
return HttpResponse(retval, mimetype="application/json")
def find(request):
idx = ''
channel = '*'
content = ''
retval = []
limit = 5
offset = 0
if request.method == "POST":
try:
data = simplejson.loads(request.raw_post_data)
channel = data.get('channel', '*').lower()
content = data.get('content', '')
idx = data.get('index', 'url')
try:
limit = int(data.get('limit', 5))
except:
limit = 5
try:
offset = int(data.get('offset', 0))
except:
offset = 0
logging.debug('channel: %s, content: %s' % (channel, content))
except Exception as e:
logging.warning('Error %s' % (e))
# if not content:
# retval=simplejson.dumps([{'id':0,'title': ''}])
# return HttpResponse(retval, mimetype="application/json")
try:
# Set query options
# date_desc = search.SortExpression(
# expression='_score',
# direction=search.SortExpression.DESCENDING,
# default_value='')
# Sort up to 1000 matching results by subject in descending order
# sort = search.SortOptions(expressions=[date_desc], limit=10)
options = search.QueryOptions(
limit=limit, # the number of results to return
offset=offset,
# cursor=cursor,
# sort_options=sort,
# returned_fields=['author', 'subject', 'summary'],
# snippeted_fields=['content']
)
if channel and channel != '*':
content = 'channel:' + channel + ' ' + content
query = search.Query(query_string=content, options=options)
index = search.Index(name=idx)
results = index.search(query)
for scored_document in results:
# process scored_document
doc_id = scored_document.doc_id
doc_url = None
doc_user = None
doc_channel = None
doc_date = None
for field in scored_document.fields:
if field.name == 'url':
doc_url = field.value
if field.name == 'user':
doc_user = field.value
if field.name == 'channel':
doc_channel = field.value
if field.name == 'date':
doc_date = field.value
# logging.debug('Search result: %s' % (scored_document))
urlinstance = Url.get_by_id(int(doc_id))
if channel == '*':
channelurlquery = ChannelUrl.query(
ChannelUrl.url == urlinstance.key)
else:
channelinstance = Channel.query(Channel.name == channel).get()
channelurlquery = ChannelUrl.query(
ChannelUrl.url == urlinstance.key,
ChannelUrl.channel == channelinstance.key)
channelurls = channelurlquery.fetch(3)
for channelurl in channelurls:
retval.append({
'id': channelurl.key.id(),
'url': urlinstance.url,
'posts': channelurl.posts()
})
except search.Error:
logging.exception('Search failed')
# logging.debug('retval %s' % (retval))
retvaljson = simplejson.dumps(retval)
return HttpResponse(retvaljson, mimetype="application/json")
def check_env_vars(vars):
for var in vars:
if not os.environ[var]:
logging.exception(f"Required env var '{var}' not set")
exit(1)