def reportsv1():
if not request.args or not request.args['key'] or request.args['key'] != IOS_API_KEY:
abort(401)
if request.method == "POST":
if not request.json or not 'google_places_id' in request.json or not 'crowd_level' in request.json:
abort(400)
report = Report(
google_places_id=request.json['google_places_id'],
crowd_level=request.json['crowd_level'])
if 'comments' in request.json:
report.populate(comments=request.json['comments'])
if 'ios_device_id' in request.json:
report.populate(ios_device_id=request.json['ios_device_id'])
if 'photo_url' in request.json:
report.populate(photo_url=request.json['photo_url'])
try:
# report.put()
return jsonify(report.to_dict())
except CapabilityDisabledError:
abort(400)
if len(request.args) == 1:
reports = Report.query().order(-Report.created_at).fetch(20)
return jsonify({"reports": [report.to_dict() for report in reports]})
if len(request.args) > 2:
abort(400)
if not request.args['google_places_id']:
abort(400)
google_places_id = request.args['google_places_id']
reports = Report.query(Report.google_places_id==google_places_id).order(-Report.created_at).fetch(20)
return jsonify({"reports": [report.to_dict() for report in reports]})
def get(self):
# Check if datasets are loaded in datastore
# Items in datastore
d = Dataset.query().count()
# Items in CDB
q = "select count(*) as c from resource_staging" + \
" where ipt is true and networks like '%VertNet%';"
c = cartodb_query(q)[0]['c']
# Number of reports stored in the datastore
num_reports = Report.query().count()
periods = Period.query()
num_periods = periods.count()
periods_done = Period.query(Period.status == "done")
num_periods_done = periods_done.count()
periods_progress = Period.query(Period.status == "in progress")
num_periods_progress = periods_progress.count()
periods_failed = Period.query(Period.status == "failed")
num_periods_failed = periods_failed.count()
resp = {
"Datastore integrity": [
{"Datasets in CartoDB": c},
{"Datasets in the Datastore": d}
],
"Report periods": [
{"Stored periods": num_periods},
{"Stored reports": num_reports},
{"Periods completed": num_periods_done},
{"Periods in progress": num_periods_progress},
{"Periods failed": num_periods_failed},
]
}
if c != d or c == 0:
dataset_setup_url = "http://%s/setup_datasets" % _HOSTNAME
resp["Datastore integrity"].append({"URL for dataset setup": dataset_setup_url})
if num_periods > 0:
links_to_periods = ["http://%s/status/period/%s" % (_HOSTNAME, x.key.id()) for x in periods.fetch()]
resp["Report periods"].append({"Links to periods": links_to_periods})
if num_periods_done > 0:
resp['Report periods'].append({'List of periods done': [x.period.strftime("%Y-%m") for x in periods_done.fetch()]})
if num_periods_progress > 0:
resp['Report periods'].append({'List of periods in progress': [x.period.strftime("%Y-%m") for x in periods_progress.fetch()]})
if num_periods_failed > 0:
resp['Report periods'].append({'List of periods failed': [x.period.strftime("%Y-%m") for x in periods_failed.fetch()]})
self.response.headers['content-type'] = "application/json"
self.response.write(json.dumps(resp))
def get(self):
from models import Report
cutoff = datetime.now() - timedelta(days=30)
old_reports = Report.query().filter(Report.dt_created < cutoff).fetch(limit=None)
n = 0
if old_reports:
for report in old_reports:
try:
report.clean_delete(self_delete=False)
except Exception, e:
logging.info(str(e))
n = len(old_reports)
ndb.delete_multi([dr.key for dr in old_reports])
def post(self):
user = users.get_current_user()
if user:
# data = json.loads(self.request.body)
reports = Report.query(Report.author == user).fetch()
# json_query_data = JSONEncoder().encode(reports)
result = {
'result': reports,
'total': len(reports)
}
self.response.headers['Content-Type'] = 'application/json'
self.response.write(JSONEncoder().encode(result))
def get(self):
from models import Report
cutoff = datetime.now() - timedelta(days=30)
old_reports = Report.query().filter(Report.dt_created < cutoff).fetch(
limit=None)
n = 0
if old_reports:
for report in old_reports:
try:
report.clean_delete(self_delete=False)
except Exception, e:
logging.info(str(e))
n = len(old_reports)
ndb.delete_multi([dr.key for dr in old_reports])
def sites():
if request.method == 'GET':
query = Site.query()
compact_sites = []
for site in query:
report = Report.query().filter(
Report.site == site.key).order(-Report.created_on).fetch(1)
compact_site = Site.to_compact(site)
compact_site.update(Report.to_dict(report[0]))
compact_sites.append(compact_site)
return jsonify({'siteList': compact_sites}), 200
elif request.method == 'POST':
request_body = request.get_json()
name = request_body.get('name', '')
url = request_body.get('url', '')
id_info = get_user_id_info(request)
email = id_info['email']
try:
report_results = PageSpeedInights.run(url)
user_key = User.query().filter(User.email == email).fetch(
keys_only=True)[0]
site = Site(name=name, url=url)
site.created_by = user_key
site.last_edited_by = user_key
site_key = site.put()
report = Report(
site=site_key,
accessibility_score=report_results['accessibility_score'],
best_practices_score=report_results['best_practices_score'],
desktop_performance_score=report_results[
'desktop_performance_score'],
mobile_performance_score=report_results[
'mobile_performance_score'],
seo_score=report_results['seo_score'],
pwa_score=report_results['pwa_score'],
)
report.put()
return jsonify({'success': True}), 200
except:
raise Exception('Page Speed Insights API returned an error')
else:
raise Exception('Method not supported')
def get(self, gbifdatasetid):
dataset_key = ndb.Key("Dataset", gbifdatasetid)
query = Report.query(Report.reported_resource == dataset_key)
query = query.order(-Report.reported_period)
report_keys = query.fetch(keys_only=True)
period_list = [{
"text":
x.id().split("|")[0][:4] + "-" + x.id().split("|")[0][4:],
"url":
x.id().split("|")[0]
} for x in report_keys]
template = JINJA_ENVIRONMENT.get_template('dataset.html')
self.response.write(
template.render(dataset=dataset_key.get(),
period_list=period_list,
periods=len(period_list)))
def get(self, gbifdatasetid):
dataset_key = ndb.Key("Dataset", gbifdatasetid)
query = Report.query(Report.reported_resource == dataset_key)
query = query.order(-Report.reported_period)
report_keys = query.fetch(keys_only=True)
period_list = [
{
"text": x.id().split("|")[0][:4]+"-"+x.id().split("|")[0][4:],
"url": x.id().split("|")[0]
} for x in report_keys]
template = JINJA_ENVIRONMENT.get_template('dataset.html')
self.response.write(template.render(
dataset=dataset_key.get(),
period_list=period_list,
periods=len(period_list)
))
def get(self):
# Checks for active Google account session
user = users.get_current_user()
if user:
reports_query = Report.query(Report.author == user).order(-Report.date)
reports = reports_query.fetch(10)
url = users.create_logout_url(self.request.uri)
url_linktext = 'Logout'
else:
url = users.create_login_url(self.request.uri)
url_linktext = 'Login'
template_values = {
'reports': reports,
'url': url,
'url_linktext': url_linktext,
}
template = JINJA_ENVIRONMENT.get_template('index.html')
self.response.write(template.render(template_values))
def post(self):
# Get parameters from memcache
memcache_keys = ["period", "testing"]
params = memcache.get_multi(memcache_keys,
key_prefix="usagestats_parser_")
# Try to get 'params' from memcache
try:
self.period = params['period']
# If not in memcache (i.e., if called directly), get from request
except KeyError:
self.period = self.request.get("period")
# If still not there, halt
if not self.period:
self.error(400)
resp = {
"status": "error",
"message": "Period parameter was not provided."
}
logging.error(resp)
self.response.write(json.dumps(resp)+"\n")
return
else:
memcache.set("usagestats_parser_period", self.period)
# If Period not already stored, halt
period_key = ndb.Key("Period", self.period)
period_entity = period_key.get()
if not period_entity:
self.error(400)
resp = {
"status": "error",
"message": "Provided period does not exist in datastore",
"data": {
"period": self.period
}
}
logging.error(resp)
self.response.write(json.dumps(resp)+"\n")
return
# Try to get 'testing' from memcache
try:
self.testing = params['testing']
# If not in memcache (i.e., if called directly), get from request
except KeyError:
self.testing = self.request.get('testing').lower() == 'true'
# Prepare list of reports to store
logging.info("Getting list of reports to send issue")
# Base query
reports_q = Report.query()
# Only Reports for current Period
reports_q = reports_q.filter(Report.reported_period == period_key)
# Only those with 'issue_sent' property set to False
reports_q = reports_q.filter(Report.issue_sent == False)
# Store final query
reports_query = reports_q
logging.info("Found %d Reports to send issue" % reports_query.count())
# Get cursor from request, if any
cursor_str = self.request.get("cursor", None)
cursor = None
if cursor_str:
cursor = ndb.Cursor(urlsafe=cursor_str)
logging.info("Cursor built: %s" % cursor)
# Initialize loop
more = True
# Loop until DeadlineExceededError
try:
# or until no more reports left
while more is True:
# Get next (or first) round of results
report, new_cursor, more = reports_query.fetch_page(
PAGE_SIZE, start_cursor=cursor
)
# Send issue
self.send_issue(report[0])
if more is True:
cursor = new_cursor
logging.info("Finished creating all issues")
resp = {
"status": "success",
"message": "Successfully finished creating all issues",
}
period_entity.status = "done"
mail.send_mail(
sender=EMAIL_SENDER,
to=EMAIL_RECIPIENT,
subject="Usage reports for period %s" % self.period,
body="""
Hey there!
Just a brief note to let you know the extraction of %s stats has successfully
finished, all reports have been stored in their respective GitHub
repositories and issues have been created. The full package.
Congrats!
""" % self.period)
# In any case, store period data, show message and finish
period_entity.put()
logging.info(resp)
self.response.write(json.dumps(resp)+"\n")
return
except DeadlineExceededError:
# Launch new instance with current (failed) cursor
taskqueue.add(url=URI_GITHUB_ISSUE,
params={"cursor": cursor.urlsafe()},
queue_name=QUEUENAME)
logging.info("Caught a DeadlineExceededError. Relaunching")
resp = {
"status": "in progress",
"message": "Caught a DeadlineExceededError."
" Relaunching with new cursor",
"data": {
"period": self.period,
"cursor": cursor.urlsafe()
}
}
logging.info(resp)
self.response.write(json.dumps(resp)+"\n")
return
def post(self):
# Create instance variable to track if parameters came from a direct request
# Or if they came through Period entity
self.params_from_request = None
params = None
s = "Version: %s\n" % __version__
s += "Arguments from POST:"
for arg in self.request.arguments():
s += '\n%s:%s' % (arg, self.request.get(arg))
logging.info(s)
# Try to get period from the request in case GitHubStore was called directly
self.period = self.request.get("period", None)
# If real period not in request, try to get parameters from StatsRun entity
# in case GetEvents was called from a previous task.
if self.period is None or len(self.period)==0:
run_key = ndb.Key("StatsRun", 5759180434571264)
run_entity = run_key.get()
self.period = run_entity.period
self.params_from_request = False
s = "Version: %s\n" % __version__
s += "Period %s determined from StatsRun entity: %s" % (self.period, params)
logging.info(s)
else:
self.params_from_request = True
s = "Version: %s\n" % __version__
s += "Period %s determined from request: %s" % (self.period, self.request)
logging.info(s)
if self.period is None or len(self.period)==0:
self.error(400)
resp = {
"status": "error",
"message": "Period parameter was not provided."
}
s = "Version: %s\n" % __version__
s += "%s" % resp
logging.error(s)
self.response.write(json.dumps(resp)+"\n")
return
# If Period not already stored, halt
period_key = ndb.Key("Period", self.period)
period_entity = period_key.get()
if not period_entity:
self.error(400)
resp = {
"status": "error",
"message": "Provided period does not exist in datastore",
"data": {
"period": self.period
}
}
logging.error(resp)
self.response.write(json.dumps(resp)+"\n")
return
# Get the remaining parameters based on the parameter source
if self.params_from_request == True:
# Get parameters from request
# 'testing' parameter
try:
self.testing = self.request.get('testing').lower() == 'true'
except Exception:
# default value for 'testing' if not provided is False
self.testing = False
# 'gbifdatasetid' parameter
try:
self.gbifdatasetid = self.request.get('gbifdatasetid').lower()
except Exception:
# default value for 'gbifdatasetid' if not provided is None
self.gbifdatasetid = None
else:
# Get parameters from Period entity
# 'testing' parameter
try:
self.testing = period_entity.testing
except Exception:
self.testing = False
# 'gbifdatasetid' parameter can't be used when called from another task
# Prepare list of reports to create issues for
# Base query
reports_q = Report.query()
# Only Reports for current Period
reports_q = reports_q.filter(Report.reported_period == period_key)
# Only those with 'issue_sent' property set to False
reports_q = reports_q.filter(Report.issue_sent == False)
# Only those with 'report_stored' property set to True
reports_q = reports_q.filter(Report.stored == True)
# And if there is a gbifdatasetid, filter on that too
if self.gbifdatasetid is not None and len(self.gbifdatasetid) > 0:
dataset_key = ndb.Key("Dataset", self.gbifdatasetid)
if dataset_key is None:
s = "Version: %s\n" % __version__
s += "gbifdatasetid %s not found in data store." % self.gbifdatasetid
logging.error(s)
return
else:
reports_q = reports_q.filter(Report.reported_resource == dataset_key)
# Store final query
reports_query = reports_q
s = "Version: %s\n" % __version__
s += "Found %d Reports to send issues for " % reports_query.count()
s += "from query %s" % reports_query
logging.info(s)
# Get cursor from request, if any
cursor_str = self.request.get("cursor", None)
cursor = None
if cursor_str:
cursor = ndb.Cursor(urlsafe=cursor_str)
s = "Version: %s\n" % __version__
s += "Cursor built: %s" % cursor
logging.info(s)
# Initialize loop
if reports_query.count==0:
more = False
else:
more = True
# Loop until DeadlineExceededError
# or until there are no more reports left to store
try:
# Keep track of dataset for which Reports have been stored in this run
datasets = []
while more is True:
s = "Version: %s\n" % __version__
s += "Issuing query: %s" % reports_query
logging.info(s)
# Get next (or first) round of results
report, new_cursor, more = reports_query.fetch_page(
PAGE_SIZE, start_cursor=cursor
)
# Check to see if there is actually another report
if report is not None and len(report) != 0:
# Send issue
self.send_issue(report[0])
gbifdatasetid = report[0].reported_resource.id()
datasets.append(gbifdatasetid)
if more is True:
cursor = new_cursor
s = "Version: %s\n" % __version__
s += "Finished creating all %d issues" % len(datasets)
logging.info(s)
resp = {
"status": "success",
"message": s,
}
period_entity.status = "done"
mail.send_mail(
sender=EMAIL_SENDER,
to=EMAIL_ADMINS,
subject="Usage reports for period %s" % self.period,
body="""
Hey there!
Just a note to let you know the GitHubIssue process for period %s
stats has successfully finished. Reports have been stored in their
respective GitHub repositories and issues have been created.
Issues submitted for (%d) datasets:
%s
Code version: %s
""" % (self.period, len(datasets), datasets, __version__) )
# In any case, store period data, show message and finish
period_entity.put()
s = "Version: %s\n" % __version__
s += "Response: %s" % resp
logging.info(s)
self.response.write(json.dumps(resp)+"\n")
return
except DeadlineExceededError:
# Launch new instance with current (failed) cursor
taskqueue.add(url=URI_GITHUB_ISSUE,
params={"cursor": cursor.urlsafe()},
queue_name=QUEUENAME)
s = "Version: %s\n" % __version__
s += "Caught a DeadlineExceededError. Relaunching."
logging.info(s)
resp = {
"status": "in progress",
"message": s,
"data": {
"period": self.period,
"cursor": cursor.urlsafe()
}
}
logging.info(resp)
self.response.write(json.dumps(resp)+"\n")
return
def initialize_extraction(self, period=None, force=None):
"""Check if Period parameter is valid, if the Period entity already exists
and create a new Period.
"""
self.response.headers['Content-Type'] = "application/json"
# Check that 'period' is provided
if not self.period:
s = "Version: %s\n" % __version__
s += "Period not found on POST body. Aborting."
logging.error(s)
self.error(400)
resp = {"status": "error", "message": s}
self.response.write(json.dumps(resp) + "\n")
return 1
# Check that 'period' is valid
if len(self.period) != 6:
s = "Version: %s\n" % __version__
s += "Malformed period. Should be YYYYMM (e.g., 201603)"
logging.error(s)
self.error(400)
resp = {"status": "error", "message": s}
self.response.write(json.dumps(resp) + "\n")
return 1
# Get existing period
period_key = ndb.Key("Period", self.period)
period_entity = period_key.get()
# If existing, abort or clear and start from scratch
if period_entity:
if self.force is not True:
s = "Version: %s\n" % __version__
s += "Period %s already exists. " % self.period
s += "Aborting. To override, use 'force=true'."
logging.error(s)
resp = {"status": "error", "message": s}
self.response.write(json.dumps(resp) + "\n")
return 1
else:
s = "Version: %s\n" % __version__
s += "Period %s already exists. " % self.period
s += "Overriding."
logging.warning(s)
# Delete Reports referencing period
r = Report.query().filter(Report.reported_period == period_key)
to_delete = r.fetch(keys_only=True)
s = "Version: %s\n" % __version__
s += "Deleting %d Report entities" % len(to_delete)
logging.info(s)
deleted = ndb.delete_multi(to_delete)
s = "Version: %s\n" % __version__
s += "%d Report entities removed" % len(deleted)
logging.info(s)
# Delete Period itself
s = "Version: %s\n" % __version__
s += "Deleting Period %s" % period_key
logging.info(s)
period_key.delete()
s = "Version: %s\n" % __version__
s += "Period %s deleted" % period_key
logging.info(s)
# Create new Period (id=YYYYMM)
s = "Version: %s\n" % __version__
s += "Creating new Period %s" % self.period
logging.info(s)
y, m = (int(self.period[:4]), int(self.period[-2:]))
p = Period(id=self.period)
p.year = y
p.month = m
p.status = 'in progress'
period_key = p.put()
# Check
if period_key:
s = "Version: %s\n" % __version__
s += "New Period %s created successfully" % self.period
s += "with key %s" % period_key
logging.info(s)
else:
self.error(500)
s = "Version: %s\n" % __version__
s += "Could not create new Period %s" % self.period
logging.error(s)
resp = {"status": "error", "message": s}
self.response.write(json.dumps(resp) + "\n")
return 1
# Clear temporary entities
keys_to_delete = ReportToProcess.query().fetch(keys_only=True)
s = "Version: %s\n" % __version__
s += "Deleting %d temporal (internal use only) entities" % len(
keys_to_delete)
logging.info(s)
ndb.delete_multi(keys_to_delete)
return 0
def get(self):
# Check if datasets are loaded in datastore
# Items in datastore
d = Dataset.query().count()
# Items in Carto
q = "select count(*) as c from resource_staging" + \
" where ipt is true and networks like '%VertNet%';"
c = carto_query(q)[0]['c']
# Number of reports stored in the datastore
num_reports = Report.query().count()
periods = Period.query()
num_periods = periods.count()
periods_done = Period.query(Period.status == "done")
num_periods_done = periods_done.count()
periods_progress = Period.query(Period.status == "in progress")
num_periods_progress = periods_progress.count()
periods_failed = Period.query(Period.status == "failed")
num_periods_failed = periods_failed.count()
resp = {
"Datastore integrity": [
{"Datasets in Carto": c},
{"Datasets in the Datastore": d}
],
"Report periods": [
{"Stored periods": num_periods},
{"Stored reports": num_reports},
{"Periods completed": num_periods_done},
{"Periods in progress": num_periods_progress},
{"Periods failed": num_periods_failed},
]
}
if c != d or c == 0:
dataset_setup_url = "http://%s/admin/setup/datasets" % _HOSTNAME
resp["Datastore integrity"].append({"URL for dataset setup": dataset_setup_url})
if num_periods > 0:
links_to_periods = ["http://%s/admin/status/period/%s" % (_HOSTNAME, x.key.id()) for x in periods.fetch()]
resp["Report periods"].append({"Links to periods": links_to_periods})
if num_periods_done > 0:
resp['Report periods'].append({'List of periods done': [x.period.strftime("%Y-%m") for x in periods_done.fetch()]})
if num_periods_progress > 0:
resp['Report periods'].append({'List of periods in progress': [x.period.strftime("%Y-%m") for x in periods_progress.fetch()]})
if num_periods_failed > 0:
resp['Report periods'].append({'List of periods failed': [x.period.strftime("%Y-%m") for x in periods_failed.fetch()]})
headers = {
'User-Agent': 'VertNet',
'Accept': 'application/vnd.github.v3+json',
'Authorization': 'token {0}'.format(apikey('ghb'))
}
# API URL https://api.github.com/rate-limit
rpc = urlfetch.create_rpc()
url = '/'.join([GH_URL, 'users', 'VertNet'])
urlfetch.set_default_fetch_deadline(60)
urlfetch.make_fetch_call(rpc, url, headers=headers)
h = rpc.get_result().headers
u = rpc.get_result().content
resp['url'] = url
user = {}
user['content']=json.loads(u)
timefloat = float(h['x-ratelimit-reset'])
expires = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(timefloat))
user['x-ratelimit-limit'] = h['x-ratelimit-limit']
user['x-ratelimit-remaining'] = h['x-ratelimit-remaining']
user['x-ratelimit-reset'] = expires
resp['user'] = user
self.response.headers['content-type'] = "application/json"
self.response.write(json.dumps(resp))
def initialize_extraction(self, period=None, force=None):
"""Check if Period parameter is valid, if the Period entity already exists
and create a new Period."""
self.response.headers['Content-Type'] = "application/json"
# Check that 'period' is provided
if not self.period:
logging.error("Period not found on POST body. Aborting.")
self.error(400)
resp = {
"status": "error",
"message": "Period not found on POST body. " +
"Aborting."
}
self.response.write(json.dumps(resp) + "\n")
return 1
# Check that 'period' is valid
if len(self.period) != 6:
self.error(400)
resp = {
"status": "error",
"message": "Malformed period. Should be YYYYMM (e.g., 201603)"
}
self.response.write(json.dumps(resp) + "\n")
return 1
# Get existing period
period_key = ndb.Key("Period", self.period)
period_entity = period_key.get()
# If existing, abort or clear and start from scratch
if period_entity:
if self.force is not True:
logging.error("Period %s already exists. " % self.period +
"Aborting. To override, use 'force=true'.")
resp = {
"status": "error",
"message": "Period %s already exists. " % self.period +
"Aborting. To override, use 'force=true'."
}
self.response.write(json.dumps(resp) + "\n")
return 1
else:
logging.warning("Period %s already exists. " % self.period +
"Overriding.")
# Delete Reports referencing period
r = Report.query().filter(Report.reported_period == period_key)
to_delete = r.fetch(keys_only=True)
logging.info("Deleting %d Report entities" % len(to_delete))
deleted = ndb.delete_multi(to_delete)
logging.info("%d Report entities removed" % len(deleted))
# Delete Period itself
logging.info("Deleting Period %s" % period_key)
period_key.delete()
logging.info("Period entity deleted")
# Create new Period (id=YYYYMM)
logging.info("Creating new Period %s" % self.period)
y, m = (int(self.period[:4]), int(self.period[-2:]))
p = Period(id=self.period)
p.year = y
p.month = m
p.status = 'in progress'
period_key = p.put()
# Check
if period_key:
logging.info("New Period %s created successfully." % self.period)
logging.info("New period's key = %s" % period_key)
else:
self.error(500)
logging.error("Could not create new Period %s" % self.period)
resp = {
"status": "error",
"message": "Could not create new Period %s" % self.period
}
self.response.write(json.dumps(resp) + "\n")
return 1
# Clear temporary entities
keys_to_delete = ReportToProcess.query().fetch(keys_only=True)
logging.info("Deleting %d temporal (internal use only) entities"
% len(keys_to_delete))
ndb.delete_multi(keys_to_delete)
return 0
