def reportsv1(): if not request.args or not request.args['key'] or request.args['key'] != IOS_API_KEY: abort(401) if request.method == "POST": if not request.json or not 'google_places_id' in request.json or not 'crowd_level' in request.json: abort(400) report = Report( google_places_id=request.json['google_places_id'], crowd_level=request.json['crowd_level']) if 'comments' in request.json: report.populate(comments=request.json['comments']) if 'ios_device_id' in request.json: report.populate(ios_device_id=request.json['ios_device_id']) if 'photo_url' in request.json: report.populate(photo_url=request.json['photo_url']) try: # report.put() return jsonify(report.to_dict()) except CapabilityDisabledError: abort(400) if len(request.args) == 1: reports = Report.query().order(-Report.created_at).fetch(20) return jsonify({"reports": [report.to_dict() for report in reports]}) if len(request.args) > 2: abort(400) if not request.args['google_places_id']: abort(400) google_places_id = request.args['google_places_id'] reports = Report.query(Report.google_places_id==google_places_id).order(-Report.created_at).fetch(20) return jsonify({"reports": [report.to_dict() for report in reports]})
def get(self): # Check if datasets are loaded in datastore # Items in datastore d = Dataset.query().count() # Items in CDB q = "select count(*) as c from resource_staging" + \ " where ipt is true and networks like '%VertNet%';" c = cartodb_query(q)[0]['c'] # Number of reports stored in the datastore num_reports = Report.query().count() periods = Period.query() num_periods = periods.count() periods_done = Period.query(Period.status == "done") num_periods_done = periods_done.count() periods_progress = Period.query(Period.status == "in progress") num_periods_progress = periods_progress.count() periods_failed = Period.query(Period.status == "failed") num_periods_failed = periods_failed.count() resp = { "Datastore integrity": [ {"Datasets in CartoDB": c}, {"Datasets in the Datastore": d} ], "Report periods": [ {"Stored periods": num_periods}, {"Stored reports": num_reports}, {"Periods completed": num_periods_done}, {"Periods in progress": num_periods_progress}, {"Periods failed": num_periods_failed}, ] } if c != d or c == 0: dataset_setup_url = "http://%s/setup_datasets" % _HOSTNAME resp["Datastore integrity"].append({"URL for dataset setup": dataset_setup_url}) if num_periods > 0: links_to_periods = ["http://%s/status/period/%s" % (_HOSTNAME, x.key.id()) for x in periods.fetch()] resp["Report periods"].append({"Links to periods": links_to_periods}) if num_periods_done > 0: resp['Report periods'].append({'List of periods done': [x.period.strftime("%Y-%m") for x in periods_done.fetch()]}) if num_periods_progress > 0: resp['Report periods'].append({'List of periods in progress': [x.period.strftime("%Y-%m") for x in periods_progress.fetch()]}) if num_periods_failed > 0: resp['Report periods'].append({'List of periods failed': [x.period.strftime("%Y-%m") for x in periods_failed.fetch()]}) self.response.headers['content-type'] = "application/json" self.response.write(json.dumps(resp))
def get(self): from models import Report cutoff = datetime.now() - timedelta(days=30) old_reports = Report.query().filter(Report.dt_created < cutoff).fetch(limit=None) n = 0 if old_reports: for report in old_reports: try: report.clean_delete(self_delete=False) except Exception, e: logging.info(str(e)) n = len(old_reports) ndb.delete_multi([dr.key for dr in old_reports])
def post(self): user = users.get_current_user() if user: # data = json.loads(self.request.body) reports = Report.query(Report.author == user).fetch() # json_query_data = JSONEncoder().encode(reports) result = { 'result': reports, 'total': len(reports) } self.response.headers['Content-Type'] = 'application/json' self.response.write(JSONEncoder().encode(result))
def get(self): from models import Report cutoff = datetime.now() - timedelta(days=30) old_reports = Report.query().filter(Report.dt_created < cutoff).fetch( limit=None) n = 0 if old_reports: for report in old_reports: try: report.clean_delete(self_delete=False) except Exception, e: logging.info(str(e)) n = len(old_reports) ndb.delete_multi([dr.key for dr in old_reports])
def sites(): if request.method == 'GET': query = Site.query() compact_sites = [] for site in query: report = Report.query().filter( Report.site == site.key).order(-Report.created_on).fetch(1) compact_site = Site.to_compact(site) compact_site.update(Report.to_dict(report[0])) compact_sites.append(compact_site) return jsonify({'siteList': compact_sites}), 200 elif request.method == 'POST': request_body = request.get_json() name = request_body.get('name', '') url = request_body.get('url', '') id_info = get_user_id_info(request) email = id_info['email'] try: report_results = PageSpeedInights.run(url) user_key = User.query().filter(User.email == email).fetch( keys_only=True)[0] site = Site(name=name, url=url) site.created_by = user_key site.last_edited_by = user_key site_key = site.put() report = Report( site=site_key, accessibility_score=report_results['accessibility_score'], best_practices_score=report_results['best_practices_score'], desktop_performance_score=report_results[ 'desktop_performance_score'], mobile_performance_score=report_results[ 'mobile_performance_score'], seo_score=report_results['seo_score'], pwa_score=report_results['pwa_score'], ) report.put() return jsonify({'success': True}), 200 except: raise Exception('Page Speed Insights API returned an error') else: raise Exception('Method not supported')
def get(self, gbifdatasetid): dataset_key = ndb.Key("Dataset", gbifdatasetid) query = Report.query(Report.reported_resource == dataset_key) query = query.order(-Report.reported_period) report_keys = query.fetch(keys_only=True) period_list = [{ "text": x.id().split("|")[0][:4] + "-" + x.id().split("|")[0][4:], "url": x.id().split("|")[0] } for x in report_keys] template = JINJA_ENVIRONMENT.get_template('dataset.html') self.response.write( template.render(dataset=dataset_key.get(), period_list=period_list, periods=len(period_list)))
def get(self, gbifdatasetid): dataset_key = ndb.Key("Dataset", gbifdatasetid) query = Report.query(Report.reported_resource == dataset_key) query = query.order(-Report.reported_period) report_keys = query.fetch(keys_only=True) period_list = [ { "text": x.id().split("|")[0][:4]+"-"+x.id().split("|")[0][4:], "url": x.id().split("|")[0] } for x in report_keys] template = JINJA_ENVIRONMENT.get_template('dataset.html') self.response.write(template.render( dataset=dataset_key.get(), period_list=period_list, periods=len(period_list) ))
def get(self): # Checks for active Google account session user = users.get_current_user() if user: reports_query = Report.query(Report.author == user).order(-Report.date) reports = reports_query.fetch(10) url = users.create_logout_url(self.request.uri) url_linktext = 'Logout' else: url = users.create_login_url(self.request.uri) url_linktext = 'Login' template_values = { 'reports': reports, 'url': url, 'url_linktext': url_linktext, } template = JINJA_ENVIRONMENT.get_template('index.html') self.response.write(template.render(template_values))
def post(self): # Get parameters from memcache memcache_keys = ["period", "testing"] params = memcache.get_multi(memcache_keys, key_prefix="usagestats_parser_") # Try to get 'params' from memcache try: self.period = params['period'] # If not in memcache (i.e., if called directly), get from request except KeyError: self.period = self.request.get("period") # If still not there, halt if not self.period: self.error(400) resp = { "status": "error", "message": "Period parameter was not provided." } logging.error(resp) self.response.write(json.dumps(resp)+"\n") return else: memcache.set("usagestats_parser_period", self.period) # If Period not already stored, halt period_key = ndb.Key("Period", self.period) period_entity = period_key.get() if not period_entity: self.error(400) resp = { "status": "error", "message": "Provided period does not exist in datastore", "data": { "period": self.period } } logging.error(resp) self.response.write(json.dumps(resp)+"\n") return # Try to get 'testing' from memcache try: self.testing = params['testing'] # If not in memcache (i.e., if called directly), get from request except KeyError: self.testing = self.request.get('testing').lower() == 'true' # Prepare list of reports to store logging.info("Getting list of reports to send issue") # Base query reports_q = Report.query() # Only Reports for current Period reports_q = reports_q.filter(Report.reported_period == period_key) # Only those with 'issue_sent' property set to False reports_q = reports_q.filter(Report.issue_sent == False) # Store final query reports_query = reports_q logging.info("Found %d Reports to send issue" % reports_query.count()) # Get cursor from request, if any cursor_str = self.request.get("cursor", None) cursor = None if cursor_str: cursor = ndb.Cursor(urlsafe=cursor_str) logging.info("Cursor built: %s" % cursor) # Initialize loop more = True # Loop until DeadlineExceededError try: # or until no more reports left while more is True: # Get next (or first) round of results report, new_cursor, more = reports_query.fetch_page( PAGE_SIZE, start_cursor=cursor ) # Send issue self.send_issue(report[0]) if more is True: cursor = new_cursor logging.info("Finished creating all issues") resp = { "status": "success", "message": "Successfully finished creating all issues", } period_entity.status = "done" mail.send_mail( sender=EMAIL_SENDER, to=EMAIL_RECIPIENT, subject="Usage reports for period %s" % self.period, body=""" Hey there! Just a brief note to let you know the extraction of %s stats has successfully finished, all reports have been stored in their respective GitHub repositories and issues have been created. The full package. Congrats! """ % self.period) # In any case, store period data, show message and finish period_entity.put() logging.info(resp) self.response.write(json.dumps(resp)+"\n") return except DeadlineExceededError: # Launch new instance with current (failed) cursor taskqueue.add(url=URI_GITHUB_ISSUE, params={"cursor": cursor.urlsafe()}, queue_name=QUEUENAME) logging.info("Caught a DeadlineExceededError. Relaunching") resp = { "status": "in progress", "message": "Caught a DeadlineExceededError." " Relaunching with new cursor", "data": { "period": self.period, "cursor": cursor.urlsafe() } } logging.info(resp) self.response.write(json.dumps(resp)+"\n") return
def post(self): # Create instance variable to track if parameters came from a direct request # Or if they came through Period entity self.params_from_request = None params = None s = "Version: %s\n" % __version__ s += "Arguments from POST:" for arg in self.request.arguments(): s += '\n%s:%s' % (arg, self.request.get(arg)) logging.info(s) # Try to get period from the request in case GitHubStore was called directly self.period = self.request.get("period", None) # If real period not in request, try to get parameters from StatsRun entity # in case GetEvents was called from a previous task. if self.period is None or len(self.period)==0: run_key = ndb.Key("StatsRun", 5759180434571264) run_entity = run_key.get() self.period = run_entity.period self.params_from_request = False s = "Version: %s\n" % __version__ s += "Period %s determined from StatsRun entity: %s" % (self.period, params) logging.info(s) else: self.params_from_request = True s = "Version: %s\n" % __version__ s += "Period %s determined from request: %s" % (self.period, self.request) logging.info(s) if self.period is None or len(self.period)==0: self.error(400) resp = { "status": "error", "message": "Period parameter was not provided." } s = "Version: %s\n" % __version__ s += "%s" % resp logging.error(s) self.response.write(json.dumps(resp)+"\n") return # If Period not already stored, halt period_key = ndb.Key("Period", self.period) period_entity = period_key.get() if not period_entity: self.error(400) resp = { "status": "error", "message": "Provided period does not exist in datastore", "data": { "period": self.period } } logging.error(resp) self.response.write(json.dumps(resp)+"\n") return # Get the remaining parameters based on the parameter source if self.params_from_request == True: # Get parameters from request # 'testing' parameter try: self.testing = self.request.get('testing').lower() == 'true' except Exception: # default value for 'testing' if not provided is False self.testing = False # 'gbifdatasetid' parameter try: self.gbifdatasetid = self.request.get('gbifdatasetid').lower() except Exception: # default value for 'gbifdatasetid' if not provided is None self.gbifdatasetid = None else: # Get parameters from Period entity # 'testing' parameter try: self.testing = period_entity.testing except Exception: self.testing = False # 'gbifdatasetid' parameter can't be used when called from another task # Prepare list of reports to create issues for # Base query reports_q = Report.query() # Only Reports for current Period reports_q = reports_q.filter(Report.reported_period == period_key) # Only those with 'issue_sent' property set to False reports_q = reports_q.filter(Report.issue_sent == False) # Only those with 'report_stored' property set to True reports_q = reports_q.filter(Report.stored == True) # And if there is a gbifdatasetid, filter on that too if self.gbifdatasetid is not None and len(self.gbifdatasetid) > 0: dataset_key = ndb.Key("Dataset", self.gbifdatasetid) if dataset_key is None: s = "Version: %s\n" % __version__ s += "gbifdatasetid %s not found in data store." % self.gbifdatasetid logging.error(s) return else: reports_q = reports_q.filter(Report.reported_resource == dataset_key) # Store final query reports_query = reports_q s = "Version: %s\n" % __version__ s += "Found %d Reports to send issues for " % reports_query.count() s += "from query %s" % reports_query logging.info(s) # Get cursor from request, if any cursor_str = self.request.get("cursor", None) cursor = None if cursor_str: cursor = ndb.Cursor(urlsafe=cursor_str) s = "Version: %s\n" % __version__ s += "Cursor built: %s" % cursor logging.info(s) # Initialize loop if reports_query.count==0: more = False else: more = True # Loop until DeadlineExceededError # or until there are no more reports left to store try: # Keep track of dataset for which Reports have been stored in this run datasets = [] while more is True: s = "Version: %s\n" % __version__ s += "Issuing query: %s" % reports_query logging.info(s) # Get next (or first) round of results report, new_cursor, more = reports_query.fetch_page( PAGE_SIZE, start_cursor=cursor ) # Check to see if there is actually another report if report is not None and len(report) != 0: # Send issue self.send_issue(report[0]) gbifdatasetid = report[0].reported_resource.id() datasets.append(gbifdatasetid) if more is True: cursor = new_cursor s = "Version: %s\n" % __version__ s += "Finished creating all %d issues" % len(datasets) logging.info(s) resp = { "status": "success", "message": s, } period_entity.status = "done" mail.send_mail( sender=EMAIL_SENDER, to=EMAIL_ADMINS, subject="Usage reports for period %s" % self.period, body=""" Hey there! Just a note to let you know the GitHubIssue process for period %s stats has successfully finished. Reports have been stored in their respective GitHub repositories and issues have been created. Issues submitted for (%d) datasets: %s Code version: %s """ % (self.period, len(datasets), datasets, __version__) ) # In any case, store period data, show message and finish period_entity.put() s = "Version: %s\n" % __version__ s += "Response: %s" % resp logging.info(s) self.response.write(json.dumps(resp)+"\n") return except DeadlineExceededError: # Launch new instance with current (failed) cursor taskqueue.add(url=URI_GITHUB_ISSUE, params={"cursor": cursor.urlsafe()}, queue_name=QUEUENAME) s = "Version: %s\n" % __version__ s += "Caught a DeadlineExceededError. Relaunching." logging.info(s) resp = { "status": "in progress", "message": s, "data": { "period": self.period, "cursor": cursor.urlsafe() } } logging.info(resp) self.response.write(json.dumps(resp)+"\n") return
def initialize_extraction(self, period=None, force=None): """Check if Period parameter is valid, if the Period entity already exists and create a new Period. """ self.response.headers['Content-Type'] = "application/json" # Check that 'period' is provided if not self.period: s = "Version: %s\n" % __version__ s += "Period not found on POST body. Aborting." logging.error(s) self.error(400) resp = {"status": "error", "message": s} self.response.write(json.dumps(resp) + "\n") return 1 # Check that 'period' is valid if len(self.period) != 6: s = "Version: %s\n" % __version__ s += "Malformed period. Should be YYYYMM (e.g., 201603)" logging.error(s) self.error(400) resp = {"status": "error", "message": s} self.response.write(json.dumps(resp) + "\n") return 1 # Get existing period period_key = ndb.Key("Period", self.period) period_entity = period_key.get() # If existing, abort or clear and start from scratch if period_entity: if self.force is not True: s = "Version: %s\n" % __version__ s += "Period %s already exists. " % self.period s += "Aborting. To override, use 'force=true'." logging.error(s) resp = {"status": "error", "message": s} self.response.write(json.dumps(resp) + "\n") return 1 else: s = "Version: %s\n" % __version__ s += "Period %s already exists. " % self.period s += "Overriding." logging.warning(s) # Delete Reports referencing period r = Report.query().filter(Report.reported_period == period_key) to_delete = r.fetch(keys_only=True) s = "Version: %s\n" % __version__ s += "Deleting %d Report entities" % len(to_delete) logging.info(s) deleted = ndb.delete_multi(to_delete) s = "Version: %s\n" % __version__ s += "%d Report entities removed" % len(deleted) logging.info(s) # Delete Period itself s = "Version: %s\n" % __version__ s += "Deleting Period %s" % period_key logging.info(s) period_key.delete() s = "Version: %s\n" % __version__ s += "Period %s deleted" % period_key logging.info(s) # Create new Period (id=YYYYMM) s = "Version: %s\n" % __version__ s += "Creating new Period %s" % self.period logging.info(s) y, m = (int(self.period[:4]), int(self.period[-2:])) p = Period(id=self.period) p.year = y p.month = m p.status = 'in progress' period_key = p.put() # Check if period_key: s = "Version: %s\n" % __version__ s += "New Period %s created successfully" % self.period s += "with key %s" % period_key logging.info(s) else: self.error(500) s = "Version: %s\n" % __version__ s += "Could not create new Period %s" % self.period logging.error(s) resp = {"status": "error", "message": s} self.response.write(json.dumps(resp) + "\n") return 1 # Clear temporary entities keys_to_delete = ReportToProcess.query().fetch(keys_only=True) s = "Version: %s\n" % __version__ s += "Deleting %d temporal (internal use only) entities" % len( keys_to_delete) logging.info(s) ndb.delete_multi(keys_to_delete) return 0
def get(self): # Check if datasets are loaded in datastore # Items in datastore d = Dataset.query().count() # Items in Carto q = "select count(*) as c from resource_staging" + \ " where ipt is true and networks like '%VertNet%';" c = carto_query(q)[0]['c'] # Number of reports stored in the datastore num_reports = Report.query().count() periods = Period.query() num_periods = periods.count() periods_done = Period.query(Period.status == "done") num_periods_done = periods_done.count() periods_progress = Period.query(Period.status == "in progress") num_periods_progress = periods_progress.count() periods_failed = Period.query(Period.status == "failed") num_periods_failed = periods_failed.count() resp = { "Datastore integrity": [ {"Datasets in Carto": c}, {"Datasets in the Datastore": d} ], "Report periods": [ {"Stored periods": num_periods}, {"Stored reports": num_reports}, {"Periods completed": num_periods_done}, {"Periods in progress": num_periods_progress}, {"Periods failed": num_periods_failed}, ] } if c != d or c == 0: dataset_setup_url = "http://%s/admin/setup/datasets" % _HOSTNAME resp["Datastore integrity"].append({"URL for dataset setup": dataset_setup_url}) if num_periods > 0: links_to_periods = ["http://%s/admin/status/period/%s" % (_HOSTNAME, x.key.id()) for x in periods.fetch()] resp["Report periods"].append({"Links to periods": links_to_periods}) if num_periods_done > 0: resp['Report periods'].append({'List of periods done': [x.period.strftime("%Y-%m") for x in periods_done.fetch()]}) if num_periods_progress > 0: resp['Report periods'].append({'List of periods in progress': [x.period.strftime("%Y-%m") for x in periods_progress.fetch()]}) if num_periods_failed > 0: resp['Report periods'].append({'List of periods failed': [x.period.strftime("%Y-%m") for x in periods_failed.fetch()]}) headers = { 'User-Agent': 'VertNet', 'Accept': 'application/vnd.github.v3+json', 'Authorization': 'token {0}'.format(apikey('ghb')) } # API URL https://api.github.com/rate-limit rpc = urlfetch.create_rpc() url = '/'.join([GH_URL, 'users', 'VertNet']) urlfetch.set_default_fetch_deadline(60) urlfetch.make_fetch_call(rpc, url, headers=headers) h = rpc.get_result().headers u = rpc.get_result().content resp['url'] = url user = {} user['content']=json.loads(u) timefloat = float(h['x-ratelimit-reset']) expires = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(timefloat)) user['x-ratelimit-limit'] = h['x-ratelimit-limit'] user['x-ratelimit-remaining'] = h['x-ratelimit-remaining'] user['x-ratelimit-reset'] = expires resp['user'] = user self.response.headers['content-type'] = "application/json" self.response.write(json.dumps(resp))
def initialize_extraction(self, period=None, force=None): """Check if Period parameter is valid, if the Period entity already exists and create a new Period.""" self.response.headers['Content-Type'] = "application/json" # Check that 'period' is provided if not self.period: logging.error("Period not found on POST body. Aborting.") self.error(400) resp = { "status": "error", "message": "Period not found on POST body. " + "Aborting." } self.response.write(json.dumps(resp) + "\n") return 1 # Check that 'period' is valid if len(self.period) != 6: self.error(400) resp = { "status": "error", "message": "Malformed period. Should be YYYYMM (e.g., 201603)" } self.response.write(json.dumps(resp) + "\n") return 1 # Get existing period period_key = ndb.Key("Period", self.period) period_entity = period_key.get() # If existing, abort or clear and start from scratch if period_entity: if self.force is not True: logging.error("Period %s already exists. " % self.period + "Aborting. To override, use 'force=true'.") resp = { "status": "error", "message": "Period %s already exists. " % self.period + "Aborting. To override, use 'force=true'." } self.response.write(json.dumps(resp) + "\n") return 1 else: logging.warning("Period %s already exists. " % self.period + "Overriding.") # Delete Reports referencing period r = Report.query().filter(Report.reported_period == period_key) to_delete = r.fetch(keys_only=True) logging.info("Deleting %d Report entities" % len(to_delete)) deleted = ndb.delete_multi(to_delete) logging.info("%d Report entities removed" % len(deleted)) # Delete Period itself logging.info("Deleting Period %s" % period_key) period_key.delete() logging.info("Period entity deleted") # Create new Period (id=YYYYMM) logging.info("Creating new Period %s" % self.period) y, m = (int(self.period[:4]), int(self.period[-2:])) p = Period(id=self.period) p.year = y p.month = m p.status = 'in progress' period_key = p.put() # Check if period_key: logging.info("New Period %s created successfully." % self.period) logging.info("New period's key = %s" % period_key) else: self.error(500) logging.error("Could not create new Period %s" % self.period) resp = { "status": "error", "message": "Could not create new Period %s" % self.period } self.response.write(json.dumps(resp) + "\n") return 1 # Clear temporary entities keys_to_delete = ReportToProcess.query().fetch(keys_only=True) logging.info("Deleting %d temporal (internal use only) entities" % len(keys_to_delete)) ndb.delete_multi(keys_to_delete) return 0