def scraper(url, resp) -> list: if resp.status == 200: # no error status: return valid URLs links = extract_next_links(url, resp) get_reports() return links else: return []
def exportcsv(bot, update): try: users[update.message.chat.id] except KeyError: undefined(bot, update) update.message.reply_text(botstring["exportcsv_start"]) reports = get_reports() start = "%s%s%s" % (users[update.message.chat.id]["begin_year"], users[update.message.chat.id]["begin_month"], users[update.message.chat.id]["begin_day"]) end = "%s%s%s" % (users[update.message.chat.id]["end_year"], users[update.message.chat.id]["end_month"], users[update.message.chat.id]["end_day"]) reports_sorted = sort_reports(reports, str(start), str(end)) if len(reports_sorted) > 0: update.message.reply_text(botstring["exportcsv_info"] % (len(reports_sorted))) json_to_csv(reports_sorted, str(update.message.chat.id)) update.message.reply_text(botstring["exportcsv"]) try: bot.send_document(chat_id=update.message.chat.id, timeout=360, document=open( '%s.csv' % (str(update.message.chat.id)), 'rb')) logging.info("The file %s.csv was sent" % (str(update.message.chat.id))) os.remove('%s.csv' % (str(update.message.chat.id))) except: logging.info("Sending of document not possible") else: logging.info("For timeframe no reports") update.message.reply_text(botstring["exportcsv_no_reports"])
def reports(): reports = report_util.get_reports() # Return as JSON if requested. if get_format(request) == 'json': return jsonify(status=200, reports=reports) return render_template('reports.html', reports=reports)
def deliver_reports(contracts_path, reports_path, template, output_path, bucket=500, testing=False): reports = get_reports(contracts_path, reports_path) popper = Popper(reports) pops = popper.pop(bucket) while pops: j = push_reports.delay(pops, template, output_path, testing) logger.info("Job id:%s | %s/%s" % ( j.id, len(pops), len(popper.items)) ) pops = popper.pop(bucket)
def exportpdf(bot, update): try: users[update.message.chat.id] except KeyError: undefined(bot, update) print(users[update.message.chat.id]) update.message.reply_text(botstring["exportpdf_start"]) reports = get_reports() start = "%s%s%s" % (users[update.message.chat.id]["begin_year"], users[update.message.chat.id]["begin_month"], users[update.message.chat.id]["begin_day"]) end = "%s%s%s" % (users[update.message.chat.id]["end_year"], users[update.message.chat.id]["end_month"], users[update.message.chat.id]["end_day"]) reports_sorted = sort_reports(reports, str(start), str(end)) if len(reports_sorted) > 0: update.message.reply_text(botstring["exportpdf_info"] % (len(reports_sorted))) mergeList = [] for report in reports_sorted: name = "%s.pdf" % ((report["report_link"][46:-1])) mergeList.append(name) link = report["report_link"] html_to_pdf(link, name, False) merge_pdf(mergeList, "pdfs", str(update.message.chat.id)) add_footer("pdfs/%s" % (update.message.chat.id), "pdfs/%s" % (update.message.chat.id)) update.message.reply_text(botstring["exportpdf"]) try: bot.send_document( chat_id=update.message.chat.id, timeout=360, document=open('pdfs/%s.pdf' % (str(update.message.chat.id)), 'rb')) logging.info("The file pdfs/%s.pdf was sent" % (str(update.message.chat.id))) if os.path.exists('pdfs/%s.pdf' % (str(update.message.chat.id))): os.remove('pdfs/%s.pdf' % (str(update.message.chat.id))) else: logging.info("The file pdfs/%s.pdf does not exist" % (str(update.message.chat.id))) except: logging.info("Sending of document not possible") else: logging.info("For timeframe no reports") update.message.reply_text(botstring["exportpdf_no_reports"])
def deliver_reports(contracts_path, reports_path, template, output_path, source, data, bucket=500): try: source_class = eval(source) except NameError: logger.error('%s source type not found' % str(source)) reports = get_reports(contracts_path, reports_path) popper = Popper(reports) pops = popper.pop(bucket) while pops: push_reports(pops, template, output_path, source_class, data) pops = popper.pop(bucket)
def update(): try: logging.info("update started") filename = "%s.db" % (str(random.randint(1, 999999999999999))) logging.info("Created new db %s" % (filename)) # filename = "sqlite:///reports.db" # filename = "mysql+mysqlconnector://gobitodic:[email protected]/gobitodic$reports" db = dataset.connect("sqlite:///%s" % (filename)) reportsdb = db["reports"] db.begin() reports = get_reports() for report in reports: reportsdb.insert(report) db.commit() db.engine.dispose() logging.info("DB updated") except Exception as e: logging.error("Update() didnt' work; Error: %s" % (e)) os.remove(filename) return filename
def statistics(bot, update): reports = get_reports() update.message.reply_text(botstring["statistics"] % (len(reports)))
def report(report_id): report = report_util.get_report(report_id) if not report: abort(404) dates = report_util.get_dates() if not dates: abort(500) min_date = report.get('minDate') max_date = report.get('maxDate') date_pattern = report.get('datePattern') max_date_metric = report.get('maxDateMetric') # TODO: If a report doesn't explicitly have a min/max date, # but all of its metrics do, take the min/max of the metrics # and set that as the report's implicit min/max date. # Omit dates for which this report has no data. if max_date_metric: max_date = report_util.get_latest_date(max_date_metric) if min_date: dates = dates[:dates.index(min_date) + 1] if max_date: dates = dates[dates.index(max_date):] if date_pattern: date_pattern = re.compile(date_pattern) dates = [d for d in dates if date_pattern.match(d)] report['dates'] = dates report['lenses'] = report_util.get_lenses() start = request.args.get('start') end = request.args.get('end') # Canonicalize single-date formats. if end and not start: start, end = end, start # Canonicalize aliases. if start == 'latest': start = dates[0] elif start == 'earliest': start = dates[-1] if end == 'latest': end = dates[0] elif end == 'earliest': end = dates[-1] # This is longhand for the snapshot (histogram) view. if start == end: end = None # This is shorthand for the trends (timeseries) view. if not start and not end: # The default date range is 24 crawls (1 year). # May be shorter if the report's minimum date is more recent. start = dates[min(24, len(dates) - 1)] end = dates[0] if start and start not in dates: abort(400) if end and end not in dates: abort(400) viz = report_util.VizTypes.HISTOGRAM if (start and not end) else report_util.VizTypes.TIMESERIES if viz == report_util.VizTypes.TIMESERIES and report.get('timeseries') and not report.get('timeseries').get('enabled'): end = None viz = report_util.VizTypes.HISTOGRAM # The default for histograms should be the latest date. if not request.args.get('start'): start = dates[0] lens_id = get_lens_id(request) lens = report_util.get_lens(lens_id) if lens: report['lens'] = lens report['view'] = get_report_view(report, request) # Determine which metrics should be enabled for this report. for metric in report['metrics']: # Get a list of reports that also contain this metric. metric['similar_reports'] = report_util.get_similar_reports(metric['id'], report_id) # Mark the lens used for this metric, if applicable. if lens: metric['lens'] = lens metric[viz] = metric.get(viz, {}) enabled = metric[viz].get('enabled', True) min_date = metric[viz].get('minDate', start) max_date = metric[viz].get('maxDate', end) # Disabled metrics should stay that way. if not enabled: continue # Disable the metric if it start/end is outside of the min/max window. enabled = start >= min_date if max_date and enabled: enabled = start <= max_date if end and enabled: enabled = end <= max_date metric[viz]['enabled'] = enabled if not request.script_root: request.script_root = url_for('report', report_id=report_id, _external=True) # Return as JSON if requested. if get_format(request) == 'json': return jsonify(status=200, report=report, start=start, end=end, viz=viz) return render_template('report/%s.html' % viz, viz=viz, reports=report_util.get_reports(), report=report, start=start, end=end)
def faq(): return render_template('faq.html', reports=report_util.get_reports(), faq=faq_util)
def about(): return render_template('about.html', reports=report_util.get_reports())
def index(): return render_template('index.html', reports=report_util.get_reports(), featured_reports=report_util.get_featured_reports(), faq=faq_util)
}, content_security_policy_nonce_in=['script-src']) def sitemap(): delattr(request, 'csp_nonce') xml = render_template('sitemap.xml') resp = app.make_response(xml) resp.mimetype = "text/xml" return resp if __name__ == '__main__': # This is used when running locally. Gunicorn is used to run the # application on Google App Engine. See entrypoint in app.yaml. update_config() report_util.get_reports() # If the 'background' command line argument is given: # python main.py background & # then run in non-debug mode, as debug mode can't be backgrounded # but debug mode is useful in general (as auto reloads on change) if len(sys.argv) > 1 and sys.argv[1] == 'background': logging.debug('Running in background mode') # Turn off HTTPS redirects (automatically turned off for debug) talisman.force_https = False app.run(host='0.0.0.0', port=8080) else: logging.debug('Running in debug mode') app.run(host='0.0.0.0', port=8080, debug=True) # [END app]
def reports(): return render_template('reports.html', reports=report_util.get_reports())
def report(report_id): report = reportutil.get_report(report_id) if not report: abort(404) dates = reportutil.get_dates() if not dates: abort(500) min_date = report.get('minDate') max_date = report.get('maxDate') # TODO: If a report doesn't explicitly have a min/max date, # but all of its metrics do, take the min/max of the metrics # and set that as the report's implicit min/max date. # Omit dates for which this report has no data. if min_date: dates = dates[:dates.index(min_date) + 1] if max_date: dates = dates[dates.index(max_date):] report['dates'] = dates start = request.args.get('start') end = request.args.get('end') # Canonicalize single-date formats. if end and not start: start, end = end, start # Canonicalize aliases. if start == 'latest': start = dates[0] elif start == 'earliest': start = dates[-1] if end == 'latest': end = dates[0] elif end == 'earliest': end = dates[-1] # This is longhand for the snapshot (histogram) view. if start == end: end = None # This is shorthand for the trends (timeseries) view. if not start and not end: # The default date range is 24 crawls (1 year). # May be shorter if the report's minimum date is more recent. start = dates[min(23, len(dates) - 1)] end = dates[0] if start and start not in dates: abort(400) if end and end not in dates: abort(400) viz = reportutil.VizTypes.HISTOGRAM if ( start and not end) else reportutil.VizTypes.TIMESERIES if viz == reportutil.VizTypes.TIMESERIES and report.get( 'timeseries') and not report.get('timeseries').get('enabled'): end = None viz = reportutil.VizTypes.HISTOGRAM # Determine which metrics should be enabled for this report. for metric in report['metrics']: # Get a list of reports that also contain this metric. metric['similar_reports'] = reportutil.get_similar_reports( metric['id'], report_id) metric[viz] = metric.get(viz, {}) enabled = metric[viz].get('enabled', True) min_date = metric[viz].get('minDate', start) max_date = metric[viz].get('maxDate', end) # Disabled metrics should stay that way. if not enabled: continue # Disable the metric if it start/end is outside of the min/max window. enabled = start >= min_date if end and enabled: enabled = end <= max_date metric[viz]['enabled'] = enabled if not request.script_root: request.script_root = url_for('report', report_id=report_id, _external=True) return render_template('report/%s.html' % viz, viz=viz, reports=reportutil.get_reports(), report=report, start=start, end=end)