def check_site(): try: params = parse_params(request.forms.decode(), domain=string_param('domain', required=True, strip=True, min_length=1, max_length=DOMAIN_MAX_LENGTH), no_rescan=boolean_param('no_rescan', default=False, empty=True, strip=True)) except ParamParseError: return template('gpc_invalid', domain=None) domain = normalise_domain(params['domain']) if not check_domain(domain): return template('gpc_invalid', domain=domain) result = es_dao.get(domain) if result is not None: if params['no_rescan'] or result['status'] == 'pending': redirect(f'/sites/{domain}') # Non-pending scans should have a scan datetime. last_scan_dt = rfc3339.parse_datetime(result['last_scan_dt']) # If the last scan hasn't expired yet, don't rescan. if rfc3339.now() < last_scan_dt + SCAN_TTL: if testing_mode: log.info('Would have redirected to existing scan for %(domain)s if on prod.', {'domain': domain}) else: redirect(f'/sites/{domain}') r = requests.post(well_known_sites_endpoint, data={'domain': domain, 'rescan': 'true'}) r.raise_for_status() redirect(f'/sites/{domain}')
def get_gcal_events(): """ Authenticates with google apis and uses the v3 calendar api to grab your events 24 hours before and after the current time. This is done ahead of time so Rodney wouldn't have to check every message. """ import httplib2 from apiclient.discovery import build from oauth2client.file import Storage from oauth2client.client import AccessTokenRefreshError from oauth2client.client import OAuth2WebServerFlow from oauth2client.tools import run FLOW = OAuth2WebServerFlow( client_id="FILL_THIS_IN", client_secret="FILL_THIS_IN", scope="https://www.googleapis.com/auth/calendar.readonly", user_agent="rodney-gcal/1.0", ) storage = Storage("gcal.dat") credentials = storage.get() if credentials is None or credentials.invalid == True: credentials = run(FLOW, storage) http = httplib2.Http(cache=".cache") http = credentials.authorize(http) service = build("calendar", "v3", http=http) yesterday = rfc.datetimetostr(rfc.now() - timedelta(1)) tomorrow = rfc.datetimetostr(rfc.now() + timedelta(1)) events = ( service.events() .list(calendarId="primary", timeMin=yesterday, timeMax=tomorrow, orderBy="startTime", singleEvents=True) .execute() ) try: print "Found {0} events between {1} and {2}".format(len(events["items"]), yesterday, tomorrow) except KeyError: print "no events" return [] return events["items"]
def run_worker(dao, **kwargs): while True: now_dt = rfc3339.now() deleted_secrets = dao.delete_expired_secrets(now_dt) if deleted_secrets: log.info('Deleted %(deleted_secrets)s secrets.', {'deleted_secrets': deleted_secrets}) time.sleep(60)
def get_secrets(): now_dt = rfc3339.now() user_id = request.session['user_id'] secrets = dao.find_secrets(user_id, now_dt) return template('secrets', service_address=service_address, secrets=secrets)
def check_gcal(event_items): """ Given a bunch of events from gcal, It looks at the time now and finds if it intersects any current events If it does it says you're free at the end of the event, so it will not be accurate if you have 2 back-to-back events. input: 'event_items' : list = events grabbed by rodney at the beginning. output : string = the string that gets sent by rodney over fbchat """ now = int(time.time()) yesterday = rfc.datetimetostr(rfc.now() - timedelta(1)) tomorrow = rfc.datetimetostr(rfc.now() + timedelta(1)) busy = False times = [] for event in event_items: estartts = rfc.strtotimestamp(event["start"]["dateTime"]) eendts = rfc.strtotimestamp(event["end"]["dateTime"]) if estartts < now < eendts: busy = True times.append(estartts) times.append(eendts) print times print busy print now if not busy: return "{0} should be free right now (according to gcal)" else: msg = "{0} is busy right now. He will be free..." free = zip(times[::2], times[1::2]) freestr = [] for (s, e) in free: if s < now < e: estr = time.strftime("%I:%M%p", time.localtime(e)) freestr.append("after {0}.\n".format(estr)) if len(freestr) == 0: msg += "Never :( try sending a text" else: msg += " ".join(freestr) return msg
def pack_kafka_payload(svc, item, refs): payload = { 'item': item, 'hist': { 'svc': svc, 'dt': rfc3339.datetimetostr(rfc3339.now()), 'refs': refs or [] } } return json.dumps(payload)
def setUp(self): self.storage = Storage(address="http://127.0.0.1:8017", user_id="user-1") self.sample = StorageSample(filecontents=b"hello world!", source="Unknown", name="testfile.txt", date=rfc3339.now().isoformat(), tags=["malware", "really nasty file ;)"], comment="just some test") self.sha256 = self.sample.sha256()
def load_latest_tweets(since): print("Querying twitter proxy for tweets since {}".format(since)) params = { "from": since, "to": rfc3339.now().isoformat(), } response = requests.get(TWITTER_URL + "/tweets", params=params).json() print("Twitter proxy responded with {}".format(response)) return response
def setUp(self): self.storage = Storage( address="http://127.0.0.1:8017", user_id="user-1" ) self.sample = StorageSample( filecontents=b"hello world!", source="Unknown", name="testfile.txt", date=rfc3339.now().isoformat(), tags=["malware","really nasty file ;)"], comment="just some test" ) self.sha256 = self.sample.sha256()
def pipe(): while True: print("Loading newest tweets") if len(store.all()) > 0: since = max(x['date'] for x in store.all()) else: since = (rfc3339.now() - timedelta(seconds=60)).isoformat() loaded = load_latest_tweets(since) post_on_slack(loaded) store.insert_multiple(loaded) time.sleep(1)
def issue_lister_cb(self): # Send issues updated in the last hour to the main channel w/o details if self.last_issue_update is None: self.last_issue_update = rfc3339.now() feed = IssueUpdateFeed().get_issues('dolphin-emu') if feed is None: log.msg("Unable to get feed") return for i in reversed(feed.entry): issue_date = rfc3339.parse_datetime(i.updated.text) if issue_date > self.last_issue_update: self.last_issue_update = issue_date issue_fmtd = u"%s %s %s" % (i.title.text, color_f_green + i.author[0].name.text + color_normal, i.link[0].href) self.msg(self.factory.channel, issue_fmtd.encode("utf-8"))
def issue_lister_cb(self): # Send issues updated in the last hour to the main channel w/o details if self.last_issue_update is None: self.last_issue_update = rfc3339.now() feed = IssueUpdateFeed().get_issues('dolphin-emu') if feed is None: log.msg("Unable to get feed") return for i in reversed(feed.entry): issue_date = rfc3339.parse_datetime(i.updated.text) if issue_date > self.last_issue_update: self.last_issue_update = issue_date issue_fmtd = u"%s %s %s" % ( i.title.text, color_f_green + i.author[0].name.text + color_normal, i.link[0].href) self.msg(self.factory.channel, issue_fmtd.encode("utf-8"))
def submit_secret(): if request.session: user_id = request.session['user_id'] else: user_id = None params = parse_params(request.forms.decode(), description=string_param('description', strip=True, max_length=100), secret=string_param('secret', required=True, max_length=2000), ttl=string_param('ttl', required=True, enum=VALID_TTLS.keys())) description = params.get('description') secret = params['secret'] ttl = params['ttl'] now_dt = rfc3339.now() secret_id = generate_id() secret = Secret(secret_id=secret_id, user_id=user_id, description=description, secret=secret, create_dt=now_dt, expire_dt=now_dt + VALID_TTLS[ttl][0]) dao.insert_secret(secret) response.status = 202 response.set_header('Location', f'{service_path}/secrets/{secret_id}') return template('submit_result', service_address=service_address, user_id=user_id, secret_id=secret_id, ttl=VALID_TTLS[ttl][1])
def get_now_dts(): # NOTE: rfc3339.now() doesn't give us fractional seconds. This is less precise, but using # fractional seconds complicates specifying datetime ranges when searching. # e.g. 23:59:59.001 is after 23:59:59, but before 00:00:00 return rfc3339.datetimetostr(rfc3339.now())
def import_trades(): """ Import trades from Bitpanda API """ global resp with Session() as session: result = session.execute( sa.select(Trade).order_by(sa.desc("timestamp"))) latest = result.scalars().first() cursor = None s = requests.session() s.headers.update({'User-Agent': 'PyBitPandaFetcher'}) s.headers.update({"Authorization": "Bearer " + APIKEY}) alltrades = [] ppppage = 1 while True: ppppage += 1 print(f"Fetching page {ppppage}") url = 'https://api.exchange.bitpanda.com/public/v1/account/trades' p = {"max_page_size": 30} if latest: p["from"] = latest.timestamp.isoformat() p["to"] = rfc3339.now().isoformat() if cursor: p["cursor"] = cursor resp = s.get(url, params=p) if resp.status_code != 200: raise ValueError("Invalid status code") j = resp.json() trades = j["trade_history"] for trade in trades: t = trade["trade"] ormtrade = Trade( id=t["trade_id"], trade_pair=t["instrument_code"], transaction_type=t["side"], #BUY, SELL amount=D(t["amount"]), price=D(t["price"]), timestamp=rfc3339.parse_datetime(t["time"])) (tradee, traded) = t["instrument_code"].split("_") # switch currencies if t["side"] == "SELL": (tradee, traded) = (traded, tradee) f = trade["fee"] if f["collection_type"] == "BEST": ormtrade.is_best_fee = True ormtrade.fee = D(f["fee_amount"]) ormtrade.fee_currency = f["fee_currency"] elif f["collection_type"] == "STANDARD": ormtrade.is_best_fee = False #fee_amount, fee_currency if f["fee_currency"] != tradee: raise ValueError( "Something appears to be wrong with the fee") ormtrade.fee = D(f["fee_amount"]) ormtrade.fee_currency = f["fee_currency"] else: raise ValueError("Unknown fee collection type") alltrades.append(ormtrade) if not "cursor" in j: break else: cursor = j["cursor"] with Session() as session: session.add_all(alltrades) session.commit() pass
def calc_ref_latencies(this_svc, ref, target_svc=None): hist = ref['hist'] if not hist: return [], 0 last_end_dt = rfc3339.now() stage_timings = _build_hist_stage_timings(hist) if target_svc: target_i = None for i, (svc, _) in enumerate(stage_timings): if svc == target_svc: target_i = i break if target_i is None: # We didn't find our target, the message mustn't have # flowed through our target, so we can't time from it. stage_timings = [] elif target_i == 0: # Target was the first svc, so time from after it. # Not quite what we're after - don't have timings from # the target itself - but close enough. stage_timings = stage_timings else: # Time from end of service before the target, so we # include timings from the target. stage_timings = stage_timings[target_i - 1:] if not stage_timings: return [], 0 # Add a stage for this service, since it's not in the ref history. # Only part way through processing, but will capture any lag at least. stage_timings.append((this_svc, last_end_dt)) first_svc, first_end_dt = stage_timings[0] total_latency = (last_end_dt - first_end_dt).total_seconds() stage_latencies = [] # The first service doesn't have a stage, as it's what produced the # first message, so we don't have a start time for it. # We start timing the second service from the end of the first service. current_stage_start_dt = first_end_dt current_stage_svc, current_stage_end_dt = stage_timings[1] for stage_svc, stage_end_dt in stage_timings[2:]: if stage_svc == current_stage_svc: current_stage_end_dt = stage_end_dt else: stage_latency = (current_stage_end_dt - current_stage_start_dt).total_seconds() stage_latencies.append((current_stage_svc, stage_latency)) current_stage_svc = stage_svc current_stage_start_dt = current_stage_end_dt current_stage_end_dt = stage_end_dt stage_latency = (current_stage_end_dt - current_stage_start_dt).total_seconds() stage_latencies.append((current_stage_svc, stage_latency)) return stage_latencies, total_latency
def populate_active_patients(): l = DBPatientsActive.query.order_by(DBPatientsActive.date.desc(), DBPatientsActive.hour.desc()).first() if l: start_date = datetime.combine(l.date, time(hour=l.hour)) + timedelta(hours=1) start_date_utc = convert_date_to_utc(start_date) until = convert_date_to_tz(now()) if start_date_utc <= now().replace(tzinfo=None, minute=0, second=0, microsecond=0): ps = DBPatient.query.filter_by(deleted=False).all() c = 0 for p in ps: c += 1 print c stop = False search_date = None a = DBPatientActivation.query.filter(DBPatientActivation.patient_id == p.id, DBPatientActivation.occurred < start_date_utc, DBPatientActivation.status.in_(['Activated', 'Reactivated'])).order_by(DBPatientActivation.occurred.desc()).first() if a: d = DBPatientActivation.query.filter(DBPatientActivation.patient_id == p.id, DBPatientActivation.status == 'Inactivated', DBPatientActivation.occurred > a.occurred).first() if d: search_date = d.occurred a_occurred_converted = convert_date_to_tz(a.occurred) end_date = convert_date_to_tz(d.occurred) for dt in rrule.rrule(rrule.HOURLY, dtstart=a_occurred_converted.replace(minute=0, second=0, microsecond=0), until=end_date): po = DBPatientsActive.query.filter_by(date=dt.date(), hour=dt.hour, unit_floor=p.unit_floor).first() if po: po.number_active += 1 db.session.merge(po) else: po = DBPatientsActive(dt.date(), dt.hour, p.unit_floor, 1) db.session.add(po) else: stop = True for dt in rrule.rrule(rrule.HOURLY, dtstart=start_date.replace(minute=0, second=0, microsecond=0), until=until): po = DBPatientsActive.query.filter_by(date=dt.date(), hour=dt.hour, unit_floor=p.unit_floor).first() if po: po.number_active += 1 db.session.merge(po) else: po = DBPatientsActive(dt.date(), dt.hour, p.unit_floor, 1) db.session.add(po) else: stop = True while not stop: a = DBPatientActivation.query.filter(DBPatientActivation.patient_id == p.id, DBPatientActivation.status.in_(['Activated', 'Reactivated']), DBPatientActivation.occurred > search_date).order_by( DBPatientActivation.occurred).first() if a: d = DBPatientActivation.query.filter(DBPatientActivation.patient_id == p.id, DBPatientActivation.status == 'Inactivated', DBPatientActivation.occurred > a.occurred).first() if d: search_date = d.occurred a_occurred_converted = convert_date_to_tz(a.occurred) end_date = convert_date_to_tz(d.occurred) for dt in rrule.rrule(rrule.HOURLY, dtstart=a_occurred_converted.replace(minute=0, second=0, microsecond=0), until=end_date): pa = DBPatientsActive.query.filter_by(date=dt.date(), hour=dt.hour, unit_floor=p.unit_floor).first() if pa: pa.number_active += 1 db.session.merge(po) else: pa = DBPatientsActive(dt.date(), dt.hour, p.unit_floor, 1) db.session.add(po) else: stop = True a_occurred_converted = convert_date_to_tz(a.occurred) for dt in rrule.rrule(rrule.HOURLY, dtstart=a_occurred_converted.replace(minute=0, second=0, microsecond=0), until=until): pa = DBPatientsActive.query.filter_by(date=dt.date(), hour=dt.hour, unit_floor=p.unit_floor).first() if po: pa.number_active += 1 db.session.merge(po) else: pa = DBPatientsActive(dt.date(), dt.hour, p.unit_floor, 1) db.session.add(po) else: stop = True db.session.commit()
def get_site(domain): domain = normalise_domain(domain) if not check_domain(domain): return template('gpc_invalid', domain=domain) # Well-Known doesn't scan www subdomains - redirect to the base domain instead. if domain_is_www_subdomain(domain): base_domain = extract_base_domain(domain) redirect(f'/sites/{base_domain}') result = es_dao.get(domain) if result is None: redirect(f'/?domain={domain}') status = result['status'] scan_data = result.get('scan_data') if status == 'pending': return template('gpc_pending', domain=domain) elif status == 'blocked': return template('gpc_blocked', domain=domain) elif status == 'failed' and not scan_data: return template('gpc_error', domain=domain) # Status should be `ok`, or `failed` but with a previously successful scan. # In either case, `scan_data` should be present. assert scan_data scheme = scan_data['scheme'] scan_dt = rfc3339.parse_datetime(scan_data['scan_dt']) if result['scan_priority'] == 0: rescan_queued = True can_rescan = False else: rescan_queued = False last_scan_dt = rfc3339.parse_datetime(result['last_scan_dt']) can_rescan = (last_scan_dt + SCAN_TTL) < rfc3339.now() error = scan_data.get('error') if error: message = None if error == 'not-found': message = 'The GPC support resource was not found.' elif error in ('unexpected-scheme-redirect', 'unexpected-status', 'client-error', 'server-error', 'unexpected-status'): message = 'Server responded unexpectedly when fetching the GPC support resource.' elif error in ('parse-error', 'json-parse-error', 'unexpected-json-root-type', 'content-too-long', 'content-length-too-long', 'bad-content'): message = 'The GPC support resource is invalid.' elif error: log.error('Unsupported GPC scan error %(error)s', {'error': error}) r = template('gpc_unknown', scheme=scheme, domain=domain, message=message, scan_dt=scan_dt, rescan_queued=rescan_queued, can_rescan=can_rescan) set_headers(r, SCAN_RESULT_HEADERS) return r else: assert scan_data['found'], 'gpc.json should have been found if no error.' gpc_data = scan_data['gpc'] warnings = scan_data.get('warnings') or [] warnings += gpc_data.get('warning_codes') or [] message = None if warnings: message_parts = [] for warning in warnings: if warning == 'wrong-content-type': message_parts.append('incorrect content type') elif warning == 'invalid-update-field': message_parts.append('invalid last update field') if message_parts: message = ' and '.join(message_parts) + '.' last_update = gpc_data['parsed'].get('lastUpdate') template_name = 'gpc_supported' if gpc_data['parsed']['gpc'] else 'gpc_unsupported' r = template(template_name, scheme=scheme, domain=domain, last_update=last_update, message=message, scan_dt=scan_dt, rescan_queued=rescan_queued, can_rescan=can_rescan) set_headers(r, SCAN_RESULT_HEADERS) return r