def check_site():
try:
params = parse_params(request.forms.decode(),
domain=string_param('domain', required=True, strip=True,
min_length=1, max_length=DOMAIN_MAX_LENGTH),
no_rescan=boolean_param('no_rescan', default=False, empty=True,
strip=True))
except ParamParseError:
return template('gpc_invalid', domain=None)
domain = normalise_domain(params['domain'])
if not check_domain(domain):
return template('gpc_invalid', domain=domain)
result = es_dao.get(domain)
if result is not None:
if params['no_rescan'] or result['status'] == 'pending':
redirect(f'/sites/{domain}')
# Non-pending scans should have a scan datetime.
last_scan_dt = rfc3339.parse_datetime(result['last_scan_dt'])
# If the last scan hasn't expired yet, don't rescan.
if rfc3339.now() < last_scan_dt + SCAN_TTL:
if testing_mode:
log.info('Would have redirected to existing scan for %(domain)s if on prod.',
{'domain': domain})
else:
redirect(f'/sites/{domain}')
r = requests.post(well_known_sites_endpoint, data={'domain': domain, 'rescan': 'true'})
r.raise_for_status()
redirect(f'/sites/{domain}')
def get_gcal_events():
"""
Authenticates with google apis and uses the v3 calendar api to grab your
events 24 hours before and after the current time. This is done ahead of
time so Rodney wouldn't have to check every message.
"""
import httplib2
from apiclient.discovery import build
from oauth2client.file import Storage
from oauth2client.client import AccessTokenRefreshError
from oauth2client.client import OAuth2WebServerFlow
from oauth2client.tools import run
FLOW = OAuth2WebServerFlow(
client_id="FILL_THIS_IN",
client_secret="FILL_THIS_IN",
scope="https://www.googleapis.com/auth/calendar.readonly",
user_agent="rodney-gcal/1.0",
)
storage = Storage("gcal.dat")
credentials = storage.get()
if credentials is None or credentials.invalid == True:
credentials = run(FLOW, storage)
http = httplib2.Http(cache=".cache")
http = credentials.authorize(http)
service = build("calendar", "v3", http=http)
yesterday = rfc.datetimetostr(rfc.now() - timedelta(1))
tomorrow = rfc.datetimetostr(rfc.now() + timedelta(1))
events = (
service.events()
.list(calendarId="primary", timeMin=yesterday, timeMax=tomorrow, orderBy="startTime", singleEvents=True)
.execute()
)
try:
print "Found {0} events between {1} and {2}".format(len(events["items"]), yesterday, tomorrow)
except KeyError:
print "no events"
return []
return events["items"]
def run_worker(dao, **kwargs):
while True:
now_dt = rfc3339.now()
deleted_secrets = dao.delete_expired_secrets(now_dt)
if deleted_secrets:
log.info('Deleted %(deleted_secrets)s secrets.',
{'deleted_secrets': deleted_secrets})
time.sleep(60)
def get_secrets():
now_dt = rfc3339.now()
user_id = request.session['user_id']
secrets = dao.find_secrets(user_id, now_dt)
return template('secrets',
service_address=service_address,
secrets=secrets)
def check_gcal(event_items):
"""
Given a bunch of events from gcal,
It looks at the time now and finds if it intersects any current events
If it does it says you're free at the end of the event, so it will not
be accurate if you have 2 back-to-back events.
input:
'event_items' : list = events grabbed by rodney at the beginning.
output : string = the string that gets sent by rodney over fbchat
"""
now = int(time.time())
yesterday = rfc.datetimetostr(rfc.now() - timedelta(1))
tomorrow = rfc.datetimetostr(rfc.now() + timedelta(1))
busy = False
times = []
for event in event_items:
estartts = rfc.strtotimestamp(event["start"]["dateTime"])
eendts = rfc.strtotimestamp(event["end"]["dateTime"])
if estartts < now < eendts:
busy = True
times.append(estartts)
times.append(eendts)
print times
print busy
print now
if not busy:
return "{0} should be free right now (according to gcal)"
else:
msg = "{0} is busy right now. He will be free..."
free = zip(times[::2], times[1::2])
freestr = []
for (s, e) in free:
if s < now < e:
estr = time.strftime("%I:%M%p", time.localtime(e))
freestr.append("after {0}.\n".format(estr))
if len(freestr) == 0:
msg += "Never :( try sending a text"
else:
msg += " ".join(freestr)
return msg
def pack_kafka_payload(svc, item, refs):
payload = {
'item': item,
'hist': {
'svc': svc,
'dt': rfc3339.datetimetostr(rfc3339.now()),
'refs': refs or []
}
}
return json.dumps(payload)
def setUp(self):
self.storage = Storage(address="http://127.0.0.1:8017",
user_id="user-1")
self.sample = StorageSample(filecontents=b"hello world!",
source="Unknown",
name="testfile.txt",
date=rfc3339.now().isoformat(),
tags=["malware", "really nasty file ;)"],
comment="just some test")
self.sha256 = self.sample.sha256()
def load_latest_tweets(since):
print("Querying twitter proxy for tweets since {}".format(since))
params = {
"from": since,
"to": rfc3339.now().isoformat(),
}
response = requests.get(TWITTER_URL + "/tweets", params=params).json()
print("Twitter proxy responded with {}".format(response))
return response
def setUp(self):
self.storage = Storage(
address="http://127.0.0.1:8017",
user_id="user-1"
)
self.sample = StorageSample(
filecontents=b"hello world!",
source="Unknown",
name="testfile.txt",
date=rfc3339.now().isoformat(),
tags=["malware","really nasty file ;)"],
comment="just some test"
)
self.sha256 = self.sample.sha256()
def pipe():
while True:
print("Loading newest tweets")
if len(store.all()) > 0:
since = max(x['date'] for x in store.all())
else:
since = (rfc3339.now() - timedelta(seconds=60)).isoformat()
loaded = load_latest_tweets(since)
post_on_slack(loaded)
store.insert_multiple(loaded)
time.sleep(1)
def issue_lister_cb(self):
# Send issues updated in the last hour to the main channel w/o details
if self.last_issue_update is None:
self.last_issue_update = rfc3339.now()
feed = IssueUpdateFeed().get_issues('dolphin-emu')
if feed is None:
log.msg("Unable to get feed")
return
for i in reversed(feed.entry):
issue_date = rfc3339.parse_datetime(i.updated.text)
if issue_date > self.last_issue_update:
self.last_issue_update = issue_date
issue_fmtd = u"%s %s %s" % (i.title.text, color_f_green +
i.author[0].name.text +
color_normal, i.link[0].href)
self.msg(self.factory.channel, issue_fmtd.encode("utf-8"))
def issue_lister_cb(self):
# Send issues updated in the last hour to the main channel w/o details
if self.last_issue_update is None:
self.last_issue_update = rfc3339.now()
feed = IssueUpdateFeed().get_issues('dolphin-emu')
if feed is None:
log.msg("Unable to get feed")
return
for i in reversed(feed.entry):
issue_date = rfc3339.parse_datetime(i.updated.text)
if issue_date > self.last_issue_update:
self.last_issue_update = issue_date
issue_fmtd = u"%s %s %s" % (
i.title.text,
color_f_green + i.author[0].name.text + color_normal,
i.link[0].href)
self.msg(self.factory.channel, issue_fmtd.encode("utf-8"))
def submit_secret():
if request.session:
user_id = request.session['user_id']
else:
user_id = None
params = parse_params(request.forms.decode(),
description=string_param('description',
strip=True,
max_length=100),
secret=string_param('secret',
required=True,
max_length=2000),
ttl=string_param('ttl',
required=True,
enum=VALID_TTLS.keys()))
description = params.get('description')
secret = params['secret']
ttl = params['ttl']
now_dt = rfc3339.now()
secret_id = generate_id()
secret = Secret(secret_id=secret_id,
user_id=user_id,
description=description,
secret=secret,
create_dt=now_dt,
expire_dt=now_dt + VALID_TTLS[ttl][0])
dao.insert_secret(secret)
response.status = 202
response.set_header('Location', f'{service_path}/secrets/{secret_id}')
return template('submit_result',
service_address=service_address,
user_id=user_id,
secret_id=secret_id,
ttl=VALID_TTLS[ttl][1])
def get_now_dts():
# NOTE: rfc3339.now() doesn't give us fractional seconds. This is less precise, but using
# fractional seconds complicates specifying datetime ranges when searching.
# e.g. 23:59:59.001 is after 23:59:59, but before 00:00:00
return rfc3339.datetimetostr(rfc3339.now())
def import_trades():
"""
Import trades from Bitpanda API
"""
global resp
with Session() as session:
result = session.execute(
sa.select(Trade).order_by(sa.desc("timestamp")))
latest = result.scalars().first()
cursor = None
s = requests.session()
s.headers.update({'User-Agent': 'PyBitPandaFetcher'})
s.headers.update({"Authorization": "Bearer " + APIKEY})
alltrades = []
ppppage = 1
while True:
ppppage += 1
print(f"Fetching page {ppppage}")
url = 'https://api.exchange.bitpanda.com/public/v1/account/trades'
p = {"max_page_size": 30}
if latest:
p["from"] = latest.timestamp.isoformat()
p["to"] = rfc3339.now().isoformat()
if cursor:
p["cursor"] = cursor
resp = s.get(url, params=p)
if resp.status_code != 200:
raise ValueError("Invalid status code")
j = resp.json()
trades = j["trade_history"]
for trade in trades:
t = trade["trade"]
ormtrade = Trade(
id=t["trade_id"],
trade_pair=t["instrument_code"],
transaction_type=t["side"], #BUY, SELL
amount=D(t["amount"]),
price=D(t["price"]),
timestamp=rfc3339.parse_datetime(t["time"]))
(tradee, traded) = t["instrument_code"].split("_")
# switch currencies
if t["side"] == "SELL":
(tradee, traded) = (traded, tradee)
f = trade["fee"]
if f["collection_type"] == "BEST":
ormtrade.is_best_fee = True
ormtrade.fee = D(f["fee_amount"])
ormtrade.fee_currency = f["fee_currency"]
elif f["collection_type"] == "STANDARD":
ormtrade.is_best_fee = False
#fee_amount, fee_currency
if f["fee_currency"] != tradee:
raise ValueError(
"Something appears to be wrong with the fee")
ormtrade.fee = D(f["fee_amount"])
ormtrade.fee_currency = f["fee_currency"]
else:
raise ValueError("Unknown fee collection type")
alltrades.append(ormtrade)
if not "cursor" in j:
break
else:
cursor = j["cursor"]
with Session() as session:
session.add_all(alltrades)
session.commit()
pass
def calc_ref_latencies(this_svc, ref, target_svc=None):
hist = ref['hist']
if not hist:
return [], 0
last_end_dt = rfc3339.now()
stage_timings = _build_hist_stage_timings(hist)
if target_svc:
target_i = None
for i, (svc, _) in enumerate(stage_timings):
if svc == target_svc:
target_i = i
break
if target_i is None:
# We didn't find our target, the message mustn't have
# flowed through our target, so we can't time from it.
stage_timings = []
elif target_i == 0:
# Target was the first svc, so time from after it.
# Not quite what we're after - don't have timings from
# the target itself - but close enough.
stage_timings = stage_timings
else:
# Time from end of service before the target, so we
# include timings from the target.
stage_timings = stage_timings[target_i - 1:]
if not stage_timings:
return [], 0
# Add a stage for this service, since it's not in the ref history.
# Only part way through processing, but will capture any lag at least.
stage_timings.append((this_svc, last_end_dt))
first_svc, first_end_dt = stage_timings[0]
total_latency = (last_end_dt - first_end_dt).total_seconds()
stage_latencies = []
# The first service doesn't have a stage, as it's what produced the
# first message, so we don't have a start time for it.
# We start timing the second service from the end of the first service.
current_stage_start_dt = first_end_dt
current_stage_svc, current_stage_end_dt = stage_timings[1]
for stage_svc, stage_end_dt in stage_timings[2:]:
if stage_svc == current_stage_svc:
current_stage_end_dt = stage_end_dt
else:
stage_latency = (current_stage_end_dt - current_stage_start_dt).total_seconds()
stage_latencies.append((current_stage_svc, stage_latency))
current_stage_svc = stage_svc
current_stage_start_dt = current_stage_end_dt
current_stage_end_dt = stage_end_dt
stage_latency = (current_stage_end_dt - current_stage_start_dt).total_seconds()
stage_latencies.append((current_stage_svc, stage_latency))
return stage_latencies, total_latency
def populate_active_patients():
l = DBPatientsActive.query.order_by(DBPatientsActive.date.desc(), DBPatientsActive.hour.desc()).first()
if l:
start_date = datetime.combine(l.date, time(hour=l.hour)) + timedelta(hours=1)
start_date_utc = convert_date_to_utc(start_date)
until = convert_date_to_tz(now())
if start_date_utc <= now().replace(tzinfo=None, minute=0, second=0, microsecond=0):
ps = DBPatient.query.filter_by(deleted=False).all()
c = 0
for p in ps:
c += 1
print c
stop = False
search_date = None
a = DBPatientActivation.query.filter(DBPatientActivation.patient_id == p.id, DBPatientActivation.occurred < start_date_utc,
DBPatientActivation.status.in_(['Activated', 'Reactivated'])).order_by(DBPatientActivation.occurred.desc()).first()
if a:
d = DBPatientActivation.query.filter(DBPatientActivation.patient_id == p.id, DBPatientActivation.status == 'Inactivated',
DBPatientActivation.occurred > a.occurred).first()
if d:
search_date = d.occurred
a_occurred_converted = convert_date_to_tz(a.occurred)
end_date = convert_date_to_tz(d.occurred)
for dt in rrule.rrule(rrule.HOURLY, dtstart=a_occurred_converted.replace(minute=0, second=0, microsecond=0), until=end_date):
po = DBPatientsActive.query.filter_by(date=dt.date(), hour=dt.hour, unit_floor=p.unit_floor).first()
if po:
po.number_active += 1
db.session.merge(po)
else:
po = DBPatientsActive(dt.date(), dt.hour, p.unit_floor, 1)
db.session.add(po)
else:
stop = True
for dt in rrule.rrule(rrule.HOURLY, dtstart=start_date.replace(minute=0, second=0, microsecond=0), until=until):
po = DBPatientsActive.query.filter_by(date=dt.date(), hour=dt.hour, unit_floor=p.unit_floor).first()
if po:
po.number_active += 1
db.session.merge(po)
else:
po = DBPatientsActive(dt.date(), dt.hour, p.unit_floor, 1)
db.session.add(po)
else:
stop = True
while not stop:
a = DBPatientActivation.query.filter(DBPatientActivation.patient_id == p.id,
DBPatientActivation.status.in_(['Activated', 'Reactivated']), DBPatientActivation.occurred > search_date).order_by(
DBPatientActivation.occurred).first()
if a:
d = DBPatientActivation.query.filter(DBPatientActivation.patient_id == p.id, DBPatientActivation.status == 'Inactivated',
DBPatientActivation.occurred > a.occurred).first()
if d:
search_date = d.occurred
a_occurred_converted = convert_date_to_tz(a.occurred)
end_date = convert_date_to_tz(d.occurred)
for dt in rrule.rrule(rrule.HOURLY, dtstart=a_occurred_converted.replace(minute=0, second=0, microsecond=0), until=end_date):
pa = DBPatientsActive.query.filter_by(date=dt.date(), hour=dt.hour, unit_floor=p.unit_floor).first()
if pa:
pa.number_active += 1
db.session.merge(po)
else:
pa = DBPatientsActive(dt.date(), dt.hour, p.unit_floor, 1)
db.session.add(po)
else:
stop = True
a_occurred_converted = convert_date_to_tz(a.occurred)
for dt in rrule.rrule(rrule.HOURLY, dtstart=a_occurred_converted.replace(minute=0, second=0, microsecond=0), until=until):
pa = DBPatientsActive.query.filter_by(date=dt.date(), hour=dt.hour, unit_floor=p.unit_floor).first()
if po:
pa.number_active += 1
db.session.merge(po)
else:
pa = DBPatientsActive(dt.date(), dt.hour, p.unit_floor, 1)
db.session.add(po)
else:
stop = True
db.session.commit()
def get_site(domain):
domain = normalise_domain(domain)
if not check_domain(domain):
return template('gpc_invalid', domain=domain)
# Well-Known doesn't scan www subdomains - redirect to the base domain instead.
if domain_is_www_subdomain(domain):
base_domain = extract_base_domain(domain)
redirect(f'/sites/{base_domain}')
result = es_dao.get(domain)
if result is None:
redirect(f'/?domain={domain}')
status = result['status']
scan_data = result.get('scan_data')
if status == 'pending':
return template('gpc_pending', domain=domain)
elif status == 'blocked':
return template('gpc_blocked', domain=domain)
elif status == 'failed' and not scan_data:
return template('gpc_error', domain=domain)
# Status should be `ok`, or `failed` but with a previously successful scan.
# In either case, `scan_data` should be present.
assert scan_data
scheme = scan_data['scheme']
scan_dt = rfc3339.parse_datetime(scan_data['scan_dt'])
if result['scan_priority'] == 0:
rescan_queued = True
can_rescan = False
else:
rescan_queued = False
last_scan_dt = rfc3339.parse_datetime(result['last_scan_dt'])
can_rescan = (last_scan_dt + SCAN_TTL) < rfc3339.now()
error = scan_data.get('error')
if error:
message = None
if error == 'not-found':
message = 'The GPC support resource was not found.'
elif error in ('unexpected-scheme-redirect', 'unexpected-status',
'client-error', 'server-error', 'unexpected-status'):
message = 'Server responded unexpectedly when fetching the GPC support resource.'
elif error in ('parse-error', 'json-parse-error', 'unexpected-json-root-type',
'content-too-long', 'content-length-too-long', 'bad-content'):
message = 'The GPC support resource is invalid.'
elif error:
log.error('Unsupported GPC scan error %(error)s', {'error': error})
r = template('gpc_unknown', scheme=scheme, domain=domain,
message=message, scan_dt=scan_dt,
rescan_queued=rescan_queued, can_rescan=can_rescan)
set_headers(r, SCAN_RESULT_HEADERS)
return r
else:
assert scan_data['found'], 'gpc.json should have been found if no error.'
gpc_data = scan_data['gpc']
warnings = scan_data.get('warnings') or []
warnings += gpc_data.get('warning_codes') or []
message = None
if warnings:
message_parts = []
for warning in warnings:
if warning == 'wrong-content-type':
message_parts.append('incorrect content type')
elif warning == 'invalid-update-field':
message_parts.append('invalid last update field')
if message_parts:
message = ' and '.join(message_parts) + '.'
last_update = gpc_data['parsed'].get('lastUpdate')
template_name = 'gpc_supported' if gpc_data['parsed']['gpc'] else 'gpc_unsupported'
r = template(template_name, scheme=scheme, domain=domain,
last_update=last_update, message=message, scan_dt=scan_dt,
rescan_queued=rescan_queued, can_rescan=can_rescan)
set_headers(r, SCAN_RESULT_HEADERS)
return r
