def check(self, fields, before, after, limit):
url = '{}/{}'.format(SUNLIGHT_URL, 'bills/search')
params = {
'fields':
','.join([
"bill_id", "bill_type", "number", "introduced_on",
"short_title", "official_title", "sponsor", "urls.congress",
"urls.opencongress"
]),
'query':
fields.get('query'),
'order':
'congress,introduced_on,number',
}
data = yield from self.get_json(url, params=params, limit=limit)
if before:
params['introduced_on__lte'] = util.epoch_to_date(before)
params['order'] = 'introduced_on__desc'
if after:
params['introduced_on__gte'] = util.epoch_to_date(after)
params['order'] = 'introduced_on__asc'
ifttt = []
for bill in data['results']:
timestamp = util.date_to_epoch(bill['introduced_on'])
record = {
'meta': {
'id': bill['bill_id'],
'timestamp': timestamp,
},
'query': fields.get('query'),
'sponsor_name': util.name(bill['sponsor']),
'code': util.bill_code(bill),
'title': util.bill_title(bill),
'introduced_on': util.readable_date(bill['introduced_on']),
'official_url': bill['urls']['congress'],
'open_congress_url': bill['urls']['opencongress'],
'date': bill['introduced_on'],
}
ifttt.append(record)
return util.JSONResponse(ifttt)
def check(self, fields, before, after, limit):
url = '{}/{}'.format(SUNLIGHT_URL, 'bills')
params = {
'fields':
','.join([
"bill_id", "bill_type", "number", "history.enacted_at",
"short_title", "official_title", "sponsor", "urls.congress",
"urls.opencongress"
]),
'history.enacted':
'true',
'order':
'history.enacted_at',
}
if before:
params['history.enacted_at__lte'] = util.epoch_to_date(before)
params['order'] = 'history.enacted_at__desc'
if after:
params['history.enacted_at__gte'] = util.epoch_to_date(after)
params['order'] = 'history.enacted_at__asc'
data = yield from self.get_json(url, params=params, limit=limit)
ifttt = []
for bill in data['results']:
timestamp = util.date_to_epoch(bill['history']['enacted_at'])
record = {
'meta': {
'id': bill['bill_id'],
'timestamp': timestamp,
},
'SponsorName': util.name(bill['sponsor']),
'Code': util.bill_code(bill),
'Title': util.bill_title(bill),
'BecameLawOn':
util.readable_date(bill['history']['enacted_at']),
'OfficialURL': bill['urls']['congress'],
'OpenCongressURL': bill['urls']['opencongress'],
'date': bill['history']['enacted_at'],
}
ifttt.append(record)
return util.JSONResponse(ifttt)
def to_dict(self):
o = list()
for dp in self.data_points:
dt = epoch_to_date(dp[TS]).strftime("%Y-%m-%dT%H:%M:%S")
item = [dt, int_if_possible(dp[VALUE]), dp[STATUS]]
o.append(item)
return o
def dump_rollup(self, start_index, limit):
q = '''
SELECT id, sensor_id, type, ts, vmax, vmin, vsum, vcount, vavg
FROM rollup
WHERE id >= {}
LIMIT {}
'''
query = q.format(start_index, limit)
rows = self.fetchall(query, self.conn)
d = list()
for row in rows:
dt = epoch_to_date(row[3]).strftime("%Y-%m-%dT%H:%M:%S")
item = {
"id": row[0],
"sensor_id": row[1],
"type": self.translate_rollup_type(row[2]),
"ts": dt,
"max": int_if_possible(row[4]),
"min": int_if_possible(row[5]),
"sum": int_if_possible(row[6]),
"count": int_if_possible(row[7]),
"avg": int_if_possible(row[8])
}
d.append(item)
return d
def rollup_job_dmy(conn, sensor_id, jobs, job_type, source_type, verbose=False, dry_run=False):
sd = sys.float_info.max
ed = sys.float_info.min
for k, _ in jobs.items():
if k < sd:
sd = k
if k > ed:
ed = k
if len(jobs) == 1:
dt = epoch_to_date(sd)
if job_type == JOB_TYPE_DAY:
dt += timedelta(days=1)
elif job_type == JOB_TYPE_MONTH:
dt += relativedelta(months=1)
else:
dt += relativedelta(months=12)
ed = date_to_epoch(dt)
query = get_dmy_query(sensor_id, job_type, source_type, sd, ed)
if verbose > 1:
log("rollup_job_dmy - {}".format(query))
try:
cursor = conn.cursor()
cursor.execute(query)
conn.commit()
except Exception as e:
s = "ERROR: rollup_job_dmy - {} - {}".format(e, query)
raise Exception(s)
def check(self, fields, before, after, limit):
url = '{}/{}'.format(SUNLIGHT_URL, 'bills/search')
params = {
'fields': ','.join(
["bill_id", "bill_type", "number", "introduced_on",
"short_title", "official_title", "sponsor",
"urls.congress", "urls.opencongress"]),
'query': fields.get('query'),
'order': 'congress,introduced_on,number',
}
data = yield from self.get_json(url, params=params, limit=limit)
if before:
params['introduced_on__lte'] = util.epoch_to_date(before)
params['order'] = 'introduced_on__desc'
if after:
params['introduced_on__gte'] = util.epoch_to_date(after)
params['order'] = 'introduced_on__asc'
ifttt = []
for bill in data['results']:
timestamp = util.date_to_epoch(bill['introduced_on'])
record = {
'meta': {
'id': bill['bill_id'],
'timestamp': timestamp,
},
'query': fields.get('query'),
'sponsor_name': util.name(bill['sponsor']),
'code': util.bill_code(bill),
'title': util.bill_title(bill),
'introduced_on': util.readable_date(bill['introduced_on']),
'official_url': bill['urls']['congress'],
'open_congress_url': bill['urls']['opencongress'],
'date': bill['introduced_on'],
}
ifttt.append(record)
return util.JSONResponse(ifttt)
def check(self, fields, before, after, limit):
url = '{}/{}'.format(SUNLIGHT_URL, 'bills')
params = {
'fields': ','.join(
["bill_id", "bill_type", "number", "history.enacted_at",
"short_title", "official_title", "sponsor",
"urls.congress", "urls.opencongress"]),
'history.enacted': 'true',
'order': 'history.enacted_at',
}
if before:
params['history.enacted_at__lte'] = util.epoch_to_date(before)
params['order'] = 'history.enacted_at__desc'
if after:
params['history.enacted_at__gte'] = util.epoch_to_date(after)
params['order'] = 'history.enacted_at__asc'
data = yield from self.get_json(url, params=params, limit=limit)
ifttt = []
for bill in data['results']:
timestamp = util.date_to_epoch(bill['history']['enacted_at'])
record = {
'meta': {
'id': bill['bill_id'],
'timestamp': timestamp,
},
'SponsorName': util.name(bill['sponsor']),
'Code': util.bill_code(bill),
'Title': util.bill_title(bill),
'BecameLawOn': util.readable_date(bill['history']['enacted_at']),
'OfficialURL': bill['urls']['congress'],
'OpenCongressURL': bill['urls']['opencongress'],
'date': bill['history']['enacted_at'],
}
ifttt.append(record)
return util.JSONResponse(ifttt)
def reduce_jobs(jobs, job_type):
new_jobs = dict()
for ts, _ in jobs.items():
dt = epoch_to_date(ts)
if job_type == JOB_TYPE_HOUR:
dt = datetime(dt.year, dt.month, dt.day)
elif job_type == JOB_TYPE_DAY:
dt = datetime(dt.year, dt.month, 1)
elif job_type == JOB_TYPE_MONTH:
dt = datetime(dt.year, 1, 1)
epoch = date_to_epoch(dt)
new_jobs[epoch] = 1
return new_jobs
def dump_interval(self, start_index, limit):
q = '''
SELECT id, sensor_id, ts, value, status
FROM series
WHERE id >= {}
LIMIT {}
'''
query = q.format(start_index, limit)
rows = self.fetchall(query, self.conn)
d = list()
for row in rows:
dt = epoch_to_date(row[2]).strftime("%Y-%m-%dT%H:%M:%S")
item = {
"id": row[0],
"sensor_id": row[1],
"ts": dt,
"value": int_if_possible(row[3]),
"status": row[4]
}
d.append(item)
return d
def check(self, fields, before, after, limit):
today = datetime.datetime.utcnow() - datetime.timedelta(hours=13)
# today = today.replace(hour=0, minute=0, second=0)
today = today.date()
url = '{}/{}'.format(SUNLIGHT_URL, 'legislators')
params = {
'fields':
','.join([
"title", "first_name", "last_name", "state", "party",
"district", "birthday", "bioguide_id", "twitter_id"
]),
'per_page':
'all',
}
data = yield from self.get_json(url, params=params)
legislators = []
for legislator in data['results']:
bday = parse(legislator['birthday'])
bday = bday.replace(year=today.year, hour=0, minute=0, second=0)
legislator['current_birthday'] = bday
bday = bday.date()
is_good = bday <= today
if is_good and before:
is_good = is_good and bday <= util.epoch_to_date(before)
if is_good and after:
is_good = is_good and bday > util.epoch_to_date(after)
if is_good:
legislators.append(legislator)
legislators = sorted(legislators, key=itemgetter('last_name'))
legislators = sorted(legislators,
key=itemgetter('current_birthday'),
reverse=True)
ifttt = []
for legislator in legislators:
birth_year = parse(legislator['birthday']).year
record = {
'meta': {
'id': '{}/{}'.format(today.year, legislator['bioguide_id']),
'timestamp': int(legislator['current_birthday'].timestamp()),
},
'name': '{title}. {first_name} {last_name}'.format(**legislator),
'state': '{state}-{district}'.format(**legislator) \
if legislator.get('district') else legislator['state'],
'party': legislator['party'],
'twitter_username': legislator.get('twitter_id') or '',
'birthday_date': util.readable_date(legislator['birthday']),
'numerical_birthday_date': legislator['birthday'],
'birth_year': birth_year,
'age': today.year - birth_year,
'date': legislator['current_birthday'].date().isoformat(),
}
ifttt.append(record)
return util.JSONResponse(ifttt[:limit or 20])
def check(self, fields, before, after, limit):
today = datetime.datetime.utcnow() - datetime.timedelta(hours=13)
# today = today.replace(hour=0, minute=0, second=0)
today = today.date()
url = '{}/{}'.format(SUNLIGHT_URL, 'legislators')
params = {
'fields': ','.join(
["title", "first_name", "last_name", "state", "party",
"district", "birthday", "bioguide_id", "twitter_id"]),
'per_page': 'all',
}
data = yield from self.get_json(url, params=params)
legislators = []
for legislator in data['results']:
bday = parse(legislator['birthday'])
bday = bday.replace(year=today.year, hour=0, minute=0, second=0)
legislator['current_birthday'] = bday
bday = bday.date()
is_good = bday <= today
if is_good and before:
is_good = is_good and bday <= util.epoch_to_date(before)
if is_good and after:
is_good = is_good and bday > util.epoch_to_date(after)
if is_good:
legislators.append(legislator)
legislators = sorted(legislators, key=itemgetter('last_name'))
legislators = sorted(legislators,
key=itemgetter('current_birthday'),
reverse=True)
ifttt = []
for legislator in legislators:
birth_year = parse(legislator['birthday']).year
record = {
'meta': {
'id': '{}/{}'.format(today.year, legislator['bioguide_id']),
'timestamp': int(legislator['current_birthday'].timestamp()),
},
'name': '{title}. {first_name} {last_name}'.format(**legislator),
'state': '{state}-{district}'.format(**legislator) \
if legislator.get('district') else legislator['state'],
'party': legislator['party'],
'twitter_username': legislator.get('twitter_id') or '',
'birthday_date': util.readable_date(legislator['birthday']),
'numerical_birthday_date': legislator['birthday'],
'birth_year': birth_year,
'age': today.year - birth_year,
'date': legislator['current_birthday'].date().isoformat(),
}
ifttt.append(record)
return util.JSONResponse(ifttt[:limit or 20])