def get_scele_events(calendar_data):
events = []
gcal = icalendar.Calendar.from_ical(calendar_data)
for component in gcal.walk():
if component.name == "VEVENT":
summary = component.get('summary')
categories = component.get('categories')
description = component.get(
'description').encode('ascii', 'ignore')
location = component.get('location')
startdt = component.get('dtstart').dt
enddt = component.get('dtend').dt
start_time = format_datetime(startdt)
end_time = format_datetime(enddt)
trello_time_zone = enddt - timedelta(hours=5)
trello_time = format_datetime(trello_time_zone)
event_name = '[{}] {}'.format(categories, summary)
event = {
'name': event_name,
'description': description,
'location': location,
'start_time': start_time,
'end_time': end_time,
'trello_time': trello_time
}
events.append(event)
return events
def save(self):
self._meta.get_field('slug').pre_save(self, not self.pk)
content = self.summary
content += """<p><a target="_top" href="%s%s">Full event details</a></p>""" % (Site.objects.get_current().domain, self.get_absolute_url(), )
if self.uri:
# existing event, update
entry = self.calendar.account.service.GetCalendarEventEntry(uri = self.edit_uri)
entry.title.text = self.title
entry.content.text = content
start_time = format_datetime(self.start_time)
end_time = format_datetime(self.end_time)
entry.when = []
entry.when.append(gdata.calendar.When(start_time = start_time, end_time = end_time))
new_entry = with_request_error_try(lambda: self.calendar.account.service.UpdateEvent(entry.GetEditLink().href, entry))
else:
entry = gdata.calendar.CalendarEventEntry()
entry.title = atom.Title(text = self.title)
entry.content = atom.Content(text = content)
if not self.start_time:
self.start_time = datetime.datetime.utcnow()
if not self.end_time:
self.end_time = self.start_time + datetime.timedelta(hours = 1)
start_time = format_datetime(self.start_time)
end_time = format_datetime(self.end_time)
entry.when.append(gdata.calendar.When(start_time = start_time, end_time = end_time))
new_entry = with_request_error_try(lambda: self.calendar.account.service.InsertEvent(entry, self.calendar.feed_uri))
self.uri = new_entry.id.text
self.edit_uri = new_entry.GetEditLink().href
self.view_uri = new_entry.GetHtmlLink().href
super(Event, self).save()
def process_list(self, api_version, data): # pylint: disable=unused-argument
"""
This method returns details for given set of Errata.
:param data: data obtained from api, we're interested in data["errata_list"]
:returns: dictionary containing detailed information for given errata list}
"""
validate(data, JSON_SCHEMA)
modified_since = data.get("modified_since", None)
modified_since_dt = parse_datetime(modified_since)
errata_to_process = data.get("errata_list", None)
page = data.get("page", None)
page_size = data.get("page_size", None)
response = {"errata_list": {}}
if modified_since:
response["modified_since"] = modified_since
if not errata_to_process:
return response
if len(errata_to_process) == 1:
# treat single-label like a regex, get all matching names
errata_to_process = self.find_errata_by_regex(errata_to_process[0])
filters = []
# if we have information about modified/published dates and receive "modified_since" in request,
# compare the dates
if modified_since:
filters.append((self._filter_modified_since, [modified_since_dt]))
errata_list = {}
errata_page_to_process, pagination_response = paginate(errata_to_process, page, page_size, filters=filters)
for errata in errata_page_to_process:
errata_detail = self.cache.errata_detail.get(errata, None)
if not errata_detail:
continue
errata_list[errata] = {
"synopsis": none2empty(errata_detail[ERRATA_SYNOPSIS]),
"summary": none2empty(errata_detail[ERRATA_SUMMARY]),
"type": none2empty(errata_detail[ERRATA_TYPE]),
"severity": none2empty(errata_detail[ERRATA_SEVERITY]),
"description": none2empty(errata_detail[ERRATA_DESCRIPTION]),
"solution": none2empty(errata_detail[ERRATA_SOLUTION]),
"issued": none2empty(format_datetime(errata_detail[ERRATA_ISSUED])),
"updated": none2empty(format_datetime(errata_detail[ERRATA_UPDATED])),
"cve_list": errata_detail[ERRATA_CVE],
"package_list": pkgidlist2packages(self.cache, errata_detail[ERRATA_PKGIDS]),
"bugzilla_list": errata_detail[ERRATA_BUGZILLA],
"reference_list": errata_detail[ERRATA_REFERENCE],
"url": none2empty(errata_detail[ERRATA_URL])
}
response["errata_list"] = errata_list
response.update(pagination_response)
return response
def set_interval(self,
begin_datetime: date.datetime,
end_datetime: date.datetime = None):
self.begin_datetime = format_datetime(begin_datetime)
if not end_datetime:
end_datetime = date.datetime.fromordinal(
begin_datetime.toordinal() + 1)
else:
self.end_datetime = end_datetime
self.end_datetime = format_datetime(end_datetime)
def default_filters(cls):
end_datetime = datetime.datetime.utcnow()
start_datetime = datetime.datetime(end_datetime.year,
end_datetime.month, 1)
return {
cls.AGGREGATE: 'data',
cls.BREAKDOWN: True,
cls.CONSOLIDATE: _consolidate.MONTHLY,
cls.START: _utils.format_datetime(start_datetime),
cls.END: _utils.format_datetime(end_datetime),
cls.FACTOR_SPS: False,
cls.GROUP_BY: _group_by.PRODUCT,
cls.IS_COST: True,
cls.SHOW_SPS: False,
}
def default_filters(cls):
end_datetime = datetime.datetime.utcnow()
start_datetime = datetime.datetime(end_datetime.year,
end_datetime.month, 1)
return {
cls.AGGREGATE: 'data',
cls.BREAKDOWN: True,
cls.CONSOLIDATE: _consolidate.MONTHLY,
cls.START: _utils.format_datetime(start_datetime),
cls.END: _utils.format_datetime(end_datetime),
cls.FACTOR_SPS: False,
cls.GROUP_BY: _group_by.PRODUCT,
cls.IS_COST: True,
cls.SHOW_SPS: False,
}
def get_log_events(self, log_group_name, log_stream_name):
all_events = []
next_token = None
proceed = True
while proceed:
# Apparently it seems that the boto3 CloudWatchLogs won't accept
# a value of None or '' for the nextToken field. Ridiculous!!!
try:
if next_token:
res = self.cwlogs \
.get_log_events(logGroupName=log_group_name,
logStreamName=log_stream_name,
startFromHead=True,
nextToken=next_token)
else:
res = self.cwlogs \
.get_log_events(logGroupName=log_group_name,
logStreamName=log_stream_name,
startFromHead=True)
except botocore.exceptions.ClientError as ex:
print "error_code=%s, err=%s" % (ex.response['Error']['Code'],
str(ex))
if ex.response['Error']['Code'] == 'ResourceNotFoundException':
if "stream" in str(ex):
log.error(
"Log stream does not exist, log_group_name=%s, log_stream_name=%s"
% (log_group_name, log_stream_name))
raise CloudWatchStreamDoesNotExist(
"log_group_name=%s, log_stream_name=%s" %
(log_group_name, log_stream_name))
elif "group" in str(ex):
log.error(
"Log group does not exist, log_group_name=%s" %
log_group_name)
raise CloudWatchLogDoesNotExist("log_group_name=%s" %
log_group_name)
else:
log.error(
"Unable to get log events, log_group=%s, log_stream=%s"
% (log_group_name, log_stream_name))
raise ex
else:
log.error(
"Unable to get log events, log_group=%s, log_stream=%s"
% (log_group_name, log_stream_name))
raise ex
events = res['events']
for e in events:
ts = datetime.fromtimestamp(e['timestamp'] / 1000)
ts = utils.format_datetime(ts)
message = json.loads(e['message'])
all_events.append({"timestamp": ts, "data": message})
if next_token == res['nextForwardToken']:
proceed = False
else:
next_token = res['nextForwardToken']
return all_events
def parse_rollcall_dtl_date(rollcall_dtl_date):
from datetime import datetime
potential_date_formats = [
"%b. %d, %Y", # JAN. 1, 1900
"%B %d, %Y", # JANUARY 1, 1900
"%b %d, %Y", # JAN 1, 1900
"%b. %d,%Y", # JAN. 1,1900
"%B %d,%Y", # JANUARY 1,1900
"%B. %d, %Y", # JANUARY. 1, 1900
"%b, %d, %Y", # JAN, 1, 1900
"%B, %d, %Y", # JANUARY, 1, 1900
"%b.%d, %Y", # JAN.1, 1900
]
# Python doesn't consider "SEPT" a valid abbreviation for September.
rollcall_dtl_date = rollcall_dtl_date.replace("SEPT.", "SEP.").replace(
"SEPT ", "SEP ")
parsed_date = None
for potential_date_format in potential_date_formats:
try:
parsed_date = datetime.strptime(rollcall_dtl_date,
potential_date_format)
except ValueError:
pass
else:
break
formatted_date = utils.format_datetime(parsed_date)
return formatted_date[:10] if formatted_date is not None else formatted_date
def es_index_artifact(artifact_id):
from tasks import es_index_doc
artifact = cache_registry.lookup(
"pinwall.artifacts.ArtifactService.artifact_by_id")(artifact_id)
user = cache_registry.lookup("pinwall.users.UserService.user_by_id")(
artifact["user_id"])
del artifact["profile_image"]
del artifact["comment_count"]
del artifact["praise_count"]
del artifact["assets"]
artifact.get("terms", []).append(user.get("fullname"))
artifact["created_at"] = format_datetime(artifact.get("created_at"))
artifact["suggest_field"] = {
"input": gen_suggest(artifact.get("name")),
"output": artifact.get("name"),
"payload": {
"id": artifact_id
}
}
es_index_doc.delay(
"artifact",
artifact["id"],
artifact,
parent_id=artifact.get("topic_id") if artifact.get("topic_id") else -1)
def parse_rollcall_dtl_date(rollcall_dtl_date):
from datetime import datetime
potential_date_formats = [
"%b. %d, %Y", # JAN. 1, 1900
"%B %d, %Y", # JANUARY 1, 1900
"%b %d, %Y", # JAN 1, 1900
"%b. %d,%Y", # JAN. 1,1900
"%B %d,%Y", # JANUARY 1,1900
"%B. %d, %Y", # JANUARY. 1, 1900
"%b, %d, %Y", # JAN, 1, 1900
"%B, %d, %Y", # JANUARY, 1, 1900
"%b.%d, %Y", # JAN.1, 1900
]
# Python doesn't consider "SEPT" a valid abbreviation for September.
rollcall_dtl_date = rollcall_dtl_date.replace("SEPT.", "SEP.").replace("SEPT ", "SEP ")
parsed_date = None
for potential_date_format in potential_date_formats:
try:
parsed_date = datetime.strptime(rollcall_dtl_date, potential_date_format)
except ValueError:
pass
else:
break
formatted_date = utils.format_datetime(parsed_date)
return formatted_date[:10] if formatted_date is not None else formatted_date
def parse_rollcall_dtl_date(rollcall_dtl_date):
from datetime import datetime
potential_date_formats = [
"%b %d, %Y", # JAN 1, 1900
"%B %d, %Y", # JANUARY 1, 1900
"%b, %d, %Y", # JAN, 1, 1900
"%B, %d, %Y", # JANUARY, 1, 1900
"%b.%d, %Y", # JAN.1, 1900
]
# Make things easier by removing periods after month abbreviations.
rollcall_dtl_date = rollcall_dtl_date.replace(". ", " ")
# Make things easier by inserting spaces after commas where they are missing.
rollcall_dtl_date = rollcall_dtl_date.replace(",1", ", 1")
# Python doesn't consider "SEPT" a valid abbreviation for September.
rollcall_dtl_date = rollcall_dtl_date.replace("SEPT ", "SEP ")
parsed_date = None
for potential_date_format in potential_date_formats:
try:
parsed_date = datetime.strptime(rollcall_dtl_date, potential_date_format)
except ValueError:
pass
else:
break
formatted_date = utils.format_datetime(parsed_date)
return formatted_date[:10] if formatted_date is not None else formatted_date
def get_membership_details(meeting_uuid: str, user_uuid: str):
validate_meeting_uuid(meeting_uuid)
validate_user_uuid(user_uuid)
meeting = find_meeting(meeting_uuid)
user = find_user(user_uuid)
membership = find_membership(meeting_uuid, user_uuid)
return {
'uuid':
meeting.uuid,
'name':
meeting.name,
'description':
meeting.description,
'datetime':
format_datetime(meeting.datetime),
'stop_name':
meeting.stop_name,
'members': [{
'nickname': membership.nickname,
'is_owner': is_owner(membership.user, meeting),
'is_you': membership.user == user,
'stop_name': membership.stop_name,
} for membership in meeting.users],
'membership': {
'is_owner': is_owner(user, meeting),
'stop_name': membership.stop_name,
}
}, 200
def parse_rollcall_dtl_date(rollcall_dtl_date):
from datetime import datetime
potential_date_formats = [
"%b %d, %Y", # JAN 1, 1900
"%B %d, %Y", # JANUARY 1, 1900
"%b, %d, %Y", # JAN, 1, 1900
"%B, %d, %Y", # JANUARY, 1, 1900
"%b.%d, %Y", # JAN.1, 1900
]
# Make things easier by removing periods after month abbreviations.
rollcall_dtl_date = rollcall_dtl_date.replace(". ", " ")
# Make things easier by inserting spaces after commas where they are missing.
rollcall_dtl_date = rollcall_dtl_date.replace(",1", ", 1")
# Python doesn't consider "SEPT" a valid abbreviation for September.
rollcall_dtl_date = rollcall_dtl_date.replace("SEPT ", "SEP ")
parsed_date = None
for potential_date_format in potential_date_formats:
try:
parsed_date = datetime.strptime(rollcall_dtl_date,
potential_date_format)
except ValueError:
pass
else:
break
formatted_date = utils.format_datetime(parsed_date)
return formatted_date[:10] if formatted_date is not None else formatted_date
def create_govtrack_xml(amdt, options):
govtrack_type_codes = {'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc'}
root = etree.Element("amendment")
root.set("session", amdt['congress'])
root.set("chamber", amdt['amendment_type'][0])
root.set("number", str(amdt['number']))
root.set("updated", utils.format_datetime(amdt['updated_at']))
make_node = utils.make_node
if amdt.get("amends_bill", None):
make_node(root, "amends", None,
type=govtrack_type_codes[amdt["amends_bill"]["bill_type"]],
number=str(amdt["amends_bill"]["number"]),
sequence=str(amdt["house_number"]) if amdt.get("house_number", None) else "")
elif amdt.get("amends_treaty", None):
make_node(root, "amends", None,
type="treaty",
number=str(amdt["amends_treaty"]["number"]))
make_node(root, "status", amdt['status'], datetime=amdt['status_at'])
if amdt['sponsor'] and amdt['sponsor']['type'] == 'person':
v = amdt['sponsor']['thomas_id']
if not options.get("govtrack", False):
make_node(root, "sponsor", None, thomas_id=v)
else:
v = str(utils.translate_legislator_id('thomas', v, 'govtrack'))
make_node(root, "sponsor", None, id=v)
elif amdt['sponsor'] and amdt['sponsor']['type'] == 'committee':
make_node(root, "sponsor", None, committee=amdt['sponsor']['name'])
else:
make_node(root, "sponsor", None)
make_node(root, "offered", None, datetime=amdt['introduced_at'])
make_node(root, "description", amdt["description"] if amdt["description"] else amdt["purpose"])
if amdt["description"]:
make_node(root, "purpose", amdt["purpose"])
actions = make_node(root, "actions", None)
for action in amdt['actions']:
a = make_node(actions,
action['type'] if action['type'] in ("vote",) else "action",
None,
datetime=action['acted_at'])
if action['type'] == 'vote':
a.set("how", action["how"])
a.set("result", action["result"])
if action.get("roll") != None:
a.set("roll", str(action["roll"]))
if action.get('text'):
make_node(a, "text", action['text'])
if action.get('in_committee'):
make_node(a, "committee", None, name=action['in_committee'])
for cr in action['references']:
make_node(a, "reference", None, ref=cr['reference'], label=cr['type'])
return etree.tostring(root, pretty_print=True)
def create_govtrack_xml(amdt, options):
govtrack_type_codes = {'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc'}
root = etree.Element("amendment")
root.set("session", amdt['congress'])
root.set("chamber", amdt['amendment_type'][0])
root.set("number", str(amdt['number']))
root.set("updated", utils.format_datetime(amdt['updated_at']))
make_node = utils.make_node
if amdt.get("amends_bill", None):
make_node(root, "amends", None,
type=govtrack_type_codes[amdt["amends_bill"]["bill_type"]],
number=str(amdt["amends_bill"]["number"]),
sequence=str(amdt["house_number"]) if amdt.get("house_number", None) else "")
elif amdt.get("amends_treaty", None):
make_node(root, "amends", None,
type="treaty",
number=str(amdt["amends_treaty"]["number"]))
make_node(root, "status", amdt['status'], datetime=amdt['status_at'])
if amdt['sponsor'] and amdt['sponsor']['type'] == 'person':
v = amdt['sponsor']['bioguide_id']
if not options.get("govtrack", False):
make_node(root, "sponsor", None, bioguide_id=v)
else:
v = str(utils.translate_legislator_id('bioguide', v, 'govtrack'))
make_node(root, "sponsor", None, id=v)
elif amdt['sponsor'] and amdt['sponsor']['type'] == 'committee':
make_node(root, "sponsor", None, committee=amdt['sponsor']['name'])
else:
make_node(root, "sponsor", None)
make_node(root, "offered", None, datetime=amdt['introduced_at'])
make_node(root, "description", amdt["description"] if amdt["description"] else amdt["purpose"])
if amdt["description"]:
make_node(root, "purpose", amdt["purpose"])
actions = make_node(root, "actions", None)
for action in amdt['actions']:
a = make_node(actions,
action['type'] if action['type'] in ("vote",) else "action",
None,
datetime=action['acted_at'])
if action['type'] == 'vote':
a.set("how", action["how"])
a.set("result", action["result"])
if action.get("roll") != None:
a.set("roll", str(action["roll"]))
if action.get('text'):
make_node(a, "text", action['text'])
if action.get('in_committee'):
make_node(a, "committee", None, name=action['in_committee'])
for cr in action['references']:
make_node(a, "reference", None, ref=cr['reference'], label=cr['type'])
return etree.tostring(root, pretty_print=True)
def get_dates(sdate, edate):
sdt = utils.parse_datetime(sdate, form='%Y-%m-%d')
edt = utils.parse_datetime(edate, form='%Y-%m-%d')
delta = edt - sdt
return [utils.format_datetime(sdt + timedelta(days=i), form='%Y-%m-%d')\
for i in range(delta.days + 1)]
def get_dates(sdate, edate):
sdt = utils.parse_datetime(sdate, form='%Y-%m-%d')
edt = utils.parse_datetime(edate, form='%Y-%m-%d')
delta = edt - sdt
return [utils.format_datetime(sdt + timedelta(days=i), form='%Y-%m-%d')\
for i in range(delta.days + 1)]
def get_keys_from_page(query, date, pagenum):
root = html.parse(listurl % (query, date, date, pagenum))
items = root.xpath('//ul[@class="list_type_1 search_list"]')[0]
blog_ids = items.xpath('./input[@name="blogId"]/@value')
log_nos = items.xpath('./input[@name="logNo"]/@value')
times = [utils.format_datetime(utils.parse_datetime(time))\
for time in items.xpath('./li/div[@class="list_data"]/span[@class="date"]/text()')]
return {(b, l): t for b, l, t in zip(blog_ids, log_nos, times)}
def get_keys_from_page(query, date, pagenum):
root = html.parse(listurl % (query, date, date, pagenum))
items = root.xpath('//ul[@class="list_type_1 search_list"]')[0]
blog_ids = items.xpath('./input[@name="blogId"]/@value')
log_nos = items.xpath('./input[@name="logNo"]/@value')
times = [utils.format_datetime(utils.parse_datetime(time))\
for time in items.xpath('./li/div[@class="list_data"]/span[@class="date"]/text()')]
return {(b, l): t for b, l, t in zip(blog_ids, log_nos, times)}
def _query_flow_day_group(imsi_list, begin_datetime, end_datetime, con=None):
flowdata = {}
if not con:
con = database('OSS_MGO').get_db()
# 处理时间跨月的情况
while begin_datetime < end_datetime:
begin_time = datetime_timestamp(begin_datetime)
end_time = datetime_timestamp(end_datetime)
pipeline = [{
'$match': {
'countTime': {
'$gte': begin_time,
'$lt': end_time
},
'imsi': {
'$in': imsi_list
}
}
}, {
'$group': {
'_id': '$imsi',
'total': {
'$sum': '$total'
},
'userTotal': {
'$sum': '$userTotal'
},
'cardTotal': {
'$sum': '$cardTotal'
}
}
}, {
'$project': {
'_id': 1,
'flow': {
'$add': ['$userTotal', '$total', '$cardTotal']
}
}
}]
col = 't_terminal_flow_count_day_{}'.format(
format_datetime(begin_datetime, '%Y%m01'))
for doc in con[col].aggregate(pipeline, allowDiskUse=True):
if doc['_id'] in flowdata.keys():
flowdata[doc['_id']] += doc['flow']
else:
flowdata.update({doc['_id']: doc['flow']})
if begin_datetime.month + 1 > 12:
begin_datetime = datetime.datetime(year=begin_datetime.year + 1,
month=1,
day=1)
else:
begin_datetime = datetime.datetime(year=begin_datetime.year,
month=begin_datetime.month + 1,
day=1)
return flowdata
def get_meeting_join_info(meeting_uuid: str):
validate_meeting_uuid(meeting_uuid)
meeting = find_meeting(meeting_uuid)
return {
'name': meeting.name,
'datetime': format_datetime(meeting.datetime),
'members_count': len(meeting.users),
'owner_nickname': get_meeting_owner_nickname(meeting)
}, 200
def process(self):
"""
This method returns details of last-processed-time from the VMaaS DB
:returns: dictionary of errata_changes/cve_changes/repository_changes/last_change timestamps
"""
answer = {}
for key, value in self.cache.dbchange.items():
answer[key] = format_datetime(value)
return answer
def es_index_user(user_id):
from tasks import es_index_doc
user = cache_registry.lookup("pinwall.users.UserService.user_by_id")(
user_id)
setting_dict = user.pop("setting")
if setting_dict is not None:
user["description"] = setting_dict.get("description")
user["lang"] = setting_dict.get("lang")
user["tz"] = setting_dict.get("tz")
user["comment_active"] = setting_dict.get("comment_active")
user["confirmed_at"] = format_datetime(user.get("confirmed_at"))
user["registered_at"] = format_datetime(user.get("registered_at"))
user["suggest_field"] = {
"input": gen_suggest(user.get("fullname")),
"output": user.get("fullname"),
"payload": {
"id": user_id
}
}
es_index_doc.delay("user", user_id, user)
def serialize(self):
debt_params = {
"debt_id": self.debt_id,
"debt_type": self.debt_type,
"title": self.title,
"description": self.description,
"photo": self.photo,
"amount": self.amount,
"interest": self.interest,
"fees": self.fees,
"photo": self.photo,
"to_whom": self.to_whom,
"debt_date": format_datetime(self.debt_date),
"date_created": format_datetime(self.date_created),
"amount_with_interest": self.amount_with_interest,
"compound_frequency": self.compound_frequency,
"is_active": self.is_active,
}
if self.date_modified:
debt_params["date_modified"] = format_datetime(self.date_modified)
return debt_params
def prepare_notification_rdf(item, event_type, rejected=None):
app = flask.current_app
parcel = item.parcel
metadata = parcel.metadata
parcel_url = (app.config['BASE_URL'] +
flask.url_for('parcel.view', name=parcel.name))
event_id = "%s#history-%d" % (parcel_url, item.id_)
full_name = auth.ldap_full_name(item.actor)
title = item.title
if full_name:
title += " by %s" % full_name
title += " (stage reference: %s)" % parcel.name
[event_type_def] = [f for f in UNS_FIELD_DEFS if f['name'] == 'event_type']
event_type_values = [k for k, v in event_type_def['range']]
if event_type not in event_type_values:
raise RuntimeError("Unknown event type %r (not in %r)" %
(event_type, event_type_values))
event_data = [
(RDF_URI['rdf_type'], RDF_URI['parcel_event']),
(RDF_URI['title'], title),
(RDF_URI['identifier'], parcel_url),
(RDF_URI['date'], format_datetime(item.time, 'uns')),
(RDF_URI['actor'], item.actor),
(RDF_URI['actor_name'], full_name),
(RDF_URI['event_type'], event_type),
]
if rejected is not None:
decision = 'rejected' if rejected else 'accepted'
event_data.append((RDF_URI['decision'], decision))
if metadata['delivery_type'] == COUNTRY:
EXCLUDE_METADATA = COUNTRY_EXCLUDE_METADATA
elif metadata['delivery_type'] == LOT:
EXCLUDE_METADATA = LOT_EXCLUDE_METADATA
else:
EXCLUDE_METADATA = STREAM_EXCLUDE_METADATA
filtered_metadata = [(uri, name, data) for uri, name, data in metadata_rdf_fields
if name not in EXCLUDE_METADATA]
for rdf_uri, metadata_name, value_map in filtered_metadata:
value = value_map.get(metadata[metadata_name], "")
event_data.append((rdf_uri, value))
return [[event_id, pred, obj] for pred, obj in event_data]
def identification():
image = None
if request.method == 'GET':
image_url = request.args.get('image_url')
if image_url:
try:
img_data = requests.get(image_url).content
except RequestException:
return jsonify({'error': 'invalid image URL'})
else:
image = img_data
else:
context = {
'init_js_script': 'Identification',
'recognition_server': FACE_RECOGNITION_SERVER
}
return render_template('identification.html', **context)
elif request.method == 'POST':
image = request.files.get('image')
if image:
image = image.stream
else:
return jsonify({'error': 'Image is not specified'})
response = recognition_api.post(
f'http://{FACE_RECOGNITION_SERVER}/identify-faces',
files={
'image': image
}).json()
persons = response.get('persons')
if persons:
for person in persons:
person_id = person['id']
if person_id:
person_info = Person.query.get(person_id)
if person_info:
person_info = person_info.to_json()
del person_info['id']
registered = person_info['registered']
person_info['registered'] = {
'utc':
registered.replace(tzinfo=timezone.utc).timestamp(),
'formatted': format_datetime(registered)
}
person['info'] = person_info
return jsonify(response)
def get_user_meetings(user_uuid: str):
validate_user_uuid(user_uuid)
user = find_user(user_uuid)
return {
'meetings': [
{
'uuid': membership.meeting.uuid,
'name': membership.meeting.name,
'nickname': membership.nickname,
'datetime': format_datetime(membership.meeting.datetime),
'members_count':
len(membership.meeting.users), # TODO: use COUNT()
} for membership in user.meetings
]
}, 200
def parse_rollcall_dtl_date(rollcall_dtl_date):
from datetime import datetime
# Match locale abbreviations.
rollcall_dtl_date = rollcall_dtl_date.replace("SEPT.", "SEP.")
rollcall_dtl_date = rollcall_dtl_date.replace("JAN ", "JANUARY ")
try:
parsed_date = datetime.strptime(rollcall_dtl_date, "%B %d, %Y")
except ValueError:
try:
parsed_date = datetime.strptime(rollcall_dtl_date, "%b. %d, %Y")
except ValueError:
parsed_date = None
formatted_date = utils.format_datetime(parsed_date)
return formatted_date[:10] if formatted_date is not None else formatted_date
def es_index_comment(comment):
from tasks import es_index_doc
user = cache_registry.lookup("pinwall.users.UserService.user_by_id")(
comment.commenter_id)
artifact = cache_registry.lookup(
"pinwall.artifacts.ArtifactService.artifact_by_id")(
comment.artifact_id)
comment_doc = {
"id": comment.id,
"user_id": comment.commenter_id,
"user_name": user.get("fullname"),
"content": comment.content,
"commented_at": format_datetime(comment.commented_at),
"artifact_id": artifact.get("id"),
"artifact_name": artifact.get("name")
}
es_index_doc.delay("comment", comment.id, comment_doc)
def es_index_topic(topic_id):
from tasks import es_index_doc
topic = cache_registry.lookup(
"pinwall.artifacts.TopicService.topic_by_id")(topic_id)
user = cache_registry.lookup("pinwall.users.UserService.user_by_id")(
topic["user_id"])
del topic["artifact_count"]
topic.get("terms", []).append(user.get("fullname"))
topic["created_at"] = format_datetime(topic.get("created_at"))
topic["suggest_field"] = {
"input": gen_suggest(topic.get("name")),
"output": topic.get("name"),
"payload": {
"id": topic_id
}
}
es_index_doc.delay("topic", topic_id, topic)
def _query_flow_hour_group(imsi_list, begin_datetime, end_datetime, con=None):
begin_time = datetime_timestamp(begin_datetime)
end_time = datetime_timestamp(end_datetime)
pipeline = [{
'$match': {
'countTime': {
'$gte': begin_time,
'$lt': end_time
},
'imsi': {
'$in': imsi_list
}
}
}, {
'$group': {
'_id': '$imsi',
'total': {
'$sum': '$total'
},
'userTotal': {
'$sum': '$userTotal'
},
'cardTotal': {
'$sum': '$cardTotal'
}
}
}, {
'$project': {
'_id': 1,
'flow': {
'$add': ['$userTotal', '$total', '$cardTotal']
}
}
}]
if not con:
con = database('OSS_MGO').get_db()
col = 't_terminal_flow_count_hour_{}'.format(
format_datetime(begin_datetime, '%Y%m01'))
flowdata = {}
for doc in con[col].aggregate(pipeline, allowDiskUse=True):
flowdata.update({doc['_id']: doc['flow']})
return flowdata
def _add_default_headers(self):
# set the connection header
connection = None
if self.upgrade:
connection = 'upgrade'
elif not self.closing if self.keepalive is None else self.keepalive:
if self.version == "1.0":
connection = 'keep-alive'
else:
if self.version == "1.1":
connection = 'close'
if connection is not None:
self.headers['Connection'] = connection
if 'Date' not in self.headers:
self.headers['Date'] = format_datetime(None)
self.headers['Server'] = self.server_version
def get_words():
u = user.get_user()
dt = cn_time_now() - datetime.timedelta(hours=config.BURY_HOURS)
models = db.Word.find({'user_id': u.id, 'add_time': {'$lt': dt}})
total = models.count()
data = []
keys = ['word', 'sound', 'phon', 'pos', 'sent', 'def', 'note']
for m in models.limit(config.PAGE_SIZE):
w = {'id': str(m['_id']), 'add_time': format_datetime(m['add_time'])}
for k in keys:
w[k] = m[k]
data.append(w)
context = {
'status': 1,
'data': data,
'total': total,
'over': total <= config.PAGE_SIZE
}
return jsonify(context)
def prepare_notification_rdf(item, event_type, rejected=None):
app = flask.current_app
parcel = item.parcel
metadata = parcel.metadata
parcel_url = (app.config['BASE_URL'] +
flask.url_for('parcel.view', name=parcel.name))
event_id = "%s#history-%d" % (parcel_url, item.id_)
full_name = auth.ldap_full_name(item.actor)
title = item.title
if full_name:
title += " by %s" % full_name
title += " (stage reference: %s)" % parcel.name
[event_type_def] = [f for f in UNS_FIELD_DEFS if f['name'] == 'event_type']
event_type_values = [k for k, v in event_type_def['range']]
if event_type not in event_type_values:
raise RuntimeError("Unknown event type %r (not in %r)" %
(event_type, event_type_values))
event_data = [
(RDF_URI['rdf_type'], RDF_URI['parcel_event']),
(RDF_URI['title'], title),
(RDF_URI['identifier'], parcel_url),
(RDF_URI['date'], format_datetime(item.time, 'uns')),
(RDF_URI['actor'], item.actor),
(RDF_URI['actor_name'], full_name),
(RDF_URI['event_type'], event_type),
]
if rejected is not None:
decision = 'rejected' if rejected else 'accepted'
event_data.append((RDF_URI['decision'], decision))
for rdf_uri, metadata_name, value_map in metadata_rdf_fields:
value = value_map.get(metadata[metadata_name], "")
event_data.append((rdf_uri, value))
return [[event_id, pred, obj] for pred, obj in event_data]
def person(email):
person = Person.query.filter_by(email=email).first()
if person:
person_info = person.to_json()
if person_info:
registered = person_info['registered']
person_info['registered'] = {
'utc': registered.replace(tzinfo=timezone.utc).timestamp(),
'formatted': format_datetime(registered)
}
images = recognition_api.get(
f'http://{FACE_RECOGNITION_SERVER}/person-images/{person.id}'
).json()['images']
context = {
'person':
person_info,
'images':
[f'http://{FACE_RECOGNITION_SERVER}/{image}' for image in images]
}
return render_template('person.html', **context)
else:
abort(404)
def get_words():
u = user.get_user()
dt = cn_time_now() - datetime.timedelta(hours=config.BURY_HOURS)
models = db.Word.find({ 'user_id': u.id, 'add_time': { '$lt': dt } })
total = models.count()
data = []
keys = [ 'word', 'sound', 'phon', 'pos', 'sent', 'def', 'note' ]
for m in models.limit(config.PAGE_SIZE):
w = {
'id': str(m['_id']),
'add_time': format_datetime(m['add_time'])
}
for k in keys:
w[k] = m[k]
data.append(w)
context = {
'status': 1,
'data': data,
'total': total,
'over': total <= config.PAGE_SIZE
}
return jsonify(context)
"source_url": document['pages'][0]['link'],
}]
bill = {
"bill_id": bill_id,
"bill_type": bill_type,
"number": bill_number,
"congress": document['congress'],
"original_bill_number": bill_no,
"session": document['session'],
"chamber": document['chamber'],
"actions": actions,
"status": bill_status,
"status_at": format_datetime(document['dates'][-1]),
"titles": [ { "type": "official", "as": "introduced", "title": bill_title } ] if bill_title else [],
"official_title": bill_title,
"description": bill_description,
"committees": committees,
"sources": sources,
"updated_at": format_datetime(datetime.datetime.fromtimestamp(time.time())),
"urls": {
"web": document['pages'][0]['link'],
"tiff": document['pages'][0]['large_image_url'],
"gif": document['pages'][0]['small_image_url'],
continue
bill_id = document['bill_type'] + str(document['bill_stable_number']) + "-" + str(document['congress'])
# A bill can appear multiple times because it may have been reprinted after
# major activity. Turn each print into action information.
bill = bills\
.setdefault(document['congress'], {})\
.setdefault(document['bill_type'], {})\
.setdefault(bill_id, {
"bill_id": bill_id,
"bill_type": document['bill_type'],
"number": str(document['bill_stable_number']),
"congress": str(document['congress']),
"updated_at": utils.format_datetime(datetime.datetime.fromtimestamp(time.time())),
"original_bill_number": "/".join(document['bill_numbers']),
"session": document['session'],
"chamber": document['chamber'],
"introduced_at": utils.format_datetime(document['dates'][0]),
# filled in below
"titles": [],
"official_title": None,
"committees": [],
"actions": [],
# not yet scraped
"sponsor": None,
def create_govtrack_xml(bill, options):
govtrack_type_codes = {'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc'}
root = etree.Element("bill")
root.set("session", bill['congress'])
root.set("type", govtrack_type_codes[bill['bill_type']])
root.set("number", bill['number'])
root.set("updated", utils.format_datetime(bill['updated_at']))
def make_node(parent, tag, text, **attrs):
if options.get("govtrack", False):
# Rewrite bioguide_id attributes as just id with GovTrack person IDs.
attrs2 = {}
for k, v in attrs.items():
if v:
if k == "bioguide_id":
# remap "bioguide_id" attributes to govtrack "id"
k = "id"
v = str(utils.translate_legislator_id('bioguide', v, 'govtrack'))
attrs2[k] = v
attrs = attrs2
return utils.make_node(parent, tag, text, **attrs)
# for American Memory Century of Lawmaking bills...
for source in bill.get("sources", []):
n = make_node(root, "source", "")
for k, v in sorted(source.items()):
if k == "source":
n.text = v
elif k == "source_url":
n.set("url", v)
else:
n.set(k, unicode(v))
if "original_bill_number" in bill:
make_node(root, "bill-number", bill["original_bill_number"])
make_node(root, "state", bill['status'], datetime=bill['status_at'])
old_status = make_node(root, "status", None)
make_node(old_status, "introduced" if bill['status'] in ("INTRODUCED", "REFERRED") else "unknown", None, datetime=bill['status_at']) # dummy for the sake of comparison
make_node(root, "introduced", None, datetime=bill['introduced_at'])
titles = make_node(root, "titles", None)
for title in bill['titles']:
n = make_node(titles, "title", title['title'])
n.set("type", title['type'])
if title['as']:
n.set("as", title['as'])
if title['is_for_portion']:
n.set("partial", "1")
if bill['sponsor']:
# TODO: Sponsored by committee?
make_node(root, "sponsor", None, bioguide_id=bill['sponsor']['bioguide_id'])
else:
make_node(root, "sponsor", None)
cosponsors = make_node(root, "cosponsors", None)
for cosp in bill['cosponsors']:
n = make_node(cosponsors, "cosponsor", None, bioguide_id=cosp["bioguide_id"])
if cosp["sponsored_at"]:
n.set("joined", cosp["sponsored_at"])
if cosp["withdrawn_at"]:
n.set("withdrawn", cosp["withdrawn_at"])
actions = make_node(root, "actions", None)
for action in bill['actions']:
a = make_node(actions,
action['type'] if action['type'] in ("vote", "vote-aux", "calendar", "topresident", "signed", "enacted", "vetoed") else "action",
None,
datetime=action['acted_at'])
if action.get("status"):
a.set("state", action["status"])
if action['type'] in ('vote', 'vote-aux'):
a.clear() # re-insert date between some of these attributes
a.set("how", action["how"])
a.set("type", action["vote_type"])
if action.get("roll") != None:
a.set("roll", action["roll"])
a.set("datetime", utils.format_datetime(action['acted_at']))
a.set("where", action["where"])
a.set("result", action["result"])
if action.get("suspension"):
a.set("suspension", "1")
if action.get("status"):
a.set("state", action["status"])
if action['type'] == 'calendar' and "calendar" in action:
a.set("calendar", action["calendar"])
if action["under"]:
a.set("under", action["under"])
if action["number"]:
a.set("number", action["number"])
if action['type'] == 'enacted':
a.clear() # re-insert date between some of these attributes
a.set("number", "%s-%s" % (bill['congress'], action["number"]))
a.set("type", action["law"])
a.set("datetime", utils.format_datetime(action['acted_at']))
if action.get("status"):
a.set("state", action["status"])
if action['type'] == 'vetoed':
if action.get("pocket"):
a.set("pocket", "1")
if action.get('text'):
make_node(a, "text", action['text'])
if action.get('in_committee'):
make_node(a, "committee", None, name=action['in_committee'])
for cr in action['references']:
make_node(a, "reference", None, ref=cr['reference'], label=cr['type'])
committees = make_node(root, "committees", None)
for cmt in bill['committees']:
make_node(committees, "committee", None, code=(cmt["committee_id"] + cmt["subcommittee_id"]) if cmt.get("subcommittee_id", None) else cmt["committee_id"], name=cmt["committee"], subcommittee=cmt.get("subcommittee").replace("Subcommittee on ", "") if cmt.get("subcommittee") else "", activity=", ".join(c.title() for c in cmt["activity"]))
relatedbills = make_node(root, "relatedbills", None)
for rb in bill['related_bills']:
if rb['type'] == "bill":
rb_bill_type, rb_number, rb_congress = utils.split_bill_id(rb['bill_id'])
make_node(relatedbills, "bill", None, session=rb_congress, type=govtrack_type_codes[rb_bill_type], number=rb_number, relation="unknown" if rb['reason'] == "related" else rb['reason'])
subjects = make_node(root, "subjects", None)
if bill['subjects_top_term']:
make_node(subjects, "term", None, name=bill['subjects_top_term'])
for s in bill['subjects']:
if s != bill['subjects_top_term']:
make_node(subjects, "term", None, name=s)
amendments = make_node(root, "amendments", None)
for amd in bill['amendments']:
make_node(amendments, "amendment", None, number=amd["chamber"] + str(amd["number"]))
if bill.get('summary'):
make_node(root, "summary", bill['summary']['text'], date=bill['summary']['date'], status=bill['summary']['as'])
committee_reports = make_node(root, "committee-reports", None)
for report in bill['committee_reports']:
make_node(committee_reports, "report", report)
return etree.tostring(root, pretty_print=True)
def action_for(item):
# acted_at
if not item.get('actionTime'):
acted_at = item.get('actionDate', '')
else:
# Although we get the action date & time in an ISO-ish format (split
# across two fields), and although we know it's in local time at the
# U.S. Capitol (i.e. U.S. Eastern), we don't know the UTC offset which
# is a part of how we used to serialize the time. So parse and then
# use pytz (via format_datetime) to re-serialize.
acted_at = utils.format_datetime(datetime.datetime.strptime(item.get('actionDate', '') + " " + item['actionTime'], "%Y-%m-%d %H:%M:%S"))
# text & references
# (amendment actions don't always have text?)
text = item['text'] if item['text'] is not None else ''
# strip out links
text = re.sub(r"</?[Aa]( \S.*?)?>", "", text)
# remove and extract references
references = []
match = re.search("\s*\(([^)]+)\)\s*$", text)
if match:
# remove the matched section
text = text[0:match.start()] + text[match.end():]
types = match.group(1)
# fix use of comma or colon instead of a semi colon between reference types
# have seen some accidental capitalization combined with accidental comma, thus the 'T'
# e.g. "text of Title VII as reported in House: CR H3075-3077, Text omission from Title VII:" (hr5384-109)
types = re.sub("[,:] ([a-zT])", r"; \1", types)
# fix "CR:"
types = re.sub("CR:", "CR", types)
# fix a missing semicolon altogether between references
# e.g. sres107-112, "consideration: CR S1877-1878 text as"
types = re.sub("(\d+) +([a-z])", r"\1; \2", types)
for reference in re.split("; ?", types):
if ": " not in reference:
type, reference = None, reference
else:
type, reference = reference.split(": ", 1)
references.append({'type': type, 'reference': reference})
# form dict
action_dict = {
'acted_at': acted_at,
'action_code': item.get('actionCode', ''),
'committees': [item['committee']['systemCode'][0:-2].upper()] if item['committee'] else None,
'references': references,
'type': 'action', # replaced by parse_bill_action if a regex matches
'text': text,
}
if not action_dict["committees"]:
# remove if empty - not present in how we used to generate the file
del action_dict["committees"]
# sometimes there are links (one case is for bills passed by a rule in a resolution, the link will point to the resolution)
if (item.get("links") or {}).get("link") is not None:
action_dict["links"] = item["links"]["link"]
return action_dict
def output_vote(vote, options):
logging.info("[%s] Writing to disk..." % vote['vote_id'])
# output JSON - so easy!
utils.write(
json.dumps(vote, sort_keys=True, indent=2, default=utils.format_datetime),
output_for_vote(vote["vote_id"], "json"),
)
# output XML
root = etree.Element("roll")
root.set("where", "house" if vote['chamber'] == "h" else "senate")
root.set("session", str(vote["congress"]))
root.set("year", str(vote["date"].year))
root.set("roll", str(vote["number"]))
root.set("source", "house.gov" if vote["chamber"] == "h" else "senate.gov")
root.set("datetime", utils.format_datetime(vote['date']))
root.set("updated", utils.format_datetime(vote['updated_at']))
def get_votes(option): return len(vote["votes"].get(option, []))
root.set("aye", str(get_votes("Yea") + get_votes("Aye")))
root.set("nay", str(get_votes("Nay") + get_votes("No")))
root.set("nv", str(get_votes("Not Voting")))
root.set("present", str(get_votes("Present")))
utils.make_node(root, "category", vote["category"])
utils.make_node(root, "type", vote["type"])
utils.make_node(root, "question", vote["question"])
utils.make_node(root, "required", vote["requires"])
utils.make_node(root, "result", vote["result"])
if "bill" in vote:
govtrack_type_codes = { 'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc' }
utils.make_node(root, "bill", None, session=str(vote["bill"]["congress"]), type=govtrack_type_codes[vote["bill"]["type"]], number=str(vote["bill"]["number"]))
if "amendment" in vote:
n = utils.make_node(root, "amendment", None)
if vote["amendment"]["type"] == "s":
n.set("ref", "regular")
n.set("session", str(vote["congress"]))
n.set("number", "s" + str(vote["amendment"]["number"]))
elif vote["amendment"]["type"] == "h-bill":
n.set("ref", "bill-serial")
n.set("session", str(vote["congress"]))
n.set("number", str(vote["amendment"]["number"]))
# well-known keys for certain vote types: +/-/P/0
option_keys = { "Aye": "+", "Yea": "+", "Nay": "-", "No": "-", "Present": "P", "Not Voting": "0" }
# preferred order of output: ayes, nays, present, then not voting, and similarly for guilty/not-guilty
# and handling other options like people's names for votes for the Speaker.
option_sort_order = ('Aye', 'Yea', 'Guilty', 'No', 'Nay', 'Not Guilty', 'OTHER', 'Present', 'Not Voting')
options_list = sorted(vote["votes"].keys(), key = lambda o : option_sort_order.index(o) if o in option_sort_order else option_sort_order.index("OTHER") )
for option in options_list:
if option not in option_keys: option_keys[option] = option
utils.make_node(root, "option", option, key=option_keys[option])
for option in options_list:
for v in vote["votes"][option]:
n = utils.make_node(root, "voter", None)
if v == "VP":
n.set("id", "0")
n.set("VP", "1")
elif not options.get("govtrack", False):
n.set("id", str(v["id"]))
else:
n.set("id", str(utils.get_govtrack_person_id("bioguide" if vote["chamber"] == "h" else "lis", v["id"])))
n.set("vote", option_keys[option])
n.set("value", option)
if v != "VP":
n.set("state", v["state"])
xmloutput = etree.tostring(root, pretty_print=True, encoding="utf8")
# mimick two hard line breaks in GovTrack's legacy output to ease running diffs
xmloutput = re.sub('(source=".*?") ', r"\1\n ", xmloutput)
xmloutput = re.sub('(updated=".*?") ', r"\1\n ", xmloutput)
utils.write(
xmloutput,
output_for_vote(vote['vote_id'], "xml")
)
def test_format_datetime(self):
import utils
d = datetime.datetime(2013,11,22,10,9,8)
self.assertEqual(utils.format_datetime(d),
'11/22/2013 10:09 am')
def output_vote(vote, options, id_type=None):
logging.info("[%s] Writing to disk..." % vote['vote_id'])
# output JSON - so easy!
utils.write(json.dumps(vote,
sort_keys=True,
indent=2,
default=utils.format_datetime),
output_for_vote(vote["vote_id"], "json"),
options=options)
# What kind of IDs are we passed for Members of Congress?
# For current data, we infer from the chamber. For historical data from voteview,
# we're passed the type in id_type, which is set to "bioguide".
if not id_type:
id_type = ("bioguide" if vote["chamber"] == "h" else "lis")
# output XML
root = etree.Element("roll")
root.set("where", "house" if vote['chamber'] == "h" else "senate")
root.set("session", str(vote["congress"]))
root.set("year", str(vote["date"].year))
root.set("roll", str(vote["number"]))
if "voteview" in vote["source_url"]:
root.set("source", "keithpoole")
else:
root.set("source",
"house.gov" if vote["chamber"] == "h" else "senate.gov")
root.set("datetime", utils.format_datetime(vote['date']))
root.set("updated", utils.format_datetime(vote['updated_at']))
def get_votes(option):
return len(vote["votes"].get(option, []))
root.set("aye", str(get_votes("Yea") + get_votes("Aye")))
root.set("nay", str(get_votes("Nay") + get_votes("No")))
root.set("nv", str(get_votes("Not Voting")))
root.set("present", str(get_votes("Present")))
utils.make_node(root, "category", vote["category"])
utils.make_node(root, "type", vote["type"])
utils.make_node(root, "question", vote["question"])
utils.make_node(root, "required", vote["requires"])
utils.make_node(root, "result", vote["result"])
if vote.get("bill"):
govtrack_type_codes = {
'hr': 'h',
's': 's',
'hres': 'hr',
'sres': 'sr',
'hjres': 'hj',
'sjres': 'sj',
'hconres': 'hc',
'sconres': 'sc'
}
utils.make_node(root,
"bill",
None,
session=str(vote["bill"]["congress"]),
type=govtrack_type_codes[vote["bill"]["type"]],
number=str(vote["bill"]["number"]))
if "amendment" in vote:
n = utils.make_node(root, "amendment", None)
if vote["amendment"]["type"] == "s":
n.set("ref", "regular")
n.set("session", str(vote["congress"]))
n.set("number", "s" + str(vote["amendment"]["number"]))
elif vote["amendment"]["type"] == "h-bill":
n.set("ref", "bill-serial")
n.set("session", str(vote["congress"]))
n.set("number", str(vote["amendment"]["number"]))
# well-known keys for certain vote types: +/-/P/0
option_keys = {
"Aye": "+",
"Yea": "+",
"Nay": "-",
"No": "-",
"Present": "P",
"Not Voting": "0",
"Guilty": "+",
"Not Guilty": "-"
}
# preferred order of output: ayes, nays, present, then not voting, and similarly for guilty/not-guilty
# and handling other options like people's names for votes for the Speaker.
option_sort_order = ('Aye', 'Yea', 'Guilty', 'No', 'Nay', 'Not Guilty',
'OTHER', 'Present', 'Not Voting')
options_list = sorted(
vote["votes"].keys(),
key=lambda o: option_sort_order.index(o)
if o in option_sort_order else option_sort_order.index("OTHER"))
for option in options_list:
if option not in option_keys:
option_keys[option] = option
utils.make_node(root, "option", option, key=option_keys[option])
for option in options_list:
for v in vote["votes"][option]:
n = utils.make_node(root, "voter", None)
if v == "VP":
n.set("id", "0")
n.set("VP", "1")
elif not options.get("govtrack", False):
n.set("id", str(v["id"]))
else:
n.set("id", str(utils.get_govtrack_person_id(id_type,
v["id"])))
n.set("vote", option_keys[option])
n.set("value", option)
if v != "VP":
n.set("state", v["state"])
if v.get("voteview_votecode_extra") is not None:
n.set("voteview_votecode_extra",
v["voteview_votecode_extra"])
xmloutput = etree.tostring(root, pretty_print=True, encoding="utf8")
# mimick two hard line breaks in GovTrack's legacy output to ease running diffs
xmloutput = re.sub('(source=".*?") ', r"\1\n ", xmloutput)
xmloutput = re.sub('(updated=".*?") ', r"\1\n ", xmloutput)
utils.write(xmloutput,
output_for_vote(vote['vote_id'], "xml"),
options=options)
def test_format_datetime_with_sections(self):
import utils
d = datetime.datetime(2013,11,22,10,9,8)
self.assertEqual(utils.format_datetime(d, with_seconds=1),
'11/22/2013 10:09:08 am')
def output_vote(vote, options):
logging.info("[%s] Writing to disk..." % vote["vote_id"])
# output JSON - so easy!
utils.write(
json.dumps(vote, sort_keys=True, indent=2, default=utils.format_datetime),
output_for_vote(vote["vote_id"], "json"),
)
# output XML
root = etree.Element("roll")
root.set("where", "house" if vote["chamber"] == "h" else "senate")
root.set("session", str(vote["congress"]))
root.set("year", str(vote["date"].year))
root.set("roll", str(vote["number"]))
root.set("datetime", utils.format_datetime(vote["date"]))
root.set("updated", utils.format_datetime(vote["updated_at"]))
def get_votes(option):
return len(vote["votes"].get(option, []))
root.set("aye", str(get_votes("Yea") + get_votes("Aye")))
root.set("nay", str(get_votes("Nay") + get_votes("No")))
root.set("present", str(get_votes("Present")))
root.set("nv", str(get_votes("Not Voting")))
root.set("source", "house.gov" if vote["chamber"] == "h" else "senate.gov")
utils.make_node(root, "category", vote["category"])
utils.make_node(root, "type", vote["type"])
utils.make_node(root, "question", vote["question"])
utils.make_node(root, "required", vote["requires"])
utils.make_node(root, "result", vote["result"])
if "bill" in vote:
govtrack_type_codes = {
"hr": "h",
"s": "s",
"hres": "hr",
"sres": "sr",
"hjres": "hj",
"sjres": "sj",
"hconres": "hc",
"sconres": "sc",
}
utils.make_node(
root,
"bill",
None,
session=str(vote["bill"]["congress"]),
type=govtrack_type_codes[vote["bill"]["type"]],
number=str(vote["bill"]["number"]),
)
if "amendment" in vote:
if vote["amendment"]["type"] == "s":
utils.make_node(root, "amendment", None, ref="regular", number="s" + str(vote["amendment"]["number"]))
elif vote["amendment"]["type"] == "h-bill":
utils.make_node(root, "amendment", None, ref="bill-serial", number=str(vote["amendment"]["number"]))
# well-known keys for certain vote types: +/-/P/0
option_keys = {"Aye": "+", "Yea": "+", "Nay": "-", "No": "-", "Present": "P", "Not Voting": "0"}
# preferred order of output: ayes, nays, present, then not voting, and similarly for guilty/not-guilty
# and handling other options like people's names for votes for the Speaker.
option_sort_order = ("Aye", "Yea", "Guilty", "No", "Nay", "Not Guilty", "OTHER", "Present", "Not Voting")
options_list = sorted(
vote["votes"].keys(),
key=lambda o: option_sort_order.index(o) if o in option_sort_order else option_sort_order.index("OTHER"),
)
for option in options_list:
if option not in option_keys:
option_keys[option] = option
utils.make_node(root, "option", option, key=option_keys[option])
for option in options_list:
for v in vote["votes"][option]:
attrs = {"vote": option_keys[option], "value": option}
if v == "VP":
attrs["id"] = "0"
attrs["VP"] = "1"
elif not options.get("govtrack", False):
attrs["id"] = str(v["id"])
attrs["state"] = v["state"]
else:
attrs["id"] = str(
utils.get_govtrack_person_id("bioguide" if vote["chamber"] == "h" else "lis", v["id"])
)
attrs["state"] = v["state"]
utils.make_node(root, "voter", None, **attrs)
utils.write(etree.tostring(root, pretty_print=True), output_for_vote(vote["vote_id"], "xml"))
def _set_filter_datetime(self, filter_key, value):
if isinstance(value, datetime.datetime):
self._filters[filter_key] = _utils.format_datetime(value)
else:
raise TypeError('Parameter %s must be a datetime.' % filter_key)
def test_format_datetime_ISO8601(self):
import utils
d = datetime.datetime(2013,11,22,10,9,8)
self.assertEqual(utils.format_datetime(d, format='ISO8601'),
'2013-11-22T10:09:08-05:00')
def output_amendment(amdt, options):
logging.info("[%s] Writing to disk..." % amdt['amendment_id'])
# output JSON - so easy!
utils.write(
json.dumps(amdt, sort_keys=True, indent=2, default=utils.format_datetime),
output_for_amdt(amdt['amendment_id'], "json")
)
# output XML
govtrack_type_codes = { 'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc' }
root = etree.Element("amendment")
root.set("session", amdt['congress'])
root.set("chamber", amdt['amendment_type'][0])
root.set("number", amdt['number'])
root.set("updated", utils.format_datetime(amdt['updated_at']))
make_node = utils.make_node
make_node(root, "amends", None,
type=govtrack_type_codes[amdt["amends_bill"]["bill_type"]],
number=str(amdt["amends_bill"]["number"]),
sequence=str(int(amdt["house_number"][1:])) if amdt["house_number"] else "") # chop off A from the house_number
make_node(root, "status", amdt['status'], datetime=amdt['status_at'])
if amdt['sponsor'] and amdt['sponsor']['type'] == 'person':
v = amdt['sponsor']['thomas_id']
if not options.get("govtrack", False):
make_node(root, "sponsor", None, thomas_id=v)
else:
v = str(utils.get_govtrack_person_id('thomas', v))
make_node(root, "sponsor", None, id=v)
elif amdt['sponsor'] and amdt['sponsor']['type'] == 'committee':
make_node(root, "sponsor", None, committee=amdt['sponsor']['name'])
else:
make_node(root, "sponsor", None)
make_node(root, "offered", None, datetime=amdt['offered_at'] if amdt['offered_at'] else amdt['submitted_at'])
if amdt["title"]: make_node(root, "title", amdt["title"])
make_node(root, "description", amdt["description"] if amdt["description"] else amdt["purpose"])
make_node(root, "purpose", amdt["purpose"])
actions = make_node(root, "actions", None)
for action in amdt['actions']:
a = make_node(actions,
action['type'] if action['type'] in ("vote",) else "action",
None,
datetime=action['acted_at'])
if action['type'] == 'vote':
a.set("how", action["how"])
a.set("result", action["result"])
if action.get("roll") != None: a.set("roll", str(action["roll"]))
if action.get('text'): make_node(a, "text", action['text'])
if action.get('in_committee'): make_node(a, "committee", None, name=action['in_committee'])
for cr in action['references']:
make_node(a, "reference", None, ref=cr['reference'], label=cr['type'])
utils.write(
etree.tostring(root, pretty_print=True),
output_for_amdt(amdt['amendment_id'], "xml")
)
def output_amendment(amdt, options):
logging.info("[%s] Writing to disk..." % amdt['amendment_id'])
# output JSON - so easy!
utils.write(
json.dumps(amdt,
sort_keys=True,
indent=2,
default=utils.format_datetime),
output_for_amdt(amdt['amendment_id'], "json"))
# output XML
govtrack_type_codes = {
'hr': 'h',
's': 's',
'hres': 'hr',
'sres': 'sr',
'hjres': 'hj',
'sjres': 'sj',
'hconres': 'hc',
'sconres': 'sc'
}
root = etree.Element("amendment")
root.set("session", amdt['congress'])
root.set("chamber", amdt['amendment_type'][0])
root.set("number", str(amdt['number']))
root.set("updated", utils.format_datetime(amdt['updated_at']))
make_node = utils.make_node
if amdt.get("amends_bill", None):
make_node(root,
"amends",
None,
type=govtrack_type_codes[amdt["amends_bill"]["bill_type"]],
number=str(amdt["amends_bill"]["number"]),
sequence=str(amdt["house_number"]) if amdt.get(
"house_number", None) else "")
elif amdt.get("amends_treaty", None):
make_node(root,
"amends",
None,
type="treaty",
number=str(amdt["amends_treaty"]["number"]))
make_node(root, "status", amdt['status'], datetime=amdt['status_at'])
if amdt['sponsor'] and amdt['sponsor']['type'] == 'person':
v = amdt['sponsor']['thomas_id']
if not options.get("govtrack", False):
make_node(root, "sponsor", None, thomas_id=v)
else:
v = str(utils.get_govtrack_person_id('thomas', v))
make_node(root, "sponsor", None, id=v)
elif amdt['sponsor'] and amdt['sponsor']['type'] == 'committee':
make_node(root, "sponsor", None, committee=amdt['sponsor']['name'])
else:
make_node(root, "sponsor", None)
make_node(root, "offered", None, datetime=amdt['introduced_at'])
make_node(root, "description",
amdt["description"] if amdt["description"] else amdt["purpose"])
if amdt["description"]: make_node(root, "purpose", amdt["purpose"])
actions = make_node(root, "actions", None)
for action in amdt['actions']:
a = make_node(
actions,
action['type'] if action['type'] in ("vote", ) else "action",
None,
datetime=action['acted_at'])
if action['type'] == 'vote':
a.set("how", action["how"])
a.set("result", action["result"])
if action.get("roll") != None: a.set("roll", str(action["roll"]))
if action.get('text'): make_node(a, "text", action['text'])
if action.get('in_committee'):
make_node(a, "committee", None, name=action['in_committee'])
for cr in action['references']:
make_node(a,
"reference",
None,
ref=cr['reference'],
label=cr['type'])
utils.write(etree.tostring(root, pretty_print=True),
output_for_amdt(amdt['amendment_id'], "xml"))
def output_vote(vote, options, id_type=None):
logging.info("[%s] Writing to disk..." % vote["vote_id"])
# output JSON - so easy!
utils.write(
json.dumps(vote, sort_keys=True, indent=2, default=utils.format_datetime),
output_for_vote(vote["vote_id"], "json"),
options=options,
)
# What kind of IDs are we passed for Members of Congress?
# For current data, we infer from the chamber. For historical data from voteview,
# we're passed the type in id_type, which is set to "bioguide".
if not id_type:
id_type = "bioguide" if vote["chamber"] == "h" else "lis"
# output XML
root = etree.Element("roll")
root.set("where", "house" if vote["chamber"] == "h" else "senate")
root.set("session", str(vote["congress"]))
root.set("year", str(vote["date"].year))
root.set("roll", str(vote["number"]))
if "voteview" in vote["source_url"]:
root.set("source", "keithpoole")
else:
root.set("source", "house.gov" if vote["chamber"] == "h" else "senate.gov")
root.set("datetime", utils.format_datetime(vote["date"]))
root.set("updated", utils.format_datetime(vote["updated_at"]))
def get_votes(option):
return len(vote["votes"].get(option, []))
root.set("aye", str(get_votes("Yea") + get_votes("Aye")))
root.set("nay", str(get_votes("Nay") + get_votes("No")))
root.set("nv", str(get_votes("Not Voting")))
root.set("present", str(get_votes("Present")))
utils.make_node(root, "category", vote["category"])
utils.make_node(root, "type", vote["type"])
utils.make_node(root, "question", vote["question"])
utils.make_node(root, "required", vote["requires"])
utils.make_node(root, "result", vote["result"])
if vote.get("bill"):
govtrack_type_codes = {
"hr": "h",
"s": "s",
"hres": "hr",
"sres": "sr",
"hjres": "hj",
"sjres": "sj",
"hconres": "hc",
"sconres": "sc",
}
utils.make_node(
root,
"bill",
None,
session=str(vote["bill"]["congress"]),
type=govtrack_type_codes[vote["bill"]["type"]],
number=str(vote["bill"]["number"]),
)
if "amendment" in vote:
n = utils.make_node(root, "amendment", None)
if vote["amendment"]["type"] == "s":
n.set("ref", "regular")
n.set("session", str(vote["congress"]))
n.set("number", "s" + str(vote["amendment"]["number"]))
elif vote["amendment"]["type"] == "h-bill":
n.set("ref", "bill-serial")
n.set("session", str(vote["congress"]))
n.set("number", str(vote["amendment"]["number"]))
# well-known keys for certain vote types: +/-/P/0
option_keys = {
"Aye": "+",
"Yea": "+",
"Nay": "-",
"No": "-",
"Present": "P",
"Not Voting": "0",
"Guilty": "+",
"Not Guilty": "-",
}
# preferred order of output: ayes, nays, present, then not voting, and similarly for guilty/not-guilty
# and handling other options like people's names for votes for the Speaker.
option_sort_order = ("Aye", "Yea", "Guilty", "No", "Nay", "Not Guilty", "OTHER", "Present", "Not Voting")
options_list = sorted(
vote["votes"].keys(),
key=lambda o: option_sort_order.index(o) if o in option_sort_order else option_sort_order.index("OTHER"),
)
for option in options_list:
if option not in option_keys:
option_keys[option] = option
utils.make_node(root, "option", option, key=option_keys[option])
for option in options_list:
for v in vote["votes"][option]:
n = utils.make_node(root, "voter", None)
if v == "VP":
n.set("id", "0")
n.set("VP", "1")
elif not options.get("govtrack", False):
n.set("id", str(v["id"]))
else:
n.set("id", str(utils.get_govtrack_person_id(id_type, v["id"])))
n.set("vote", option_keys[option])
n.set("value", option)
if v != "VP":
n.set("state", v["state"])
if v.get("voteview_votecode_extra") is not None:
n.set("voteview_votecode_extra", v["voteview_votecode_extra"])
xmloutput = etree.tostring(root, pretty_print=True, encoding="utf8")
# mimick two hard line breaks in GovTrack's legacy output to ease running diffs
xmloutput = re.sub('(source=".*?") ', r"\1\n ", xmloutput)
xmloutput = re.sub('(updated=".*?") ', r"\1\n ", xmloutput)
utils.write(xmloutput, output_for_vote(vote["vote_id"], "xml"), options=options)
def test_format_datetime_ISO8601_without_time(self):
import utils
d = datetime.datetime(2013,11,22)
self.assertEqual(utils.format_datetime(d, format='ISO8601'),
'2013-11-22T00:00:00-05:00')
