def get_scele_events(calendar_data): events = [] gcal = icalendar.Calendar.from_ical(calendar_data) for component in gcal.walk(): if component.name == "VEVENT": summary = component.get('summary') categories = component.get('categories') description = component.get( 'description').encode('ascii', 'ignore') location = component.get('location') startdt = component.get('dtstart').dt enddt = component.get('dtend').dt start_time = format_datetime(startdt) end_time = format_datetime(enddt) trello_time_zone = enddt - timedelta(hours=5) trello_time = format_datetime(trello_time_zone) event_name = '[{}] {}'.format(categories, summary) event = { 'name': event_name, 'description': description, 'location': location, 'start_time': start_time, 'end_time': end_time, 'trello_time': trello_time } events.append(event) return events
def save(self): self._meta.get_field('slug').pre_save(self, not self.pk) content = self.summary content += """<p><a target="_top" href="%s%s">Full event details</a></p>""" % (Site.objects.get_current().domain, self.get_absolute_url(), ) if self.uri: # existing event, update entry = self.calendar.account.service.GetCalendarEventEntry(uri = self.edit_uri) entry.title.text = self.title entry.content.text = content start_time = format_datetime(self.start_time) end_time = format_datetime(self.end_time) entry.when = [] entry.when.append(gdata.calendar.When(start_time = start_time, end_time = end_time)) new_entry = with_request_error_try(lambda: self.calendar.account.service.UpdateEvent(entry.GetEditLink().href, entry)) else: entry = gdata.calendar.CalendarEventEntry() entry.title = atom.Title(text = self.title) entry.content = atom.Content(text = content) if not self.start_time: self.start_time = datetime.datetime.utcnow() if not self.end_time: self.end_time = self.start_time + datetime.timedelta(hours = 1) start_time = format_datetime(self.start_time) end_time = format_datetime(self.end_time) entry.when.append(gdata.calendar.When(start_time = start_time, end_time = end_time)) new_entry = with_request_error_try(lambda: self.calendar.account.service.InsertEvent(entry, self.calendar.feed_uri)) self.uri = new_entry.id.text self.edit_uri = new_entry.GetEditLink().href self.view_uri = new_entry.GetHtmlLink().href super(Event, self).save()
def process_list(self, api_version, data): # pylint: disable=unused-argument """ This method returns details for given set of Errata. :param data: data obtained from api, we're interested in data["errata_list"] :returns: dictionary containing detailed information for given errata list} """ validate(data, JSON_SCHEMA) modified_since = data.get("modified_since", None) modified_since_dt = parse_datetime(modified_since) errata_to_process = data.get("errata_list", None) page = data.get("page", None) page_size = data.get("page_size", None) response = {"errata_list": {}} if modified_since: response["modified_since"] = modified_since if not errata_to_process: return response if len(errata_to_process) == 1: # treat single-label like a regex, get all matching names errata_to_process = self.find_errata_by_regex(errata_to_process[0]) filters = [] # if we have information about modified/published dates and receive "modified_since" in request, # compare the dates if modified_since: filters.append((self._filter_modified_since, [modified_since_dt])) errata_list = {} errata_page_to_process, pagination_response = paginate(errata_to_process, page, page_size, filters=filters) for errata in errata_page_to_process: errata_detail = self.cache.errata_detail.get(errata, None) if not errata_detail: continue errata_list[errata] = { "synopsis": none2empty(errata_detail[ERRATA_SYNOPSIS]), "summary": none2empty(errata_detail[ERRATA_SUMMARY]), "type": none2empty(errata_detail[ERRATA_TYPE]), "severity": none2empty(errata_detail[ERRATA_SEVERITY]), "description": none2empty(errata_detail[ERRATA_DESCRIPTION]), "solution": none2empty(errata_detail[ERRATA_SOLUTION]), "issued": none2empty(format_datetime(errata_detail[ERRATA_ISSUED])), "updated": none2empty(format_datetime(errata_detail[ERRATA_UPDATED])), "cve_list": errata_detail[ERRATA_CVE], "package_list": pkgidlist2packages(self.cache, errata_detail[ERRATA_PKGIDS]), "bugzilla_list": errata_detail[ERRATA_BUGZILLA], "reference_list": errata_detail[ERRATA_REFERENCE], "url": none2empty(errata_detail[ERRATA_URL]) } response["errata_list"] = errata_list response.update(pagination_response) return response
def set_interval(self, begin_datetime: date.datetime, end_datetime: date.datetime = None): self.begin_datetime = format_datetime(begin_datetime) if not end_datetime: end_datetime = date.datetime.fromordinal( begin_datetime.toordinal() + 1) else: self.end_datetime = end_datetime self.end_datetime = format_datetime(end_datetime)
def default_filters(cls): end_datetime = datetime.datetime.utcnow() start_datetime = datetime.datetime(end_datetime.year, end_datetime.month, 1) return { cls.AGGREGATE: 'data', cls.BREAKDOWN: True, cls.CONSOLIDATE: _consolidate.MONTHLY, cls.START: _utils.format_datetime(start_datetime), cls.END: _utils.format_datetime(end_datetime), cls.FACTOR_SPS: False, cls.GROUP_BY: _group_by.PRODUCT, cls.IS_COST: True, cls.SHOW_SPS: False, }
def get_log_events(self, log_group_name, log_stream_name): all_events = [] next_token = None proceed = True while proceed: # Apparently it seems that the boto3 CloudWatchLogs won't accept # a value of None or '' for the nextToken field. Ridiculous!!! try: if next_token: res = self.cwlogs \ .get_log_events(logGroupName=log_group_name, logStreamName=log_stream_name, startFromHead=True, nextToken=next_token) else: res = self.cwlogs \ .get_log_events(logGroupName=log_group_name, logStreamName=log_stream_name, startFromHead=True) except botocore.exceptions.ClientError as ex: print "error_code=%s, err=%s" % (ex.response['Error']['Code'], str(ex)) if ex.response['Error']['Code'] == 'ResourceNotFoundException': if "stream" in str(ex): log.error( "Log stream does not exist, log_group_name=%s, log_stream_name=%s" % (log_group_name, log_stream_name)) raise CloudWatchStreamDoesNotExist( "log_group_name=%s, log_stream_name=%s" % (log_group_name, log_stream_name)) elif "group" in str(ex): log.error( "Log group does not exist, log_group_name=%s" % log_group_name) raise CloudWatchLogDoesNotExist("log_group_name=%s" % log_group_name) else: log.error( "Unable to get log events, log_group=%s, log_stream=%s" % (log_group_name, log_stream_name)) raise ex else: log.error( "Unable to get log events, log_group=%s, log_stream=%s" % (log_group_name, log_stream_name)) raise ex events = res['events'] for e in events: ts = datetime.fromtimestamp(e['timestamp'] / 1000) ts = utils.format_datetime(ts) message = json.loads(e['message']) all_events.append({"timestamp": ts, "data": message}) if next_token == res['nextForwardToken']: proceed = False else: next_token = res['nextForwardToken'] return all_events
def parse_rollcall_dtl_date(rollcall_dtl_date): from datetime import datetime potential_date_formats = [ "%b. %d, %Y", # JAN. 1, 1900 "%B %d, %Y", # JANUARY 1, 1900 "%b %d, %Y", # JAN 1, 1900 "%b. %d,%Y", # JAN. 1,1900 "%B %d,%Y", # JANUARY 1,1900 "%B. %d, %Y", # JANUARY. 1, 1900 "%b, %d, %Y", # JAN, 1, 1900 "%B, %d, %Y", # JANUARY, 1, 1900 "%b.%d, %Y", # JAN.1, 1900 ] # Python doesn't consider "SEPT" a valid abbreviation for September. rollcall_dtl_date = rollcall_dtl_date.replace("SEPT.", "SEP.").replace( "SEPT ", "SEP ") parsed_date = None for potential_date_format in potential_date_formats: try: parsed_date = datetime.strptime(rollcall_dtl_date, potential_date_format) except ValueError: pass else: break formatted_date = utils.format_datetime(parsed_date) return formatted_date[:10] if formatted_date is not None else formatted_date
def es_index_artifact(artifact_id): from tasks import es_index_doc artifact = cache_registry.lookup( "pinwall.artifacts.ArtifactService.artifact_by_id")(artifact_id) user = cache_registry.lookup("pinwall.users.UserService.user_by_id")( artifact["user_id"]) del artifact["profile_image"] del artifact["comment_count"] del artifact["praise_count"] del artifact["assets"] artifact.get("terms", []).append(user.get("fullname")) artifact["created_at"] = format_datetime(artifact.get("created_at")) artifact["suggest_field"] = { "input": gen_suggest(artifact.get("name")), "output": artifact.get("name"), "payload": { "id": artifact_id } } es_index_doc.delay( "artifact", artifact["id"], artifact, parent_id=artifact.get("topic_id") if artifact.get("topic_id") else -1)
def parse_rollcall_dtl_date(rollcall_dtl_date): from datetime import datetime potential_date_formats = [ "%b. %d, %Y", # JAN. 1, 1900 "%B %d, %Y", # JANUARY 1, 1900 "%b %d, %Y", # JAN 1, 1900 "%b. %d,%Y", # JAN. 1,1900 "%B %d,%Y", # JANUARY 1,1900 "%B. %d, %Y", # JANUARY. 1, 1900 "%b, %d, %Y", # JAN, 1, 1900 "%B, %d, %Y", # JANUARY, 1, 1900 "%b.%d, %Y", # JAN.1, 1900 ] # Python doesn't consider "SEPT" a valid abbreviation for September. rollcall_dtl_date = rollcall_dtl_date.replace("SEPT.", "SEP.").replace("SEPT ", "SEP ") parsed_date = None for potential_date_format in potential_date_formats: try: parsed_date = datetime.strptime(rollcall_dtl_date, potential_date_format) except ValueError: pass else: break formatted_date = utils.format_datetime(parsed_date) return formatted_date[:10] if formatted_date is not None else formatted_date
def parse_rollcall_dtl_date(rollcall_dtl_date): from datetime import datetime potential_date_formats = [ "%b %d, %Y", # JAN 1, 1900 "%B %d, %Y", # JANUARY 1, 1900 "%b, %d, %Y", # JAN, 1, 1900 "%B, %d, %Y", # JANUARY, 1, 1900 "%b.%d, %Y", # JAN.1, 1900 ] # Make things easier by removing periods after month abbreviations. rollcall_dtl_date = rollcall_dtl_date.replace(". ", " ") # Make things easier by inserting spaces after commas where they are missing. rollcall_dtl_date = rollcall_dtl_date.replace(",1", ", 1") # Python doesn't consider "SEPT" a valid abbreviation for September. rollcall_dtl_date = rollcall_dtl_date.replace("SEPT ", "SEP ") parsed_date = None for potential_date_format in potential_date_formats: try: parsed_date = datetime.strptime(rollcall_dtl_date, potential_date_format) except ValueError: pass else: break formatted_date = utils.format_datetime(parsed_date) return formatted_date[:10] if formatted_date is not None else formatted_date
def get_membership_details(meeting_uuid: str, user_uuid: str): validate_meeting_uuid(meeting_uuid) validate_user_uuid(user_uuid) meeting = find_meeting(meeting_uuid) user = find_user(user_uuid) membership = find_membership(meeting_uuid, user_uuid) return { 'uuid': meeting.uuid, 'name': meeting.name, 'description': meeting.description, 'datetime': format_datetime(meeting.datetime), 'stop_name': meeting.stop_name, 'members': [{ 'nickname': membership.nickname, 'is_owner': is_owner(membership.user, meeting), 'is_you': membership.user == user, 'stop_name': membership.stop_name, } for membership in meeting.users], 'membership': { 'is_owner': is_owner(user, meeting), 'stop_name': membership.stop_name, } }, 200
def create_govtrack_xml(amdt, options): govtrack_type_codes = {'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc'} root = etree.Element("amendment") root.set("session", amdt['congress']) root.set("chamber", amdt['amendment_type'][0]) root.set("number", str(amdt['number'])) root.set("updated", utils.format_datetime(amdt['updated_at'])) make_node = utils.make_node if amdt.get("amends_bill", None): make_node(root, "amends", None, type=govtrack_type_codes[amdt["amends_bill"]["bill_type"]], number=str(amdt["amends_bill"]["number"]), sequence=str(amdt["house_number"]) if amdt.get("house_number", None) else "") elif amdt.get("amends_treaty", None): make_node(root, "amends", None, type="treaty", number=str(amdt["amends_treaty"]["number"])) make_node(root, "status", amdt['status'], datetime=amdt['status_at']) if amdt['sponsor'] and amdt['sponsor']['type'] == 'person': v = amdt['sponsor']['thomas_id'] if not options.get("govtrack", False): make_node(root, "sponsor", None, thomas_id=v) else: v = str(utils.translate_legislator_id('thomas', v, 'govtrack')) make_node(root, "sponsor", None, id=v) elif amdt['sponsor'] and amdt['sponsor']['type'] == 'committee': make_node(root, "sponsor", None, committee=amdt['sponsor']['name']) else: make_node(root, "sponsor", None) make_node(root, "offered", None, datetime=amdt['introduced_at']) make_node(root, "description", amdt["description"] if amdt["description"] else amdt["purpose"]) if amdt["description"]: make_node(root, "purpose", amdt["purpose"]) actions = make_node(root, "actions", None) for action in amdt['actions']: a = make_node(actions, action['type'] if action['type'] in ("vote",) else "action", None, datetime=action['acted_at']) if action['type'] == 'vote': a.set("how", action["how"]) a.set("result", action["result"]) if action.get("roll") != None: a.set("roll", str(action["roll"])) if action.get('text'): make_node(a, "text", action['text']) if action.get('in_committee'): make_node(a, "committee", None, name=action['in_committee']) for cr in action['references']: make_node(a, "reference", None, ref=cr['reference'], label=cr['type']) return etree.tostring(root, pretty_print=True)
def create_govtrack_xml(amdt, options): govtrack_type_codes = {'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc'} root = etree.Element("amendment") root.set("session", amdt['congress']) root.set("chamber", amdt['amendment_type'][0]) root.set("number", str(amdt['number'])) root.set("updated", utils.format_datetime(amdt['updated_at'])) make_node = utils.make_node if amdt.get("amends_bill", None): make_node(root, "amends", None, type=govtrack_type_codes[amdt["amends_bill"]["bill_type"]], number=str(amdt["amends_bill"]["number"]), sequence=str(amdt["house_number"]) if amdt.get("house_number", None) else "") elif amdt.get("amends_treaty", None): make_node(root, "amends", None, type="treaty", number=str(amdt["amends_treaty"]["number"])) make_node(root, "status", amdt['status'], datetime=amdt['status_at']) if amdt['sponsor'] and amdt['sponsor']['type'] == 'person': v = amdt['sponsor']['bioguide_id'] if not options.get("govtrack", False): make_node(root, "sponsor", None, bioguide_id=v) else: v = str(utils.translate_legislator_id('bioguide', v, 'govtrack')) make_node(root, "sponsor", None, id=v) elif amdt['sponsor'] and amdt['sponsor']['type'] == 'committee': make_node(root, "sponsor", None, committee=amdt['sponsor']['name']) else: make_node(root, "sponsor", None) make_node(root, "offered", None, datetime=amdt['introduced_at']) make_node(root, "description", amdt["description"] if amdt["description"] else amdt["purpose"]) if amdt["description"]: make_node(root, "purpose", amdt["purpose"]) actions = make_node(root, "actions", None) for action in amdt['actions']: a = make_node(actions, action['type'] if action['type'] in ("vote",) else "action", None, datetime=action['acted_at']) if action['type'] == 'vote': a.set("how", action["how"]) a.set("result", action["result"]) if action.get("roll") != None: a.set("roll", str(action["roll"])) if action.get('text'): make_node(a, "text", action['text']) if action.get('in_committee'): make_node(a, "committee", None, name=action['in_committee']) for cr in action['references']: make_node(a, "reference", None, ref=cr['reference'], label=cr['type']) return etree.tostring(root, pretty_print=True)
def get_dates(sdate, edate): sdt = utils.parse_datetime(sdate, form='%Y-%m-%d') edt = utils.parse_datetime(edate, form='%Y-%m-%d') delta = edt - sdt return [utils.format_datetime(sdt + timedelta(days=i), form='%Y-%m-%d')\ for i in range(delta.days + 1)]
def get_keys_from_page(query, date, pagenum): root = html.parse(listurl % (query, date, date, pagenum)) items = root.xpath('//ul[@class="list_type_1 search_list"]')[0] blog_ids = items.xpath('./input[@name="blogId"]/@value') log_nos = items.xpath('./input[@name="logNo"]/@value') times = [utils.format_datetime(utils.parse_datetime(time))\ for time in items.xpath('./li/div[@class="list_data"]/span[@class="date"]/text()')] return {(b, l): t for b, l, t in zip(blog_ids, log_nos, times)}
def _query_flow_day_group(imsi_list, begin_datetime, end_datetime, con=None): flowdata = {} if not con: con = database('OSS_MGO').get_db() # 处理时间跨月的情况 while begin_datetime < end_datetime: begin_time = datetime_timestamp(begin_datetime) end_time = datetime_timestamp(end_datetime) pipeline = [{ '$match': { 'countTime': { '$gte': begin_time, '$lt': end_time }, 'imsi': { '$in': imsi_list } } }, { '$group': { '_id': '$imsi', 'total': { '$sum': '$total' }, 'userTotal': { '$sum': '$userTotal' }, 'cardTotal': { '$sum': '$cardTotal' } } }, { '$project': { '_id': 1, 'flow': { '$add': ['$userTotal', '$total', '$cardTotal'] } } }] col = 't_terminal_flow_count_day_{}'.format( format_datetime(begin_datetime, '%Y%m01')) for doc in con[col].aggregate(pipeline, allowDiskUse=True): if doc['_id'] in flowdata.keys(): flowdata[doc['_id']] += doc['flow'] else: flowdata.update({doc['_id']: doc['flow']}) if begin_datetime.month + 1 > 12: begin_datetime = datetime.datetime(year=begin_datetime.year + 1, month=1, day=1) else: begin_datetime = datetime.datetime(year=begin_datetime.year, month=begin_datetime.month + 1, day=1) return flowdata
def get_meeting_join_info(meeting_uuid: str): validate_meeting_uuid(meeting_uuid) meeting = find_meeting(meeting_uuid) return { 'name': meeting.name, 'datetime': format_datetime(meeting.datetime), 'members_count': len(meeting.users), 'owner_nickname': get_meeting_owner_nickname(meeting) }, 200
def process(self): """ This method returns details of last-processed-time from the VMaaS DB :returns: dictionary of errata_changes/cve_changes/repository_changes/last_change timestamps """ answer = {} for key, value in self.cache.dbchange.items(): answer[key] = format_datetime(value) return answer
def es_index_user(user_id): from tasks import es_index_doc user = cache_registry.lookup("pinwall.users.UserService.user_by_id")( user_id) setting_dict = user.pop("setting") if setting_dict is not None: user["description"] = setting_dict.get("description") user["lang"] = setting_dict.get("lang") user["tz"] = setting_dict.get("tz") user["comment_active"] = setting_dict.get("comment_active") user["confirmed_at"] = format_datetime(user.get("confirmed_at")) user["registered_at"] = format_datetime(user.get("registered_at")) user["suggest_field"] = { "input": gen_suggest(user.get("fullname")), "output": user.get("fullname"), "payload": { "id": user_id } } es_index_doc.delay("user", user_id, user)
def serialize(self): debt_params = { "debt_id": self.debt_id, "debt_type": self.debt_type, "title": self.title, "description": self.description, "photo": self.photo, "amount": self.amount, "interest": self.interest, "fees": self.fees, "photo": self.photo, "to_whom": self.to_whom, "debt_date": format_datetime(self.debt_date), "date_created": format_datetime(self.date_created), "amount_with_interest": self.amount_with_interest, "compound_frequency": self.compound_frequency, "is_active": self.is_active, } if self.date_modified: debt_params["date_modified"] = format_datetime(self.date_modified) return debt_params
def prepare_notification_rdf(item, event_type, rejected=None): app = flask.current_app parcel = item.parcel metadata = parcel.metadata parcel_url = (app.config['BASE_URL'] + flask.url_for('parcel.view', name=parcel.name)) event_id = "%s#history-%d" % (parcel_url, item.id_) full_name = auth.ldap_full_name(item.actor) title = item.title if full_name: title += " by %s" % full_name title += " (stage reference: %s)" % parcel.name [event_type_def] = [f for f in UNS_FIELD_DEFS if f['name'] == 'event_type'] event_type_values = [k for k, v in event_type_def['range']] if event_type not in event_type_values: raise RuntimeError("Unknown event type %r (not in %r)" % (event_type, event_type_values)) event_data = [ (RDF_URI['rdf_type'], RDF_URI['parcel_event']), (RDF_URI['title'], title), (RDF_URI['identifier'], parcel_url), (RDF_URI['date'], format_datetime(item.time, 'uns')), (RDF_URI['actor'], item.actor), (RDF_URI['actor_name'], full_name), (RDF_URI['event_type'], event_type), ] if rejected is not None: decision = 'rejected' if rejected else 'accepted' event_data.append((RDF_URI['decision'], decision)) if metadata['delivery_type'] == COUNTRY: EXCLUDE_METADATA = COUNTRY_EXCLUDE_METADATA elif metadata['delivery_type'] == LOT: EXCLUDE_METADATA = LOT_EXCLUDE_METADATA else: EXCLUDE_METADATA = STREAM_EXCLUDE_METADATA filtered_metadata = [(uri, name, data) for uri, name, data in metadata_rdf_fields if name not in EXCLUDE_METADATA] for rdf_uri, metadata_name, value_map in filtered_metadata: value = value_map.get(metadata[metadata_name], "") event_data.append((rdf_uri, value)) return [[event_id, pred, obj] for pred, obj in event_data]
def identification(): image = None if request.method == 'GET': image_url = request.args.get('image_url') if image_url: try: img_data = requests.get(image_url).content except RequestException: return jsonify({'error': 'invalid image URL'}) else: image = img_data else: context = { 'init_js_script': 'Identification', 'recognition_server': FACE_RECOGNITION_SERVER } return render_template('identification.html', **context) elif request.method == 'POST': image = request.files.get('image') if image: image = image.stream else: return jsonify({'error': 'Image is not specified'}) response = recognition_api.post( f'http://{FACE_RECOGNITION_SERVER}/identify-faces', files={ 'image': image }).json() persons = response.get('persons') if persons: for person in persons: person_id = person['id'] if person_id: person_info = Person.query.get(person_id) if person_info: person_info = person_info.to_json() del person_info['id'] registered = person_info['registered'] person_info['registered'] = { 'utc': registered.replace(tzinfo=timezone.utc).timestamp(), 'formatted': format_datetime(registered) } person['info'] = person_info return jsonify(response)
def get_user_meetings(user_uuid: str): validate_user_uuid(user_uuid) user = find_user(user_uuid) return { 'meetings': [ { 'uuid': membership.meeting.uuid, 'name': membership.meeting.name, 'nickname': membership.nickname, 'datetime': format_datetime(membership.meeting.datetime), 'members_count': len(membership.meeting.users), # TODO: use COUNT() } for membership in user.meetings ] }, 200
def parse_rollcall_dtl_date(rollcall_dtl_date): from datetime import datetime # Match locale abbreviations. rollcall_dtl_date = rollcall_dtl_date.replace("SEPT.", "SEP.") rollcall_dtl_date = rollcall_dtl_date.replace("JAN ", "JANUARY ") try: parsed_date = datetime.strptime(rollcall_dtl_date, "%B %d, %Y") except ValueError: try: parsed_date = datetime.strptime(rollcall_dtl_date, "%b. %d, %Y") except ValueError: parsed_date = None formatted_date = utils.format_datetime(parsed_date) return formatted_date[:10] if formatted_date is not None else formatted_date
def es_index_comment(comment): from tasks import es_index_doc user = cache_registry.lookup("pinwall.users.UserService.user_by_id")( comment.commenter_id) artifact = cache_registry.lookup( "pinwall.artifacts.ArtifactService.artifact_by_id")( comment.artifact_id) comment_doc = { "id": comment.id, "user_id": comment.commenter_id, "user_name": user.get("fullname"), "content": comment.content, "commented_at": format_datetime(comment.commented_at), "artifact_id": artifact.get("id"), "artifact_name": artifact.get("name") } es_index_doc.delay("comment", comment.id, comment_doc)
def es_index_topic(topic_id): from tasks import es_index_doc topic = cache_registry.lookup( "pinwall.artifacts.TopicService.topic_by_id")(topic_id) user = cache_registry.lookup("pinwall.users.UserService.user_by_id")( topic["user_id"]) del topic["artifact_count"] topic.get("terms", []).append(user.get("fullname")) topic["created_at"] = format_datetime(topic.get("created_at")) topic["suggest_field"] = { "input": gen_suggest(topic.get("name")), "output": topic.get("name"), "payload": { "id": topic_id } } es_index_doc.delay("topic", topic_id, topic)
def _query_flow_hour_group(imsi_list, begin_datetime, end_datetime, con=None): begin_time = datetime_timestamp(begin_datetime) end_time = datetime_timestamp(end_datetime) pipeline = [{ '$match': { 'countTime': { '$gte': begin_time, '$lt': end_time }, 'imsi': { '$in': imsi_list } } }, { '$group': { '_id': '$imsi', 'total': { '$sum': '$total' }, 'userTotal': { '$sum': '$userTotal' }, 'cardTotal': { '$sum': '$cardTotal' } } }, { '$project': { '_id': 1, 'flow': { '$add': ['$userTotal', '$total', '$cardTotal'] } } }] if not con: con = database('OSS_MGO').get_db() col = 't_terminal_flow_count_hour_{}'.format( format_datetime(begin_datetime, '%Y%m01')) flowdata = {} for doc in con[col].aggregate(pipeline, allowDiskUse=True): flowdata.update({doc['_id']: doc['flow']}) return flowdata
def _add_default_headers(self): # set the connection header connection = None if self.upgrade: connection = 'upgrade' elif not self.closing if self.keepalive is None else self.keepalive: if self.version == "1.0": connection = 'keep-alive' else: if self.version == "1.1": connection = 'close' if connection is not None: self.headers['Connection'] = connection if 'Date' not in self.headers: self.headers['Date'] = format_datetime(None) self.headers['Server'] = self.server_version
def get_words(): u = user.get_user() dt = cn_time_now() - datetime.timedelta(hours=config.BURY_HOURS) models = db.Word.find({'user_id': u.id, 'add_time': {'$lt': dt}}) total = models.count() data = [] keys = ['word', 'sound', 'phon', 'pos', 'sent', 'def', 'note'] for m in models.limit(config.PAGE_SIZE): w = {'id': str(m['_id']), 'add_time': format_datetime(m['add_time'])} for k in keys: w[k] = m[k] data.append(w) context = { 'status': 1, 'data': data, 'total': total, 'over': total <= config.PAGE_SIZE } return jsonify(context)
def prepare_notification_rdf(item, event_type, rejected=None): app = flask.current_app parcel = item.parcel metadata = parcel.metadata parcel_url = (app.config['BASE_URL'] + flask.url_for('parcel.view', name=parcel.name)) event_id = "%s#history-%d" % (parcel_url, item.id_) full_name = auth.ldap_full_name(item.actor) title = item.title if full_name: title += " by %s" % full_name title += " (stage reference: %s)" % parcel.name [event_type_def] = [f for f in UNS_FIELD_DEFS if f['name'] == 'event_type'] event_type_values = [k for k, v in event_type_def['range']] if event_type not in event_type_values: raise RuntimeError("Unknown event type %r (not in %r)" % (event_type, event_type_values)) event_data = [ (RDF_URI['rdf_type'], RDF_URI['parcel_event']), (RDF_URI['title'], title), (RDF_URI['identifier'], parcel_url), (RDF_URI['date'], format_datetime(item.time, 'uns')), (RDF_URI['actor'], item.actor), (RDF_URI['actor_name'], full_name), (RDF_URI['event_type'], event_type), ] if rejected is not None: decision = 'rejected' if rejected else 'accepted' event_data.append((RDF_URI['decision'], decision)) for rdf_uri, metadata_name, value_map in metadata_rdf_fields: value = value_map.get(metadata[metadata_name], "") event_data.append((rdf_uri, value)) return [[event_id, pred, obj] for pred, obj in event_data]
def person(email): person = Person.query.filter_by(email=email).first() if person: person_info = person.to_json() if person_info: registered = person_info['registered'] person_info['registered'] = { 'utc': registered.replace(tzinfo=timezone.utc).timestamp(), 'formatted': format_datetime(registered) } images = recognition_api.get( f'http://{FACE_RECOGNITION_SERVER}/person-images/{person.id}' ).json()['images'] context = { 'person': person_info, 'images': [f'http://{FACE_RECOGNITION_SERVER}/{image}' for image in images] } return render_template('person.html', **context) else: abort(404)
def get_words(): u = user.get_user() dt = cn_time_now() - datetime.timedelta(hours=config.BURY_HOURS) models = db.Word.find({ 'user_id': u.id, 'add_time': { '$lt': dt } }) total = models.count() data = [] keys = [ 'word', 'sound', 'phon', 'pos', 'sent', 'def', 'note' ] for m in models.limit(config.PAGE_SIZE): w = { 'id': str(m['_id']), 'add_time': format_datetime(m['add_time']) } for k in keys: w[k] = m[k] data.append(w) context = { 'status': 1, 'data': data, 'total': total, 'over': total <= config.PAGE_SIZE } return jsonify(context)
"source_url": document['pages'][0]['link'], }] bill = { "bill_id": bill_id, "bill_type": bill_type, "number": bill_number, "congress": document['congress'], "original_bill_number": bill_no, "session": document['session'], "chamber": document['chamber'], "actions": actions, "status": bill_status, "status_at": format_datetime(document['dates'][-1]), "titles": [ { "type": "official", "as": "introduced", "title": bill_title } ] if bill_title else [], "official_title": bill_title, "description": bill_description, "committees": committees, "sources": sources, "updated_at": format_datetime(datetime.datetime.fromtimestamp(time.time())), "urls": { "web": document['pages'][0]['link'], "tiff": document['pages'][0]['large_image_url'], "gif": document['pages'][0]['small_image_url'],
continue bill_id = document['bill_type'] + str(document['bill_stable_number']) + "-" + str(document['congress']) # A bill can appear multiple times because it may have been reprinted after # major activity. Turn each print into action information. bill = bills\ .setdefault(document['congress'], {})\ .setdefault(document['bill_type'], {})\ .setdefault(bill_id, { "bill_id": bill_id, "bill_type": document['bill_type'], "number": str(document['bill_stable_number']), "congress": str(document['congress']), "updated_at": utils.format_datetime(datetime.datetime.fromtimestamp(time.time())), "original_bill_number": "/".join(document['bill_numbers']), "session": document['session'], "chamber": document['chamber'], "introduced_at": utils.format_datetime(document['dates'][0]), # filled in below "titles": [], "official_title": None, "committees": [], "actions": [], # not yet scraped "sponsor": None,
def create_govtrack_xml(bill, options): govtrack_type_codes = {'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc'} root = etree.Element("bill") root.set("session", bill['congress']) root.set("type", govtrack_type_codes[bill['bill_type']]) root.set("number", bill['number']) root.set("updated", utils.format_datetime(bill['updated_at'])) def make_node(parent, tag, text, **attrs): if options.get("govtrack", False): # Rewrite bioguide_id attributes as just id with GovTrack person IDs. attrs2 = {} for k, v in attrs.items(): if v: if k == "bioguide_id": # remap "bioguide_id" attributes to govtrack "id" k = "id" v = str(utils.translate_legislator_id('bioguide', v, 'govtrack')) attrs2[k] = v attrs = attrs2 return utils.make_node(parent, tag, text, **attrs) # for American Memory Century of Lawmaking bills... for source in bill.get("sources", []): n = make_node(root, "source", "") for k, v in sorted(source.items()): if k == "source": n.text = v elif k == "source_url": n.set("url", v) else: n.set(k, unicode(v)) if "original_bill_number" in bill: make_node(root, "bill-number", bill["original_bill_number"]) make_node(root, "state", bill['status'], datetime=bill['status_at']) old_status = make_node(root, "status", None) make_node(old_status, "introduced" if bill['status'] in ("INTRODUCED", "REFERRED") else "unknown", None, datetime=bill['status_at']) # dummy for the sake of comparison make_node(root, "introduced", None, datetime=bill['introduced_at']) titles = make_node(root, "titles", None) for title in bill['titles']: n = make_node(titles, "title", title['title']) n.set("type", title['type']) if title['as']: n.set("as", title['as']) if title['is_for_portion']: n.set("partial", "1") if bill['sponsor']: # TODO: Sponsored by committee? make_node(root, "sponsor", None, bioguide_id=bill['sponsor']['bioguide_id']) else: make_node(root, "sponsor", None) cosponsors = make_node(root, "cosponsors", None) for cosp in bill['cosponsors']: n = make_node(cosponsors, "cosponsor", None, bioguide_id=cosp["bioguide_id"]) if cosp["sponsored_at"]: n.set("joined", cosp["sponsored_at"]) if cosp["withdrawn_at"]: n.set("withdrawn", cosp["withdrawn_at"]) actions = make_node(root, "actions", None) for action in bill['actions']: a = make_node(actions, action['type'] if action['type'] in ("vote", "vote-aux", "calendar", "topresident", "signed", "enacted", "vetoed") else "action", None, datetime=action['acted_at']) if action.get("status"): a.set("state", action["status"]) if action['type'] in ('vote', 'vote-aux'): a.clear() # re-insert date between some of these attributes a.set("how", action["how"]) a.set("type", action["vote_type"]) if action.get("roll") != None: a.set("roll", action["roll"]) a.set("datetime", utils.format_datetime(action['acted_at'])) a.set("where", action["where"]) a.set("result", action["result"]) if action.get("suspension"): a.set("suspension", "1") if action.get("status"): a.set("state", action["status"]) if action['type'] == 'calendar' and "calendar" in action: a.set("calendar", action["calendar"]) if action["under"]: a.set("under", action["under"]) if action["number"]: a.set("number", action["number"]) if action['type'] == 'enacted': a.clear() # re-insert date between some of these attributes a.set("number", "%s-%s" % (bill['congress'], action["number"])) a.set("type", action["law"]) a.set("datetime", utils.format_datetime(action['acted_at'])) if action.get("status"): a.set("state", action["status"]) if action['type'] == 'vetoed': if action.get("pocket"): a.set("pocket", "1") if action.get('text'): make_node(a, "text", action['text']) if action.get('in_committee'): make_node(a, "committee", None, name=action['in_committee']) for cr in action['references']: make_node(a, "reference", None, ref=cr['reference'], label=cr['type']) committees = make_node(root, "committees", None) for cmt in bill['committees']: make_node(committees, "committee", None, code=(cmt["committee_id"] + cmt["subcommittee_id"]) if cmt.get("subcommittee_id", None) else cmt["committee_id"], name=cmt["committee"], subcommittee=cmt.get("subcommittee").replace("Subcommittee on ", "") if cmt.get("subcommittee") else "", activity=", ".join(c.title() for c in cmt["activity"])) relatedbills = make_node(root, "relatedbills", None) for rb in bill['related_bills']: if rb['type'] == "bill": rb_bill_type, rb_number, rb_congress = utils.split_bill_id(rb['bill_id']) make_node(relatedbills, "bill", None, session=rb_congress, type=govtrack_type_codes[rb_bill_type], number=rb_number, relation="unknown" if rb['reason'] == "related" else rb['reason']) subjects = make_node(root, "subjects", None) if bill['subjects_top_term']: make_node(subjects, "term", None, name=bill['subjects_top_term']) for s in bill['subjects']: if s != bill['subjects_top_term']: make_node(subjects, "term", None, name=s) amendments = make_node(root, "amendments", None) for amd in bill['amendments']: make_node(amendments, "amendment", None, number=amd["chamber"] + str(amd["number"])) if bill.get('summary'): make_node(root, "summary", bill['summary']['text'], date=bill['summary']['date'], status=bill['summary']['as']) committee_reports = make_node(root, "committee-reports", None) for report in bill['committee_reports']: make_node(committee_reports, "report", report) return etree.tostring(root, pretty_print=True)
def action_for(item): # acted_at if not item.get('actionTime'): acted_at = item.get('actionDate', '') else: # Although we get the action date & time in an ISO-ish format (split # across two fields), and although we know it's in local time at the # U.S. Capitol (i.e. U.S. Eastern), we don't know the UTC offset which # is a part of how we used to serialize the time. So parse and then # use pytz (via format_datetime) to re-serialize. acted_at = utils.format_datetime(datetime.datetime.strptime(item.get('actionDate', '') + " " + item['actionTime'], "%Y-%m-%d %H:%M:%S")) # text & references # (amendment actions don't always have text?) text = item['text'] if item['text'] is not None else '' # strip out links text = re.sub(r"</?[Aa]( \S.*?)?>", "", text) # remove and extract references references = [] match = re.search("\s*\(([^)]+)\)\s*$", text) if match: # remove the matched section text = text[0:match.start()] + text[match.end():] types = match.group(1) # fix use of comma or colon instead of a semi colon between reference types # have seen some accidental capitalization combined with accidental comma, thus the 'T' # e.g. "text of Title VII as reported in House: CR H3075-3077, Text omission from Title VII:" (hr5384-109) types = re.sub("[,:] ([a-zT])", r"; \1", types) # fix "CR:" types = re.sub("CR:", "CR", types) # fix a missing semicolon altogether between references # e.g. sres107-112, "consideration: CR S1877-1878 text as" types = re.sub("(\d+) +([a-z])", r"\1; \2", types) for reference in re.split("; ?", types): if ": " not in reference: type, reference = None, reference else: type, reference = reference.split(": ", 1) references.append({'type': type, 'reference': reference}) # form dict action_dict = { 'acted_at': acted_at, 'action_code': item.get('actionCode', ''), 'committees': [item['committee']['systemCode'][0:-2].upper()] if item['committee'] else None, 'references': references, 'type': 'action', # replaced by parse_bill_action if a regex matches 'text': text, } if not action_dict["committees"]: # remove if empty - not present in how we used to generate the file del action_dict["committees"] # sometimes there are links (one case is for bills passed by a rule in a resolution, the link will point to the resolution) if (item.get("links") or {}).get("link") is not None: action_dict["links"] = item["links"]["link"] return action_dict
def output_vote(vote, options): logging.info("[%s] Writing to disk..." % vote['vote_id']) # output JSON - so easy! utils.write( json.dumps(vote, sort_keys=True, indent=2, default=utils.format_datetime), output_for_vote(vote["vote_id"], "json"), ) # output XML root = etree.Element("roll") root.set("where", "house" if vote['chamber'] == "h" else "senate") root.set("session", str(vote["congress"])) root.set("year", str(vote["date"].year)) root.set("roll", str(vote["number"])) root.set("source", "house.gov" if vote["chamber"] == "h" else "senate.gov") root.set("datetime", utils.format_datetime(vote['date'])) root.set("updated", utils.format_datetime(vote['updated_at'])) def get_votes(option): return len(vote["votes"].get(option, [])) root.set("aye", str(get_votes("Yea") + get_votes("Aye"))) root.set("nay", str(get_votes("Nay") + get_votes("No"))) root.set("nv", str(get_votes("Not Voting"))) root.set("present", str(get_votes("Present"))) utils.make_node(root, "category", vote["category"]) utils.make_node(root, "type", vote["type"]) utils.make_node(root, "question", vote["question"]) utils.make_node(root, "required", vote["requires"]) utils.make_node(root, "result", vote["result"]) if "bill" in vote: govtrack_type_codes = { 'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc' } utils.make_node(root, "bill", None, session=str(vote["bill"]["congress"]), type=govtrack_type_codes[vote["bill"]["type"]], number=str(vote["bill"]["number"])) if "amendment" in vote: n = utils.make_node(root, "amendment", None) if vote["amendment"]["type"] == "s": n.set("ref", "regular") n.set("session", str(vote["congress"])) n.set("number", "s" + str(vote["amendment"]["number"])) elif vote["amendment"]["type"] == "h-bill": n.set("ref", "bill-serial") n.set("session", str(vote["congress"])) n.set("number", str(vote["amendment"]["number"])) # well-known keys for certain vote types: +/-/P/0 option_keys = { "Aye": "+", "Yea": "+", "Nay": "-", "No": "-", "Present": "P", "Not Voting": "0" } # preferred order of output: ayes, nays, present, then not voting, and similarly for guilty/not-guilty # and handling other options like people's names for votes for the Speaker. option_sort_order = ('Aye', 'Yea', 'Guilty', 'No', 'Nay', 'Not Guilty', 'OTHER', 'Present', 'Not Voting') options_list = sorted(vote["votes"].keys(), key = lambda o : option_sort_order.index(o) if o in option_sort_order else option_sort_order.index("OTHER") ) for option in options_list: if option not in option_keys: option_keys[option] = option utils.make_node(root, "option", option, key=option_keys[option]) for option in options_list: for v in vote["votes"][option]: n = utils.make_node(root, "voter", None) if v == "VP": n.set("id", "0") n.set("VP", "1") elif not options.get("govtrack", False): n.set("id", str(v["id"])) else: n.set("id", str(utils.get_govtrack_person_id("bioguide" if vote["chamber"] == "h" else "lis", v["id"]))) n.set("vote", option_keys[option]) n.set("value", option) if v != "VP": n.set("state", v["state"]) xmloutput = etree.tostring(root, pretty_print=True, encoding="utf8") # mimick two hard line breaks in GovTrack's legacy output to ease running diffs xmloutput = re.sub('(source=".*?") ', r"\1\n ", xmloutput) xmloutput = re.sub('(updated=".*?") ', r"\1\n ", xmloutput) utils.write( xmloutput, output_for_vote(vote['vote_id'], "xml") )
def test_format_datetime(self): import utils d = datetime.datetime(2013,11,22,10,9,8) self.assertEqual(utils.format_datetime(d), '11/22/2013 10:09 am')
def output_vote(vote, options, id_type=None): logging.info("[%s] Writing to disk..." % vote['vote_id']) # output JSON - so easy! utils.write(json.dumps(vote, sort_keys=True, indent=2, default=utils.format_datetime), output_for_vote(vote["vote_id"], "json"), options=options) # What kind of IDs are we passed for Members of Congress? # For current data, we infer from the chamber. For historical data from voteview, # we're passed the type in id_type, which is set to "bioguide". if not id_type: id_type = ("bioguide" if vote["chamber"] == "h" else "lis") # output XML root = etree.Element("roll") root.set("where", "house" if vote['chamber'] == "h" else "senate") root.set("session", str(vote["congress"])) root.set("year", str(vote["date"].year)) root.set("roll", str(vote["number"])) if "voteview" in vote["source_url"]: root.set("source", "keithpoole") else: root.set("source", "house.gov" if vote["chamber"] == "h" else "senate.gov") root.set("datetime", utils.format_datetime(vote['date'])) root.set("updated", utils.format_datetime(vote['updated_at'])) def get_votes(option): return len(vote["votes"].get(option, [])) root.set("aye", str(get_votes("Yea") + get_votes("Aye"))) root.set("nay", str(get_votes("Nay") + get_votes("No"))) root.set("nv", str(get_votes("Not Voting"))) root.set("present", str(get_votes("Present"))) utils.make_node(root, "category", vote["category"]) utils.make_node(root, "type", vote["type"]) utils.make_node(root, "question", vote["question"]) utils.make_node(root, "required", vote["requires"]) utils.make_node(root, "result", vote["result"]) if vote.get("bill"): govtrack_type_codes = { 'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc' } utils.make_node(root, "bill", None, session=str(vote["bill"]["congress"]), type=govtrack_type_codes[vote["bill"]["type"]], number=str(vote["bill"]["number"])) if "amendment" in vote: n = utils.make_node(root, "amendment", None) if vote["amendment"]["type"] == "s": n.set("ref", "regular") n.set("session", str(vote["congress"])) n.set("number", "s" + str(vote["amendment"]["number"])) elif vote["amendment"]["type"] == "h-bill": n.set("ref", "bill-serial") n.set("session", str(vote["congress"])) n.set("number", str(vote["amendment"]["number"])) # well-known keys for certain vote types: +/-/P/0 option_keys = { "Aye": "+", "Yea": "+", "Nay": "-", "No": "-", "Present": "P", "Not Voting": "0", "Guilty": "+", "Not Guilty": "-" } # preferred order of output: ayes, nays, present, then not voting, and similarly for guilty/not-guilty # and handling other options like people's names for votes for the Speaker. option_sort_order = ('Aye', 'Yea', 'Guilty', 'No', 'Nay', 'Not Guilty', 'OTHER', 'Present', 'Not Voting') options_list = sorted( vote["votes"].keys(), key=lambda o: option_sort_order.index(o) if o in option_sort_order else option_sort_order.index("OTHER")) for option in options_list: if option not in option_keys: option_keys[option] = option utils.make_node(root, "option", option, key=option_keys[option]) for option in options_list: for v in vote["votes"][option]: n = utils.make_node(root, "voter", None) if v == "VP": n.set("id", "0") n.set("VP", "1") elif not options.get("govtrack", False): n.set("id", str(v["id"])) else: n.set("id", str(utils.get_govtrack_person_id(id_type, v["id"]))) n.set("vote", option_keys[option]) n.set("value", option) if v != "VP": n.set("state", v["state"]) if v.get("voteview_votecode_extra") is not None: n.set("voteview_votecode_extra", v["voteview_votecode_extra"]) xmloutput = etree.tostring(root, pretty_print=True, encoding="utf8") # mimick two hard line breaks in GovTrack's legacy output to ease running diffs xmloutput = re.sub('(source=".*?") ', r"\1\n ", xmloutput) xmloutput = re.sub('(updated=".*?") ', r"\1\n ", xmloutput) utils.write(xmloutput, output_for_vote(vote['vote_id'], "xml"), options=options)
def test_format_datetime_with_sections(self): import utils d = datetime.datetime(2013,11,22,10,9,8) self.assertEqual(utils.format_datetime(d, with_seconds=1), '11/22/2013 10:09:08 am')
def output_vote(vote, options): logging.info("[%s] Writing to disk..." % vote["vote_id"]) # output JSON - so easy! utils.write( json.dumps(vote, sort_keys=True, indent=2, default=utils.format_datetime), output_for_vote(vote["vote_id"], "json"), ) # output XML root = etree.Element("roll") root.set("where", "house" if vote["chamber"] == "h" else "senate") root.set("session", str(vote["congress"])) root.set("year", str(vote["date"].year)) root.set("roll", str(vote["number"])) root.set("datetime", utils.format_datetime(vote["date"])) root.set("updated", utils.format_datetime(vote["updated_at"])) def get_votes(option): return len(vote["votes"].get(option, [])) root.set("aye", str(get_votes("Yea") + get_votes("Aye"))) root.set("nay", str(get_votes("Nay") + get_votes("No"))) root.set("present", str(get_votes("Present"))) root.set("nv", str(get_votes("Not Voting"))) root.set("source", "house.gov" if vote["chamber"] == "h" else "senate.gov") utils.make_node(root, "category", vote["category"]) utils.make_node(root, "type", vote["type"]) utils.make_node(root, "question", vote["question"]) utils.make_node(root, "required", vote["requires"]) utils.make_node(root, "result", vote["result"]) if "bill" in vote: govtrack_type_codes = { "hr": "h", "s": "s", "hres": "hr", "sres": "sr", "hjres": "hj", "sjres": "sj", "hconres": "hc", "sconres": "sc", } utils.make_node( root, "bill", None, session=str(vote["bill"]["congress"]), type=govtrack_type_codes[vote["bill"]["type"]], number=str(vote["bill"]["number"]), ) if "amendment" in vote: if vote["amendment"]["type"] == "s": utils.make_node(root, "amendment", None, ref="regular", number="s" + str(vote["amendment"]["number"])) elif vote["amendment"]["type"] == "h-bill": utils.make_node(root, "amendment", None, ref="bill-serial", number=str(vote["amendment"]["number"])) # well-known keys for certain vote types: +/-/P/0 option_keys = {"Aye": "+", "Yea": "+", "Nay": "-", "No": "-", "Present": "P", "Not Voting": "0"} # preferred order of output: ayes, nays, present, then not voting, and similarly for guilty/not-guilty # and handling other options like people's names for votes for the Speaker. option_sort_order = ("Aye", "Yea", "Guilty", "No", "Nay", "Not Guilty", "OTHER", "Present", "Not Voting") options_list = sorted( vote["votes"].keys(), key=lambda o: option_sort_order.index(o) if o in option_sort_order else option_sort_order.index("OTHER"), ) for option in options_list: if option not in option_keys: option_keys[option] = option utils.make_node(root, "option", option, key=option_keys[option]) for option in options_list: for v in vote["votes"][option]: attrs = {"vote": option_keys[option], "value": option} if v == "VP": attrs["id"] = "0" attrs["VP"] = "1" elif not options.get("govtrack", False): attrs["id"] = str(v["id"]) attrs["state"] = v["state"] else: attrs["id"] = str( utils.get_govtrack_person_id("bioguide" if vote["chamber"] == "h" else "lis", v["id"]) ) attrs["state"] = v["state"] utils.make_node(root, "voter", None, **attrs) utils.write(etree.tostring(root, pretty_print=True), output_for_vote(vote["vote_id"], "xml"))
def _set_filter_datetime(self, filter_key, value): if isinstance(value, datetime.datetime): self._filters[filter_key] = _utils.format_datetime(value) else: raise TypeError('Parameter %s must be a datetime.' % filter_key)
def test_format_datetime_ISO8601(self): import utils d = datetime.datetime(2013,11,22,10,9,8) self.assertEqual(utils.format_datetime(d, format='ISO8601'), '2013-11-22T10:09:08-05:00')
def output_amendment(amdt, options): logging.info("[%s] Writing to disk..." % amdt['amendment_id']) # output JSON - so easy! utils.write( json.dumps(amdt, sort_keys=True, indent=2, default=utils.format_datetime), output_for_amdt(amdt['amendment_id'], "json") ) # output XML govtrack_type_codes = { 'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc' } root = etree.Element("amendment") root.set("session", amdt['congress']) root.set("chamber", amdt['amendment_type'][0]) root.set("number", amdt['number']) root.set("updated", utils.format_datetime(amdt['updated_at'])) make_node = utils.make_node make_node(root, "amends", None, type=govtrack_type_codes[amdt["amends_bill"]["bill_type"]], number=str(amdt["amends_bill"]["number"]), sequence=str(int(amdt["house_number"][1:])) if amdt["house_number"] else "") # chop off A from the house_number make_node(root, "status", amdt['status'], datetime=amdt['status_at']) if amdt['sponsor'] and amdt['sponsor']['type'] == 'person': v = amdt['sponsor']['thomas_id'] if not options.get("govtrack", False): make_node(root, "sponsor", None, thomas_id=v) else: v = str(utils.get_govtrack_person_id('thomas', v)) make_node(root, "sponsor", None, id=v) elif amdt['sponsor'] and amdt['sponsor']['type'] == 'committee': make_node(root, "sponsor", None, committee=amdt['sponsor']['name']) else: make_node(root, "sponsor", None) make_node(root, "offered", None, datetime=amdt['offered_at'] if amdt['offered_at'] else amdt['submitted_at']) if amdt["title"]: make_node(root, "title", amdt["title"]) make_node(root, "description", amdt["description"] if amdt["description"] else amdt["purpose"]) make_node(root, "purpose", amdt["purpose"]) actions = make_node(root, "actions", None) for action in amdt['actions']: a = make_node(actions, action['type'] if action['type'] in ("vote",) else "action", None, datetime=action['acted_at']) if action['type'] == 'vote': a.set("how", action["how"]) a.set("result", action["result"]) if action.get("roll") != None: a.set("roll", str(action["roll"])) if action.get('text'): make_node(a, "text", action['text']) if action.get('in_committee'): make_node(a, "committee", None, name=action['in_committee']) for cr in action['references']: make_node(a, "reference", None, ref=cr['reference'], label=cr['type']) utils.write( etree.tostring(root, pretty_print=True), output_for_amdt(amdt['amendment_id'], "xml") )
def output_amendment(amdt, options): logging.info("[%s] Writing to disk..." % amdt['amendment_id']) # output JSON - so easy! utils.write( json.dumps(amdt, sort_keys=True, indent=2, default=utils.format_datetime), output_for_amdt(amdt['amendment_id'], "json")) # output XML govtrack_type_codes = { 'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc' } root = etree.Element("amendment") root.set("session", amdt['congress']) root.set("chamber", amdt['amendment_type'][0]) root.set("number", str(amdt['number'])) root.set("updated", utils.format_datetime(amdt['updated_at'])) make_node = utils.make_node if amdt.get("amends_bill", None): make_node(root, "amends", None, type=govtrack_type_codes[amdt["amends_bill"]["bill_type"]], number=str(amdt["amends_bill"]["number"]), sequence=str(amdt["house_number"]) if amdt.get( "house_number", None) else "") elif amdt.get("amends_treaty", None): make_node(root, "amends", None, type="treaty", number=str(amdt["amends_treaty"]["number"])) make_node(root, "status", amdt['status'], datetime=amdt['status_at']) if amdt['sponsor'] and amdt['sponsor']['type'] == 'person': v = amdt['sponsor']['thomas_id'] if not options.get("govtrack", False): make_node(root, "sponsor", None, thomas_id=v) else: v = str(utils.get_govtrack_person_id('thomas', v)) make_node(root, "sponsor", None, id=v) elif amdt['sponsor'] and amdt['sponsor']['type'] == 'committee': make_node(root, "sponsor", None, committee=amdt['sponsor']['name']) else: make_node(root, "sponsor", None) make_node(root, "offered", None, datetime=amdt['introduced_at']) make_node(root, "description", amdt["description"] if amdt["description"] else amdt["purpose"]) if amdt["description"]: make_node(root, "purpose", amdt["purpose"]) actions = make_node(root, "actions", None) for action in amdt['actions']: a = make_node( actions, action['type'] if action['type'] in ("vote", ) else "action", None, datetime=action['acted_at']) if action['type'] == 'vote': a.set("how", action["how"]) a.set("result", action["result"]) if action.get("roll") != None: a.set("roll", str(action["roll"])) if action.get('text'): make_node(a, "text", action['text']) if action.get('in_committee'): make_node(a, "committee", None, name=action['in_committee']) for cr in action['references']: make_node(a, "reference", None, ref=cr['reference'], label=cr['type']) utils.write(etree.tostring(root, pretty_print=True), output_for_amdt(amdt['amendment_id'], "xml"))
def output_vote(vote, options, id_type=None): logging.info("[%s] Writing to disk..." % vote["vote_id"]) # output JSON - so easy! utils.write( json.dumps(vote, sort_keys=True, indent=2, default=utils.format_datetime), output_for_vote(vote["vote_id"], "json"), options=options, ) # What kind of IDs are we passed for Members of Congress? # For current data, we infer from the chamber. For historical data from voteview, # we're passed the type in id_type, which is set to "bioguide". if not id_type: id_type = "bioguide" if vote["chamber"] == "h" else "lis" # output XML root = etree.Element("roll") root.set("where", "house" if vote["chamber"] == "h" else "senate") root.set("session", str(vote["congress"])) root.set("year", str(vote["date"].year)) root.set("roll", str(vote["number"])) if "voteview" in vote["source_url"]: root.set("source", "keithpoole") else: root.set("source", "house.gov" if vote["chamber"] == "h" else "senate.gov") root.set("datetime", utils.format_datetime(vote["date"])) root.set("updated", utils.format_datetime(vote["updated_at"])) def get_votes(option): return len(vote["votes"].get(option, [])) root.set("aye", str(get_votes("Yea") + get_votes("Aye"))) root.set("nay", str(get_votes("Nay") + get_votes("No"))) root.set("nv", str(get_votes("Not Voting"))) root.set("present", str(get_votes("Present"))) utils.make_node(root, "category", vote["category"]) utils.make_node(root, "type", vote["type"]) utils.make_node(root, "question", vote["question"]) utils.make_node(root, "required", vote["requires"]) utils.make_node(root, "result", vote["result"]) if vote.get("bill"): govtrack_type_codes = { "hr": "h", "s": "s", "hres": "hr", "sres": "sr", "hjres": "hj", "sjres": "sj", "hconres": "hc", "sconres": "sc", } utils.make_node( root, "bill", None, session=str(vote["bill"]["congress"]), type=govtrack_type_codes[vote["bill"]["type"]], number=str(vote["bill"]["number"]), ) if "amendment" in vote: n = utils.make_node(root, "amendment", None) if vote["amendment"]["type"] == "s": n.set("ref", "regular") n.set("session", str(vote["congress"])) n.set("number", "s" + str(vote["amendment"]["number"])) elif vote["amendment"]["type"] == "h-bill": n.set("ref", "bill-serial") n.set("session", str(vote["congress"])) n.set("number", str(vote["amendment"]["number"])) # well-known keys for certain vote types: +/-/P/0 option_keys = { "Aye": "+", "Yea": "+", "Nay": "-", "No": "-", "Present": "P", "Not Voting": "0", "Guilty": "+", "Not Guilty": "-", } # preferred order of output: ayes, nays, present, then not voting, and similarly for guilty/not-guilty # and handling other options like people's names for votes for the Speaker. option_sort_order = ("Aye", "Yea", "Guilty", "No", "Nay", "Not Guilty", "OTHER", "Present", "Not Voting") options_list = sorted( vote["votes"].keys(), key=lambda o: option_sort_order.index(o) if o in option_sort_order else option_sort_order.index("OTHER"), ) for option in options_list: if option not in option_keys: option_keys[option] = option utils.make_node(root, "option", option, key=option_keys[option]) for option in options_list: for v in vote["votes"][option]: n = utils.make_node(root, "voter", None) if v == "VP": n.set("id", "0") n.set("VP", "1") elif not options.get("govtrack", False): n.set("id", str(v["id"])) else: n.set("id", str(utils.get_govtrack_person_id(id_type, v["id"]))) n.set("vote", option_keys[option]) n.set("value", option) if v != "VP": n.set("state", v["state"]) if v.get("voteview_votecode_extra") is not None: n.set("voteview_votecode_extra", v["voteview_votecode_extra"]) xmloutput = etree.tostring(root, pretty_print=True, encoding="utf8") # mimick two hard line breaks in GovTrack's legacy output to ease running diffs xmloutput = re.sub('(source=".*?") ', r"\1\n ", xmloutput) xmloutput = re.sub('(updated=".*?") ', r"\1\n ", xmloutput) utils.write(xmloutput, output_for_vote(vote["vote_id"], "xml"), options=options)
def test_format_datetime_ISO8601_without_time(self): import utils d = datetime.datetime(2013,11,22) self.assertEqual(utils.format_datetime(d, format='ISO8601'), '2013-11-22T00:00:00-05:00')