def parse_case(case): url = case.get('url') name = combine_name(*reversed(case.get('name').split(', '))) updated = dateutil_parse(case.get('last_updated')) record = { 'uid': make_id('interpol', url.split('/')[-1]), 'source_url': url, 'name': name, 'summary': case.get('reason'), 'updated_at': updated.date().isoformat(), 'place_of_birth': case.get('place_of_birth'), 'gender': case.get('sex', '').lower(), 'first_name': case.get('forename'), 'last_name': case.get('present_family_name'), 'nationality': source.normalize_country(case.get('nationality')), 'identities': [], 'addresses': [], 'other_names': [] } record.update(SOURCE) birth = case.get('date_of_birth').split(' ')[0] try: dt = datetime.strptime(birth, '%Y').date().isoformat() record['date_of_birth'] = dt except Exception: try: dt = datetime.strptime(birth, '%d/%m/%Y').date().isoformat() record['date_of_birth'] = dt except Exception as ex: log.exception(ex) source.emit(record)
def parse_entry(doc, target, sanctions, places): node = target.find('./individual') if node is None: node = target.find('./entity') if node is None: node = target.find('./object') # TODO: build out support for these! return record = { 'uid': make_id('ch', 'seco', target.get('ssid')), 'type': node.tag, 'updated_at': doc.getroot().get('date'), 'program': sanctions.get(target.get('sanctions-set-id')), 'function': node.findtext('./other-information'), 'summary': node.findtext('./justification'), 'other_names': [], 'addresses': [], 'identities': [] } record.update(PUBLISHER) for ident in node.findall('./identity'): parse_identity(record, ident, places) # from pprint import pprint # pprint(record) source.emit(record)
def parse_case(case): url = case.get('url') name = combine_name(*reversed(case.get('name').split(', '))) updated = dateutil_parse(case.get('last_updated')) record = { 'uid': make_id('interpol', url.split('/')[-1]), 'source_url': url, 'name': name, 'summary': case.get('reason'), 'updated_at': updated.date().isoformat(), 'place_of_birth': case.get('place_of_birth'), 'gender': case.get('sex', '').lower(), 'first_name': case.get('forename'), 'last_name': case.get('present_family_name'), 'nationality': normalize_country(case.get('nationality')), 'identities': [], 'addresses': [], 'other_names': [] } record.update(SOURCE) birth = case.get('date_of_birth').split(' ')[0] try: dt = datetime.strptime(birth, '%Y').date().isoformat() record['date_of_birth'] = dt except Exception: try: dt = datetime.strptime(birth, '%d/%m/%Y').date().isoformat() record['date_of_birth'] = dt except Exception as ex: log.exception(ex) source.emit(record)
def parse_common(node, type_): program_ref = '%s (%s)' % (node.findtext('./UN_LIST_TYPE').strip(), node.findtext('./REFERENCE_NUMBER').strip()) record = { 'uid': make_id('un', 'sc', node.findtext('./DATAID')), 'type': type_, 'program': program_ref, 'summary': node.findtext('./COMMENTS1'), 'name': combine_name(node.findtext('./FIRST_NAME'), node.findtext('./SECOND_NAME'), node.findtext('./THIRD_NAME')), 'function': node.findtext('./DESIGNATION/VALUE'), 'updated_at': node.findtext('./LISTED_ON'), 'nationality': normalize_country(node.findtext('./NATIONALITY/VALUE')), 'other_names': [], 'addresses': [], 'identities': [] } record.update(BASE) orig = node.findtext('./NAME_ORIGINAL_SCRIPT') if orig is not None: record['name'] = orig last_updated = node.findtext('./LAST_DAY_UPDATED/VALUE') if last_updated is not None: record['updated_at'] = last_updated if ':' in record['updated_at']: record['updated_at'] = record['updated_at'].rsplit('-', 1)[0] # print etree.tostring(node, pretty_print=True) return record
def worldleaders_parse(json_file): with open(json_file, 'r') as fh: data = json.load(fh) for leader in data.get('leaders'): if not len(leader.get('name')): log.warning('No name on entity: %r', leader) continue country = source.normalize_country(leader.get('country')) summary = leader.get('title') if leader.get('component'): summary = '%s (%s)' % (summary, leader.get('component')) entity = { 'uid': make_id('us', 'cia', 'worldleaders', country, leader.get('name')), 'name': leader.get('name'), 'nationality': country, 'type': 'individual', 'summary': summary, 'updated_at': parse_date(leader.get('last_update')), 'source_url': leader.get('url') } entity.update(PUBLISHER) source.emit(entity)
def parse_politician(source, country, legislature, data): # from pprint import pprint # pprint(policitian) # TODO: add politician code = normalize_country(country.get('code')) entity = { 'uid': make_id('evpo', data.get('id').split('-')[-1]), 'name': data.get('name'), 'type': 'individual', 'addresses': [{'country': code}], 'updated_at': parse_ts(legislature.get('lastmod')), 'source_url': data.get('source_url'), 'source': '%s (%s)' % (legislature['name'], country['name']) } entity.update(PUBLISHER) source.emit(entity)
def wbdeb_parse(html_file): doc = html.parse(html_file) for table in doc.findall('//table'): if 'List of Debarred' not in table.get('summary', ''): continue rows = table.findall('.//tr') print table.get('summary'), len(rows) for row in rows: tds = row.findall('./td') if len(tds) != 6: continue values = [clean_value(td) for td in tds] uid = sha1() for value in values: uid.update(value.encode('utf-8')) uid = uid.hexdigest()[:10] names = clean_name(values[0]) if not len(names): log.warning("No name: %r", values) continue record = { 'uid': make_id('wb', 'debarred', uid), 'name': names[0], 'nationality': source.normalize_country(values[2]), 'program': values[5], 'addresses': [{ 'text': values[1], 'country': source.normalize_country(values[2]) }], 'other_names': [], 'updated_at': dateutil_parse(values[3]).date().isoformat() } for name in names[1:]: record['other_names'].append({'other_name': name}) record.update(SOURCE) source.emit(record)
def parse_row(row): record = SOURCE.copy() record.update({ 'uid': make_id('us', 'bis', row.get('Effective_Date'), row.get('Name')), 'name': row.get('Name'), 'program': row.get('FR_Citation'), 'summary': row.get('Action'), 'updated_at': row.get('Last Update'), 'nationality': normalize_country(row.get('Country')), 'addresses': [{ 'address1': row.get('Street_Address'), 'postal_code': row.get('Postal_Code'), 'region': row.get('State'), 'city': row.get('City'), 'country': normalize_country(row.get('Country')) }] }) source.emit(record)
def parse_politician(source, country, legislature, data): # from pprint import pprint # pprint(policitian) # TODO: add politician code = source.normalize_country(country.get('code')) entity = { 'uid': make_id('evpo', data.get('id').split('-')[-1]), 'name': data.get('name'), 'type': 'individual', 'addresses': [{ 'country': code }], 'updated_at': parse_ts(legislature.get('lastmod')), 'source_url': data.get('source_url'), 'source': '%s (%s)' % (legislature['name'], country['name']) } entity.update(PUBLISHER) source.emit(entity)
def wbdeb_parse(html_file): doc = html.parse(html_file) for table in doc.findall('//table'): if 'List of Debarred' not in table.get('summary', ''): continue rows = table.findall('.//tr') print table.get('summary'), len(rows) for row in rows: tds = row.findall('./td') if len(tds) != 6: continue values = [clean_value(td) for td in tds] uid = sha1() for value in values: uid.update(value.encode('utf-8')) uid = uid.hexdigest()[:10] names = clean_name(values[0]) if not len(names): log.warning("No name: %r", values) continue record = { 'uid': make_id('wb', 'debarred', uid), 'name': names[0], 'nationality': normalize_country(values[2]), 'program': values[5], 'addresses': [{ 'text': values[1], 'country': normalize_country(values[2]) }], 'other_names': [], 'updated_at': dateutil_parse(values[3]).date().isoformat() } for name in names[1:]: record['other_names'].append({ 'other_name': name }) record.update(SOURCE) source.emit(record)
def parse_common(node, type_): program_ref = '%s (%s)' % (node.findtext('./UN_LIST_TYPE').strip(), node.findtext('./REFERENCE_NUMBER').strip()) record = { 'uid': make_id('un', 'sc', node.findtext('./DATAID')), 'type': type_, 'program': program_ref, 'summary': node.findtext('./COMMENTS1'), 'name': combine_name(node.findtext('./FIRST_NAME'), node.findtext('./SECOND_NAME'), node.findtext('./THIRD_NAME')), 'function': node.findtext('./DESIGNATION/VALUE'), 'updated_at': node.findtext('./LISTED_ON'), 'nationality': source.normalize_country(node.findtext('./NATIONALITY/VALUE')), 'other_names': [], 'addresses': [], 'identities': [] } record.update(BASE) orig = node.findtext('./NAME_ORIGINAL_SCRIPT') if orig is not None: record['name'] = orig last_updated = node.findtext('./LAST_DAY_UPDATED/VALUE') if last_updated is not None: record['updated_at'] = last_updated if ':' in record['updated_at']: record['updated_at'] = record['updated_at'].rsplit('-', 1)[0] # print etree.tostring(node, pretty_print=True) return record
def worldleaders_parse(json_file): with open(json_file, 'r') as fh: data = json.load(fh) for leader in data.get('leaders'): if not len(leader.get('name')): log.warning('No name on entity: %(title)s (%(country)s)', leader) continue country = normalize_country(leader.get('country')) summary = leader.get('title') if leader.get('component'): summary = '%s (%s)' % (summary, leader.get('component')) entity = { 'uid': make_id('us', 'cia', 'worldleaders', country, leader.get('name')), 'name': leader.get('name'), 'type': 'individual', 'summary': summary, 'addresses': [{'country': country}], 'updated_at': parse_date(leader.get('last_update')), 'source_url': leader.get('url') } entity.update(PUBLISHER) source.emit(entity)
def parse_row(row): record = SOURCE.copy() record.update({ 'uid': make_id('us', 'bis', row.get('Effective_Date'), row.get('Name')), 'name': row.get('Name'), 'program': row.get('FR_Citation'), 'summary': row.get('Action'), 'updated_at': row.get('Last Update'), 'nationality': source.normalize_country(row.get('Country')), 'addresses': [{ 'address1': row.get('Street_Address'), 'postal_code': row.get('Postal_Code'), 'region': row.get('State'), 'city': row.get('City'), 'country': source.normalize_country(row.get('Country')) }] }) source.emit(record)
def parse_entry(source, record, entry): uid = entry.findtext('uid') record.update({ 'uid': make_id('us', 'ofac', uid), 'type': 'individual', 'program': entry.findtext('./programList/program'), 'summary': entry.findtext('./remarks'), 'first_name': entry.findtext('./firstName'), 'last_name': entry.findtext('./lastName'), 'name': combine_name(entry.findtext('./firstName'), entry.findtext('./lastName')) }) is_entity = entry.findtext('./sdnType') != 'Individual' if is_entity: record['type'] = 'entity' record.pop('last_name', None) record['other_names'] = [] for aka in entry.findall('./akaList/aka'): data = { 'type': aka.findtext('./type'), 'quality': aka.findtext('./category'), 'first_name': aka.findtext('./firstName'), 'last_name': aka.findtext('./lastName'), 'other_name': combine_name(aka.findtext('./firstName'), aka.findtext('./lastName')) } if is_entity: data.pop('last_name', None) record['other_names'].append(data) record['identities'] = [] for ident in entry.findall('./idList/id'): data = { 'type': ident.findtext('./idType'), 'number': ident.findtext('./idNumber'), 'country': normalize_country(ident.findtext('./idCountry')) } record['identities'].append(data) record['addresses'] = [] for address in entry.findall('./addressList/address'): data = { 'address1': address.findtext('./address1'), 'address2': address.findtext('./address2'), 'city': address.findtext('./city'), 'country': normalize_country(address.findtext('./country')) } record['addresses'].append(data) for pob in entry.findall('./placeOfBirthList/placeOfBirthItem'): if pob.findtext('./mainEntry') == 'true': record['place_of_birth'] = pob.findtext('./placeOfBirth') for pob in entry.findall('./dateOfBirthList/dateOfBirthItem'): if pob.findtext('./mainEntry') == 'true': dt = pob.findtext('./dateOfBirth') record['date_of_birth'] = parse_date(dt) # print etree.tostring(entry, pretty_print=True) if is_entity: record.pop('last_name', None) source.emit(record)
def parse_entry(group, rows): record = SOURCE.copy() record.update({ 'uid': make_id('gb', 'hmt', group), 'identities': [], 'addresses': [], 'other_names': [] }) for row in rows: record.update({ 'type': row.pop('Group Type').lower(), 'date_of_birth': parse_date(row.pop('DOB')), 'place_of_birth': row.pop('Town of Birth'), 'country_of_birth': source.normalize_country(row.pop('Country of Birth')), 'nationality': source.normalize_country(row.get('Nationality')), 'program': row.pop('Regime'), 'summary': row.pop('Other Information'), 'updated_at': parse_date(row.pop('Last Updated')), 'function': row.pop('Position') }) names = { 'first_name': row.get('Name 1'), 'second_name': row.get('Name 2'), 'middle_name': row.get('Name 3'), 'last_name': row.get('Name 6') } name = [ row.pop('Title'), row.pop('Name 1'), row.pop('Name 2'), row.pop('Name 3'), row.pop('Name 4'), row.pop('Name 5'), row.pop('Name 6') ] name = combine_name(*name) if 'name' not in record: record['name'] = name record.update(names) else: names['other_name'] = name names['type'] = row.pop('Alias Type') record['other_names'].append(names) addr = [ row.pop('Address 1'), row.pop('Address 2'), row.pop('Address 3'), row.pop('Address 4'), row.pop('Address 5'), row.pop('Address 6') ] addr = combine_name(*addr) if len(addr): record['addresses'].append({ 'text': addr, 'postal_code': row.pop('Post/Zip Code') }) if row.get('Passport Details'): record['identities'].append({ 'type': 'Passport', 'number': row.pop('Passport Details'), 'country': source.normalize_country(row.get('Nationality')) }) if row.get('NI Number'): record['identities'].append({ 'type': 'NI', 'number': row.pop('NI Number'), 'country': source.normalize_country(row.get('Country')) }) # from pprint import pprint # pprint(row) source.emit(record)
def parse_entry(record, entry): uid = entry.get('Id') record.update({ 'uid': make_id('eu', 'eeas', uid), 'type': 'individual', 'updated_at': entry.get('reg_date'), 'source_url': entry.get('pdf_link'), 'program': entry.get('programme'), 'summary': entry.get('remark') }) is_entity = entry.get('Type') != 'P' if is_entity: record['type'] = 'entity' record['other_names'] = [] for aka in entry.findall('./NAME'): data = { 'first_name': aka.findtext('./FIRSTNAME'), 'last_name': aka.findtext('./LASTNAME'), 'middle_name': aka.findtext('./MIDDLENAME'), 'other_name': aka.findtext('./WHOLENAME') } funct = aka.findtext('./FUNCTION') if funct and len(funct) > len(record.get('function', '')): record['function'] = funct gender = aka.findtext('./GENDER') if gender == 'M': data['gender'] = 'male' if gender == 'F': data['gender'] = 'female' if 'name' not in record: record['name'] = data.pop('other_name') record.update(data) else: record['other_names'].append(data) record['identities'] = [] for passport in entry.findall('./PASSPORT'): data = { 'type': 'Passport', 'number': passport.findtext('./NUMBER'), 'country': normalize_country(passport.findtext('./COUNTRY')) } record['identities'].append(data) record['addresses'] = [] for address in entry.findall('./ADDRESS'): data = { 'address1': address.findtext('./STREET'), 'address2': address.findtext('./NUMBER'), 'city': address.findtext('./CITY'), 'postal_code': address.findtext('./ZIPCODE'), 'country': normalize_country(address.findtext('./COUNTRY')) } record['addresses'].append(data) for birth in entry.findall('./BIRTH'): place = birth.findtext('./PLACE') if place and len(place) > len(record.get('place_of_birth', '')): record['place_of_birth'] = place date_ = parse_date(birth.findtext('./DATE')) if date_ and len(date_) > len(record.get('date_of_birth', '')): record['date_of_birth'] = date_ country = normalize_country(birth.findtext('./COUNTRY')) if country and len(country) > len(record.get('country_of_birth', '')): record['country_of_birth'] = country # print etree.tostring(entry, pretty_print=True) source.emit(record)
def parse_entry(record, entry): uid = entry.get('Id') record.update({ 'uid': make_id('eu', 'eeas', uid), 'type': 'individual', 'updated_at': entry.get('reg_date'), 'source_url': entry.get('pdf_link'), 'program': entry.get('programme'), 'summary': entry.get('remark') }) is_entity = entry.get('Type') != 'P' if is_entity: record['type'] = 'entity' record['other_names'] = [] for aka in entry.findall('./NAME'): data = { 'first_name': aka.findtext('./FIRSTNAME'), 'last_name': aka.findtext('./LASTNAME'), 'middle_name': aka.findtext('./MIDDLENAME'), 'other_name': aka.findtext('./WHOLENAME') } funct = aka.findtext('./FUNCTION') if funct and len(funct) > len(record.get('function', '')): record['function'] = funct gender = aka.findtext('./GENDER') if gender == 'M': data['gender'] = 'male' if gender == 'F': data['gender'] = 'female' if 'name' not in record: record['name'] = data.pop('other_name') record.update(data) else: record['other_names'].append(data) record['identities'] = [] for passport in entry.findall('./PASSPORT'): data = { 'type': 'Passport', 'number': passport.findtext('./NUMBER'), 'country': source.normalize_country(passport.findtext('./COUNTRY')) } record['identities'].append(data) record['addresses'] = [] for address in entry.findall('./ADDRESS'): data = { 'address1': address.findtext('./STREET'), 'address2': address.findtext('./NUMBER'), 'city': address.findtext('./CITY'), 'postal_code': address.findtext('./ZIPCODE'), 'country': source.normalize_country(address.findtext('./COUNTRY')) } record['addresses'].append(data) for birth in entry.findall('./BIRTH'): place = birth.findtext('./PLACE') if place and len(place) > len(record.get('place_of_birth', '')): record['place_of_birth'] = place date_ = parse_date(birth.findtext('./DATE')) if date_ and len(date_) > len(record.get('date_of_birth', '')): record['date_of_birth'] = date_ country = source.normalize_country(birth.findtext('./COUNTRY')) if country and len(country) > len(record.get('country_of_birth', '')): record['country_of_birth'] = country # print etree.tostring(entry, pretty_print=True) source.emit(record)
def parse_entry(entry): uid = entry.findtext('number-entry') record = { 'uid': make_id('ua', 'sdfm', uid), 'type': 'individual', 'publisher': 'Ukraine SDFM', 'publisher_url': 'http://www.sdfm.gov.ua/', 'source': 'Blacklist', 'source_id': 'UA-SDFM-SANC', 'source_url': 'http://www.sdfm.gov.ua/articles.php?cat_id=87&lang=en', 'program': entry.findtext('./program-entry'), 'summary': entry.findtext('./comments') } date_entry = entry.findtext('./date-entry') if date_entry: date_entry = datetime.strptime(date_entry, '%Y%m%d') record['updated_at'] = date_entry.date().isoformat() is_entity = entry.findtext('./type-entry') != '1' if is_entity: record['type'] = 'entity' record['other_names'] = [] for aka in entry.findall('./aka-list'): data = get_names(aka) if aka.findtext('type-aka') == 'N': record['name'] = data.pop('other_name', None) record.update(data) else: data['type'] = aka.findtext('./type-aka') data['quality'] = aka.findtext('./quality-aka') if data['quality'] == '1': data['quality'] = 'strong' if data['quality'] == '2': data['quality'] = 'weak' # if is_entity: # data.pop('last_name', None) record['other_names'].append(data) record['identities'] = [] for doc in entry.findall('./document-list'): data = { 'text': doc.findtext('./document-reg'), 'number': doc.findtext('./document-id'), 'country': normalize_country(doc.findtext('./document-country')) } record['identities'].append(data) for doc in entry.findall('./id-number-list'): data = {'text': doc.text.strip()} record['identities'].append(data) record['addresses'] = [] for address in entry.findall('./address-list'): data = { 'text': address.findtext('./address') } record['addresses'].append(data) # FIXME: handle multiple for pob in entry.findall('./place-of-birth-list'): if 'place_of_birth' in record: log.debug('Multiple places of birth: %(name)s', record) record['place_of_birth'] = pob.text.strip() # FIXME: handle multiple for dob in entry.findall('./date-of-birth-list'): if 'place_of_birth' in record: log.debug('Multiple dates of birth: %(name)s', record) record['date_of_birth'] = parse_date(dob.text) # print etree.tostring(entry, pretty_print=True) source.emit(record)
def parse_entry(group, rows): record = SOURCE.copy() record.update({"uid": make_id("gb", "hmt", group), "identities": [], "addresses": [], "other_names": []}) for row in rows: record.update( { "type": row.pop("Group Type").lower(), "date_of_birth": parse_date(row.pop("DOB")), "place_of_birth": row.pop("Town of Birth"), "country_of_birth": normalize_country(row.pop("Country of Birth")), "nationality": normalize_country(row.get("Nationality")), "program": row.pop("Regime"), "summary": row.pop("Other Information"), "updated_at": parse_date(row.pop("Last Updated")), "function": row.pop("Position"), } ) names = { "first_name": row.get("Name 1"), "second_name": row.get("Name 2"), "middle_name": row.get("Name 3"), "last_name": row.get("Name 6"), } name = [ row.pop("Title"), row.pop("Name 1"), row.pop("Name 2"), row.pop("Name 3"), row.pop("Name 4"), row.pop("Name 5"), row.pop("Name 6"), ] name = combine_name(*name) if "name" not in record: record["name"] = name record.update(names) else: names["other_name"] = name names["type"] = row.pop("Alias Type") record["other_names"].append(names) addr = [ row.pop("Address 1"), row.pop("Address 2"), row.pop("Address 3"), row.pop("Address 4"), row.pop("Address 5"), row.pop("Address 6"), ] addr = combine_name(*addr) if len(addr): record["addresses"].append({"text": addr, "postal_code": row.pop("Post/Zip Code")}) if row.get("Passport Details"): record["identities"].append( { "type": "Passport", "number": row.pop("Passport Details"), "country": normalize_country(row.get("Nationality")), } ) if row.get("NI Number"): record["identities"].append( {"type": "NI", "number": row.pop("NI Number"), "country": normalize_country(row.get("Country"))} ) # from pprint import pprint # pprint(row) source.emit(record)
def parse_entry(source, record, entry): uid = entry.findtext('uid') record.update({ 'uid': make_id('us', 'ofac', uid), 'type': 'individual', 'program': entry.findtext('./programList/program'), 'summary': entry.findtext('./remarks'), 'first_name': entry.findtext('./firstName'), 'last_name': entry.findtext('./lastName'), 'name': combine_name(entry.findtext('./firstName'), entry.findtext('./lastName')) }) is_entity = entry.findtext('./sdnType') != 'Individual' if is_entity: record['type'] = 'entity' record.pop('last_name', None) record['other_names'] = [] for aka in entry.findall('./akaList/aka'): data = { 'type': aka.findtext('./type'), 'quality': aka.findtext('./category'), 'first_name': aka.findtext('./firstName'), 'last_name': aka.findtext('./lastName'), 'other_name': combine_name(aka.findtext('./firstName'), aka.findtext('./lastName')) } if is_entity: data.pop('last_name', None) record['other_names'].append(data) record['identities'] = [] for ident in entry.findall('./idList/id'): data = { 'type': ident.findtext('./idType'), 'number': ident.findtext('./idNumber'), 'country': source.normalize_country(ident.findtext('./idCountry')) } record['identities'].append(data) record['addresses'] = [] for address in entry.findall('./addressList/address'): data = { 'address1': address.findtext('./address1'), 'address2': address.findtext('./address2'), 'city': address.findtext('./city'), 'country': source.normalize_country(address.findtext('./country')) } record['addresses'].append(data) for pob in entry.findall('./placeOfBirthList/placeOfBirthItem'): if pob.findtext('./mainEntry') == 'true': record['place_of_birth'] = pob.findtext('./placeOfBirth') for pob in entry.findall('./dateOfBirthList/dateOfBirthItem'): if pob.findtext('./mainEntry') == 'true': dt = pob.findtext('./dateOfBirth') record['date_of_birth'] = parse_date(dt) # print etree.tostring(entry, pretty_print=True) if is_entity: record.pop('last_name', None) source.emit(record)
def parse_entry(entry): uid = entry.findtext('number-entry') record = { 'uid': make_id('ua', 'sdfm', uid), 'type': 'individual', 'publisher': 'Ukraine SDFM', 'publisher_url': 'http://www.sdfm.gov.ua/', 'source': 'Blacklist', 'source_id': 'UA-SDFM-SANC', 'source_url': 'http://www.sdfm.gov.ua/articles.php?cat_id=87&lang=en', 'program': entry.findtext('./program-entry'), 'summary': entry.findtext('./comments') } date_entry = entry.findtext('./date-entry') if date_entry: date_entry = datetime.strptime(date_entry, '%Y%m%d') record['updated_at'] = date_entry.date().isoformat() if entry.findtext('./type-entry') == '1': record['type'] = 'entity' record['other_names'] = [] for aka in entry.findall('./aka-list'): data = get_names(aka) if aka.findtext('type-aka') == 'N': record['name'] = data.pop('other_name', None) record.update(data) else: data['type'] = aka.findtext('./type-aka') data['quality'] = aka.findtext('./quality-aka') if data['quality'] == '1': data['quality'] = 'strong' if data['quality'] == '2': data['quality'] = 'weak' # if is_entity: # data.pop('last_name', None) record['other_names'].append(data) record['identities'] = [] for doc in entry.findall('./document-list'): data = { 'text': doc.findtext('./document-reg'), 'number': doc.findtext('./document-id'), 'country': source.normalize_country(doc.findtext('./document-country')) } record['identities'].append(data) for doc in entry.findall('./id-number-list'): data = {'text': doc.text.strip()} record['identities'].append(data) record['addresses'] = [] for address in entry.findall('./address-list'): data = {'text': address.findtext('./address')} record['addresses'].append(data) # FIXME: handle multiple for pob in entry.findall('./place-of-birth-list'): if 'place_of_birth' in record: log.debug('Multiple places of birth: %(name)s', record) record['place_of_birth'] = pob.text.strip() # FIXME: handle multiple for dob in entry.findall('./date-of-birth-list'): if 'place_of_birth' in record: log.debug('Multiple dates of birth: %(name)s', record) record['date_of_birth'] = parse_date(dob.text) # print etree.tostring(entry, pretty_print=True) source.emit(record)