def parse(context, data): emitter = EntityEmitter(context) entity = emitter.make("LegalEntity") name = data.get("SUPP_NAME") ent_id = data.get("SUPP_ID") city = data.get("SUPP_CITY") address = data.get("SUPP_ADDR") start_date = clean_date(data.get("DEBAR_FROM_DATE")) entity.make_id("WBDEBAR", name, ent_id) names = clean_name(name) entity.add("name", names[0]) entity.add("address", address) entity.add("address", city) entity.add("country", data.get("COUNTRY_NAME")) for name in names[1:]: entity.add("alias", name) sanction = emitter.make("Sanction") sanction.make_id("Sanction", entity.id) sanction.add("authority", "World Bank Debarrment") sanction.add("program", data.get("DEBAR_REASON")) sanction.add("startDate", start_date) sanction.add("endDate", clean_date(data.get("DEBAR_TO_DATE"))) sanction.add("sourceUrl", SOURCE) emitter.emit(entity) emitter.emit(sanction) emitter.finalize()
def parse(context, data): emitter = EntityEmitter(context) url = data.get('url') country = normalize_country(data.get('country')) with context.http.rehash(data) as res: doc = res.html # updated_at = doc.findtext('.//span[@id="lastUpdateDate"]') output = doc.find('.//div[@id="countryOutput"]') if output is None: return # component = None for row in output.findall('.//li'): # next_comp = row.findtext('./td[@class="componentName"]/strong') # if next_comp is not None: # component = next_comp # continue function = element_text(row.find('.//span[@class="title"]')) if function is None: continue name = element_text(row.find('.//span[@class="cos_name"]')) if name is None: continue person = emitter.make('Person') person.make_id(country, name, function) person.add('name', name) person.add('country', country) person.add('position', function) person.add('sourceUrl', url) emitter.emit(person) emitter.finalize()
def parse_notice(context, data): with context.http.rehash(data) as res: res = res.json first_name = res['forename'] or '' last_name = res['name'] or '' dob = res['date_of_birth'] nationalities = res['nationalities'] place_of_birth = res['place_of_birth'] warrants = [ (warrant['charge'], warrant['issuing_country_id']) for warrant in res['arrest_warrants'] # noqa ] gender = SEXES.get(res['sex_id']) emitter = EntityEmitter(context) entity = emitter.make('Person') entity.make_id(first_name, last_name, res['entity_id']) entity.add('name', first_name + ' ' + last_name) entity.add('firstName', first_name) entity.add('lastName', last_name) entity.add('nationality', nationalities) for charge, country in warrants: entity.add('program', country) entity.add('summary', charge) entity.add('gender', gender) entity.add('birthPlace', place_of_birth) entity.add('birthDate', parse_date(dob)) entity.add('sourceUrl', res['_links']['self']['href']) entity.add('keywords', 'REDNOTICE') entity.add('keywords', 'CRIME') emitter.emit(entity) emitter.finalize()
def crawl_country(context, path, country): emitter = EntityEmitter(context) source_url = UI_URL % path context.log.info("Crawling country: %s", country) res = requests.get(DATA_URL % path) data = res.json().get("result", {}).get("data", {}).get("page", {}) blocks = data.get("acf", {}).get("blocks", [{}])[0] content = blocks.get("free_form_content", []).get("content") doc = html.fromstring(content) function = None for el in doc.getchildren(): text = el.text_content().strip() if el.tag == "h2": continue if el.tag == "h3": function = text continue name = text.replace("(Acting)", "") person = emitter.make("Person") person.make_id(source_url, name, function) person.add("name", name) person.add("country", country) person.add("position", function) person.add("sourceUrl", source_url) person.add("topics", "role.pep") # pprint(person.to_dict()) emitter.emit(person) emitter.finalize()
def parse_notice(context, data): with context.http.rehash(data) as res: res = res.json first_name = res["forename"] or "" last_name = res["name"] or "" dob = res["date_of_birth"] nationalities = res["nationalities"] place_of_birth = res["place_of_birth"] warrants = [ (warrant["charge"], warrant["issuing_country_id"]) for warrant in res["arrest_warrants"] # noqa ] gender = SEXES.get(res["sex_id"]) emitter = EntityEmitter(context) entity = emitter.make("Person") entity.make_id("INTERPOL", first_name, last_name, res["entity_id"]) entity.add("name", first_name + " " + last_name) entity.add("firstName", first_name) entity.add("lastName", last_name) entity.add("nationality", nationalities) for charge, country in warrants: entity.add("program", country) entity.add("summary", charge) entity.add("gender", gender) entity.add("birthPlace", place_of_birth) entity.add("birthDate", parse_date(dob)) entity.add("sourceUrl", res["_links"]["self"]["href"]) entity.add("keywords", "REDNOTICE") entity.add("topics", "crime") emitter.emit(entity) emitter.finalize()
def parse(context, data): emitter = EntityEmitter(context) entity = emitter.make('LegalEntity') name = data.get('SUPP_NAME') ent_id = data.get('SUPP_ID') reason = data.get('DEBAR_REASON') country = data.get('COUNTRY_NAME') city = data.get('SUPP_CITY') address = data.get('SUPP_ADDR') start_date = data.get('DEBAR_FROM_DATE') end_date = data.get('DEBAR_TO_DATE') entity.make_id(name, ent_id, country) names = clean_name(name) entity.add('name', names[0]) entity.add('address', address) entity.add('address', city) entity.add('country', normalize_country(country)) for name in names[1:]: entity.add('alias', name) sanction = emitter.make('Sanction') sanction.make_id('Sanction', entity.id) sanction.add('authority', 'World Bank Debarrment') sanction.add('program', reason) sanction.add('startDate', clean_date(start_date)) sanction.add('endDate', clean_date(end_date)) sanction.add('sourceUrl', SOURCE) emitter.emit(entity) emitter.emit(sanction) emitter.finalize()
def officer(context, data): emitter = EntityEmitter(context) officer_id = data.get("officer_id") url = API_URL % officer_id with context.http.get(url, auth=AUTH) as res: if res.status_code != 200: context.log.info("CoH error: %r", res.json) return data = res.json # pprint(data) person = emitter.make("Person") person.make_id(officer_id) source_url = urljoin(WEB_URL, data.get("links", {}).get("self", "/")) person.add("sourceUrl", source_url) last_name = data.pop("surname", None) person.add("lastName", last_name) forename = data.pop("forename", None) person.add("firstName", forename) other_forenames = data.pop("other_forenames", None) person.add("middleName", other_forenames) person.add("name", jointext(forename, other_forenames, last_name)) person.add("title", data.pop("title", None)) person.add("nationality", data.pop("nationality", None)) person.add("birthDate", data.pop("date_of_birth", None)) person.add("topics", "crime") for disqual in data.pop("disqualifications", []): case = disqual.get("case_identifier") sanction = emitter.make("Sanction") sanction.make_id(person.id, case) sanction.add("entity", person) sanction.add("authority", "UK Companies House") sanction.add("program", case) from_date = disqual.pop("disqualified_from", None) person.context["created_at"] = from_date sanction.add("startDate", from_date) sanction.add("endDate", disqual.pop("disqualified_until", None)) emitter.emit(sanction) address = disqual.pop("address", {}) locality = address.get("locality") locality = jointext(locality, address.get("postal_code")) street = address.get("address_line_1") premises = address.get("premises") street = jointext(street, premises) address = jointext( street, address.get("address_line_2"), locality, address.get("region"), sep=", ", ) person.add("address", address) emitter.emit(person)
def store(context, data): emitter = EntityEmitter(context) entity = model.get_proxy(data["entity"]) documentation = emitter.make("Documentation") documentation.make_id("Documentation", data["entity"]["id"], data["aleph_id"]) documentation.add("entity", entity) documentation.add("document", data["aleph_id"]) emitter.emit(documentation) emitter.finalize()
def officer(context, data): emitter = EntityEmitter(context) officer_id = data.get('officer_id') url = API_URL % officer_id with context.http.get(url, auth=AUTH) as res: if res.status_code != 200: return data = res.json person = emitter.make('Person') person.make_id(officer_id) source_url = urljoin(WEB_URL, data.get('links', {}).get('self', '/')) person.add('sourceUrl', source_url) last_name = data.pop('surname', None) person.add('lastName', last_name) forename = data.pop('forename', None) person.add('firstName', forename) other_forenames = data.pop('other_forenames', None) person.add('middleName', other_forenames) person.add('name', jointext(forename, other_forenames, last_name)) person.add('title', data.pop('title', None)) nationality = normalize_country(data.pop('nationality', None)) person.add('nationality', nationality) person.add('birthDate', data.pop('date_of_birth', None)) for disqual in data.pop('disqualifications', []): case = disqual.get('case_identifier') sanction = emitter.make('Sanction') sanction.make_id(person.id, case) sanction.add('entity', person) sanction.add('authority', 'UK Companies House') sanction.add('program', case) sanction.add('startDate', disqual.pop('disqualified_from', None)) sanction.add('endDate', disqual.pop('disqualified_until', None)) emitter.emit(sanction) address = disqual.pop('address', {}) locality = address.get('locality') locality = jointext(locality, address.get('postal_code')) street = address.get('address_line_1') premises = address.get('premises') street = jointext(street, premises) address = jointext(street, address.get('address_line_2'), locality, address.get('region'), sep=', ') person.add('address', address) emitter.emit(person)
def parse_reference(emitter: EntityEmitter, row: dict): first_name = row.pop("firstname") last_name = row.pop("lastname") birth_date = row.pop("birthdate") entity = emitter.make("Person") entity.make_id("FREEZING", "{}{}{}".format(first_name, last_name, birth_date)) entity.add("status", NATURES.get(row.pop("nature"))) entity.add("birthDate", birth_date) entity.add("birthPlace", row.pop("birthplace")) entity.add("name", "{} {}".format(first_name, last_name)) entity.add("alias", row.pop("aliases")) entity.add("keywords", "ASSETS_FREEZING") emitter.emit(entity)
def parse_notice(context, data): with context.http.rehash(data) as res: res = res.json first_name = res["forename"] or "" last_name = res["name"] or "" dob = res["date_of_birth"] nationalities = res["nationalities"] place_of_birth = res["place_of_birth"] gender = SEXES.get(res["sex_id"]) emitter = EntityEmitter(context) entity = emitter.make("Person") entity.make_id("INTERPOL", first_name, last_name, res["entity_id"]) entity.add("name", first_name + " " + last_name) entity.add("firstName", first_name) entity.add("lastName", last_name) entity.add("nationality", nationalities) entity.add("gender", gender) entity.add("birthPlace", place_of_birth) entity.add("birthDate", parse_date(dob)) entity.add("sourceUrl", res["_links"]["self"]["href"]) entity.add("keywords", "YELLOWNOTICE") entity.add("topics", "researched") emitter.emit(entity) emitter.finalize()
def parse(context, data): emitter = EntityEmitter(context) url = data.get("url") country = data.get("country") with context.http.rehash(data) as res: doc = res.html output = doc.find('.//div[@id="countryOutput"]') if output is None: return # component = None for row in output.findall(".//li"): # next_comp = row.findtext('./td[@class="componentName"]/strong') # if next_comp is not None: # component = next_comp # continue function = element_text(row.find('.//span[@class="title"]')) if function is None: continue name = element_text(row.find('.//span[@class="cos_name"]')) if name is None: continue person = emitter.make("Person") person.make_id(url, country, name, function) person.add("name", name) person.add("country", country) person.add("position", function) person.add("sourceUrl", url) person.add("topics", "role.pep") updated_at = doc.findtext('.//span[@id="lastUpdateDate"]') updated_at = parse_updated(updated_at) if updated_at is not None: person.add("modifiedAt", updated_at) person.context["updated_at"] = updated_at.isoformat() emitter.emit(person) emitter.finalize()
def feature(context, data): feature = data.pop("feature") record_id = feature.get("FeatureId") record = convert_data(feature) record["PortalTitle"] = data.pop("portal_title") record["PortalURL"] = data.pop("portal_url") record["LayerName"] = data.pop("name") record["Interest"] = record.get("Interest") record["Area"] = record.get("Area") emitter = EntityEmitter(context) commodities = parse_commodities(record) concession = emitter.make("License") concession.make_id(record_id) concession.add("name", record.get("Type")) concession.add("type", [record.get("Type"), record.get("TypeGroup")]) concession.add("country", record.get("Jurisdic")) concession.add("sourceUrl", record.get("PortalURL")) concession.add("description", record.get("Interest")) concession.add("amount", record.get("AreaValue")) area = record.get("Area") area_unit = record.get("AreaUnit") if area_unit is not None: area = "%s %s" % (area, area_unit) concession.add("area", area) concession.add("commodities", commodities) concession.add("notes", record.get("Comments")) emitter.emit(concession) parties = 0 for field, party, share in parse_parties(record): entity = emitter.make("LegalEntity") entity.make_id(party) entity.add("name", party) entity.add("sourceUrl", record.get("PortalURL")) ownership = emitter.make("Ownership") ownership.make_id(record_id, party, share) ownership.add("owner", entity) ownership.add("asset", concession) ownership.add("status", record.get("Status")) ownership.add("percentage", share) ownership.add("startDate", record.get("DteGranted")) ownership.add("endDate", record.get("DteExpires")) ownership.add("sourceUrl", record.get("PortalURL")) ownership.add("recordId", record.get("Code")) emitter.emit(entity) emitter.emit(ownership) parties += 1 if parties == 0: context.emit_warning("No parties: %s - %s" % (record["LayerName"], record_id)) emitter.finalize()
def parse(context, data): url = data["url"] response = context.http.rehash(data) html = response.html emitter = EntityEmitter(context) person = emitter.make("Person") title = _get_itemprop(html, 'http://schema.org/honorificPrefix') firstName = _get_itemprop(html, "http://schema.org/givenName") familyName = _get_itemprop(html, "http://schema.org/familyName") if not firstName or not familyName: return context.log.info("Parsing Person '" + firstName + " " + familyName + "' found at: " + url) birthDate = _extract_birth_date(_get_itemprop(html, "birthDate")) birthPlace = _get_itemprop(html, "birthPlace") telephone = _get_itemprop(html, "http://schema.org/telephone") faxNumber = _get_itemprop(html, "http://schema.org/faxNumber") image = _extract_img_src(html) email = _get_itemprop(html, "http://schema.org/email", "*") _extract_personal_websites(context, person, html) person.add("title", title) person.add("firstName", firstName) person.add("lastName", familyName) person.add("name", " ".join([firstName, familyName])) person.add("birthDate", birthDate) person.add("birthPlace", birthPlace) person.add("country", "at") _extract_social_media(html, person, context) _extract_addresses(context, html, person) person.add("phone", telephone) person.add("email", email) person.add("sourceUrl", url) person.make_id(url) _parse_info_table(emitter, context, person, html, make_mandates, "mandate") _parse_info_table(emitter, context, person, html, _make_societies, "vereine") _parse_info_table(emitter, context, person, html, _make_work_and_affiliates, "firmenfunktionen") party = _make_party(context, data, emitter, html) emitter.emit(person) if not party: emitter.finalize() return emitter.emit(party) membership = emitter.make("Membership") membership.make_id(person.id, party.id) membership.add("member", person.id) membership.add("organization", party.id) membership.add("sourceUrl", url) emitter.emit(membership) emitter.finalize()
def parse(context, data): emitter = EntityEmitter(context) url = data.get("url") with context.http.rehash(data) as res: doc = res.html divs = doc.findall('.//div[@class="regular-details"]/div') image_link = "{}{}".format(ROOT_URL, divs[0].find(".//img").attrib["src"]) infos = {} infos["phone"] = [] for li in divs[1].findall('.//ul[@class="no-bullet"]/li'): children = li.getchildren() title = children[0] if len(children) > 1: infos[title.text.strip()[0:-1]] = [] for child in children: if child.tag == "a": infos[title.text.strip()[0:-1]].append(child.text) if child.tag == "ul": for li_in in child.findall("./li"): infos[title.text.strip()[0:-1]].append(li_in.text) if child.tag == "b": for item in title.xpath( "following-sibling::*/text()|following-sibling::text()" ): item = item.strip() if item: infos[title.text.strip()[0:-1]].append(item) if child.tag == "img": infos[title.text.strip()[0:-1]].append( "image: {}{}".format(ROOT_URL, child.attrib["src"])) elif title.tag == "b" or title.tag == "i": if title.tag == "i" and not title.attrib: infos["description"] = title.text else: for item in title.xpath( "following-sibling::*/text()|following-sibling::text()" ): item = item.strip() if item: if title.tag == "b": infos[title.text.strip()[0:-1]] = item elif title.tag == "i": phone_type = "phone" if title.attrib["class"] == "fa fa-fax": phone_type = "fax" infos["phone"].append("{}: {}".format( phone_type, item)) first_name = infos["Full name"].split(", ")[1] last_name = infos["Full name"].split(", ")[0] if "Languages" in infos: infos["Languages"] = [ info.strip() for info in infos["Languages"].split(",") ] person = emitter.make("Person") person.make_id(url, first_name, last_name) person.add("sourceUrl", url) person.add("firstName", first_name) person.add("lastName", last_name) person.add("name", infos.pop("Full name")) person.add("description", infos.get("description")) street = infos.get("Street", "") city = infos.get("City", "") postal_code = infos.get("Postal code", "") country = infos.get("Country", "") person.add("address", "{} {} {} {}".format(street, city, postal_code, country)) person.add("email", email=infos.get("Emails")) person.add("country", infos.get("Represented Country")) person.add("phone", infos.get("phone")) # TODO: make political party into an entity # TODO: don't have any left-over JSON stuff :) infos["Photo"] = image_link person.add("notes", json.dumps({key: value for key, value in infos.items()})) emitter.emit(person) emitter.finalize()