def parse(context, data):
emitter = EntityEmitter(context)
entity = emitter.make("LegalEntity")
name = data.get("SUPP_NAME")
ent_id = data.get("SUPP_ID")
city = data.get("SUPP_CITY")
address = data.get("SUPP_ADDR")
start_date = clean_date(data.get("DEBAR_FROM_DATE"))
entity.make_id("WBDEBAR", name, ent_id)
names = clean_name(name)
entity.add("name", names[0])
entity.add("address", address)
entity.add("address", city)
entity.add("country", data.get("COUNTRY_NAME"))
for name in names[1:]:
entity.add("alias", name)
sanction = emitter.make("Sanction")
sanction.make_id("Sanction", entity.id)
sanction.add("authority", "World Bank Debarrment")
sanction.add("program", data.get("DEBAR_REASON"))
sanction.add("startDate", start_date)
sanction.add("endDate", clean_date(data.get("DEBAR_TO_DATE")))
sanction.add("sourceUrl", SOURCE)
emitter.emit(entity)
emitter.emit(sanction)
emitter.finalize()
def parse(context, data):
emitter = EntityEmitter(context)
entity = emitter.make('LegalEntity')
name = data.get('SUPP_NAME')
ent_id = data.get('SUPP_ID')
reason = data.get('DEBAR_REASON')
country = data.get('COUNTRY_NAME')
city = data.get('SUPP_CITY')
address = data.get('SUPP_ADDR')
start_date = data.get('DEBAR_FROM_DATE')
end_date = data.get('DEBAR_TO_DATE')
entity.make_id(name, ent_id, country)
names = clean_name(name)
entity.add('name', names[0])
entity.add('address', address)
entity.add('address', city)
entity.add('country', normalize_country(country))
for name in names[1:]:
entity.add('alias', name)
sanction = emitter.make('Sanction')
sanction.make_id('Sanction', entity.id)
sanction.add('authority', 'World Bank Debarrment')
sanction.add('program', reason)
sanction.add('startDate', clean_date(start_date))
sanction.add('endDate', clean_date(end_date))
sanction.add('sourceUrl', SOURCE)
emitter.emit(entity)
emitter.emit(sanction)
emitter.finalize()
def officer(context, data):
emitter = EntityEmitter(context)
officer_id = data.get("officer_id")
url = API_URL % officer_id
with context.http.get(url, auth=AUTH) as res:
if res.status_code != 200:
context.log.info("CoH error: %r", res.json)
return
data = res.json
# pprint(data)
person = emitter.make("Person")
person.make_id(officer_id)
source_url = urljoin(WEB_URL, data.get("links", {}).get("self", "/"))
person.add("sourceUrl", source_url)
last_name = data.pop("surname", None)
person.add("lastName", last_name)
forename = data.pop("forename", None)
person.add("firstName", forename)
other_forenames = data.pop("other_forenames", None)
person.add("middleName", other_forenames)
person.add("name", jointext(forename, other_forenames, last_name))
person.add("title", data.pop("title", None))
person.add("nationality", data.pop("nationality", None))
person.add("birthDate", data.pop("date_of_birth", None))
person.add("topics", "crime")
for disqual in data.pop("disqualifications", []):
case = disqual.get("case_identifier")
sanction = emitter.make("Sanction")
sanction.make_id(person.id, case)
sanction.add("entity", person)
sanction.add("authority", "UK Companies House")
sanction.add("program", case)
from_date = disqual.pop("disqualified_from", None)
person.context["created_at"] = from_date
sanction.add("startDate", from_date)
sanction.add("endDate", disqual.pop("disqualified_until", None))
emitter.emit(sanction)
address = disqual.pop("address", {})
locality = address.get("locality")
locality = jointext(locality, address.get("postal_code"))
street = address.get("address_line_1")
premises = address.get("premises")
street = jointext(street, premises)
address = jointext(
street,
address.get("address_line_2"),
locality,
address.get("region"),
sep=", ",
)
person.add("address", address)
emitter.emit(person)
def feature(context, data):
feature = data.pop("feature")
record_id = feature.get("FeatureId")
record = convert_data(feature)
record["PortalTitle"] = data.pop("portal_title")
record["PortalURL"] = data.pop("portal_url")
record["LayerName"] = data.pop("name")
record["Interest"] = record.get("Interest")
record["Area"] = record.get("Area")
emitter = EntityEmitter(context)
commodities = parse_commodities(record)
concession = emitter.make("License")
concession.make_id(record_id)
concession.add("name", record.get("Type"))
concession.add("type", [record.get("Type"), record.get("TypeGroup")])
concession.add("country", record.get("Jurisdic"))
concession.add("sourceUrl", record.get("PortalURL"))
concession.add("description", record.get("Interest"))
concession.add("amount", record.get("AreaValue"))
area = record.get("Area")
area_unit = record.get("AreaUnit")
if area_unit is not None:
area = "%s %s" % (area, area_unit)
concession.add("area", area)
concession.add("commodities", commodities)
concession.add("notes", record.get("Comments"))
emitter.emit(concession)
parties = 0
for field, party, share in parse_parties(record):
entity = emitter.make("LegalEntity")
entity.make_id(party)
entity.add("name", party)
entity.add("sourceUrl", record.get("PortalURL"))
ownership = emitter.make("Ownership")
ownership.make_id(record_id, party, share)
ownership.add("owner", entity)
ownership.add("asset", concession)
ownership.add("status", record.get("Status"))
ownership.add("percentage", share)
ownership.add("startDate", record.get("DteGranted"))
ownership.add("endDate", record.get("DteExpires"))
ownership.add("sourceUrl", record.get("PortalURL"))
ownership.add("recordId", record.get("Code"))
emitter.emit(entity)
emitter.emit(ownership)
parties += 1
if parties == 0:
context.emit_warning("No parties: %s - %s" %
(record["LayerName"], record_id))
emitter.finalize()
def parse(context, data):
emitter = EntityEmitter(context)
url = data.get('url')
country = normalize_country(data.get('country'))
with context.http.rehash(data) as res:
doc = res.html
# updated_at = doc.findtext('.//span[@id="lastUpdateDate"]')
output = doc.find('.//div[@id="countryOutput"]')
if output is None:
return
# component = None
for row in output.findall('.//li'):
# next_comp = row.findtext('./td[@class="componentName"]/strong')
# if next_comp is not None:
# component = next_comp
# continue
function = element_text(row.find('.//span[@class="title"]'))
if function is None:
continue
name = element_text(row.find('.//span[@class="cos_name"]'))
if name is None:
continue
person = emitter.make('Person')
person.make_id(country, name, function)
person.add('name', name)
person.add('country', country)
person.add('position', function)
person.add('sourceUrl', url)
emitter.emit(person)
emitter.finalize()
def parse_notice(context, data):
with context.http.rehash(data) as res:
res = res.json
first_name = res['forename'] or ''
last_name = res['name'] or ''
dob = res['date_of_birth']
nationalities = res['nationalities']
place_of_birth = res['place_of_birth']
warrants = [
(warrant['charge'], warrant['issuing_country_id'])
for warrant in res['arrest_warrants'] # noqa
]
gender = SEXES.get(res['sex_id'])
emitter = EntityEmitter(context)
entity = emitter.make('Person')
entity.make_id(first_name, last_name, res['entity_id'])
entity.add('name', first_name + ' ' + last_name)
entity.add('firstName', first_name)
entity.add('lastName', last_name)
entity.add('nationality', nationalities)
for charge, country in warrants:
entity.add('program', country)
entity.add('summary', charge)
entity.add('gender', gender)
entity.add('birthPlace', place_of_birth)
entity.add('birthDate', parse_date(dob))
entity.add('sourceUrl', res['_links']['self']['href'])
entity.add('keywords', 'REDNOTICE')
entity.add('keywords', 'CRIME')
emitter.emit(entity)
emitter.finalize()
def parse_notice(context, data):
with context.http.rehash(data) as res:
res = res.json
first_name = res["forename"] or ""
last_name = res["name"] or ""
dob = res["date_of_birth"]
nationalities = res["nationalities"]
place_of_birth = res["place_of_birth"]
warrants = [
(warrant["charge"], warrant["issuing_country_id"])
for warrant in res["arrest_warrants"] # noqa
]
gender = SEXES.get(res["sex_id"])
emitter = EntityEmitter(context)
entity = emitter.make("Person")
entity.make_id("INTERPOL", first_name, last_name, res["entity_id"])
entity.add("name", first_name + " " + last_name)
entity.add("firstName", first_name)
entity.add("lastName", last_name)
entity.add("nationality", nationalities)
for charge, country in warrants:
entity.add("program", country)
entity.add("summary", charge)
entity.add("gender", gender)
entity.add("birthPlace", place_of_birth)
entity.add("birthDate", parse_date(dob))
entity.add("sourceUrl", res["_links"]["self"]["href"])
entity.add("keywords", "REDNOTICE")
entity.add("topics", "crime")
emitter.emit(entity)
emitter.finalize()
def crawl_country(context, path, country):
emitter = EntityEmitter(context)
source_url = UI_URL % path
context.log.info("Crawling country: %s", country)
res = requests.get(DATA_URL % path)
data = res.json().get("result", {}).get("data", {}).get("page", {})
blocks = data.get("acf", {}).get("blocks", [{}])[0]
content = blocks.get("free_form_content", []).get("content")
doc = html.fromstring(content)
function = None
for el in doc.getchildren():
text = el.text_content().strip()
if el.tag == "h2":
continue
if el.tag == "h3":
function = text
continue
name = text.replace("(Acting)", "")
person = emitter.make("Person")
person.make_id(source_url, name, function)
person.add("name", name)
person.add("country", country)
person.add("position", function)
person.add("sourceUrl", source_url)
person.add("topics", "role.pep")
# pprint(person.to_dict())
emitter.emit(person)
emitter.finalize()
def officer(context, data):
emitter = EntityEmitter(context)
officer_id = data.get('officer_id')
url = API_URL % officer_id
with context.http.get(url, auth=AUTH) as res:
if res.status_code != 200:
return
data = res.json
person = emitter.make('Person')
person.make_id(officer_id)
source_url = urljoin(WEB_URL, data.get('links', {}).get('self', '/'))
person.add('sourceUrl', source_url)
last_name = data.pop('surname', None)
person.add('lastName', last_name)
forename = data.pop('forename', None)
person.add('firstName', forename)
other_forenames = data.pop('other_forenames', None)
person.add('middleName', other_forenames)
person.add('name', jointext(forename, other_forenames, last_name))
person.add('title', data.pop('title', None))
nationality = normalize_country(data.pop('nationality', None))
person.add('nationality', nationality)
person.add('birthDate', data.pop('date_of_birth', None))
for disqual in data.pop('disqualifications', []):
case = disqual.get('case_identifier')
sanction = emitter.make('Sanction')
sanction.make_id(person.id, case)
sanction.add('entity', person)
sanction.add('authority', 'UK Companies House')
sanction.add('program', case)
sanction.add('startDate', disqual.pop('disqualified_from', None))
sanction.add('endDate', disqual.pop('disqualified_until', None))
emitter.emit(sanction)
address = disqual.pop('address', {})
locality = address.get('locality')
locality = jointext(locality, address.get('postal_code'))
street = address.get('address_line_1')
premises = address.get('premises')
street = jointext(street, premises)
address = jointext(street, address.get('address_line_2'),
locality, address.get('region'), sep=', ')
person.add('address', address)
emitter.emit(person)
def store(context, data):
emitter = EntityEmitter(context)
entity = model.get_proxy(data["entity"])
documentation = emitter.make("Documentation")
documentation.make_id("Documentation", data["entity"]["id"],
data["aleph_id"])
documentation.add("entity", entity)
documentation.add("document", data["aleph_id"])
emitter.emit(documentation)
emitter.finalize()
def parse_reference(emitter: EntityEmitter, row: dict):
first_name = row.pop("firstname")
last_name = row.pop("lastname")
birth_date = row.pop("birthdate")
entity = emitter.make("Person")
entity.make_id("FREEZING", "{}{}{}".format(first_name, last_name,
birth_date))
entity.add("status", NATURES.get(row.pop("nature")))
entity.add("birthDate", birth_date)
entity.add("birthPlace", row.pop("birthplace"))
entity.add("name", "{} {}".format(first_name, last_name))
entity.add("alias", row.pop("aliases"))
entity.add("keywords", "ASSETS_FREEZING")
emitter.emit(entity)
def parse_notice(context, data):
with context.http.rehash(data) as res:
res = res.json
first_name = res["forename"] or ""
last_name = res["name"] or ""
dob = res["date_of_birth"]
nationalities = res["nationalities"]
place_of_birth = res["place_of_birth"]
gender = SEXES.get(res["sex_id"])
emitter = EntityEmitter(context)
entity = emitter.make("Person")
entity.make_id("INTERPOL", first_name, last_name, res["entity_id"])
entity.add("name", first_name + " " + last_name)
entity.add("firstName", first_name)
entity.add("lastName", last_name)
entity.add("nationality", nationalities)
entity.add("gender", gender)
entity.add("birthPlace", place_of_birth)
entity.add("birthDate", parse_date(dob))
entity.add("sourceUrl", res["_links"]["self"]["href"])
entity.add("keywords", "YELLOWNOTICE")
entity.add("topics", "researched")
emitter.emit(entity)
emitter.finalize()
def parse(context, data):
emitter = EntityEmitter(context)
url = data.get("url")
country = data.get("country")
with context.http.rehash(data) as res:
doc = res.html
output = doc.find('.//div[@id="countryOutput"]')
if output is None:
return
# component = None
for row in output.findall(".//li"):
# next_comp = row.findtext('./td[@class="componentName"]/strong')
# if next_comp is not None:
# component = next_comp
# continue
function = element_text(row.find('.//span[@class="title"]'))
if function is None:
continue
name = element_text(row.find('.//span[@class="cos_name"]'))
if name is None:
continue
person = emitter.make("Person")
person.make_id(url, country, name, function)
person.add("name", name)
person.add("country", country)
person.add("position", function)
person.add("sourceUrl", url)
person.add("topics", "role.pep")
updated_at = doc.findtext('.//span[@id="lastUpdateDate"]')
updated_at = parse_updated(updated_at)
if updated_at is not None:
person.add("modifiedAt", updated_at)
person.context["updated_at"] = updated_at.isoformat()
emitter.emit(person)
emitter.finalize()
def parse(context, data):
url = data["url"]
response = context.http.rehash(data)
html = response.html
emitter = EntityEmitter(context)
person = emitter.make("Person")
title = _get_itemprop(html, 'http://schema.org/honorificPrefix')
firstName = _get_itemprop(html, "http://schema.org/givenName")
familyName = _get_itemprop(html, "http://schema.org/familyName")
if not firstName or not familyName:
return
context.log.info("Parsing Person '" + firstName + " " + familyName +
"' found at: " + url)
birthDate = _extract_birth_date(_get_itemprop(html, "birthDate"))
birthPlace = _get_itemprop(html, "birthPlace")
telephone = _get_itemprop(html, "http://schema.org/telephone")
faxNumber = _get_itemprop(html, "http://schema.org/faxNumber")
image = _extract_img_src(html)
email = _get_itemprop(html, "http://schema.org/email", "*")
_extract_personal_websites(context, person, html)
person.add("title", title)
person.add("firstName", firstName)
person.add("lastName", familyName)
person.add("name", " ".join([firstName, familyName]))
person.add("birthDate", birthDate)
person.add("birthPlace", birthPlace)
person.add("country", "at")
_extract_social_media(html, person, context)
_extract_addresses(context, html, person)
person.add("phone", telephone)
person.add("email", email)
person.add("sourceUrl", url)
person.make_id(url)
_parse_info_table(emitter, context, person, html, make_mandates, "mandate")
_parse_info_table(emitter, context, person, html, _make_societies,
"vereine")
_parse_info_table(emitter, context, person, html,
_make_work_and_affiliates, "firmenfunktionen")
party = _make_party(context, data, emitter, html)
emitter.emit(person)
if not party:
emitter.finalize()
return
emitter.emit(party)
membership = emitter.make("Membership")
membership.make_id(person.id, party.id)
membership.add("member", person.id)
membership.add("organization", party.id)
membership.add("sourceUrl", url)
emitter.emit(membership)
emitter.finalize()
def parse(context, data):
emitter = EntityEmitter(context)
url = data.get("url")
with context.http.rehash(data) as res:
doc = res.html
divs = doc.findall('.//div[@class="regular-details"]/div')
image_link = "{}{}".format(ROOT_URL,
divs[0].find(".//img").attrib["src"])
infos = {}
infos["phone"] = []
for li in divs[1].findall('.//ul[@class="no-bullet"]/li'):
children = li.getchildren()
title = children[0]
if len(children) > 1:
infos[title.text.strip()[0:-1]] = []
for child in children:
if child.tag == "a":
infos[title.text.strip()[0:-1]].append(child.text)
if child.tag == "ul":
for li_in in child.findall("./li"):
infos[title.text.strip()[0:-1]].append(li_in.text)
if child.tag == "b":
for item in title.xpath(
"following-sibling::*/text()|following-sibling::text()"
):
item = item.strip()
if item:
infos[title.text.strip()[0:-1]].append(item)
if child.tag == "img":
infos[title.text.strip()[0:-1]].append(
"image: {}{}".format(ROOT_URL,
child.attrib["src"]))
elif title.tag == "b" or title.tag == "i":
if title.tag == "i" and not title.attrib:
infos["description"] = title.text
else:
for item in title.xpath(
"following-sibling::*/text()|following-sibling::text()"
):
item = item.strip()
if item:
if title.tag == "b":
infos[title.text.strip()[0:-1]] = item
elif title.tag == "i":
phone_type = "phone"
if title.attrib["class"] == "fa fa-fax":
phone_type = "fax"
infos["phone"].append("{}: {}".format(
phone_type, item))
first_name = infos["Full name"].split(", ")[1]
last_name = infos["Full name"].split(", ")[0]
if "Languages" in infos:
infos["Languages"] = [
info.strip() for info in infos["Languages"].split(",")
]
person = emitter.make("Person")
person.make_id(url, first_name, last_name)
person.add("sourceUrl", url)
person.add("firstName", first_name)
person.add("lastName", last_name)
person.add("name", infos.pop("Full name"))
person.add("description", infos.get("description"))
street = infos.get("Street", "")
city = infos.get("City", "")
postal_code = infos.get("Postal code", "")
country = infos.get("Country", "")
person.add("address", "{} {} {} {}".format(street, city, postal_code,
country))
person.add("email", email=infos.get("Emails"))
person.add("country", infos.get("Represented Country"))
person.add("phone", infos.get("phone"))
# TODO: make political party into an entity
# TODO: don't have any left-over JSON stuff :)
infos["Photo"] = image_link
person.add("notes",
json.dumps({key: value
for key, value in infos.items()}))
emitter.emit(person)
emitter.finalize()
