def main():
main_obj = get_all_persons(_main_url)
for entity in get_entities(main_obj):
# helpers.check(entity)
helpers.emit(entity)
def main():
main_obj = get_all_persons(_main_url)
for entity in get_entities(main_obj):
# helpers.check(entity)
helpers.emit(entity)
def main():
for url in URLS:
main_obj = get_all_persons(url)
for entity in get_entities(main_obj):
# helpers.check(entity)
helpers.emit(entity)
def _generate_entities(data):
"""for each scrapable page, yield an entity"""
i = 0
while i < len(data):
release_date = datetime.strptime(data[i].text, '%m/%d/%Y')
release_date = release_date.strftime('%Y-%m-%d')
name = data[i + 1].text
url = data[i + 1].find_element_by_tag_name('a').get_attribute("/href")
href = data[i + 2].find_element_by_tag_name('a').get_attribute("/href")
related = []
if href:
doc = BeautifulSoup(helpers.fetch_string(href), "html.parser")
tds = doc.find_all("td", class_='ms-vb')
for td in tds:
try:
related.append(td.find('a')['href'])
except AttributeError:
pass
related_documents = ' '.join(related)
fields = [{
"name": "Release date",
"value": release_date
}, {
"tag": "url",
"value": url
}, {
"name": "Related documents",
"value": related_documents
}]
i += 3
my_id = helpers.make_id(name)
if len(my_id) > 99:
my_id = my_id[:99]
entity = {
"_meta": {
"id": my_id,
"entity_type": "company"
},
"fields": fields,
"name": name,
}
helpers.emit(entity)
def _generate_entities(data):
"""for each scrapable page, yield an entity"""
i = 0
while i < len(data):
release_date = datetime.strptime(data[i].text, '%m/%d/%Y')
release_date = release_date.strftime('%Y-%m-%d')
name = data[i+1].text
url = data[i+1].find_element_by_tag_name('a').get_attribute("/href")
href = data[i+2].find_element_by_tag_name('a').get_attribute("/href")
related = []
if href:
doc = BeautifulSoup(helpers.fetch_string(href), "html.parser")
tds = doc.find_all("td", class_='ms-vb')
for td in tds:
try:
related.append(td.find('a')['href'])
except AttributeError:
pass
related_documents = ' '.join(related)
fields = [{"name": "Release date", "value": release_date},
{"tag": "url", "value": url},
{"name": "Related documents", "value": related_documents}]
i += 3
my_id = helpers.make_id(name)
if len(my_id) > 99:
my_id = my_id[:99]
entity = {
"_meta": {
"id": my_id,
"entity_type": "company"
},
"fields": fields,
"name": name,
}
helpers.emit(entity)
def build_document(member):
role_and_name = role_name_splitter(member.select("h2 a")[0].get_text())
years_served = member.select("ul.memberServed li")[0].get_text()
party = member.select("div.memberProfile table tr")[2].select("td")[0].get_text()
serve_state = member.select("div.memberProfile table tr")[0].select("td")[0].get_text()
entity = {
"_meta": {
"id": hashlib.sha224((re.sub("[^a-zA-Z0-9]", "", role_and_name[1] + serve_state))).hexdigest(),
"entity_type": "person"
},
"name": " ".join(role_and_name[1:]).strip(),
"types": ["pep"],
"fields": [
{"name": "Comment", "value": "Member of US Congress"},
{"name": "Role", "value": role_and_name[0]},
{"name": "Years Served", "value": years_served},
{"name": "Political Party", "value": party},
{"name": "State", "value": serve_state}
]
}
helpers.emit(entity)
def build_document(member):
role_and_name = role_name_splitter(member.select("h2 a")[0].get_text())
years_served = member.select("ul.memberServed li")[0].get_text()
party = member.select("div.memberProfile table tr")[2].select(
"td")[0].get_text()
serve_state = member.select("div.memberProfile table tr")[0].select(
"td")[0].get_text()
entity = {
"_meta": {
"id":
hashlib.sha224(
(re.sub("[^a-zA-Z0-9]", "",
role_and_name[1] + serve_state))).hexdigest(),
"entity_type":
"person"
},
"name":
" ".join(role_and_name[1:]).strip(),
"types": ["pep"],
"fields": [{
"name": "Comment",
"value": "Member of US Congress"
}, {
"name": "Role",
"value": role_and_name[0]
}, {
"name": "Years Served",
"value": years_served
}, {
"name": "Political Party",
"value": party
}, {
"name": "State",
"value": serve_state
}]
}
helpers.emit(entity)
def emit_meminfo():
helpers.emit(
subprocess.check_output(['free', '--human'], universal_newlines=True))
def emit_meminfo():
helpers.emit(subprocess.check_output(['free', '--human'],
universal_newlines=True))
def main():
main_obj = get_rows(_base_url)
for entity in get_entities(main_obj):
# helpers.check(entity)
helpers.emit(entity)
def main():
main_obj = get_all_persons(_DOMAIN)
for entity in get_entities(main_obj):
helpers.emit(entity)
def main():
for entity in _get_people(_get_parties(_base_url)):
# helpers.check(entity)
helpers.emit(entity)
def main():
for entity in _generate_entities():
helpers.emit(entity)
def main():
for entity in _get_people(_get_parties(_base_url)):
# helpers.check(entity)
helpers.emit(entity)
def main():
main_obj = get_rows([_main_url, _second_url])
for entity in get_entities(main_obj):
helpers.emit(entity)
def main():
main_obj = get_rows(_base_url)
for entity in get_entities(main_obj):
# helpers.check(entity)
helpers.emit(entity)