class _TweetLoader(ItemLoader):
CSS_BASE = 'div[role="main"] div.permalink-inner'
default_input_processor = proc.MapCompose(gettext)
default_output_processor = proc.Join('')
created_at_in = proc.TakeFirst()
created_at_out = proc.Compose(only, to_date)
user_in = proc.Identity()
user_out = proc.Compose(only)
body_in = proc.Identity()
body_out = proc.Compose(only)
media_in = proc.Identity()
media_out = proc.Compose(only)
parents_in = \
children_in = proc.Compose(extract_tweet_meta)
parents_out = \
children_out = proc.Identity()
retweet_count_in = \
favorite_count_in = \
reply_count_in = proc.Compose(only)
label_in = proc.Identity()
label_out = proc.TakeFirst()
user = None
body = None
stats = None
def load_item(self):
self.add_value('user', self.user.load_item())
self.add_value('body', self.body.load_item())
return super().load_item()
def parse_iframe(self, response):
content = response.css('.SummaryCard-content')
media_item_loader = MediaItemLoader(Media(), content)
media_item_loader.add_value('url', response.url)
media_item_loader.add_selector('content', 'p::text')
media_item_loader.add_selector('title', 'h2.TwitterCard-title::text')
media_item_loader.add_selector('content_source', '.SummaryCard-content span::text')
media_item_loader.add_selector('data_source', '::attr(data-src)')
self.add_value('media', media_item_loader.load_item())
yield self.load_item()
class PisoFiCrawlerItemLoader(ItemLoader):
default_item_class = PisoFiCrawlerItem
# Input processors
default_input_processor = strip_whitespace
# Output processors
default_output_processor = processors.TakeFirst()
name_out = name_output_processor
ip_out = ip_output_processor
description_out = description_output_processor
class SalesTransactionCrawlerItemLoader(ItemLoader):
default_item_class = SalesTransactionCrawlerItem
# Input processors
default_input_processor = strip_whitespace
# Output processors
default_output_processor = processors.TakeFirst()
mac_addr_out = mac_addr_output_processor
transaction_type_out = transaction_type_output_processor
vendo_out = vendo_output_processor
amount_out = amount_output_processor
def parse_advisors(self, response):
company_url = response.xpath(
'//*[@id="profile_header_heading"]/a/@href').extract_first()
employee_selector = response.css('div.advisors').xpath('.//ul/li')
for sel in employee_selector:
loader = ItemLoader(item=BoardMember(), selector=sel)
loader.default_input_processor = processors.MapCompose(
w3lib.html.remove_tags)
loader.default_output_processor = processors.TakeFirst()
loader.add_value('company_url', company_url)
loader.add_xpath('person_url', './/h4/a/@href')
loader.add_xpath('title', './/h5/text()')
yield loader.load_item()
def parse_partners(self, response):
company_url = response.xpath(
'//*[@id="profile_header_heading"]/a/@href').extract_first()
partner_selectors = response.css('div.partners').xpath(
'.//ul/li//h4/a')
for sel in partner_selectors:
loader = ItemLoader(item=Partner(), selector=sel)
loader.default_input_processor = processors.MapCompose(
w3lib.html.remove_tags)
loader.default_output_processor = processors.TakeFirst()
loader.add_value('focal_company_url', company_url)
loader.add_xpath('partner_url', '@href')
yield loader.load_item()
def parse_acquisitions(self, response):
company_url = response.xpath(
'//*[@id="profile_header_heading"]/a/@href').extract_first()
acq_selectors = response.css('div.acquisitions').xpath(
'.//tr[not(th)]')
for sel in acq_selectors:
loader = ItemLoader(item=Acquisition(), selector=sel)
loader.default_input_processor = processors.MapCompose(
w3lib.html.remove_tags)
loader.default_output_processor = processors.TakeFirst()
loader.add_value('focal_company_url', company_url)
loader.add_xpath('date', 'td[1]/text()')
loader.add_xpath('acquired_url', 'td[2]/a/@href')
yield loader.load_item()
def parse_organization(self, response):
loader = ItemLoader(item=Organization(), response=response)
loader.default_input_processor = processors.MapCompose(
w3lib.html.remove_tags)
loader.default_output_processor = processors.TakeFirst()
loader.add_xpath('name', '//*[@id="profile_header_heading"]/a/text()')
loader.add_value('url', response.url)
# loader.add_value('ipo_stock', None) # TODO!
# TODO: supposed to get person url for founders!
# Fields expected: headquarters, description, founders, categories,
# website, founded (date), and aliases
keys = response.css('div.definition-list').xpath('dt/text()')
values = response.css('div.definition-list').xpath('dd')
for i in range(len(keys)):
key = keys[i].extract()
key = key[:key.find(':')].lower()
try:
loader.add_value(key, values[i].extract())
except KeyError as e:
# Ignore if key is not in the Item's field
pass
loader.add_xpath('facebook',
'(//a[contains(@class,"facebook")])[1]/@href')
loader.add_xpath('twitter',
'(//a[contains(@class,"twitter")])[1]/@href')
loader.add_xpath('linkedin',
'(//a[contains(@class,"linkedin")])[1]/@href')
yield loader.load_item()
for item in self.parse_acquisitions(response):
yield item
for item in self.parse_employees(response):
yield item
for item in self.parse_competitors(response):
yield item
for item in self.parse_partners(response):
yield item
for item in self.parse_advisors(response):
yield item
def parse_person(self, response):
loader = ItemLoader(item=Person(), response=response)
loader.default_input_processor = processors.MapCompose(
w3lib.html.remove_tags)
loader.default_output_processor = processors.TakeFirst()
loader.add_xpath('name', '//*[@id="profile_header_heading"]/a/text()')
loader.add_value('url', response.url)
loader.add_xpath(
'primary_role',
'//*[@id="info-card-overview-content"]/div/dl/div/dd')
# Fields expected: born, gender, location, website
overview = response.xpath(
'//*[@id="info-card-overview-content"]/div/dl/dt/text()')
overview_loader = loader.nested_xpath(
'//*[@id="info-card-overview-content"]/div/dl')
for i in range(len(overview)):
key = overview[i].extract()
key = key[:key.find(':')].lower()
try:
overview_loader.add_xpath(key, 'dd[{}]/text()'.format(i + 1))
except KeyError as e:
# Ignore if key is not in the Item's field
pass
loader.add_xpath('facebook',
'(//a[contains(@class,"facebook")])[1]/@href')
loader.add_xpath('twitter',
'(//a[contains(@class,"twitter")])[1]/@href')
loader.add_xpath('linkedin',
'(//a[contains(@class,"linkedin")])[1]/@href')
loader.add_xpath('description', '//*[@id="description"]/span/div')
loader.add_css('current_jobs', '.current_job')
loader.add_css('past_jobs', '.past_job')
loader.nested_css('.advisory_roles').add_xpath('board_advisors',
'.//ul/li')
loader.nested_css('table.investors').add_xpath(
'investments', './/tr[not(@class="thead")]')
loader.nested_css('.education').add_xpath('education', './/ul/li')
return loader.load_item()
class PublicationLoader(ItemLoader):
default_output_processor = processors.TakeFirst()
class LinkLoader(ItemLoader):
default_output_processor = processors.TakeFirst()
title_in = processors.MapCompose(helpers.rws, helpers.beautify_romanian)
class ECBItemLoader(ItemLoader):
default_output_processor = processors.TakeFirst()
class StatusLoader(ItemLoader):
default_output_processor = processors.TakeFirst()
status_in = processors.MapCompose(helpers.rws, helpers.beautify_romanian)
class ActivityLoader(ItemLoader):
default_output_processor = processors.TakeFirst()
name_in = processors.MapCompose(helpers.rws, helpers.beautify_romanian)
dictionary_in = processors.Identity()
class PleneryTimeLoader(ItemLoader):
default_output_processor = processors.TakeFirst()
name_in = processors.MapCompose(helpers.rws, helpers.beautify_romanian)
class MediaItemLoader(ItemLoader): default_input_processor = proc.MapCompose(gettext) default_output_processor = proc.TakeFirst() url_in = proc.Identity()
class UserItemLoader(ItemLoader):
default_output_processor = proc.Join('')
is_verified_out = proc.TakeFirst()