def get_data(self):
if not self.dataParser:
self.dataParser = LinkedinPageParser()
# Feed the parser
self.dataParser.data = self._get_page()
# Check if the person work in the target regions
if self.region:
page_region = self.dataParser.get_region()
if page_region not in self.region:
print 'Person not in target region, no data return, no further network return.'
return None
print 'Getting data from :', self.url
print
education = self.dataParser.get_education_info()
work = self.dataParser.get_work_info()
skills = self.dataParser.get_skill_info()
network = self.dataParser.get_network()
p_dict = {
'edu': education,
'work': work,
'skills': skills,
'network': network,
'url': self.url
}
return p_dict
class LinkedinCrawler:
def __init__(self):
self.name = 'LinkedinCrawler'
self.region = None
self.url = None
self.delay = None
# Private Property
self.dataParser = None
def get_data(self):
if not self.dataParser:
self.dataParser = LinkedinPageParser()
# Feed the parser
self.dataParser.data = self._get_page()
# Check if the person work in the target regions
if self.region:
page_region = self.dataParser.get_region()
if page_region not in self.region:
print 'Person not in target region, no data return, no further network return.'
return None
print 'Getting data from :', self.url
print
education = self.dataParser.get_education_info()
work = self.dataParser.get_work_info()
skills = self.dataParser.get_skill_info()
network = self.dataParser.get_network()
p_dict = {
'edu': education,
'work': work,
'skills': skills,
'network': network,
'url': self.url
}
return p_dict
def _get_page(self):
if self.url:
r = requests.get(self.url, timeout=30)
return r.text
else:
print self.name, ' Method: _get_page: Error - No url is given for page'
class LinkedinCrawler:
def __init__(self):
self.name = "LinkedinCrawler"
self.region = None
self.url = None
self.delay = None
# Private Property
self.dataParser = None
def get_data(self):
if not self.dataParser:
self.dataParser = LinkedinPageParser()
# Feed the parser
self.dataParser.data = self._get_page()
# Check if the person work in the target regions
if self.region:
page_region = self.dataParser.get_region()
if page_region not in self.region:
print "Person not in target region, no data return, no further network return."
return None
print "Getting data from :", self.url
print
education = self.dataParser.get_education_info()
work = self.dataParser.get_work_info()
skills = self.dataParser.get_skill_info()
network = self.dataParser.get_network()
p_dict = {"edu": education, "work": work, "skills": skills, "network": network, "url": self.url}
return p_dict
def _get_page(self):
if self.url:
r = requests.get(self.url, timeout=30)
return r.text
else:
print self.name, " Method: _get_page: Error - No url is given for page"
def get_data(self):
if not self.dataParser:
self.dataParser = LinkedinPageParser()
# Feed the parser
self.dataParser.data = self._get_page()
# Check if the person work in the target regions
if self.region:
page_region = self.dataParser.get_region()
if page_region not in self.region:
print "Person not in target region, no data return, no further network return."
return None
print "Getting data from :", self.url
print
education = self.dataParser.get_education_info()
work = self.dataParser.get_work_info()
skills = self.dataParser.get_skill_info()
network = self.dataParser.get_network()
p_dict = {"edu": education, "work": work, "skills": skills, "network": network, "url": self.url}
return p_dict
