def profile_worker(self):
"""
Given a prefilled profile queue this worker will pop an id
and fetch the associated profile
:return:
"""
while not self._profile_queue.empty():
profile_id = self._profile_queue.get()
try:
# Fetch the profile
profile = self._crawler.get_profile_by_id(profile_id)
self._profiles.append(profile)
log.debug(
"The profile for profile_id {profile_id} has been fetched".
format(profile_id=profile_id))
# Mark task as done
self._profile_queue.task_done()
except Exception as e:
log.warn(
"Failed to fetch the profile for profile_id {profile_id}".
format(profile_id=profile_id))
self._profile_queue.task_done()
def profile_worker(self):
"""
Given a prefilled profile queue this worker will pop an id
and fetch the associated profile
:return:
"""
while not self._profile_queue.empty():
profile_id = self._profile_queue.get()
try:
# Fetch the profile
profile = self._crawler.get_profile_by_id(profile_id)
self._profiles.append(profile)
log.debug("The profile for profile_id {profile_id} has been fetched".format(
profile_id=profile_id
))
# Mark task as done
self._profile_queue.task_done()
except Exception as e:
log.warn("Failed to fetch the profile for profile_id {profile_id}".format(
profile_id=profile_id
))
self._profile_queue.task_done()
def document_worker(self):
"""
Given a prefilled profile_documents queue this worker will pop an id
and fetch the associated documents
:return:
"""
while not self._profile_documents_queue.empty():
profile_id = self._profile_documents_queue.get()
try:
# Fetch the document
documents = self._crawler.get_documents_by_profile_id(
profile_id)
self._profile_documents[profile_id] = documents
log.debug(
"{num} documents have been fetched for profile_id {profile_id}"
.format(num=len(documents), profile_id=profile_id))
# Mark task as done
self._profile_documents_queue.task_done()
except Exception as e:
log.warn(
"Failed to fetch documents for profile_id {profile_id}".
format(profile_id=profile_id))
self._profile_documents_queue.task_done()
def document_worker(self):
"""
Given a prefilled profile_documents queue this worker will pop an id
and fetch the associated documents
:return:
"""
while not self._profile_documents_queue.empty():
profile_id = self._profile_documents_queue.get()
try:
# Fetch the document
documents = self._crawler.get_documents_by_profile_id(profile_id)
self._profile_documents[profile_id] = documents
log.debug("{num} documents have been fetched for profile_id {profile_id}".format(
num=len(documents),
profile_id=profile_id
))
# Mark task as done
self._profile_documents_queue.task_done()
except Exception as e:
log.warn("Failed to fetch documents for profile_id {profile_id}".format(
profile_id=profile_id
))
self._profile_documents_queue.task_done()
