def profile_worker(self): """ Given a prefilled profile queue this worker will pop an id and fetch the associated profile :return: """ while not self._profile_queue.empty(): profile_id = self._profile_queue.get() try: # Fetch the profile profile = self._crawler.get_profile_by_id(profile_id) self._profiles.append(profile) log.debug( "The profile for profile_id {profile_id} has been fetched". format(profile_id=profile_id)) # Mark task as done self._profile_queue.task_done() except Exception as e: log.warn( "Failed to fetch the profile for profile_id {profile_id}". format(profile_id=profile_id)) self._profile_queue.task_done()
def profile_worker(self): """ Given a prefilled profile queue this worker will pop an id and fetch the associated profile :return: """ while not self._profile_queue.empty(): profile_id = self._profile_queue.get() try: # Fetch the profile profile = self._crawler.get_profile_by_id(profile_id) self._profiles.append(profile) log.debug("The profile for profile_id {profile_id} has been fetched".format( profile_id=profile_id )) # Mark task as done self._profile_queue.task_done() except Exception as e: log.warn("Failed to fetch the profile for profile_id {profile_id}".format( profile_id=profile_id )) self._profile_queue.task_done()
def document_worker(self): """ Given a prefilled profile_documents queue this worker will pop an id and fetch the associated documents :return: """ while not self._profile_documents_queue.empty(): profile_id = self._profile_documents_queue.get() try: # Fetch the document documents = self._crawler.get_documents_by_profile_id( profile_id) self._profile_documents[profile_id] = documents log.debug( "{num} documents have been fetched for profile_id {profile_id}" .format(num=len(documents), profile_id=profile_id)) # Mark task as done self._profile_documents_queue.task_done() except Exception as e: log.warn( "Failed to fetch documents for profile_id {profile_id}". format(profile_id=profile_id)) self._profile_documents_queue.task_done()
def document_worker(self): """ Given a prefilled profile_documents queue this worker will pop an id and fetch the associated documents :return: """ while not self._profile_documents_queue.empty(): profile_id = self._profile_documents_queue.get() try: # Fetch the document documents = self._crawler.get_documents_by_profile_id(profile_id) self._profile_documents[profile_id] = documents log.debug("{num} documents have been fetched for profile_id {profile_id}".format( num=len(documents), profile_id=profile_id )) # Mark task as done self._profile_documents_queue.task_done() except Exception as e: log.warn("Failed to fetch documents for profile_id {profile_id}".format( profile_id=profile_id )) self._profile_documents_queue.task_done()