def test_cache_is_reused_once_expired(self): get_client(job_board=JOB_BOARD_CHOICES[0]) # NB the real one self.assertEqual(len(responses.calls), 1) self.cache.clear() get_client(job_board=JOB_BOARD_CHOICES[0]) # NB the real one self.assertEqual(len(responses.calls), 2)
def test_creating_normal_client_calls_url(self): with open(wsdl_file_path, "r") as f: xml = f.read() responses.add( responses.GET, settings.TALENTLINK_API_WSDL, xml, status=200, content_type="text/xml", ) get_client(job_board=JOB_BOARD_CHOICES[0]) self.assertEqual(len(responses.calls), 1)
def import_attachments_for_job(job, client=None): doc_imported = 0 if not client: client = get_client(job_board=job.homepage.job_board) # This will return list of attachments with # 'content', 'description', 'fileName', 'id', 'mimeType' attachments_response = client.service.getAttachments(job.talentlink_id) for attachment in attachments_response: if attachment["id"] and attachment["fileName"]: doc, created = CustomDocument.objects.get_or_create( talentlink_attachment_id=attachment["id"]) if created: doc.title = (attachment["description"] or attachment["fileName"].split(".")[0]) doc.file = ContentFile( attachment["content"], name=attachment["fileName"], ) doc.save() doc_imported += 1 job.attachments.add(doc) job.save() return doc_imported
def get_client(self, job_board=JOB_BOARD_CHOICES[0]): wsdl_file_path = pathlib.Path(__file__).parent / "wsdl.xml" wsdl_file_url = f"file://{wsdl_file_path}" return get_client(wsdl=wsdl_file_url, job_board=job_board)
def test_cache_is_used(self): get_client(job_board=JOB_BOARD_CHOICES[0]) # NB the real one self.assertEqual(len(responses.calls), 1) get_client(job_board=JOB_BOARD_CHOICES[0]) # NB the real one self.assertEqual(len(responses.calls), 1)
def test_cache_is_set(self): get_client(job_board=JOB_BOARD_CHOICES[0]) # NB the real one self.assertNotEqual(self.cache.get(settings.TALENTLINK_API_WSDL), None)
def handle(self, *args, **options): # We import jobs for each Recruitment Homepage, including draft ones # so it is possible to import jobs before going live. if not RecruitmentHomePage.objects.exists(): msg = f"Please create a RecruitmentHomePage page before running the import." self.stdout.write(self.style.ERROR(msg)) for homepage in RecruitmentHomePage.objects.all(): self.stdout.write( f"Starting import for recruitment site with homepage id {homepage.id}:" ) job_board = homepage.job_board try: client = get_client(job_board=job_board) except AttributeError as exception: msg = exception.args[0] + "\n" msg += f"Error fetching jobs for '{homepage.job_board}' job board. Skipping import for this site." self.stdout.write(self.style.ERROR(msg)) continue page = 1 results = True num_updated = 0 num_created = 0 doc_imported = 0 image_imported = 0 errors = [] import_timestamp = now() while results: self.stdout.write( f"Fetching page {page} from job board '{job_board}'") response = client.service.getAdvertisementsByPage( pageNumber=page, showJobLocation=True) results = response["advertisements"] page += 1 if results: self.stdout.write( f"{len(results['advertisement'])} advertisements") for ad in response["advertisements"]["advertisement"]: try: job = TalentLinkJob.objects.get( talentlink_id=ad["id"]) created = False except TalentLinkJob.DoesNotExist: job = TalentLinkJob(talentlink_id=ad["id"], homepage=homepage) created = True try: job = update_job_from_ad( job, ad, homepage=homepage, defaults={"last_imported": import_timestamp}, import_categories=options["import_categories"], ) except Exception as e: msg = ( f"Error occurred while processing job {ad['id']} {ad['jobNumber']}:\n" + str(e)) errors.append(msg) else: if created: num_created += 1 else: num_updated += 1 # Fetch attachments via a different call try: doc_imported += import_attachments_for_job( job, client) except Exception as e: msg = ( "Error occurred while importing attachments " + f"for job {ad['id']} {ad['jobNumber']}:\n" + str(e)) errors.append(msg) # Fetch logo image via a different call image_found = False try: # This will return a assignedImageDto object with # 'id', 'url', 'position' logo_response = client.service.getAdvertisementImages( job.talentlink_id) for image in logo_response: if image["position"] == "Logo": talentlink_image_id = image["id"] # Only update image if it is changed or new try: logo_image = CustomImage.objects.get( talentlink_image_id= talentlink_image_id) except CustomImage.DoesNotExist: logo_image = import_image_from_url( title=job.title, url=image["url"], filename="logo " + str(job.talentlink_id), talentlink_image_id= talentlink_image_id, collection_name="jobs_logo", ) image_imported += 1 finally: job.logo = logo_image job.save() image_found = True break except Exception as e: msg = ( f"Error occurred while importing logo image for job {ad['id']} {ad['jobNumber']}:\n" + str(e)) errors.append(msg) # Remove logo from existing job if it is gone from this import if (not created) and ( not image_found) and job.logo: # Also delete logo image if not used anywhere else too if (TalentLinkJob.objects.filter( logo=job.logo).count() == 1): job.logo.delete() job.logo = None job.save() # Check for outdated jobs num_deleted = 0 try: num_deleted = delete_jobs(imported_before=import_timestamp, homepage=homepage) except Exception as e: msg = f"Error occurred while deleting jobs:\n" + str(e) errors.append(msg) self.stdout.write("No more results") self.stdout.write(f"{num_updated} existing jobs updated") self.stdout.write(f"{num_created} new jobs created") self.stdout.write(f"{doc_imported} new documents imported") self.stdout.write(f"{image_imported} new images imported") self.stdout.write(f"{num_deleted} jobs deleted") self.stdout.write(self.style.ERROR(f"{len(errors)} errors")) for msg in errors: self.stdout.write(self.style.ERROR(msg))