def process_request(self, download_job): if settings.IS_LOCAL: # Locally, we do not use SQS csv_generation.generate_csvs(download_job=download_job) else: # Send a SQS message that will be processed by another server which will eventually run # csv_generation.write_csvs(**kwargs) (see download_sqs_worker.py) write_to_log( message='Passing download_job {} to SQS'.format(download_job.download_job_id), download_job=download_job ) queue = get_sqs_queue_resource(queue_name=settings.BULK_DOWNLOAD_SQS_QUEUE_NAME) queue.send_message(MessageBody=str(download_job.download_job_id))
def process_request(self, download_job): if settings.IS_LOCAL: # Locally, we do not use SQS csv_generation.generate_csvs(download_job=download_job) else: # Send a SQS message that will be processed by another server which will eventually run # csv_generation.write_csvs(**kwargs) (see download_sqs_worker.py) write_to_log(message='Passing download_job {} to SQS'.format( download_job.download_job_id), download_job=download_job) queue = get_sqs_queue_resource( queue_name=settings.BULK_DOWNLOAD_SQS_QUEUE_NAME) queue.send_message(MessageBody=str(download_job.download_job_id))
def handle(self, *args, **options): """Run the application.""" queue = get_sqs_queue_resource( queue_name=settings.BULK_DOWNLOAD_SQS_QUEUE_NAME) write_to_log(message="Starting SQS polling") while True: second_attempt = True try: # Grabs one (or more) messages from the queue messages = queue.receive_messages( WaitTimeSeconds=10, MessageAttributeNames=["All"], VisibilityTimeout=DEFAULT_VISIBILITY_TIMEOUT) for message in messages: write_to_log( message="Message Received: {}".format(message)) if message.body is not None: # Retrieve and update the job download_job = DownloadJob.objects.filter( download_job_id=int(message.body)).first() second_attempt = download_job.error_message is not None # Retrieve the data and write to the CSV(s) csv_generation.generate_csvs(download_job=download_job, sqs_message=message) # If successful, we do not want to run again; delete message.delete() except Exception as e: logger.error(e) write_to_log(message=str(e), download_job=download_job, is_error=True) if download_job: download_job.error_message = str(e) download_job.job_status_id = JOB_STATUS_DICT[ "failed" if second_attempt else "ready"] download_job.save() finally: # Set visibility to 0 so that another attempt can be made to process in SQS immediately, instead of # waiting for the timeout window to expire for message in messages: try: message.change_visibility(VisibilityTimeout=0) except botocore.exceptions.ClientError: # TODO: check existence instead of catching error continue
def poll_sqs_for_message(queue_name, wait_time): """ Returns 0 or 1 message from the queue""" try: queue = get_sqs_queue_resource(queue_name=queue_name) sqs_message = queue.receive_messages( WaitTimeSeconds=wait_time, MessageAttributeNames=["All"], VisibilityTimeout=DEFAULT_VISIBILITY_TIMEOUT, MaxNumberOfMessages=1, ) except botocore.exceptions.ClientError: write_to_log(message="SQS connection issue. Investigate settings", is_error=True) raise SystemExit(1) return sqs_message[0] if sqs_message else None
def poll_sqs_for_message(queue_name, wait_time): """ Returns 0 or 1 message from the queue""" try: queue = get_sqs_queue_resource(queue_name=queue_name) sqs_message = queue.receive_messages( WaitTimeSeconds=wait_time, MessageAttributeNames=["All"], VisibilityTimeout=DEFAULT_VISIBILITY_TIMEOUT, MaxNumberOfMessages=1, ) except botocore.exceptions.ClientError: write_to_log(message="SQS connection issue. Investigate settings", is_error=True) raise SystemExit(1) return sqs_message[0] if sqs_message else None
def download(self, file_name, award_levels, award_types=None, agency=None, sub_agency=None, date_type=None, start_date=None, end_date=None, columns=[], file_format="csv", monthly_download=False, cleanup=False, use_sqs=False): date_range = {} if start_date: date_range['start_date'] = start_date if end_date: date_range['end_date'] = end_date json_request = { 'constraint_type': 'year', 'award_levels': award_levels, 'filters': { 'award_types': award_types, 'agency': str(agency), 'date_type': date_type, 'date_range': date_range, }, 'columns': columns, 'file_format': file_format } download_viewset = YearLimitedDownloadViewSet() download_viewset.process_filters(json_request) validated_request = validate_award_request(json_request) download_job = DownloadJob.objects.create(job_status_id=JOB_STATUS_DICT['ready'], file_name=file_name, json_request=json.dumps(order_nested_object(validated_request)), monthly_download=True) if not use_sqs: # Note: Because of the line below, it's advised to only run this script on a separate instance as this will # modify your bulk download settings. settings.BULK_DOWNLOAD_S3_BUCKET_NAME = settings.MONTHLY_DOWNLOAD_S3_BUCKET_NAME csv_generation.generate_csvs(download_job=download_job) if cleanup: # Get all the files that have the same prefix except for the update date file_name_prefix = file_name[:-12] # subtracting the 'YYYYMMDD.zip' for key in self.bucket.objects.filter(Prefix=file_name_prefix): if key.key == file_name: # ignore the one we just uploaded continue key.delete() logger.info('Deleting {} from bucket'.format(key.key)) else: queue = get_sqs_queue_resource(queue_name=settings.BULK_DOWNLOAD_SQS_QUEUE_NAME) queue.send_message(MessageBody=str(download_job.download_job_id))
def download(self, file_name, award_levels, award_types=None, agency=None, sub_agency=None, date_type=None, start_date=None, end_date=None, columns=[], file_format="csv", monthly_download=False, cleanup=False, use_sqs=False): date_range = {} if start_date: date_range['start_date'] = start_date if end_date: date_range['end_date'] = end_date json_request = { 'constraint_type': 'year', 'award_levels': award_levels, 'filters': { 'award_types': award_types, 'agency': str(agency), 'date_type': date_type, 'date_range': date_range, }, 'columns': columns, 'file_format': file_format } download_viewset = YearLimitedDownloadViewSet() download_viewset.process_filters(json_request) validated_request = download_viewset.validate_award_request(json_request) download_job = DownloadJob.objects.create(job_status_id=JOB_STATUS_DICT['ready'], file_name=file_name, json_request=json.dumps(order_nested_object(validated_request)), monthly_download=True) if not use_sqs: # Note: Because of the line below, it's advised to only run this script on a separate instance as this will # modify your bulk download settings. settings.BULK_DOWNLOAD_S3_BUCKET_NAME = settings.MONTHLY_DOWNLOAD_S3_BUCKET_NAME csv_generation.generate_csvs(download_job=download_job) if cleanup: # Get all the files that have the same prefix except for the update date file_name_prefix = file_name[:-12] # subtracting the 'YYYYMMDD.zip' for key in self.bucket.objects.filter(Prefix=file_name_prefix): if key.key == file_name: # ignore the one we just uploaded continue key.delete() logger.info('Deleting {} from bucket'.format(key.key)) else: queue = get_sqs_queue_resource(queue_name=settings.BULK_DOWNLOAD_SQS_QUEUE_NAME) queue.send_message(MessageBody=str(download_job.download_job_id))
def push_job_to_queue(self): # Candidate for separate object or file queue = get_sqs_queue_resource( queue_name=settings.BULK_DOWNLOAD_SQS_QUEUE_NAME) queue.send_message(MessageBody=str(self.download_job.download_job_id))