async def do_task_loop(self, task): parameters = task["parameters"] job_id = task["jobId"] task_id = task["id"] max_retries = self.__config['max_attempts'] min_zoom_level = parameters["minZoom"] max_zoom_level = parameters["maxZoom"] zoom_levels = '{0}-{1}'.format(min_zoom_level, max_zoom_level) current_retry = task['attempts'] success = False while (current_retry < max_retries and not success): current_retry = current_retry + 1 success, reason = self.execute_task(task, zoom_levels) if success: self.log.info( 'Successfully finished {0}, Committing from queue'.format( utilities.task_format_log(task))) await self.queue_handler.ack(job_id, task_id) else: self.log.error( 'Failed executing task with {0}, current attempt is: {1}'. format(utilities.task_format_log(task), current_retry)) await self.queue_handler.reject(job_id, task_id, True, reason) if current_retry >= max_retries and not success: await self.queue_handler.reject(job_id, task_id, False)
async def handle_tasks(self): try: while True: task = await self.queue_handler.dequeue( self.__config["queue"]["dequeue_interval_seconds"] ) # dequeue task self.log.info("Task received - {0}".format(task)) task_id = task["id"] job_id = task["jobId"] parameters = task["parameters"] is_valid, reason = utilities.validate_data(parameters) if not is_valid: self.log.error( "Validation error - could not process request and could not save status to DB, Committing task with: {0}" .format(utilities.task_format_log(task))) await self.queue_handler.reject(job_id, task_id, False, reason) continue await self.do_task_loop(task) request_connector.post_end_process(job_id, task_id) except Exception as e: raise e
def buildvrt_utility(self, task, zoom_levels): if not (task["parameters"]["fileNames"] and task["parameters"]["originDirectory"]): raise VRTError("jobData didn't have source files data, for {0}" .format(utilities.task_format_log(task))) vrt_config = { 'VRTNodata': self.__config["gdal"]["vrt"]["no_data"], 'outputSRS': self.__config["gdal"]["vrt"]["output_srs"], 'resampleAlg': self.__config["gdal"]["vrt"]["resample_algo"], 'addAlpha': self.__config["gdal"]["vrt"]["add_alpha"] } self.log.info("Starting process GDAL-BUILD-VRT on {0} and zoom-levels: {1}" .format(utilities.task_format_log(task), zoom_levels)) mount_path = self.__config['source_mount'] files = [path.join(mount_path, task["parameters"]['originDirectory'], file) for file in task["parameters"]['fileNames']] vrt_result = gdal.BuildVRT(self.vrt_file_location(task["parameters"]['discreteId']), files, **vrt_config) if vrt_result != None: vrt_result.FlushCache() vrt_result = None else: raise VRTError("Could not create VRT File")
def execute_task(self, task, zoom_levels): try: self.__worker.buildvrt_utility(task, zoom_levels) self.__worker.gdal2tiles_utility(task, zoom_levels) if self.__config['storage_provider'].upper() == StorageProvider.S3: self.__worker.remove_s3_temp_files(task) self.__worker.remove_vrt_file(task) success_reason = "Task Completed" return True, success_reason except Exception as error: self.log.error( 'An error occured while processing {0} with error: {1}'.format( utilities.task_format_log(task), error)) return False, str(error)
def gdal2tiles_utility(self, task, zoom_levels): options = { 'resampling': self.__config['gdal']['resampling'], 'tmscompatible': self.__config['gdal']['tms_compatible'], 'profile': self.__config['gdal']['profile'], 'nb_processes': self.__config['gdal']['process_count'], 'srcnodata': self.__config['gdal']['src_nodata'], 'zoom': zoom_levels, 'verbose': self.__config['gdal']['verbose'] } discreteId = task['parameters']['discreteId'] layerRelativePath = task['parameters']['layerRelativePath'] tiles_path = '{0}/{1}'.format(self.tiles_folder_location, layerRelativePath) self.log.info("Starting process GDAL2TILES on {0} and zoom-levels: {1}" .format(utilities.task_format_log(task), zoom_levels)) generate_tiles(self.vrt_file_location(discreteId), tiles_path, **options)
def remove_s3_temp_files(self, task): tiles_location = '{0}/{1}'.format(self.tiles_folder_location, task['parameters']['discreteId']) self.log.info('Removing folder {0} on {1}'.format(tiles_location, utilities.task_format_log(task))) shutil.rmtree(tiles_location)
def remove_vrt_file(self, task): vrt_path = self.vrt_file_location(task['parameters']['discreteId']) self.log.info('Removing vrt file from path "{0}", {1}'.format(vrt_path, utilities.task_format_log(task))) remove_file(vrt_path)