async def do_task_loop(self, task):
parameters = task["parameters"]
job_id = task["jobId"]
task_id = task["id"]
max_retries = self.__config['max_attempts']
min_zoom_level = parameters["minZoom"]
max_zoom_level = parameters["maxZoom"]
zoom_levels = '{0}-{1}'.format(min_zoom_level, max_zoom_level)
current_retry = task['attempts']
success = False
while (current_retry < max_retries and not success):
current_retry = current_retry + 1
success, reason = self.execute_task(task, zoom_levels)
if success:
self.log.info(
'Successfully finished {0}, Committing from queue'.format(
utilities.task_format_log(task)))
await self.queue_handler.ack(job_id, task_id)
else:
self.log.error(
'Failed executing task with {0}, current attempt is: {1}'.
format(utilities.task_format_log(task), current_retry))
await self.queue_handler.reject(job_id, task_id, True, reason)
if current_retry >= max_retries and not success:
await self.queue_handler.reject(job_id, task_id, False)
async def handle_tasks(self):
try:
while True:
task = await self.queue_handler.dequeue(
self.__config["queue"]["dequeue_interval_seconds"]
) # dequeue task
self.log.info("Task received - {0}".format(task))
task_id = task["id"]
job_id = task["jobId"]
parameters = task["parameters"]
is_valid, reason = utilities.validate_data(parameters)
if not is_valid:
self.log.error(
"Validation error - could not process request and could not save status to DB, Committing task with: {0}"
.format(utilities.task_format_log(task)))
await self.queue_handler.reject(job_id, task_id, False,
reason)
continue
await self.do_task_loop(task)
request_connector.post_end_process(job_id, task_id)
except Exception as e:
raise e
def buildvrt_utility(self, task, zoom_levels):
if not (task["parameters"]["fileNames"] and task["parameters"]["originDirectory"]):
raise VRTError("jobData didn't have source files data, for {0}"
.format(utilities.task_format_log(task)))
vrt_config = {
'VRTNodata': self.__config["gdal"]["vrt"]["no_data"],
'outputSRS': self.__config["gdal"]["vrt"]["output_srs"],
'resampleAlg': self.__config["gdal"]["vrt"]["resample_algo"],
'addAlpha': self.__config["gdal"]["vrt"]["add_alpha"]
}
self.log.info("Starting process GDAL-BUILD-VRT on {0} and zoom-levels: {1}"
.format(utilities.task_format_log(task), zoom_levels))
mount_path = self.__config['source_mount']
files = [path.join(mount_path, task["parameters"]['originDirectory'], file) for file in task["parameters"]['fileNames']]
vrt_result = gdal.BuildVRT(self.vrt_file_location(task["parameters"]['discreteId']), files, **vrt_config)
if vrt_result != None:
vrt_result.FlushCache()
vrt_result = None
else:
raise VRTError("Could not create VRT File")
def execute_task(self, task, zoom_levels):
try:
self.__worker.buildvrt_utility(task, zoom_levels)
self.__worker.gdal2tiles_utility(task, zoom_levels)
if self.__config['storage_provider'].upper() == StorageProvider.S3:
self.__worker.remove_s3_temp_files(task)
self.__worker.remove_vrt_file(task)
success_reason = "Task Completed"
return True, success_reason
except Exception as error:
self.log.error(
'An error occured while processing {0} with error: {1}'.format(
utilities.task_format_log(task), error))
return False, str(error)
def gdal2tiles_utility(self, task, zoom_levels):
options = {
'resampling': self.__config['gdal']['resampling'],
'tmscompatible': self.__config['gdal']['tms_compatible'],
'profile': self.__config['gdal']['profile'],
'nb_processes': self.__config['gdal']['process_count'],
'srcnodata': self.__config['gdal']['src_nodata'],
'zoom': zoom_levels,
'verbose': self.__config['gdal']['verbose']
}
discreteId = task['parameters']['discreteId']
layerRelativePath = task['parameters']['layerRelativePath']
tiles_path = '{0}/{1}'.format(self.tiles_folder_location, layerRelativePath)
self.log.info("Starting process GDAL2TILES on {0} and zoom-levels: {1}"
.format(utilities.task_format_log(task), zoom_levels))
generate_tiles(self.vrt_file_location(discreteId), tiles_path, **options)
def remove_s3_temp_files(self, task):
tiles_location = '{0}/{1}'.format(self.tiles_folder_location, task['parameters']['discreteId'])
self.log.info('Removing folder {0} on {1}'.format(tiles_location, utilities.task_format_log(task)))
shutil.rmtree(tiles_location)
def remove_vrt_file(self, task):
vrt_path = self.vrt_file_location(task['parameters']['discreteId'])
self.log.info('Removing vrt file from path "{0}", {1}'.format(vrt_path, utilities.task_format_log(task)))
remove_file(vrt_path)