def xunzip(activity, basicactivity):
global config
# Create the task for this scene
activity['task'] = 'unzipScene'
activity['message'] = ''
inst = activity['inst']
rp = activity['rp']
logging.warning('xunzip - {}'.format(activity))
# Check if this task has been already done
count = 0
allexist = True
for band in list(activity['files']):
zfile = activity['files'][band]
tfile = zfile.replace('Level-2', 'TIFF').replace('.zip', '')
count += 1
if not os.path.exists(tfile):
logging.warning('xunzip - {}/{} - {} -> {} not exist'.format(
count, len(activity['files']), zfile, tfile))
allexist = False
else:
logging.warning('xunzip - {}/{} - {} -> {} exist'.format(
count, len(activity['files']), zfile, tfile))
"""
sql = "SELECT * FROM Scene s, Product p WHERE s.SceneId = '{0}' AND p.SceneId = '{0}' AND p.Dataset = '{1}'".format(activity['sceneid'],activity['dataset'])
presult = db_fetchall(sql)
logging.warning('xunzip {}/{} Products {}'.format(activity['sceneid'],activity['dataset'],len(presult)))
"""
sql = "SELECT * FROM Scene s, Asset p WHERE s.SceneId = '{0}' AND p.SceneId = '{0}' AND p.Dataset = '{1}'".format(
activity['sceneid'], activity['dataset'])
presult = db_fetchone(sql)
if presult is not None:
logging.warning('xunzip {}/{} Products {}'.format(
activity['sceneid'], activity['dataset'], len(presult['Assets'])))
assets = json.loads(presult['Assets'])
if len(assets) == len(activity['files']) and allexist:
basicactivity['done'] += 1
basicactivity['total'] += 1
return
db_launch(activity)
# Check if number of zips correspond to number of bands
if len(activity['files']) != len(config['publish'][rp][inst]):
activity['message'] = 'Not all files found {} != {}'.format(
len(activity['files']), len(config['publish'][rp][inst]))
db_error(activity)
basicactivity['missing'] += 1
basicactivity['total'] += 1
return
taskschain = chain([tasks.unzipScene.s(activity)])
taskschain.apply_async()
basicactivity['started'] += 1
def xpositioning(activity,basicactivity):
# Create the task for this scene
activity['task'] = 'positioningScene'
activity['sceneid'] = activity['sceneid']+'-'+activity['band']
logging.warning('xpositioning - {}'.format(activity))
db_launch(activity)
if activity['band'] not in activity['files']:
activity['message'] = 'Band {} file missing for scene in {}'.format(activity['band'],activity['tiffdir'])
db_error(activity)
basicactivity['missing'] += 1
basicactivity['total'] += 1
return
taskschain = chain([tasks.positioningScene.s(activity)])
taskschain.apply_async()
basicactivity['started'] += 1
def xpublish(activity,basicactivity):
global config
logging.warning('xpublish - being called activity {}'.format(activity))
sat = activity['sat']
inst = activity['inst']
rp = activity['rp']
level = activity['level']
# Create the tasks list for this scene
mytasks = []
myfirsttask = True
# If positioning was requested
if activity['positioning'] is not None:
activity['task'] = 'positioningScene'
db_launch(activity)
if activity['band'] is not None and activity['band'] not in activity['files']:
activity['message'] = 'Band {} file missing for scene in {}'.format(activity['band'],activity['tiffdir'])
db_error(activity)
basicactivity['missing'] += 1
basicactivity['total'] += 1
return
mytask = tasks.positioningScene.s(copy.deepcopy(activity))
mytask.apply_async()
# Check if there is a quality band
qualitymissing = True if 'quality' not in activity['files'] else False
# Check if this scene/dataset has already been registered
if inst == 'XXXWPM':
sql = "SELECT * FROM Product WHERE SceneId LIKE '{0}%%' AND Dataset = '{1}'".format(activity['sceneid'],activity['dataset'])
presult = db_fetchall(sql)
logging.warning('xpublish {}/{} Products {}'.format(activity['sceneid'],activity['dataset'],len(presult)))
if len(presult) == 5*len(activity['files']):
basicactivity['done'] += 1
basicactivity['total'] += 1
return
else:
"""
sql = "SELECT * FROM Scene s, Product p WHERE s.SceneId = '{0}' AND p.SceneId = '{0}' AND p.Dataset = '{1}'".format(activity['sceneid'],activity['dataset'])
presult = db_fetchall(sql)
logging.warning('xpublish {}/{} Products {}'.format(activity['sceneid'],activity['dataset'],len(presult)))
if len(presult) == len(activity['files']):
"""
sql = "SELECT * FROM Scene s, Asset p WHERE s.SceneId = '{0}' AND p.SceneId = '{0}' AND p.Dataset = '{1}'".format(activity['sceneid'],activity['dataset'])
presult = db_fetchone(sql)
logging.warning('xpublish sql {} - {}'.format(sql,presult))
if presult is not None:
logging.warning('xpublish {}/{} Products {}'.format(activity['sceneid'],activity['dataset'],len(presult['Assets'])))
assets = json.loads(presult['Assets'])
pngname = presult['thumbnail']
logging.warning('xpublish {}/{} thumbnail {}'.format(activity['sceneid'],activity['dataset'],presult['thumbnail']))
if len(assets) == len(activity['files']) and os.path.exists(pngname):
basicactivity['done'] += 1
basicactivity['total'] += 1
return
# Lets start publishing
activity['task'] = 'publish'
activity['message'] = ''
activity['status'] = ''
# If the exact number of files was not found, log error and continue
db_launch(activity)
if rp == 'SR':
# quality file may not be missing
if qualitymissing:
activity['message'] = 'Quality file missing for scene in {}'.format(activity['tiffdir'])
db_error(activity)
basicactivity['missing'] += 1
basicactivity['total'] += 1
return
# Besides NDVI and EVI, files may not be missing
if len(activity['files']) < len(config['publish'][rp][inst])-2:
activity['message'] = 'Not all files found {} < {}'.format(len(activity['files']),len(config['publish'][rp][inst])-2)
db_error(activity)
basicactivity['missing'] += 1
basicactivity['total'] += 1
return
# Check when rp is DN
else:
if len(activity['files']) != len(config['publish'][rp][inst]):
activity['message'] = 'Not all files found {} != {}'.format(len(activity['files']), len(config['publish'][rp][inst]))
db_error(activity)
basicactivity['missing'] += 1
basicactivity['total'] += 1
return
db_end(activity)
# Filling the tasks list
# If SR, generate the EVI and NDVI bands
if rp == 'SR':
activity['task'] = 'generateVI'
db_launch(activity)
mytask = tasks.generateVI.s(copy.deepcopy(activity)) if myfirsttask else tasks.generateVI.s()
mytasks.append(mytask)
myfirsttask = False
# If WPM, generate the the tiles. generateTiles task will launch generateQL and publishOneMS3 for each tile
if inst == 'XXXWPM':
activity['task'] = 'generateTiles'
db_launch(activity)
mytask = tasks.generateTiles.s(copy.deepcopy(activity)) if myfirsttask else tasks.generateTiles.s()
mytasks.append(mytask)
myfirsttask = False
else:
if 'pngname' not in activity:
activity['task'] = 'generateQL'
db_launch(activity)
mytask = tasks.generateQL.s(copy.deepcopy(activity)) if myfirsttask else tasks.generateQL.s()
mytasks.append(mytask)
myfirsttask = False
activity['task'] = 'publishOneMS3'
db_launch(activity)
mytask = tasks.publishOneMS3.s(copy.deepcopy(activity)) if myfirsttask else tasks.publishOneMS3.s()
mytasks.append(mytask)
myfirsttask = False
# If upload was requested and product is SR, upload it. For WPM the upload will not be done yet
if activity['upload'] is not None and rp == 'SR' and inst != 'WPM':
activity['task'] = 'uploadScene'
db_launch(activity)
mytask = tasks.uploadScene.s()
mytasks.append(mytask)
myfirsttask = False
basicactivity['total'] += 1
basicactivity['started'] += 1
if activity['drd'] not in basicactivity['cq']:
basicactivity['cq'].append(activity['drd'])
logging.warning('xpublish - calling taskschain {}'.format(basicactivity))
if len(mytasks) > 0:
taskschain = chain(mytasks)
taskschain.apply_async()
return