def registering():
basicactivity = {}
retcode = buildBasicActivity(basicactivity)
if retcode != 0:
return json.dumps(basicactivity, indent=2)
updateRegisteringActivity(basicactivity)
basicactivity['started'] = 0
filestructure = traverse(basicactivity)
scenes = {}
logging.warning('registering - filestructure\n'+json.dumps(filestructure, indent=2))
for sat in filestructure:
logging.warning('registering - sat {}'.format(sat))
for ym in filestructure[sat]:
logging.warning('registering - ym {}'.format(ym))
for drd in filestructure[sat][ym]:
(satx,inst,yyyy_mm,date,year,jday) = decodeDRDFilename(drd)
logging.warning('registering - drd {}'.format(drd))
for bpr in filestructure[sat][ym][drd]:
pathrow = ''.join(bpr.split('_')[0:2])
logging.warning('registering - pathrow {}'.format(pathrow))
for levdatum in filestructure[sat][ym][drd][bpr]:
level = levdatum[0:1]
logging.warning('registering - level {}'.format(level))
for rp in filestructure[sat][ym][drd][bpr][levdatum]:
dataset = '{}_{}_L{}_{}'.format(sat,inst,level,rp)
key = dataset+'_'+pathrow
if key not in scenes:
scenes[key] = []
sceneid = '{}_{}{}{}'.format(sat,inst,pathrow,date.replace('-',''))
scene = {'sceneid':sceneid,'dataset':dataset,'rp':rp, \
'files':filestructure[sat][ym][drd][bpr][levdatum][rp]['files'], \
'metadata':filestructure[sat][ym][drd][bpr][levdatum][rp]['metadata'] \
}
logging.warning('registering - rp {}'.format(rp))
#logging.warning('registering - append {}'.format(filestructure[sat][ym][[drd]][levdatum][rp]))
scenes[key].append(scene)
# Group the scenes by pair
for key in scenes:
nscenes = len(scenes[key])
if nscenes < 2: continue
for idx1 in range(nscenes-1):
idx2 = idx1 + 1
scene1 = scenes[key][idx1]
scene2 = scenes[key][idx2]
# Build the activity
activity = copy.deepcopy(basicactivity)
activity['task'] = 'registeringScene'
activity['sceneid'] = scene1['sceneid']+'-'+scene2['sceneid']
activity['dataset'] = scene1['dataset']
activity['scene1'] = scene1
activity['scene2'] = scene2
db_launch(activity)
mytask = tasks.registeringScene.s(copy.deepcopy(activity))
mytask.apply_async()
logging.warning('registering - scenes\n'+json.dumps(scenes, indent=2))
return json.dumps(scenes, indent=2)
def xunzip(activity, basicactivity):
global config
# Create the task for this scene
activity['task'] = 'unzipScene'
activity['message'] = ''
inst = activity['inst']
rp = activity['rp']
logging.warning('xunzip - {}'.format(activity))
# Check if this task has been already done
count = 0
allexist = True
for band in list(activity['files']):
zfile = activity['files'][band]
tfile = zfile.replace('Level-2', 'TIFF').replace('.zip', '')
count += 1
if not os.path.exists(tfile):
logging.warning('xunzip - {}/{} - {} -> {} not exist'.format(
count, len(activity['files']), zfile, tfile))
allexist = False
else:
logging.warning('xunzip - {}/{} - {} -> {} exist'.format(
count, len(activity['files']), zfile, tfile))
"""
sql = "SELECT * FROM Scene s, Product p WHERE s.SceneId = '{0}' AND p.SceneId = '{0}' AND p.Dataset = '{1}'".format(activity['sceneid'],activity['dataset'])
presult = db_fetchall(sql)
logging.warning('xunzip {}/{} Products {}'.format(activity['sceneid'],activity['dataset'],len(presult)))
"""
sql = "SELECT * FROM Scene s, Asset p WHERE s.SceneId = '{0}' AND p.SceneId = '{0}' AND p.Dataset = '{1}'".format(
activity['sceneid'], activity['dataset'])
presult = db_fetchone(sql)
if presult is not None:
logging.warning('xunzip {}/{} Products {}'.format(
activity['sceneid'], activity['dataset'], len(presult['Assets'])))
assets = json.loads(presult['Assets'])
if len(assets) == len(activity['files']) and allexist:
basicactivity['done'] += 1
basicactivity['total'] += 1
return
db_launch(activity)
# Check if number of zips correspond to number of bands
if len(activity['files']) != len(config['publish'][rp][inst]):
activity['message'] = 'Not all files found {} != {}'.format(
len(activity['files']), len(config['publish'][rp][inst]))
db_error(activity)
basicactivity['missing'] += 1
basicactivity['total'] += 1
return
taskschain = chain([tasks.unzipScene.s(activity)])
taskschain.apply_async()
basicactivity['started'] += 1
def xpositioning(activity,basicactivity):
# Create the task for this scene
activity['task'] = 'positioningScene'
activity['sceneid'] = activity['sceneid']+'-'+activity['band']
logging.warning('xpositioning - {}'.format(activity))
db_launch(activity)
if activity['band'] not in activity['files']:
activity['message'] = 'Band {} file missing for scene in {}'.format(activity['band'],activity['tiffdir'])
db_error(activity)
basicactivity['missing'] += 1
basicactivity['total'] += 1
return
taskschain = chain([tasks.positioningScene.s(activity)])
taskschain.apply_async()
basicactivity['started'] += 1
def xupload(activity,basicactivity):
# If upload was requested and product is SR, upload it. For WPM the upload will not be done yet
logging.warning('xupload - called xupload {}'.format(activity))
inst = activity['inst']
rp = activity['rp']
if rp == 'SR' and inst != 'WPM':
activity['task'] = 'uploadScene'
db_launch(activity)
logging.warning('xupload - calling taskschain {}'.format(activity))
taskschain = chain([tasks.uploadScene.s(activity)])
taskschain.apply_async()
basicactivity['total'] += 1
basicactivity['started'] += 1
logging.warning('xupload - returning xupload {}'.format(activity))
return
def xpublish(activity,basicactivity):
global config
logging.warning('xpublish - being called activity {}'.format(activity))
sat = activity['sat']
inst = activity['inst']
rp = activity['rp']
level = activity['level']
# Create the tasks list for this scene
mytasks = []
myfirsttask = True
# If positioning was requested
if activity['positioning'] is not None:
activity['task'] = 'positioningScene'
db_launch(activity)
if activity['band'] is not None and activity['band'] not in activity['files']:
activity['message'] = 'Band {} file missing for scene in {}'.format(activity['band'],activity['tiffdir'])
db_error(activity)
basicactivity['missing'] += 1
basicactivity['total'] += 1
return
mytask = tasks.positioningScene.s(copy.deepcopy(activity))
mytask.apply_async()
# Check if there is a quality band
qualitymissing = True if 'quality' not in activity['files'] else False
# Check if this scene/dataset has already been registered
if inst == 'XXXWPM':
sql = "SELECT * FROM Product WHERE SceneId LIKE '{0}%%' AND Dataset = '{1}'".format(activity['sceneid'],activity['dataset'])
presult = db_fetchall(sql)
logging.warning('xpublish {}/{} Products {}'.format(activity['sceneid'],activity['dataset'],len(presult)))
if len(presult) == 5*len(activity['files']):
basicactivity['done'] += 1
basicactivity['total'] += 1
return
else:
"""
sql = "SELECT * FROM Scene s, Product p WHERE s.SceneId = '{0}' AND p.SceneId = '{0}' AND p.Dataset = '{1}'".format(activity['sceneid'],activity['dataset'])
presult = db_fetchall(sql)
logging.warning('xpublish {}/{} Products {}'.format(activity['sceneid'],activity['dataset'],len(presult)))
if len(presult) == len(activity['files']):
"""
sql = "SELECT * FROM Scene s, Asset p WHERE s.SceneId = '{0}' AND p.SceneId = '{0}' AND p.Dataset = '{1}'".format(activity['sceneid'],activity['dataset'])
presult = db_fetchone(sql)
logging.warning('xpublish sql {} - {}'.format(sql,presult))
if presult is not None:
logging.warning('xpublish {}/{} Products {}'.format(activity['sceneid'],activity['dataset'],len(presult['Assets'])))
assets = json.loads(presult['Assets'])
pngname = presult['thumbnail']
logging.warning('xpublish {}/{} thumbnail {}'.format(activity['sceneid'],activity['dataset'],presult['thumbnail']))
if len(assets) == len(activity['files']) and os.path.exists(pngname):
basicactivity['done'] += 1
basicactivity['total'] += 1
return
# Lets start publishing
activity['task'] = 'publish'
activity['message'] = ''
activity['status'] = ''
# If the exact number of files was not found, log error and continue
db_launch(activity)
if rp == 'SR':
# quality file may not be missing
if qualitymissing:
activity['message'] = 'Quality file missing for scene in {}'.format(activity['tiffdir'])
db_error(activity)
basicactivity['missing'] += 1
basicactivity['total'] += 1
return
# Besides NDVI and EVI, files may not be missing
if len(activity['files']) < len(config['publish'][rp][inst])-2:
activity['message'] = 'Not all files found {} < {}'.format(len(activity['files']),len(config['publish'][rp][inst])-2)
db_error(activity)
basicactivity['missing'] += 1
basicactivity['total'] += 1
return
# Check when rp is DN
else:
if len(activity['files']) != len(config['publish'][rp][inst]):
activity['message'] = 'Not all files found {} != {}'.format(len(activity['files']), len(config['publish'][rp][inst]))
db_error(activity)
basicactivity['missing'] += 1
basicactivity['total'] += 1
return
db_end(activity)
# Filling the tasks list
# If SR, generate the EVI and NDVI bands
if rp == 'SR':
activity['task'] = 'generateVI'
db_launch(activity)
mytask = tasks.generateVI.s(copy.deepcopy(activity)) if myfirsttask else tasks.generateVI.s()
mytasks.append(mytask)
myfirsttask = False
# If WPM, generate the the tiles. generateTiles task will launch generateQL and publishOneMS3 for each tile
if inst == 'XXXWPM':
activity['task'] = 'generateTiles'
db_launch(activity)
mytask = tasks.generateTiles.s(copy.deepcopy(activity)) if myfirsttask else tasks.generateTiles.s()
mytasks.append(mytask)
myfirsttask = False
else:
if 'pngname' not in activity:
activity['task'] = 'generateQL'
db_launch(activity)
mytask = tasks.generateQL.s(copy.deepcopy(activity)) if myfirsttask else tasks.generateQL.s()
mytasks.append(mytask)
myfirsttask = False
activity['task'] = 'publishOneMS3'
db_launch(activity)
mytask = tasks.publishOneMS3.s(copy.deepcopy(activity)) if myfirsttask else tasks.publishOneMS3.s()
mytasks.append(mytask)
myfirsttask = False
# If upload was requested and product is SR, upload it. For WPM the upload will not be done yet
if activity['upload'] is not None and rp == 'SR' and inst != 'WPM':
activity['task'] = 'uploadScene'
db_launch(activity)
mytask = tasks.uploadScene.s()
mytasks.append(mytask)
myfirsttask = False
basicactivity['total'] += 1
basicactivity['started'] += 1
if activity['drd'] not in basicactivity['cq']:
basicactivity['cq'].append(activity['drd'])
logging.warning('xpublish - calling taskschain {}'.format(basicactivity))
if len(mytasks) > 0:
taskschain = chain(mytasks)
taskschain.apply_async()
return