def process_obj():
process = Process()
json_dict = {
'data2isis': {
'from_': 'some_image.img',
'to': 'some_image.cub'
},
'spiceinit': {
'from_': 'some_image.cub'
}
}
json_str = json.dumps(json_dict)
process.JSON2Process(json_str)
return process
def main(user_args):
key = user_args.key
namespace = user_args.namespace
if namespace is None:
namespace = default_namespace
workarea = scratch + key + '/'
RQ_file = RedisQueue(key + '_FileQueue', namespace)
RQ_work = RedisQueue(key + '_WorkQueue', namespace)
RQ_zip = RedisQueue(key + '_ZIP', namespace)
RQ_loggy = RedisQueue(key + '_loggy', namespace)
RQ_final = RedisQueue('FinalQueue', namespace)
RQ_recipe = RedisQueue(key + '_recipe', namespace)
RHash = RedisHash(key + '_info')
RHerror = RedisHash(key + '_error')
RQ_lock = RedisLock(lock_obj)
RQ_lock.add({'MAP': '1'})
if int(RQ_file.QueueSize()) == 0:
print("No Files Found in Redis Queue")
elif RQ_lock.available('MAP'):
jobFile = RQ_file.Qfile2Qwork(RQ_file.getQueueName(),
RQ_work.getQueueName())
# Setup system logging
basename = os.path.splitext(os.path.basename(jobFile))[0]
logger = logging.getLogger(key + '.' + basename)
logger.setLevel(logging.INFO)
logFileHandle = logging.FileHandler(pds_log + '/Service.log')
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s, %(message)s')
logFileHandle.setFormatter(formatter)
logger.addHandler(logFileHandle)
logger.info('Starting MAP Processing')
loggyOBJ = Loggy(basename)
# File Naming
infile = workarea + \
os.path.splitext(os.path.basename(jobFile))[0] + '.input.cub'
outfile = workarea + \
os.path.splitext(os.path.basename(jobFile))[0] + '.output.cub'
# Recipe Stuff
status = 'success'
for element in RQ_recipe.RecipeGet():
if status == 'error':
break
elif status == 'success':
processOBJ = Process()
process = processOBJ.JSON2Process(element)
if 'gdal_translate' not in processOBJ.getProcessName():
if 'cubeatt-band' in processOBJ.getProcessName():
if '+' in jobFile:
processOBJ.updateParameter('from_', jobFile)
processOBJ.updateParameter('to', outfile)
processOBJ.ChangeProcess('cubeatt')
else:
continue
elif 'map2map' in processOBJ.getProcessName():
if '+' in jobFile:
processOBJ.updateParameter('from_', infile)
else:
processOBJ.updateParameter('from_', jobFile)
processOBJ.updateParameter('to', outfile)
elif 'cubeatt-bit' in processOBJ.getProcessName():
if RHash.OutBit() == 'unsignedbyte':
temp_outfile = outfile + '+lsb+tile+attached+unsignedbyte+1:254'
elif RHash.OutBit() == 'signedword':
temp_outfile = outfile + '+lsb+tile+attached+signedword+-32765:32765'
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', temp_outfile)
processOBJ.ChangeProcess('cubeatt')
elif 'isis2pds' in processOBJ.getProcessName():
# finalfile = infile.replace('.input.cub', '_final.img')
finalfile = workarea + RHash.getMAPname() + '.img'
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', finalfile)
else:
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', outfile)
print(processOBJ.getProcess())
for k, v in processOBJ.getProcess().items():
func = getattr(isis, k)
subloggyOBJ = SubLoggy(k)
try:
func(**v)
logger.info('Process %s :: Success', k)
subloggyOBJ.setStatus('SUCCESS')
subloggyOBJ.setCommand(processOBJ.LogCommandline())
subloggyOBJ.setHelpLink(processOBJ.LogHelpLink())
loggyOBJ.AddProcess(subloggyOBJ.getSLprocess())
if os.path.isfile(outfile):
os.rename(outfile, infile)
status = 'success'
except ProcessError as e:
logger.error('Process %s :: Error', k)
logger.error(e)
status = 'error'
eSTR = 'Error Executing ' + k + \
' Standard Error: ' + str(e)
RHerror.addError(
os.path.splitext(os.path.basename(jobFile))[0],
eSTR)
subloggyOBJ.setStatus('ERROR')
subloggyOBJ.setCommand(processOBJ.LogCommandline())
subloggyOBJ.setHelpLink(processOBJ.LogHelpLink())
subloggyOBJ.errorOut(eSTR)
loggyOBJ.AddProcess(subloggyOBJ.getSLprocess())
RQ_work.QueueRemove(jobFile)
break
else:
GDALcmd = ""
for process, v, in processOBJ.getProcess().items():
subloggyOBJ = SubLoggy(process)
GDALcmd += process
for dict_key, value in v.items():
GDALcmd += ' ' + dict_key + ' ' + value
img_format = RHash.Format()
if img_format == 'GeoTiff-BigTiff':
fileext = 'tif'
elif img_format == 'GeoJPEG-2000':
fileext = 'jp2'
elif img_format == 'JPEG':
fileext = 'jpg'
elif img_format == 'PNG':
fileext = 'png'
elif img_format == 'GIF':
fileext = 'gif'
logGDALcmd = GDALcmd + ' ' + basename + '.input.cub ' + RHash.getMAPname(
) + '.' + fileext
finalfile = workarea + RHash.getMAPname() + '.' + fileext
GDALcmd += ' ' + infile + ' ' + finalfile
print(GDALcmd)
try:
subprocess.call(GDALcmd, shell=True)
logger.info('Process GDAL translate :: Success')
status = 'success'
subloggyOBJ.setStatus('SUCCESS')
subloggyOBJ.setCommand(logGDALcmd)
subloggyOBJ.setHelpLink(
'www.gdal.org/gdal_translate.html')
loggyOBJ.AddProcess(subloggyOBJ.getSLprocess())
os.remove(infile)
except OSError as e:
logger.error('Process GDAL translate :: Error')
logger.error(e)
status = 'error'
RHerror.addError(
os.path.splitext(os.path.basename(jobFile))[0],
'Process GDAL translate :: Error')
subloggyOBJ.setStatus('ERROR')
subloggyOBJ.setCommand(logGDALcmd)
subloggyOBJ.setHelpLink(
'http://www.gdal.org/gdal_translate.html')
subloggyOBJ.errorOut(e)
loggyOBJ.AddProcess(subloggyOBJ.getSLprocess())
if status == 'success':
if RHash.Format() == 'ISIS3':
finalfile = workarea + RHash.getMAPname() + '.cub'
shutil.move(infile, finalfile)
if RHash.getStatus() != 'ERROR':
RHash.Status('SUCCESS')
try:
RQ_zip.QueueAdd(finalfile)
logger.info('File Added to ZIP Queue')
except:
logger.error('File NOT Added to ZIP Queue')
try:
RQ_loggy.QueueAdd(loggyOBJ.Loggy2json())
logger.info('JSON Added to Loggy Queue')
except:
logger.error('JSON NOT Added to Loggy Queue')
RQ_work.QueueRemove(jobFile)
elif status == 'error':
RHash.Status('ERROR')
if os.path.isfile(infile):
os.remove(infile)
if RQ_file.QueueSize() == 0 and RQ_work.QueueSize() == 0:
try:
RQ_final.QueueAdd(key)
logger.info('Key %s Added to Final Queue: Success', key)
logger.info('Job Complete')
except:
logger.error('Key NOT Added to Final Queue')
else:
logger.warning('Queues Not Empty: filequeue = %s work queue = %s',
str(RQ_file.QueueSize()), str(RQ_work.QueueSize()))
def main(user_args):
log_level = user_args.log_level
logger = logging.getLogger('Browse_Process')
level = logging.getLevelName(log_level)
logger.setLevel(level)
logFileHandle = logging.FileHandler(pds_log + 'Process.log')
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s, %(message)s')
logFileHandle.setFormatter(formatter)
logger.addHandler(logFileHandle)
RQ_main = RedisQueue('Browse_ReadyQueue')
RQ_error = RedisQueue(upc_error_queue)
RQ_lock = RedisLock(lock_obj)
RQ_lock.add({RQ_main.id_name: '1'})
PDSinfoDICT = json.load(open(pds_info, 'r'))
pds_session_maker, pds_engine = db_connect(pds_db)
pds_session = pds_session_maker()
upc_session_maker, upc_engine = db_connect(upc_db)
upc_session = upc_session_maker()
tid = get_tid('fullimageurl', upc_session)
while int(RQ_main.QueueSize()) > 0 and RQ_lock.available(RQ_main.id_name):
item = literal_eval(RQ_main.QueueGet())
inputfile = item[0]
fid = item[1]
archive = item[2]
if os.path.isfile(inputfile):
logger.info('Starting Process: %s', inputfile)
finalpath = makedir(inputfile)
recipeOBJ = Recipe()
recipeOBJ.addMissionJson(archive, 'reduced')
infile = workarea + os.path.splitext(
os.path.basename(inputfile))[0] + '.Binput.cub'
outfile = workarea + os.path.splitext(
os.path.basename(inputfile))[0] + '.Boutput.cub'
status = 'success'
for item in recipeOBJ.getProcesses():
if status == 'error':
logger.error("Error processing %s", inputfile)
break
elif status == 'success':
processOBJ = Process()
processOBJ.ProcessFromRecipe(item, recipeOBJ.getRecipe())
if '2isis' in item:
processOBJ.updateParameter('from_', inputfile)
processOBJ.updateParameter('to', outfile)
elif item == 'spiceinit':
processOBJ.updateParameter('from_', infile)
elif item == 'cubeatt':
label = pvl.load(infile)
bands = PDSinfoDICT[archive]['bandorder']
query_bands = label['IsisCube']['BandBin'][
PDSinfoDICT[archive]['bandbinQuery']]
# Create a set from the list / single value
try:
query_band_set = set(query_bands)
except:
query_band_set = set([query_bands])
# Iterate through 'bands' and grab the first value that is present in the
# set defined by 'bandbinquery' -- if not present, default to 1
exband = next(
(band for band in bands if band in query_band_set),
1)
band_infile = infile + '+' + str(exband)
processOBJ.updateParameter('from_', band_infile)
processOBJ.updateParameter('to', outfile)
elif item == 'ctxevenodd':
label = pvl.load(infile)
SS = label['IsisCube']['Instrument']['SpatialSumming']
if SS != 1:
break
else:
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', outfile)
elif item == 'reduce':
label = pvl.load(infile)
Nline = label['IsisCube']['Core']['Dimensions'][
'Lines']
Nsample = label['IsisCube']['Core']['Dimensions'][
'Samples']
Nline = int(Nline)
Nsample = int(Nsample)
Sfactor = scaleFactor(Nline, Nsample, recip_json)
processOBJ.updateParameter('lscale', Sfactor)
processOBJ.updateParameter('sscale', Sfactor)
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', outfile)
elif item == 'isis2std':
final_outfile = finalpath + '/' + os.path.splitext(
os.path.basename(inputfile))[0] + '.browse.jpg'
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', final_outfile)
else:
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', outfile)
for k, v in processOBJ.getProcess().items():
func = getattr(isis, k)
try:
func(**v)
logger.info('Process %s :: Success', k)
if os.path.isfile(outfile):
if '.cub' in outfile:
os.rename(outfile, infile)
status = 'success'
if '2isis' in item:
isisSerial = getISISid(infile)
except ProcessError as e:
print(e)
logger.error('Process %s :: Error', k)
status = 'error'
if status == 'success':
DB_addURL(upc_session, isisSerial, final_outfile, tid)
os.remove(infile)
logger.info('Browse Process Success: %s', inputfile)
AddProcessDB(pds_session, fid, 't')
else:
RQ_error.QueueAdd(
f'Unable to locate or access {inputfile} during browse processing'
)
logger.error('File %s Not Found', inputfile)
upc_session.close()
pds_session.close()
upc_engine.dispose()
pds_engine.dispose()
def main(user_args):
key = user_args.key
norun = user_args.norun
namespace = user_args.namespace
if namespace is None:
namespace = default_namespace
# Set up logging
logger = logging.getLogger(key)
logger.setLevel(logging.INFO)
logFileHandle = logging.FileHandler(pds_log + 'Service.log')
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s, %(message)s')
logFileHandle.setFormatter(formatter)
logger.addHandler(logFileHandle)
RQ_lock = RedisLock(lock_obj)
RQ_lock.add({'Services': '1'})
if not RQ_lock.available('Services'):
exit()
# Connect to database and access 'jobs' table
DBQO = PDS_DBquery('JOBS')
if key is None:
# If no key is specified, grab the first key
try:
key = DBQO.jobKey()
try:
key = key.decode('utf-8')
except:
pass
# If the queue is empty, it'll throw a type error.
except TypeError:
logger.debug('No keys found in clusterjobs database')
exit(1)
try:
# Set the 'queued' column to current time i.e. prep for processing
DBQO.setJobsQueued(key)
except KeyError as e:
logger.error('%s', e)
exit(1)
logger.info('Starting Process')
xmlOBJ = jobXML(DBQO.jobXML4Key(key))
# Make directory if it doesn't exist
directory = scratch + key
if not os.path.exists(directory):
os.makedirs(directory)
logger.info('Working Area: %s', directory)
# Set up Redis Hash for ground range
RedisH = RedisHash(key + '_info')
RedisH.RemoveAll()
RedisErrorH = RedisHash(key + '_error')
RedisErrorH.RemoveAll()
RedisH_DICT = {}
RedisH_DICT['service'] = xmlOBJ.getProcess()
RedisH_DICT['fileformat'] = xmlOBJ.getOutFormat()
RedisH_DICT['outbit'] = xmlOBJ.getOutBit()
if xmlOBJ.getRangeType() is not None:
RedisH_DICT['grtype'] = xmlOBJ.getRangeType()
RedisH_DICT['minlat'] = xmlOBJ.getMinLat()
RedisH_DICT['maxlat'] = xmlOBJ.getMaxLat()
RedisH_DICT['minlon'] = xmlOBJ.getMinLon()
RedisH_DICT['maxlon'] = xmlOBJ.getMaxLon()
if RedisH.IsInHash('service'):
pass
else:
RedisH.AddHash(RedisH_DICT)
if RedisH.IsInHash('service'):
logger.info('Redis info Hash: Success')
else:
logger.error('Redis info Hash Not Found')
# End ground range
RQ_recipe = RedisQueue(key + '_recipe', namespace)
RQ_recipe.RemoveAll()
RQ_file = RedisQueue(key + '_FileQueue', namespace)
RQ_file.RemoveAll()
RQ_WorkQueue = RedisQueue(key + '_WorkQueue', namespace)
RQ_WorkQueue.RemoveAll()
RQ_loggy = RedisQueue(key + '_loggy', namespace)
RQ_loggy.RemoveAll()
RQ_zip = RedisQueue(key + '_ZIP', namespace)
RQ_zip.RemoveAll()
if xmlOBJ.getProcess() == 'POW':
fileList = xmlOBJ.getFileListWB()
elif xmlOBJ.getProcess() == 'MAP2':
fileList = xmlOBJ.getMFileListWB()
for List_file in fileList:
# Input and output file naming and path stuff
if xmlOBJ.getProcess() == 'POW':
if xmlOBJ.getInst() == 'THEMIS_IR':
Input_file = List_file.replace('odtie1_', 'odtir1_')
Input_file = Input_file.replace('xxedr', 'xxrdr')
Input_file = Input_file.replace('EDR.QUB', 'RDR.QUB')
Input_file = Input_file.replace(
'http://pdsimage.wr.usgs.gov/Missions/', archive_base)
elif xmlOBJ.getInst() == 'ISSNA':
Input_file = List_file.replace('.IMG', '.LBL')
Input_file = Input_file.replace(
'http://pdsimage.wr.usgs.gov/Missions/', archive_base)
elif xmlOBJ.getInst() == 'ISSWA':
Input_file = List_file.replace('.IMG', '.LBL')
Input_file = Input_file.replace(
'http://pdsimage.wr.usgs.gov/Missions/', archive_base)
elif xmlOBJ.getInst() == 'SOLID STATE IMAGING SYSTEM':
Input_file = List_file.replace('.img', '.lbl')
Input_file = Input_file.replace(
'http://pdsimage.wr.usgs.gov/Missions/', archive_base)
else:
Input_file = List_file.replace(
'http://pdsimage.wr.usgs.gov/Missions/', archive_base)
elif xmlOBJ.getProcess() == 'MAP2':
Input_file = List_file.replace('file://pds_san', '/pds_san')
if '+' in Input_file:
tempsplit = Input_file.split('+')
tempFile = tempsplit[0]
else:
tempFile = Input_file
label = pvl.load(tempFile)
# Output final file naming
Tbasename = os.path.splitext(os.path.basename(tempFile))[0]
splitBase = Tbasename.split('_')
labP = xmlOBJ.getProjection()
if labP == 'INPUT':
lab_proj = label['IsisCube']['Mapping']['ProjectionName'][0:4]
else:
lab_proj = labP[0:4]
if xmlOBJ.getClat() is None or xmlOBJ.getClon() is None:
basefinal = splitBase[0] + splitBase[1] + \
splitBase[2] + '_MAP2_' + lab_proj.upper()
else:
lab_clat = float(xmlOBJ.getClat())
if lab_clat >= 0:
labH = 'N'
elif lab_clat < 0:
labH = 'S'
lab_clon = float(xmlOBJ.getClon())
basefinal = splitBase[0] + splitBase[1] + splitBase[
2] + '_MAP2_' + str(lab_clat) + labH + str(
lab_clon) + '_' + lab_proj.upper()
RedisH.MAPname(basefinal)
try:
RQ_file.QueueAdd(Input_file)
logger.info('File %s Added to Redis Queue', Input_file)
except Exception as e:
logger.warn('File %s NOT Added to Redis Queue', Input_file)
print('Redis Queue Error', e)
RedisH.FileCount(RQ_file.QueueSize())
logger.info('Count of Files Queue: %s', str(RQ_file.QueueSize()))
# Map Template Stuff
logger.info('Making Map File')
mapOBJ = MakeMap()
if xmlOBJ.getProcess() == 'MAP2' and xmlOBJ.getProjection() == 'INPUT':
proj = label['IsisCube']['Mapping']['ProjectionName']
mapOBJ.Projection(proj)
else:
mapOBJ.Projection(xmlOBJ.getProjection())
if xmlOBJ.getClon() is not None:
mapOBJ.CLon(float(xmlOBJ.getClon()))
if xmlOBJ.getClat() is not None:
mapOBJ.CLat(float(xmlOBJ.getClat()))
if xmlOBJ.getFirstParallel() is not None:
mapOBJ.FirstParallel(float(xmlOBJ.getFirstParallel()))
if xmlOBJ.getSecondParallel() is not None:
mapOBJ.SecondParallel(float(xmlOBJ.getSecondParallel()))
if xmlOBJ.getResolution() is not None:
mapOBJ.PixelRes(float(xmlOBJ.getResolution()))
if xmlOBJ.getTargetName() is not None:
mapOBJ.Target(xmlOBJ.getTargetName())
if xmlOBJ.getERadius() is not None:
mapOBJ.ERadius(float(xmlOBJ.getERadius()))
if xmlOBJ.getPRadius() is not None:
mapOBJ.PRadius(float(xmlOBJ.getPRadius()))
if xmlOBJ.getLatType() is not None:
mapOBJ.LatType(xmlOBJ.getLatType())
if xmlOBJ.getLonDirection() is not None:
mapOBJ.LonDirection(xmlOBJ.getLonDirection())
if xmlOBJ.getLonDomain() is not None:
mapOBJ.LonDomain(int(xmlOBJ.getLonDomain()))
if xmlOBJ.getProcess() == 'MAP2':
if xmlOBJ.getMinLat() is not None:
mapOBJ.MinLat(float(xmlOBJ.getMinLat()))
if xmlOBJ.getMaxLat() is not None:
mapOBJ.MaxLat(float(xmlOBJ.getMaxLat()))
if xmlOBJ.getMinLon() is not None:
mapOBJ.MinLon(float(xmlOBJ.getMinLon()))
if xmlOBJ.getMaxLon() is not None:
mapOBJ.MaxLon(float(xmlOBJ.getMaxLon()))
mapOBJ.Map2pvl()
MAPfile = directory + "/" + key + '.map'
mapOBJ.Map2File(MAPfile)
try:
f = open(MAPfile)
f.close
logger.info('Map File Creation: Success')
except IOError as e:
logger.error('Map File %s Not Found', MAPfile)
# ** End Map Template Stuff **
logger.info('Building Recipe')
recipeOBJ = Recipe()
if xmlOBJ.getProcess() == 'POW':
recipeOBJ.AddJsonFile(recipe_base + xmlOBJ.getCleanName() + '.json',
"pow")
elif xmlOBJ.getProcess() == 'MAP2':
recipeOBJ.AddJsonFile(recipe_base + "map2_process.json", "map")
# Test for stretch and add to recipe
# if MAP2 and 8 or 16 bit run stretch to set range
if xmlOBJ.getOutBit() == 'input':
testBitType = str(label['IsisCube']['Core']['Pixels']['Type']).upper()
else:
testBitType = xmlOBJ.getOutBit().upper()
if xmlOBJ.getProcess() == 'MAP2' and xmlOBJ.STR_Type() is None:
if str(label['IsisCube']['Core']['Pixels']['Type']).upper(
) != xmlOBJ.getOutBit().upper() and str(
label['IsisCube']['Core']['Pixels']['Type']).upper() != 'REAL':
if str(label['IsisCube']['Core']['Pixels']
['Type']).upper() == 'SIGNEDWORD':
strpairs = '0:-32765 0:-32765 100:32765 100:32765'
elif str(label['IsisCube']['Core']['Pixels']
['Type']).upper() == 'UNSIGNEDBYTE':
strpairs = '0:1 0:1 100:254 100:254'
STRprocessOBJ = Process()
STRprocessOBJ.newProcess('stretch')
STRprocessOBJ.AddParameter('from_', 'value')
STRprocessOBJ.AddParameter('to', 'value')
STRprocessOBJ.AddParameter('usepercentages', 'yes')
STRprocessOBJ.AddParameter('pairs', strpairs)
recipeOBJ.AddProcess(STRprocessOBJ.getProcess())
strType = xmlOBJ.STR_Type()
if strType == 'StretchPercent' and xmlOBJ.STR_PercentMin(
) is not None and xmlOBJ.STR_PercentMax(
) is not None and testBitType != 'REAL':
if float(xmlOBJ.STR_PercentMin()) != 0 and float(
xmlOBJ.STR_PercentMax()) != 100:
if testBitType == 'UNSIGNEDBYTE':
strpairs = '0:1 ' + xmlOBJ.STR_PercentMin() + ':1 ' + \
xmlOBJ.STR_PercentMax() + ':254 100:254'
elif testBitType == 'SIGNEDWORD':
strpairs = '0:-32765 ' + xmlOBJ.STR_PercentMin() + ':-32765 ' + \
xmlOBJ.STR_PercentMax() + ':32765 100:32765'
STRprocessOBJ = Process()
STRprocessOBJ.newProcess('stretch')
STRprocessOBJ.AddParameter('from_', 'value')
STRprocessOBJ.AddParameter('to', 'value')
STRprocessOBJ.AddParameter('usepercentages', 'yes')
STRprocessOBJ.AddParameter('pairs', strpairs)
recipeOBJ.AddProcess(STRprocessOBJ.getProcess())
elif strType == 'GaussStretch':
STRprocessOBJ = Process()
STRprocessOBJ.newProcess('gaussstretch')
STRprocessOBJ.AddParameter('from_', 'value')
STRprocessOBJ.AddParameter('to', 'value')
STRprocessOBJ.AddParameter('gsigma', xmlOBJ.STR_GaussSigma())
recipeOBJ.AddProcess(STRprocessOBJ.getProcess())
elif strType == 'HistogramEqualization':
STRprocessOBJ = Process()
STRprocessOBJ.newProcess('histeq')
STRprocessOBJ.AddParameter('from_', 'value')
STRprocessOBJ.AddParameter('to', 'value')
if xmlOBJ.STR_PercentMin() is None:
STRprocessOBJ.AddParameter('minper', '0')
else:
STRprocessOBJ.AddParameter('minper', xmlOBJ.STR_PercentMin())
if xmlOBJ.STR_PercentMax() is None:
STRprocessOBJ.AddParameter('maxper', '100')
else:
STRprocessOBJ.AddParameter('maxper', xmlOBJ.STR_PercentMax())
recipeOBJ.AddProcess(STRprocessOBJ.getProcess())
elif strType == 'SigmaStretch':
STRprocessOBJ = Process()
STRprocessOBJ.newProcess('sigmastretch')
STRprocessOBJ.AddParameter('from_', 'value')
STRprocessOBJ.AddParameter('to', 'value')
STRprocessOBJ.AddParameter('variance', xmlOBJ.STR_SigmaVariance())
recipeOBJ.AddProcess(STRprocessOBJ.getProcess())
# Test for output bit type and add to recipe
if xmlOBJ.getProcess() == 'POW':
if xmlOBJ.getOutBit().upper() == 'UNSIGNEDBYTE' or xmlOBJ.getOutBit(
).upper() == 'SIGNEDWORD':
CAprocessOBJ = Process()
CAprocessOBJ.newProcess('cubeatt-bit')
CAprocessOBJ.AddParameter('from_', 'value')
CAprocessOBJ.AddParameter('to', 'value')
recipeOBJ.AddProcess(CAprocessOBJ.getProcess())
elif xmlOBJ.getProcess() == 'MAP2':
if xmlOBJ.getOutBit().upper() != 'INPUT':
if xmlOBJ.getOutBit().upper(
) == 'UNSIGNEDBYTE' or xmlOBJ.getOutBit().upper() == 'SIGNEDWORD':
if str(label['IsisCube']['Core']['Pixels']
['Type']).upper() != xmlOBJ.getOutBit().upper():
CAprocessOBJ = Process()
CAprocessOBJ.newProcess('cubeatt-bit')
CAprocessOBJ.AddParameter('from_', 'value')
CAprocessOBJ.AddParameter('to', 'value')
recipeOBJ.AddProcess(CAprocessOBJ.getProcess())
# Add Grid(MAP2)
if xmlOBJ.getGridInterval() is not None:
GprocessOBJ = Process()
GprocessOBJ.newProcess('grid')
GprocessOBJ.AddParameter('from_', 'value')
GprocessOBJ.AddParameter('to', 'value')
GprocessOBJ.AddParameter('latinc', xmlOBJ.getGridInterval())
GprocessOBJ.AddParameter('loninc', xmlOBJ.getGridInterval())
GprocessOBJ.AddParameter('outline', 'yes')
GprocessOBJ.AddParameter('boundary', 'yes')
GprocessOBJ.AddParameter('linewidth', '3')
recipeOBJ.AddProcess(GprocessOBJ.getProcess())
# OUTPUT FORMAT
# Test for GDAL and add to recipe
Oformat = xmlOBJ.getOutFormat()
if Oformat == 'GeoTiff-BigTiff' or Oformat == 'GeoJPEG-2000' or Oformat == 'JPEG' or Oformat == 'PNG':
if Oformat == 'GeoJPEG-2000':
Oformat = 'JP2KAK'
if Oformat == 'GeoTiff-BigTiff':
Oformat = 'GTiff'
GDALprocessOBJ = Process()
GDALprocessOBJ.newProcess('gdal_translate')
if xmlOBJ.getOutBit() != 'input':
GDALprocessOBJ.AddParameter(
'-ot', GDALprocessOBJ.GDAL_OBit(xmlOBJ.getOutBit()))
GDALprocessOBJ.AddParameter('-of', Oformat)
if Oformat == 'GTiff' or Oformat == 'JP2KAK' or Oformat == 'JPEG':
GDALprocessOBJ.AddParameter('-co',
GDALprocessOBJ.GDAL_Creation(Oformat))
recipeOBJ.AddProcess(GDALprocessOBJ.getProcess())
# set up pds2isis and add to recipe
elif Oformat == 'PDS':
pdsProcessOBJ = Process()
pdsProcessOBJ.newProcess('isis2pds')
pdsProcessOBJ.AddParameter('from_', 'value')
pdsProcessOBJ.AddParameter('to', 'value')
if xmlOBJ.getOutBit() == 'unsignedbyte':
pdsProcessOBJ.AddParameter('bittype', '8bit')
elif xmlOBJ.getOutBit() == 'signedword':
pdsProcessOBJ.AddParameter('bittype', 's16bit')
recipeOBJ.AddProcess(pdsProcessOBJ.getProcess())
for item in recipeOBJ.getProcesses():
processOBJ = Process()
processOBJ.ProcessFromRecipe(item, recipeOBJ.getRecipe())
if item == 'cam2map':
processOBJ.updateParameter('map', MAPfile)
if xmlOBJ.getResolution() is None:
processOBJ.updateParameter('pixres', 'CAMERA')
else:
processOBJ.updateParameter('pixres', 'MAP')
if xmlOBJ.getRangeType() is None:
processOBJ.updateParameter('defaultrange', 'MINIMIZE')
elif xmlOBJ.getRangeType() == 'smart' or xmlOBJ.getRangeType(
) == 'fill':
processOBJ.updateParameter('defaultrange', 'CAMERA')
processOBJ.AddParameter('trim', 'YES')
elif item == 'map2map':
processOBJ.updateParameter('map', MAPfile)
if xmlOBJ.getResolution() is None:
processOBJ.updateParameter('pixres', 'FROM')
else:
processOBJ.updateParameter('pixres', 'MAP')
if xmlOBJ.OutputGeometry() is not None:
processOBJ.updateParameter('defaultrange', 'MAP')
processOBJ.AddParameter('trim', 'YES')
else:
processOBJ.updateParameter('defaultrange', 'FROM')
processJSON = processOBJ.Process2JSON()
try:
RQ_recipe.QueueAdd(processJSON)
logger.info('Recipe Element Added to Redis: %s : Success', item)
except Exception as e:
logger.warn('Recipe Element NOT Added to Redis: %s', item)
# HPC job stuff
logger.info('HPC Cluster job Submission Starting')
jobOBJ = HPCjob()
jobOBJ.setJobName(key + '_Service')
jobOBJ.setStdOut(slurm_log + key + '_%A_%a.out')
jobOBJ.setStdError(slurm_log + key + '_%A_%a.err')
jobOBJ.setWallClock('24:00:00')
jobOBJ.setMemory('24576')
jobOBJ.setPartition('pds')
JAsize = RQ_file.QueueSize()
jobOBJ.setJobArray(JAsize)
logger.info('Job Array Size : %s', str(JAsize))
# @TODO replace with source activate <env>
#jobOBJ.addPath('/usgs/apps/anaconda/bin')
# Whether or not we use the default namespace, this guarantees that the POW/MAP queues will match the namespace
# used in the job manager.
if xmlOBJ.getProcess() == 'POW':
cmd = cmd_dir + "pow_process.py -k {} -n {}".format(key, namespace)
elif xmlOBJ.getProcess() == 'MAP2':
cmd = cmd_dir + "map_process.py -k {} -n {}".format(key, namespace)
logger.info('HPC Command: %s', cmd)
jobOBJ.setCommand(cmd)
SBfile = directory + '/' + key + '.sbatch'
jobOBJ.MakeJobFile(SBfile)
try:
sb = open(SBfile)
sb.close
logger.info('SBATCH File Creation: Success')
except IOError as e:
logger.error('SBATCH File %s Not Found', SBfile)
if norun:
logger.info('No-run mode, will not submit HPC job.')
else:
try:
jobOBJ.Run()
logger.info('Job Submission to HPC: Success')
DBQO.setJobsStarted(key)
except IOError as e:
logger.error('Jobs NOT Submitted to HPC')
def main(key, namespace=None):
key = user_args.key
namespace = user_args.namespace
print(namespace)
if namespace is None:
namespace = default_namespace
workarea = scratch + key + '/'
RQ_file = RedisQueue(key + '_FileQueue', namespace)
RQ_work = RedisQueue(key + '_WorkQueue', namespace)
RQ_zip = RedisQueue(key + '_ZIP', namespace)
RQ_loggy = RedisQueue(key + '_loggy', namespace)
RQ_final = RedisQueue('FinalQueue', namespace)
RQ_recipe = RedisQueue(key + '_recipe', namespace)
RHash = RedisHash(key + '_info')
RHerror = RedisHash(key + '_error')
RQ_lock = RedisLock(lock_obj)
RQ_lock.add({'POW': '1'})
if int(RQ_file.QueueSize()) == 0:
print("No Files Found in Redis Queue")
elif RQ_lock.available('POW'):
print(RQ_file.getQueueName())
jobFile = RQ_file.Qfile2Qwork(RQ_file.getQueueName(),
RQ_work.getQueueName())
# Setup system logging
basename = os.path.splitext(os.path.basename(jobFile))[0]
logger = logging.getLogger(key + '.' + basename)
logger.setLevel(logging.INFO)
logFileHandle = logging.FileHandler(pds_log + '/Service.log')
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s, %(message)s')
logFileHandle.setFormatter(formatter)
logger.addHandler(logFileHandle)
logger.info('Starting POW Processing')
# set up loggy
loggyOBJ = Loggy(basename)
# File Naming
if '+' in jobFile:
bandSplit = jobFile.split('+')
inputFile = bandSplit[0]
else:
inputFile = jobFile
infile = workarea + \
os.path.splitext(os.path.basename(jobFile))[0] + '.input.cub'
outfile = workarea + \
os.path.splitext(os.path.basename(jobFile))[0] + '.output.cub'
status = 'success'
for element in RQ_recipe.RecipeGet():
if status == 'error':
break
elif status == 'success':
processOBJ = Process()
process = processOBJ.JSON2Process(element)
if 'gdal_translate' not in processOBJ.getProcessName():
print(processOBJ.getProcessName())
if '2isis' in processOBJ.getProcessName():
processOBJ.updateParameter('from_', inputFile)
processOBJ.updateParameter('to', outfile)
elif 'cubeatt-band' in processOBJ.getProcessName():
if '+' in jobFile:
infileB = infile + '+' + bandSplit[1]
processOBJ.updateParameter('from_', infileB)
processOBJ.updateParameter('to', outfile)
processOBJ.ChangeProcess('cubeatt')
else:
continue
elif 'cubeatt-bit' in processOBJ.getProcessName():
if RHash.OutBit() == 'unsignedbyte':
temp_outfile = outfile + '+lsb+tile+attached+unsignedbyte+1:254'
elif RHash.OutBit() == 'signedword':
temp_outfile = outfile + '+lsb+tile+attached+signedword+-32765:32765'
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', temp_outfile)
processOBJ.ChangeProcess('cubeatt')
elif 'spice' in processOBJ.getProcessName():
processOBJ.updateParameter('from_', infile)
elif 'ctxevenodd' in processOBJ.getProcessName():
label = pvl.load(infile)
SS = label['IsisCube']['Instrument']['SpatialSumming']
print(SS)
if SS != 1:
continue
else:
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', outfile)
elif 'mocevenodd' in processOBJ.getProcessName():
label = pvl.load(infile)
CTS = label['IsisCube']['Instrument'][
'CrosstrackSumming']
print(CTS)
if CTS != 1:
continue
else:
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', outfile)
elif 'mocnoise50' in processOBJ.getProcessName():
label = pvl.load(infile)
CTS = label['IsisCube']['Instrument'][
'CrosstrackSumming']
if CTS != 1:
continue
else:
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', outfile)
elif 'cam2map' in processOBJ.getProcessName():
processOBJ.updateParameter('from', infile)
processOBJ.updateParameter('to', outfile)
if RHash.getGRtype() == 'smart' or RHash.getGRtype(
) == 'fill':
subloggyOBJ = SubLoggy('cam2map')
camrangeOUT = workarea + basename + '_camrange.txt'
isis.camrange(from_=infile, to=camrangeOUT)
cam = pvl.load(camrangeOUT)
if cam['UniversalGroundRange']['MaximumLatitude'] < float(RHash.getMinLat()) or \
cam['UniversalGroundRange']['MinimumLatitude'] > float(RHash.getMaxLat()) or \
cam['UniversalGroundRange']['MaximumLongitude'] < float(RHash.getMinLon()) or \
cam['UniversalGroundRange']['MinimumLongitude'] > float(RHash.getMaxLon()):
status = 'error'
eSTR = "Error Ground Range Outside Extent Range"
RHerror.addError(
os.path.splitext(
os.path.basename(jobFile))[0], eSTR)
subloggyOBJ.setStatus('ERROR')
subloggyOBJ.errorOut(eSTR)
loggyOBJ.AddProcess(subloggyOBJ.getSLprocess())
break
elif RHash.getGRtype() == 'smart':
if cam['UniversalGroundRange'][
'MinimumLatitude'] > float(
RHash.getMinLat()):
minlat = cam['UniversalGroundRange'][
'MinimumLatitude']
else:
minlat = RHash.getMinLat()
if cam['UniversalGroundRange'][
'MaximumLatitude'] < float(
RHash.getMaxLat()):
maxlat = cam['UniversalGroundRange'][
'MaximumLatitude']
else:
maxlat = RHash.getMaxLat()
if cam['UniversalGroundRange'][
'MinimumLongitude'] > float(
RHash.getMinLon()):
minlon = cam['UniversalGroundRange'][
'MinimumLongitude']
else:
minlon = RHash.getMinLon()
if cam['UniversalGroundRange'][
'MaximumLongitude'] < float(
RHash.getMaxLon()):
maxlon = cam['UniversalGroundRange'][
'MaximumLongitude']
else:
maxlon = RHash.getMaxLon()
elif RHash.getGRtype() == 'fill':
minlat = RHash.getMinLat()
maxlat = RHash.getMaxLat()
minlon = RHash.getMinLon()
maxlon = RHash.getMaxLon()
processOBJ.AddParameter('minlat', minlat)
processOBJ.AddParameter('maxlat', maxlat)
processOBJ.AddParameter('minlon', minlon)
processOBJ.AddParameter('maxlon', maxlon)
#os.remove(camrangeOUT)
elif 'isis2pds' in processOBJ.getProcessName():
finalfile = infile.replace('.input.cub', '_final.img')
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', finalfile)
else:
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', outfile)
print(processOBJ.getProcess())
for k, v in processOBJ.getProcess().items():
func = getattr(isis, k)
subloggyOBJ = SubLoggy(k)
try:
func(**v)
logger.info('Process %s :: Success', k)
subloggyOBJ.setStatus('SUCCESS')
subloggyOBJ.setCommand(processOBJ.LogCommandline())
subloggyOBJ.setHelpLink(processOBJ.LogHelpLink())
loggyOBJ.AddProcess(subloggyOBJ.getSLprocess())
if os.path.isfile(outfile):
os.rename(outfile, infile)
status = 'success'
except ProcessError as e:
logger.error('Process %s :: Error', k)
logger.error(e)
status = 'error'
eSTR = 'Error Executing ' + k + \
' Standard Error: ' + str(e)
RHerror.addError(
os.path.splitext(os.path.basename(jobFile))[0],
eSTR)
subloggyOBJ.setStatus('ERROR')
subloggyOBJ.setCommand(processOBJ.LogCommandline())
subloggyOBJ.setHelpLink(processOBJ.LogHelpLink())
subloggyOBJ.errorOut(eSTR)
loggyOBJ.AddProcess(subloggyOBJ.getSLprocess())
else:
GDALcmd = ""
for process, v, in processOBJ.getProcess().items():
subloggyOBJ = SubLoggy(process)
GDALcmd += process
for dict_key, value in v.items():
GDALcmd += ' ' + dict_key + ' ' + value
frmt = RHash.Format()
if frmt == 'GeoTiff-BigTiff':
fileext = 'tif'
elif frmt == 'GeoJPEG-2000':
fileext = 'jp2'
elif frmt == 'JPEG':
fileext = 'jpg'
elif frmt == 'PNG':
fileext = 'png'
elif frmt == 'GIF':
fileext = 'gif'
logGDALcmd = GDALcmd + ' ' + basename + \
'.input.cub ' + basename + '_final.' + fileext
finalfile = infile.replace('.input.cub',
'_final.' + fileext)
GDALcmd += ' ' + infile + ' ' + finalfile
print(GDALcmd)
result = subprocess.call(GDALcmd, shell=True)
if result == 0:
logger.info('Process GDAL translate :: Success')
status = 'success'
subloggyOBJ.setStatus('SUCCESS')
subloggyOBJ.setCommand(logGDALcmd)
subloggyOBJ.setHelpLink(
'http://www.gdal.org/gdal_translate.html')
loggyOBJ.AddProcess(subloggyOBJ.getSLprocess())
#os.remove(infile)
else:
errmsg = 'Error Executing GDAL translate: Error'
logger.error(errmsg)
status = 'error'
RHerror.addError(
os.path.splitext(os.path.basename(jobFile))[0],
errmsg)
subloggyOBJ.setStatus('ERROR')
subloggyOBJ.setCommand(logGDALcmd)
subloggyOBJ.setHelpLink(
'http://www.gdal.org/gdal_translate.html')
subloggyOBJ.errorOut('Process GDAL translate :: Error')
loggyOBJ.AddProcess(subloggyOBJ.getSLprocess())
if status == 'success':
if RHash.Format() == 'ISIS3':
finalfile = infile.replace('.input.cub', '_final.cub')
shutil.move(infile, finalfile)
if RHash.getStatus() != 'ERROR':
RHash.Status('SUCCESS')
try:
RQ_zip.QueueAdd(finalfile)
logger.info('File Added to ZIP Queue')
except:
logger.error('File NOT Added to ZIP Queue')
elif status == 'error':
RHash.Status('ERROR')
if os.path.isfile(infile):
#os.remove(infile)
pass
try:
RQ_loggy.QueueAdd(loggyOBJ.Loggy2json())
RQ_work.QueueRemove(jobFile)
logger.info('JSON Added to Loggy Queue')
except:
logger.error('JSON NOT Added to Loggy Queue')
if RQ_file.QueueSize() == 0 and RQ_work.QueueSize() == 0:
try:
RQ_final.QueueAdd(key)
logger.info('Key %s Added to Final Queue: Success', key)
logger.info(
'Both Queues Empty: filequeue = %s work queue = %s',
str(RQ_file.QueueSize()), str(RQ_work.QueueSize()))
logger.info('JOB Complete')
except:
logger.error('Key NOT Added to Final Queue')
elif RQ_file.QueueSize() == 0 and RQ_work.QueueSize() != 0:
logger.warning(
'Work Queue Not Empty: filequeue = %s work queue = %s',
str(RQ_file.QueueSize()), str(RQ_work.QueueSize()))
def generate_isis_processes(inputfile, archive, logger):
logger.info('Starting Process: %s', inputfile)
recipeOBJ = Recipe()
recipeOBJ.addMissionJson(archive, 'upc')
infile = os.path.splitext(inputfile)[0] + '.UPCinput.cub'
logger.debug("Beginning processing on %s\n", inputfile)
outfile = os.path.splitext(inputfile)[0] + '.UPCoutput.cub'
caminfoOUT = os.path.splitext(inputfile)[0] + '_caminfo.pvl'
processes = []
# Iterate through each process listed in the recipe
for item in recipeOBJ.getProcesses():
# If any of the processes failed, discontinue processing
processOBJ = Process()
processOBJ.ProcessFromRecipe(item, recipeOBJ.getRecipe())
# Handle processing based on string description.
if '2isis' in item:
processOBJ.updateParameter('from_', inputfile)
processOBJ.updateParameter('to', outfile)
elif item == 'thmproc':
processOBJ.updateParameter('from_', inputfile)
processOBJ.updateParameter('to', outfile)
thmproc_odd = str(workarea) + str(os.path.splitext(
os.path.basename(inputfile))[0]) + '.UPCoutput.raw.odd.cub'
thmproc_even = str(workarea) + str(
os.path.splitext(os.path.basename(
inputfile))[0]) + '.UPCoutput.raw.even.cub'
elif item == 'handmos':
processOBJ.updateParameter('from_', thmproc_even)
processOBJ.updateParameter('mosaic', thmproc_odd)
elif item == 'spiceinit':
processOBJ.updateParameter('from_', infile)
elif item == 'cubeatt':
band_infile = infile + '+' + str(1)
processOBJ.updateParameter('from_', band_infile)
processOBJ.updateParameter('to', outfile)
elif item == 'footprintinit':
processOBJ.updateParameter('from_', infile)
elif item == 'caminfo':
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', caminfoOUT)
else:
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', outfile)
processes.append(processOBJ)
pwd = os.getcwd()
return processes, infile, caminfoOUT, pwd
def empty_process_obj():
process = Process()
return process