def preprocessImage(filename):
print "Star time: %s" % time.ctime(time.time())
noneExtensionFilename = filename.replace(inputFileExtension, "")
print "output:" + noneExtensionFilename + outputFileExtension
command = "gpt %sPreProcess.xml -Pfilename=\"%s%s\" -Poutputfilename=\"%s%s%s\"" % (
processXMLPath, inputPath, filename, outputPath + noneExtensionFilename + "/", noneExtensionFilename,
outputFileExtension)
print command
try:
os.system(command)
except:
print "Error: unable to start preprocessing command, preprocessImage()"
traceback.print_exc(file=sys.stdout)
raise PreprocessedCommandException(
"Error trying to run the preprocessing GPT command for the file %s" % filename)
# here is necessary to add the error management procedure
endTime = time.time()
print "End time: %s" % time.ctime(time.time())
# start to upload the preprocessed resulting image
outputFileName = filename.replace(inputFileExtension, "")
zip_filename = zipDirectory(outputPath, outputFileName)
uploadFile(outputPath + zip_filename,
BUCKET_NAME_PROCESSED_IMAGES + "/" + BUCKET_FOLDER_NAME_PREPROCESSED_IMAGES)
# end of uploading
processStatusJSon = processStatusPath + "processing.json"
data = readProcessStatusInJson(processStatusJSon)
processingStatus = data[filename]
deleteElementJson(processStatusJSon, data, filename)
writeProcessStatusInJson(processStatusJSon, data)
processStatusJSon = processStatusPath + "processed.json"
data = readProcessStatusInJson(processStatusJSon)
jsonData = {
"status: ": ProcessStatus.PROCESSED,
"starttime": processingStatus["starttime"],
"endtime": endTime
}
addProcessStatusDataToJson(filename, data, jsonData)
writeProcessStatusInJson(processStatusJSon, data)
updateFileMetadata(BUCKET_NAME_RAW_IMAGES, filename, {META_DATA_STATUS_KEY: ProcessStatus.PROCESSED})
def readFiles(input_filename):
attempts = 0
try:
if input_filename is None:
return
deleteFolder(inputPath, "")
deleteFolder(outputPath, "")
deleteFolder(SNAP_TEMP_PATH, "")
createFolder(inputPath)
createFolder(outputPath)
createFolder(SNAP_TEMP_PATH + "gpt")
filename = input_filename
# if receipt_handle is None:
# return
# if attempts >= MAX_PREPROCESSING_ATTEMPTS:
# return
# deleteMessage(receipt_handle)
print "downloading raw image %s" % input_filename
attemptsString = getFileMetadata(BUCKET_NAME_RAW_IMAGES, input_filename, META_DATA_ATTEMPTS_KEY)
if attemptsString is None:
attempts = 0
else:
attempts = int(attemptsString)
attempts = attempts + 1
updateFileMetadata(BUCKET_NAME_RAW_IMAGES, input_filename,
{
META_DATA_STATUS_KEY: ProcessStatus.PROCESSING,
META_DATA_ATTEMPTS_KEY: "%s" % attempts
})
downloadFile(inputPath, input_filename, BUCKET_NAME_RAW_IMAGES)
if input_filename.endswith(".SAFE"):
# filename = zipImageFile(inputPath, input_filename.replace(".SAFE", ""), ".SAFE")
filename = input_filename.replace(".SAFE", ".zip")
os.rename(inputPath + input_filename, inputPath + filename)
# deleteFile(inputPath, input_filename)
print "starting preprocessing"
if ce.checkMissingFiles(inputPath, filename):
startTime = time.time()
processStatusJSon = processStatusPath + "processing.json"
data = readProcessStatusInJson(processStatusJSon)
jsonData = {
"status: ": ProcessStatus.PROCESSING,
"starttime": startTime
}
addProcessStatusDataToJson(filename, data, jsonData)
writeProcessStatusInJson(processStatusJSon, data)
outputFilename = preprocessImage(input_filename, filename)
delay = 60
time.sleep(delay)
except OrbitNotIncludedException as err:
error_message = "Orbit error: {0}".format(err)
print(error_message)
sendNotification(err, error_message, input_filename, attempts)
updateFileMetadata(BUCKET_NAME_RAW_IMAGES, input_filename,
{META_DATA_STATUS_KEY: ProcessStatus.ERROR})
send_email(MAIL_ACCOUNT_SENDER, MAIL_ACCOUNT_PASSWORD, MAIL_ACCOUNT_RECIVER, "Copernicus Processing - ERROR",
filename,
err.message,
filename[17:21] + "/" + filename[21:23] + "/" + filename[23:25], "errors.json")
except VVBandNotIncludedException as err:
error_message = "VV Band error: {0}".format(err)
print(error_message)
sendNotification(err, error_message, input_filename, attempts)
updateFileMetadata(BUCKET_NAME_RAW_IMAGES, input_filename,
{META_DATA_STATUS_KEY: ProcessStatus.ERROR})
send_email(MAIL_ACCOUNT_SENDER, MAIL_ACCOUNT_PASSWORD, MAIL_ACCOUNT_RECIVER, "Copernicus Processing - ERROR",
filename,
err.message,
filename[17:21] + "/" + filename[21:23] + "/" + filename[23:25], "errors.json")
except VHBandNotIncludedException as err:
error_message = "VH error: {0}".format(err)
print(error_message)
sendNotification(err, error_message, input_filename, attempts)
updateFileMetadata(BUCKET_NAME_RAW_IMAGES, input_filename,
{META_DATA_STATUS_KEY: ProcessStatus.ERROR})
send_email(MAIL_ACCOUNT_SENDER, MAIL_ACCOUNT_PASSWORD, MAIL_ACCOUNT_RECIVER, "Copernicus Processing - ERROR",
filename,
err.message,
filename[17:21] + "/" + filename[21:23] + "/" + filename[23:25], "errors.json")
except PreprocessedCommandException as err:
error_message = "Preprocessing command error: {0}".format(err)
print(error_message)
sendNotification(err, error_message, input_filename, attempts)
updateFileMetadata(BUCKET_NAME_RAW_IMAGES, input_filename,
{META_DATA_STATUS_KEY: ProcessStatus.ERROR})
send_email(MAIL_ACCOUNT_SENDER, MAIL_ACCOUNT_PASSWORD, MAIL_ACCOUNT_RECIVER, "Copernicus Processing - ERROR",
filename,
err.message,
filename[17:21] + "/" + filename[21:23] + "/" + filename[23:25], "errors.json")
except:
error_message = "Error: unable to preprocess"
print error_message
traceback.print_exc(file=sys.stdout)
sendNotification(Exception(), error_message, input_filename, attempts)
updateFileMetadata(BUCKET_NAME_RAW_IMAGES, input_filename,
{META_DATA_STATUS_KEY: ProcessStatus.ERROR})
send_email(MAIL_ACCOUNT_SENDER, MAIL_ACCOUNT_PASSWORD, MAIL_ACCOUNT_RECIVER, "Copernicus Processing - ERROR",
filename,
error_message,
filename[17:21] + "/" + filename[21:23] + "/" + filename[23:25], "errors.json")
finally:
print "Finally"
def preprocessImage(input_filename, filename):
print "Star time: %s" % time.ctime(time.time())
noneExtensionFilename = filename.replace(zipFileExtension, "")
print "output:" + noneExtensionFilename + outputFileExtension
command = "sh /home/ubuntu/snap/bin/gpt %sPreProcess.xml -Pfilename=\"%s%s\" -Poutputfilename=\"%s%s%s\"" % (
# linux
# command = "gpt %sPreProcess.xml -Pfilename=\"%s%s\" -Poutputfilename=\"%s%s%s\"" % ( # windows
processXMLPath, inputPath, filename, outputPath + noneExtensionFilename + "/", noneExtensionFilename,
outputFileExtension)
print command
##MAIL STARTING PROCESSING
send_email(MAIL_ACCOUNT_SENDER, MAIL_ACCOUNT_PASSWORD, MAIL_ACCOUNT_RECIVER, "Copernicus Processing - Starting",
filename,
"\"STARTING PROCESSING\"",
filename[17:21] + "/" + filename[21:23] + "/" + filename[23:25], "processing.json")
try:
os.system(command)
except:
print "Error: unable to start preprocessing command, preprocessImage()"
traceback.print_exc(file=sys.stdout)
raise PreprocessedCommandException(
"Error trying to run the preprocessing GPT command for the file %s" % filename)
# here is necessary to add the error management procedure
endTime = time.time()
print "End time: %s" % time.ctime(time.time())
##MAIL COMPLETED
send_email(MAIL_ACCOUNT_SENDER, MAIL_ACCOUNT_PASSWORD, MAIL_ACCOUNT_RECIVER, "Copernicus Processing - Ending",
filename,
"\"COMPLETED\"",
filename[17:21] + "/" + filename[21:23] + "/" + filename[23:25], "processed.json")
# start to upload the preprocessed resulting image
outputFileName = filename.replace(zipFileExtension, "")
zip_filename = zipDirectory(outputPath, outputFileName)
uploadFile(outputPath + zip_filename,
BUCKET_NAME_PROCESSED_IMAGES + "/" + BUCKET_FOLDER_NAME_PREPROCESSED_IMAGES)
# end of uploading
processStatusJSon = processStatusPath + "processing.json"
data = readProcessStatusInJson(processStatusJSon)
processingStatus = data[filename]
deleteElementJson(processStatusJSon, data, filename)
writeProcessStatusInJson(processStatusJSon, data)
processStatusJSon = processStatusPath + "processed.json"
data = readProcessStatusInJson(processStatusJSon)
jsonData = {
"status: ": ProcessStatus.PROCESSED,
"starttime": processingStatus["starttime"],
"endtime": endTime
}
addProcessStatusDataToJson(filename, data, jsonData)
writeProcessStatusInJson(processStatusJSon, data)
updateFileMetadata(BUCKET_NAME_RAW_IMAGES, input_filename, {META_DATA_STATUS_KEY: ProcessStatus.PROCESSED})
return noneExtensionFilename + outputFileExtension
def readFiles(filename):
try:
if filename is None:
return
# if receipt_handle is None:
# return
# if attempts >= MAX_PREPROCESSING_ATTEMPTS:
# return
# deleteMessage(receipt_handle)
print "downloading raw image %s" % filename
attemptsString = getFileMetadata(BUCKET_NAME_RAW_IMAGES, filename, META_DATA_ATTEMPTS_KEY)
if attemptsString is None:
attempts = 0
else:
attempts = int(attemptsString)
updateFileMetadata(BUCKET_NAME_RAW_IMAGES, filename,
{
META_DATA_STATUS_KEY: ProcessStatus.PROCESSING,
META_DATA_ATTEMPTS_KEY: "%s" % (attempts + 1)
})
downloadFile(inputPath, filename, BUCKET_NAME_RAW_IMAGES)
zipfile()
print "starting preprocessing"
# listdir = os.listdir(inputPath)
# for filename in listdir:
if ce.checkMissingFiles(inputPath, filename):
startTime = time.time()
processStatusJSon = processStatusPath + "processing.json"
data = readProcessStatusInJson(processStatusJSon)
jsonData = {
"status: ": ProcessStatus.PROCESSING,
"starttime": startTime
}
addProcessStatusDataToJson(filename, data, jsonData)
writeProcessStatusInJson(processStatusJSon, data)
preprocessImage(filename)
except OrbitNotIncludedException as err:
error_message = "Orbit error: {0}".format(err)
print(error_message)
sendNotification(err, error_message, filename, attempts)
updateFileMetadata(BUCKET_NAME_RAW_IMAGES, filename,
{META_DATA_STATUS_KEY: ProcessStatus.ERROR})
except VVBandNotIncludedException as err:
error_message = "VV Band error: {0}".format(err)
print(error_message)
sendNotification(err, error_message, filename, attempts)
updateFileMetadata(BUCKET_NAME_RAW_IMAGES, filename,
{META_DATA_STATUS_KEY: ProcessStatus.ERROR})
except VHBandNotIncludedException as err:
error_message = "VH error: {0}".format(err)
print(error_message)
sendNotification(err, error_message, filename, attempts)
updateFileMetadata(BUCKET_NAME_RAW_IMAGES, filename,
{META_DATA_STATUS_KEY: ProcessStatus.ERROR})
except PreprocessedCommandException as err:
error_message = "Preprocessing command error: {0}".format(err)
print(error_message)
sendNotification(err, error_message, filename, attempts)
updateFileMetadata(BUCKET_NAME_RAW_IMAGES, filename,
{META_DATA_STATUS_KEY: ProcessStatus.ERROR})
except:
error_message = "Error: unable to preprocess"
print error_message
traceback.print_exc(file=sys.stdout)
sendNotification(Exception(), error_message, filename, attempts)
updateFileMetadata(BUCKET_NAME_RAW_IMAGES, filename,
{META_DATA_STATUS_KEY: ProcessStatus.ERROR})