def preprocessImage(filename): print "Star time: %s" % time.ctime(time.time()) noneExtensionFilename = filename.replace(inputFileExtension, "") print "output:" + noneExtensionFilename + outputFileExtension command = "gpt %sPreProcess.xml -Pfilename=\"%s%s\" -Poutputfilename=\"%s%s%s\"" % ( processXMLPath, inputPath, filename, outputPath + noneExtensionFilename + "/", noneExtensionFilename, outputFileExtension) print command try: os.system(command) except: print "Error: unable to start preprocessing command, preprocessImage()" traceback.print_exc(file=sys.stdout) raise PreprocessedCommandException( "Error trying to run the preprocessing GPT command for the file %s" % filename) # here is necessary to add the error management procedure endTime = time.time() print "End time: %s" % time.ctime(time.time()) # start to upload the preprocessed resulting image outputFileName = filename.replace(inputFileExtension, "") zip_filename = zipDirectory(outputPath, outputFileName) uploadFile(outputPath + zip_filename, BUCKET_NAME_PROCESSED_IMAGES + "/" + BUCKET_FOLDER_NAME_PREPROCESSED_IMAGES) # end of uploading processStatusJSon = processStatusPath + "processing.json" data = readProcessStatusInJson(processStatusJSon) processingStatus = data[filename] deleteElementJson(processStatusJSon, data, filename) writeProcessStatusInJson(processStatusJSon, data) processStatusJSon = processStatusPath + "processed.json" data = readProcessStatusInJson(processStatusJSon) jsonData = { "status: ": ProcessStatus.PROCESSED, "starttime": processingStatus["starttime"], "endtime": endTime } addProcessStatusDataToJson(filename, data, jsonData) writeProcessStatusInJson(processStatusJSon, data) updateFileMetadata(BUCKET_NAME_RAW_IMAGES, filename, {META_DATA_STATUS_KEY: ProcessStatus.PROCESSED})
def readFiles(input_filename): attempts = 0 try: if input_filename is None: return deleteFolder(inputPath, "") deleteFolder(outputPath, "") deleteFolder(SNAP_TEMP_PATH, "") createFolder(inputPath) createFolder(outputPath) createFolder(SNAP_TEMP_PATH + "gpt") filename = input_filename # if receipt_handle is None: # return # if attempts >= MAX_PREPROCESSING_ATTEMPTS: # return # deleteMessage(receipt_handle) print "downloading raw image %s" % input_filename attemptsString = getFileMetadata(BUCKET_NAME_RAW_IMAGES, input_filename, META_DATA_ATTEMPTS_KEY) if attemptsString is None: attempts = 0 else: attempts = int(attemptsString) attempts = attempts + 1 updateFileMetadata(BUCKET_NAME_RAW_IMAGES, input_filename, { META_DATA_STATUS_KEY: ProcessStatus.PROCESSING, META_DATA_ATTEMPTS_KEY: "%s" % attempts }) downloadFile(inputPath, input_filename, BUCKET_NAME_RAW_IMAGES) if input_filename.endswith(".SAFE"): # filename = zipImageFile(inputPath, input_filename.replace(".SAFE", ""), ".SAFE") filename = input_filename.replace(".SAFE", ".zip") os.rename(inputPath + input_filename, inputPath + filename) # deleteFile(inputPath, input_filename) print "starting preprocessing" if ce.checkMissingFiles(inputPath, filename): startTime = time.time() processStatusJSon = processStatusPath + "processing.json" data = readProcessStatusInJson(processStatusJSon) jsonData = { "status: ": ProcessStatus.PROCESSING, "starttime": startTime } addProcessStatusDataToJson(filename, data, jsonData) writeProcessStatusInJson(processStatusJSon, data) outputFilename = preprocessImage(input_filename, filename) delay = 60 time.sleep(delay) except OrbitNotIncludedException as err: error_message = "Orbit error: {0}".format(err) print(error_message) sendNotification(err, error_message, input_filename, attempts) updateFileMetadata(BUCKET_NAME_RAW_IMAGES, input_filename, {META_DATA_STATUS_KEY: ProcessStatus.ERROR}) send_email(MAIL_ACCOUNT_SENDER, MAIL_ACCOUNT_PASSWORD, MAIL_ACCOUNT_RECIVER, "Copernicus Processing - ERROR", filename, err.message, filename[17:21] + "/" + filename[21:23] + "/" + filename[23:25], "errors.json") except VVBandNotIncludedException as err: error_message = "VV Band error: {0}".format(err) print(error_message) sendNotification(err, error_message, input_filename, attempts) updateFileMetadata(BUCKET_NAME_RAW_IMAGES, input_filename, {META_DATA_STATUS_KEY: ProcessStatus.ERROR}) send_email(MAIL_ACCOUNT_SENDER, MAIL_ACCOUNT_PASSWORD, MAIL_ACCOUNT_RECIVER, "Copernicus Processing - ERROR", filename, err.message, filename[17:21] + "/" + filename[21:23] + "/" + filename[23:25], "errors.json") except VHBandNotIncludedException as err: error_message = "VH error: {0}".format(err) print(error_message) sendNotification(err, error_message, input_filename, attempts) updateFileMetadata(BUCKET_NAME_RAW_IMAGES, input_filename, {META_DATA_STATUS_KEY: ProcessStatus.ERROR}) send_email(MAIL_ACCOUNT_SENDER, MAIL_ACCOUNT_PASSWORD, MAIL_ACCOUNT_RECIVER, "Copernicus Processing - ERROR", filename, err.message, filename[17:21] + "/" + filename[21:23] + "/" + filename[23:25], "errors.json") except PreprocessedCommandException as err: error_message = "Preprocessing command error: {0}".format(err) print(error_message) sendNotification(err, error_message, input_filename, attempts) updateFileMetadata(BUCKET_NAME_RAW_IMAGES, input_filename, {META_DATA_STATUS_KEY: ProcessStatus.ERROR}) send_email(MAIL_ACCOUNT_SENDER, MAIL_ACCOUNT_PASSWORD, MAIL_ACCOUNT_RECIVER, "Copernicus Processing - ERROR", filename, err.message, filename[17:21] + "/" + filename[21:23] + "/" + filename[23:25], "errors.json") except: error_message = "Error: unable to preprocess" print error_message traceback.print_exc(file=sys.stdout) sendNotification(Exception(), error_message, input_filename, attempts) updateFileMetadata(BUCKET_NAME_RAW_IMAGES, input_filename, {META_DATA_STATUS_KEY: ProcessStatus.ERROR}) send_email(MAIL_ACCOUNT_SENDER, MAIL_ACCOUNT_PASSWORD, MAIL_ACCOUNT_RECIVER, "Copernicus Processing - ERROR", filename, error_message, filename[17:21] + "/" + filename[21:23] + "/" + filename[23:25], "errors.json") finally: print "Finally"
def preprocessImage(input_filename, filename): print "Star time: %s" % time.ctime(time.time()) noneExtensionFilename = filename.replace(zipFileExtension, "") print "output:" + noneExtensionFilename + outputFileExtension command = "sh /home/ubuntu/snap/bin/gpt %sPreProcess.xml -Pfilename=\"%s%s\" -Poutputfilename=\"%s%s%s\"" % ( # linux # command = "gpt %sPreProcess.xml -Pfilename=\"%s%s\" -Poutputfilename=\"%s%s%s\"" % ( # windows processXMLPath, inputPath, filename, outputPath + noneExtensionFilename + "/", noneExtensionFilename, outputFileExtension) print command ##MAIL STARTING PROCESSING send_email(MAIL_ACCOUNT_SENDER, MAIL_ACCOUNT_PASSWORD, MAIL_ACCOUNT_RECIVER, "Copernicus Processing - Starting", filename, "\"STARTING PROCESSING\"", filename[17:21] + "/" + filename[21:23] + "/" + filename[23:25], "processing.json") try: os.system(command) except: print "Error: unable to start preprocessing command, preprocessImage()" traceback.print_exc(file=sys.stdout) raise PreprocessedCommandException( "Error trying to run the preprocessing GPT command for the file %s" % filename) # here is necessary to add the error management procedure endTime = time.time() print "End time: %s" % time.ctime(time.time()) ##MAIL COMPLETED send_email(MAIL_ACCOUNT_SENDER, MAIL_ACCOUNT_PASSWORD, MAIL_ACCOUNT_RECIVER, "Copernicus Processing - Ending", filename, "\"COMPLETED\"", filename[17:21] + "/" + filename[21:23] + "/" + filename[23:25], "processed.json") # start to upload the preprocessed resulting image outputFileName = filename.replace(zipFileExtension, "") zip_filename = zipDirectory(outputPath, outputFileName) uploadFile(outputPath + zip_filename, BUCKET_NAME_PROCESSED_IMAGES + "/" + BUCKET_FOLDER_NAME_PREPROCESSED_IMAGES) # end of uploading processStatusJSon = processStatusPath + "processing.json" data = readProcessStatusInJson(processStatusJSon) processingStatus = data[filename] deleteElementJson(processStatusJSon, data, filename) writeProcessStatusInJson(processStatusJSon, data) processStatusJSon = processStatusPath + "processed.json" data = readProcessStatusInJson(processStatusJSon) jsonData = { "status: ": ProcessStatus.PROCESSED, "starttime": processingStatus["starttime"], "endtime": endTime } addProcessStatusDataToJson(filename, data, jsonData) writeProcessStatusInJson(processStatusJSon, data) updateFileMetadata(BUCKET_NAME_RAW_IMAGES, input_filename, {META_DATA_STATUS_KEY: ProcessStatus.PROCESSED}) return noneExtensionFilename + outputFileExtension
def readFiles(filename): try: if filename is None: return # if receipt_handle is None: # return # if attempts >= MAX_PREPROCESSING_ATTEMPTS: # return # deleteMessage(receipt_handle) print "downloading raw image %s" % filename attemptsString = getFileMetadata(BUCKET_NAME_RAW_IMAGES, filename, META_DATA_ATTEMPTS_KEY) if attemptsString is None: attempts = 0 else: attempts = int(attemptsString) updateFileMetadata(BUCKET_NAME_RAW_IMAGES, filename, { META_DATA_STATUS_KEY: ProcessStatus.PROCESSING, META_DATA_ATTEMPTS_KEY: "%s" % (attempts + 1) }) downloadFile(inputPath, filename, BUCKET_NAME_RAW_IMAGES) zipfile() print "starting preprocessing" # listdir = os.listdir(inputPath) # for filename in listdir: if ce.checkMissingFiles(inputPath, filename): startTime = time.time() processStatusJSon = processStatusPath + "processing.json" data = readProcessStatusInJson(processStatusJSon) jsonData = { "status: ": ProcessStatus.PROCESSING, "starttime": startTime } addProcessStatusDataToJson(filename, data, jsonData) writeProcessStatusInJson(processStatusJSon, data) preprocessImage(filename) except OrbitNotIncludedException as err: error_message = "Orbit error: {0}".format(err) print(error_message) sendNotification(err, error_message, filename, attempts) updateFileMetadata(BUCKET_NAME_RAW_IMAGES, filename, {META_DATA_STATUS_KEY: ProcessStatus.ERROR}) except VVBandNotIncludedException as err: error_message = "VV Band error: {0}".format(err) print(error_message) sendNotification(err, error_message, filename, attempts) updateFileMetadata(BUCKET_NAME_RAW_IMAGES, filename, {META_DATA_STATUS_KEY: ProcessStatus.ERROR}) except VHBandNotIncludedException as err: error_message = "VH error: {0}".format(err) print(error_message) sendNotification(err, error_message, filename, attempts) updateFileMetadata(BUCKET_NAME_RAW_IMAGES, filename, {META_DATA_STATUS_KEY: ProcessStatus.ERROR}) except PreprocessedCommandException as err: error_message = "Preprocessing command error: {0}".format(err) print(error_message) sendNotification(err, error_message, filename, attempts) updateFileMetadata(BUCKET_NAME_RAW_IMAGES, filename, {META_DATA_STATUS_KEY: ProcessStatus.ERROR}) except: error_message = "Error: unable to preprocess" print error_message traceback.print_exc(file=sys.stdout) sendNotification(Exception(), error_message, filename, attempts) updateFileMetadata(BUCKET_NAME_RAW_IMAGES, filename, {META_DATA_STATUS_KEY: ProcessStatus.ERROR})