def process_dataset(parameters):
global extractorName, workerScript, inputDirectory, outputDirectory
# Find input files in dataset
files = get_all_files(parameters)
# Download files to input directory
for fileExt in files:
files[fileExt]['path'] = extractors.download_file(
channel=parameters['channel'],
header=parameters['header'],
host=parameters['host'],
key=parameters['secretKey'],
fileid=files[fileExt]['id'],
# What's this argument for?
intermediatefileid=files[fileExt]['id'],
ext=fileExt)
# Restore temp filenames to original - script requires specific name formatting so tmp names aren't suitable
files[fileExt]['old_path'] = files[fileExt]['path']
files[fileExt]['path'] = os.path.join(inputDirectory,
files[fileExt]['filename'])
os.rename(files[fileExt]['old_path'], files[fileExt]['path'])
print 'found %s file: %s' % (fileExt, files[fileExt]['path'])
# Invoke terraref.sh
outFilePath = os.path.join(outputDirectory,
get_output_filename(files['_raw']['filename']))
print 'invoking terraref.sh to create: %s' % outFilePath
returncode = subprocess.call([
"bash", workerScript, "-d", "1", "-I", inputDirectory, "-O",
outputDirectory
])
print 'done creating output file (%s)' % (returncode)
if returncode != 0:
print 'terraref.sh encountered an error'
# Verify outfile exists and upload to clowder
if os.path.exists(outFilePath):
print 'output file detected'
if returncode == 0:
print 'uploading output file...'
extractors.upload_file_to_dataset(filepath=outFilePath,
parameters=parameters)
print 'done uploading'
# Clean up the output file.
os.remove(outFilePath)
else:
print 'no output file was produced'
print 'cleaning up...'
# Clean up the input files.
for fileExt in files:
os.remove(files[fileExt]['path'])
print 'done cleaning'
def process_dataset(parameters):
global outputDir
metafile, bin_file, metadata = None, None, None
# Get left/right files and metadata
for f in parameters['files']:
# First check metadata attached to dataset in Clowder for item of interest
if f.endswith('_dataset_metadata.json'):
all_dsmd = getFlir.load_json(f)
for curr_dsmd in all_dsmd:
if 'content' in curr_dsmd and 'lemnatec_measurement_metadata' in curr_dsmd[
'content']:
metafile = f
metadata = curr_dsmd['content']
# Otherwise, check if metadata was uploaded as a .json file
elif f.endswith('_metadata.json') and f.find(
'/_metadata.json') == -1 and metafile is None:
metafile = f
metadata = getFlir.load_json(metafile)
elif f.endswith('_ir.bin'):
bin_file = f
if None in [metafile, bin_file, metadata]:
getFlir.fail('Could not find all of ir.bin/metadata.')
return
print("...bin_file: %s" % bin_file)
print("...metafile: %s" % metafile)
dsname = parameters["datasetInfo"]["name"]
if dsname.find(" - ") > -1:
timestamp = dsname.split(" - ")[1]
else:
timestamp = "dsname"
if timestamp.find("__") > -1:
datestamp = timestamp.split("__")[0]
else:
datestamp = ""
out_dir = os.path.join(outputDir, datestamp, timestamp)
print("...output directory: %s" % out_dir)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
#Determine output paths
binbase = os.path.basename(bin_file)[:-7]
png_path = os.path.join(out_dir, binbase + '.png')
tiff_path = os.path.join(out_dir, binbase + '.tif')
print("...png: %s" % (png_path))
print("...tif: %s" % (tiff_path))
print("Creating png image")
raw_data = getFlir.load_flir_data(bin_file) # get raw data from bin file
im_color = getFlir.create_png(raw_data, png_path) # create png
print("Uploading output PNGs to dataset")
extractors.upload_file_to_dataset(png_path, parameters)
print("getting information from json file for geoTIFF")
center_position, scan_time, fov = getFlir.parse_metadata(metadata)
if center_position is None or scan_time is None or fov is None:
print("error getting metadata; skipping geoTIFF")
else:
gps_bounds = getFlir.get_bounding_box(
center_position,
fov) # get bounding box using gantry position and fov of camera
print("Creating geoTIFF images")
tc = getFlir.rawData_to_temperature(raw_data, scan_time,
metadata) # get temperature
getFlir.create_geotiff_with_temperature(im_color, tc, gps_bounds,
tiff_path) # create geotiff
print("Uploading output geoTIFFs to dataset")
extractors.upload_file_to_dataset(tiff_path, parameters)
# Tell Clowder this is completed so subsequent file updates don't daisy-chain
metadata = {
"@context": {
"@vocab":
"https://clowder.ncsa.illinois.edu/clowder/assets/docs/api/index.html#!/files/uploadToDataset"
},
"dataset_id": parameters["datasetId"],
"content": {
"status": "COMPLETED"
},
"agent": {
"@type": "cat:extractor",
"extractor_id":
parameters['host'] + "/api/extractors/" + extractorName
}
}
extractors.upload_dataset_metadata_jsonld(mdata=metadata,
parameters=parameters)
def process_dataset(parameters):
metafile, img_left, img_right, metadata = None, None, None, None
# Get left/right files and metadata
for f in parameters['files']:
# First check metadata attached to dataset in Clowder for item of interest
if f.endswith('_dataset_metadata.json'):
all_dsmd = bin2tiff.load_json(f)
for curr_dsmd in all_dsmd:
if 'content' in curr_dsmd and 'lemnatec_measurement_metadata' in curr_dsmd['content']:
metafile = f
metadata = curr_dsmd['content']
# Otherwise, check if metadata was uploaded as a .json file
elif f.endswith('_metadata.json') and f.find('/_metadata.json') == -1 and metafile is None:
metafile = f
metadata = bin2tiff.load_json(metafile)
elif f.endswith('_left.bin'):
img_left = f
elif f.endswith('_right.bin'):
img_right = f
if None in [metafile, img_left, img_right, metadata]:
bin2tiff.fail('Could not find all of left/right/metadata.')
return
print("img_left: %s" % img_left)
print("img_right: %s" % img_right)
print("metafile: %s" % metafile)
temp_out_dir = metafile.replace(os.path.basename(metafile), "")
print("Determining image shapes")
left_shape = bin2tiff.get_image_shape(metadata, 'left')
right_shape = bin2tiff.get_image_shape(metadata, 'right')
center_position = bin2tiff.get_position(metadata) # (x, y, z) in meters
fov = bin2tiff.get_fov(metadata, center_position[2], left_shape) # (fov_x, fov_y) in meters; need to pass in the camera height to get correct fov
left_position = [center_position[0]+bin2tiff.STEREO_OFFSET, center_position[1], center_position[2]]
right_position = [center_position[0]-bin2tiff.STEREO_OFFSET, center_position[1], center_position[2]]
left_gps_bounds = bin2tiff.get_bounding_box(left_position, fov) # (lat_max, lat_min, lng_max, lng_min) in decimal degrees
right_gps_bounds = bin2tiff.get_bounding_box(right_position, fov)
print("Creating demosaicked images")
left_out = os.path.join(temp_out_dir, img_left[:-4] + '.jpg')
left_image = bin2tiff.process_image(left_shape, img_left, left_out)
right_out = os.path.join(temp_out_dir, img_right[:-4] + '.jpg')
right_image = bin2tiff.process_image(right_shape, img_right, right_out)
print("Uploading output JPGs to dataset")
extractors.upload_file_to_dataset(left_out, parameters)
extractors.upload_file_to_dataset(right_out, parameters)
print("Creating geoTIFF images")
left_tiff_out = os.path.join(temp_out_dir, img_left[:-4] + '.tif')
bin2tiff.create_geotiff('left', left_image, left_gps_bounds, left_tiff_out)
right_tiff_out = os.path.join(temp_out_dir, img_right[:-4] + '.tif')
bin2tiff.create_geotiff('right', right_image, right_gps_bounds, right_tiff_out)
print("Uploading output geoTIFFs to dataset")
extractors.upload_file_to_dataset(left_tiff_out, parameters)
extractors.upload_file_to_dataset(right_tiff_out, parameters)
# Tell Clowder this is completed so subsequent file updates don't daisy-chain
metadata = {
"@context": {
"@vocab": "https://clowder.ncsa.illinois.edu/clowder/assets/docs/api/index.html#!/files/uploadToDataset"
},
"dataset_id": parameters["datasetId"],
"content": {"status": "COMPLETED"},
"agent": {
"@type": "cat:extractor",
"extractor_id": parameters['host'] + "/api/extractors/" + extractorName
}
}
extractors.upload_dataset_metadata_jsonld(mdata=metadata, parameters=parameters)