def test_paths(station, sensor, timestamp, expected):
s = Sensors(TERRAREF_BASE, station, sensor)
path = s.get_sensor_path(timestamp)
results = os.path.join(TERRAREF_BASE, expected)
assert path == results
def fullFieldMosaicStitcher(extractor, connector, host, secret_key, resource, rulemap):
results = {}
full_field_ready = False
# full-field queues must have at least this percent of the raw datasets present to trigger
tolerance_pct = 100
# full-field queues must have at least this many datasets to trigger
min_datasets = 200
# Determine output dataset
dsname = resource["dataset_info"]["name"]
sensor = dsname.split(" - ")[0]
# Map sensor display names to the GeoTIFF stitching target in those sensor datasets,
# including directory to look for date subfolder to count # of datasets on that date
if os.path.exists('/projects/arpae/terraref/sites'):
TERRAREF_BASE = '/projects/arpae/terraref/sites'
elif os.path.exists('/home/clowder/sites'):
TERRAREF_BASE = '/home/clowder/sites'
else:
TERRAREF_BASE = '/home/extractor/sites'
sensor_lookup = Sensors(TERRAREF_BASE, 'ua-mac')
stitchable_sensors = {
sensor_lookup.get_display_name('rgb_geotiff'): {
"target": "_left.tif",
"raw_dir": os.path.join(*(sensor_lookup.get_sensor_path('', sensor='stereoTop').split("/")[:-2]))
},
sensor_lookup.get_display_name('ir_geotiff'): {
"target": ".tif",
"raw_dir": os.path.join(*(sensor_lookup.get_sensor_path('', sensor='flirIrCamera').split("/")[:-2]))
},
sensor_lookup.get_display_name('laser3d_heightmap'): {
"target": "_west.tif",
"raw_dir": os.path.join(*(sensor_lookup.get_sensor_path('', sensor='scanner3DTop').split("/")[:-2]))
},
'scanner3DTop': {
"target": "_west.tif",
"raw_dir": os.path.join(*(sensor_lookup.get_sensor_path('', sensor='scanner3DTop').split("/")[:-2]))
}
}
if sensor in stitchable_sensors.keys():
timestamp = dsname.split(" - ")[1]
date = timestamp.split("__")[0]
progress_key = "Full Field -- " + sensor + " - " + date
# Is there actually a new left geoTIFF to add to the stack?
target_id = None
for f in resource['files']:
if f['filename'].endswith(stitchable_sensors[sensor]["target"]):
target_id = f['id']
target_path = f['filepath']
if not target_id:
# If not, no need to trigger anything for now.
logging.info("no target geoTIFF found in %s" % dsname)
for trig_extractor in rulemap["extractors"]:
results[trig_extractor] = {
"process": False,
"parameters": {}
}
return results
logging.info("[%s] found target: %s" % (progress_key, target_id))
# Fetch all existing file IDs that would be fed into this field mosaic
progress = rule_utils.retrieveProgressFromDB(progress_key)
# Is current ID already included in the list? If not, add it
submit_record = False
if 'ids' in progress:
ds_count = len(progress['ids'].keys())
if target_id not in progress['ids'].keys():
submit_record = True
ds_count += 1
else:
# Already seen this geoTIFF, so skip for now.
logging.info("previously logged target geoTIFF from %s" % dsname)
for trig_extractor in rulemap["extractors"]:
results[trig_extractor] = {
"process": False,
"parameters": {}
}
else:
submit_record = True
ds_count = 1
if submit_record:
for trig_extractor in rulemap["extractors"]:
rule_utils.submitProgressToDB("fullFieldMosaicStitcher", trig_extractor, progress_key, target_id, target_path)
if ds_count >= min_datasets:
# Check to see if list of geotiffs is same length as list of raw datasets
root_dir = stitchable_sensors[sensor]["raw_dir"]
if len(connector.mounted_paths) > 0:
for source_path in connector.mounted_paths:
if root_dir.startswith(source_path):
root_dir = root_dir.replace(source_path, connector.mounted_paths[source_path])
date_directory = os.path.join(root_dir, date)
date_directory = ("/"+date_directory if not date_directory.startswith("/") else "")
raw_file_count = float(subprocess.check_output("ls %s | wc -l" % date_directory,
shell=True).strip())
logging.info("found %s raw files in %s" % (int(raw_file_count), date_directory))
if raw_file_count == 0:
raise Exception("problem communicating with file system")
else:
# If we have enough raw files accounted for and more than min_datasets, trigger
prog_pct = (len(progress['ids'])/raw_file_count)*100
if prog_pct >= tolerance_pct:
full_field_ready = True
else:
logging.info("found %s/%s necessary geotiffs (%s%%)" % (len(progress['ids']), int(raw_file_count),
"{0:.2f}".format(prog_pct)))
for trig_extractor in rulemap["extractors"]:
results[trig_extractor] = {
"process": full_field_ready,
"parameters": {}
}
if full_field_ready:
results[trig_extractor]["parameters"]["output_dataset"] = "Full Field - "+date
# Write output ID list to a text file
output_dir = os.path.dirname(sensor_lookup.get_sensor_path(date, 'fullfield'))
logging.info("writing %s_file_ids.json to %s" % (sensor, output_dir))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
output_file = os.path.join(output_dir, sensor+"_file_paths.json")
# Sort IDs by file path before writing to disk
# TODO: Eventually alternate every other image so we have half complete and half "underneath"
paths = []
for fid in progress['ids'].keys():
paths.append(progress['ids'][fid])
with open(output_file, 'w') as out:
json.dump(sorted(paths), out)
results[trig_extractor]["parameters"]["file_paths"] = output_file
else:
for trig_extractor in rulemap["extractors"]:
results[trig_extractor] = {
"process": False,
"parameters": {}
}
return results
alt_out = "/gpfs_scratch/vnir_netcdf"
with open(args.input, 'r') as inp:
#for date in os.listdir(vnir_raw):
for date in inp:
date = date.rstrip()
date_dir = os.path.join(vnir_raw, date)
for timestamp in os.listdir(date_dir):
# e.g. .../ua-mac/raw_data/VNIR/2016-04-05/2016-04-05__00-27-08-740/
ds_dir = os.path.join(date_dir, timestamp)
for hyperfile in os.listdir(ds_dir):
if hyperfile.endswith("_raw"):
print("FOUND RAW FILE: %s" % hyperfile)
# Found a raw file - do we already have inputs?
lv1_out = sensor.get_sensor_path(timestamp)
if not os.path.isfile(lv1_out):
# Invoke terraref.sh
script_path = "/projects/arpae/terraref/shared/extractors/extractors-hyperspectral/hyperspectral/hyperspectral_workflow.sh"
alt_out_dir = os.path.join(alt_out, date, timestamp)
cmd = ["bash", script_path, "-d", "1", "-h", "--new_clb_mth",
"-i", os.path.join(ds_dir, hyperfile),
"-o", os.path.join(alt_out_dir, os.path.basename(lv1_out))]
if not os.path.isdir(alt_out_dir):
os.makedirs(alt_out_dir)
try:
print(" ".join(cmd))
subprocess.check_call(cmd)
except:
print("ERROR ON: %s" % os.path.join(ds_dir, hyperfile))