def upload_to_geostreams(file, clowder_id):
conn = Connector(
None, mounted_paths={"/home/clowder/sites": "/home/clowder/sites"})
successful_plots = 0
with open(file, 'rb') as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
centroid_lonlat = [row['lon'], row['lat']]
time_fmt = row['dp_time']
timestamp = row['timestamp']
dpmetadata = {"source": row['source'], "value": row['value']}
trait = row['trait']
create_datapoint_with_dependencies(
conn, host, secret_key, trait,
(centroid_lonlat[1], centroid_lonlat[0]), time_fmt, time_fmt,
dpmetadata, timestamp)
successful_plots += 1
# Extractor metadata
extractor_info = {
"extractor_name": "terra.geostreams",
"extractor_version": "1.0",
"extractor_author": "Max Burnette <*****@*****.**>",
"extractor_description": "Geostreams CSV uploader",
"extractor_repo": "https://github.com/terraref/computing-pipeline.git"
}
# Add metadata to original dataset indicating this was run
ext_meta = build_metadata(host, extractor_info, clowder_id, {
"plots_processed": successful_plots,
}, 'file')
upload_metadata(conn, host, secret_key, clowder_id, ext_meta)
def upload_to_bety(file, clowder_id):
conn = Connector(
None, mounted_paths={"/home/clowder/sites": "/home/clowder/sites"})
submit_traits(file, betykey=bety_key)
# Extractor metadata
extractor_info = {
"extractor_name": "terra.betydb",
"extractor_version": "1.0",
"extractor_author": "Max Burnette <*****@*****.**>",
"extractor_description": "BETYdb CSV uploader",
"extractor_repo": "https://github.com/terraref/computing-pipeline.git"
}
# Add metadata to original dataset indicating this was run
ext_meta = build_metadata(
host, extractor_info, clowder_id, {
"betydb_link":
"https://terraref.ncsa.illinois.edu/bety/api/v1/variables?name=canopy_cover"
}, 'file')
upload_metadata(conn, host, secret_key, clowder_id, ext_meta)
from terrautils.sensors import Sensors
import utils
import counts
config = {}
app_dir = '/home/filecounter'
SCAN_LOCK = False
count_defs = counts.SENSOR_COUNT_DEFINITIONS
DEFAULT_COUNT_START = None
DEFAULT_COUNT_END = None
CLOWDER_HOST = "https://terraref.ncsa.illinois.edu/clowder/"
CLOWDER_KEY = os.getenv('CLOWDER_KEY', False)
CONN = Connector("", {}, mounted_paths={"/home/clowder/sites":"/home/clowder/sites"})
# UTILITIES ----------------------------
def update_nested_dict(existing, new):
"""Nested update of python dictionaries for config parsing
Adapted from http://stackoverflow.com/questions/3232943/update-value-of-a-nested-dictionary-of-varying-depth
"""
for k, v in new.iteritems():
if isinstance(existing, collections.Mapping):
if isinstance(v, collections.Mapping):
r = update_nested_dict(existing.get(k, {}), v)
existing[k] = r
else:
existing[k] = new[k]
else:
import os
import datetime
import json
from pyclowder.connectors import Connector
from pyclowder.datasets import upload_metadata
from pyclowder.files import upload_to_dataset
from terrautils.extractors import build_dataset_hierarchy, build_metadata
from terrautils.metadata import clean_metadata
# CONNECTION SETTINGS
CLOWDER_HOST = "http://141.142.22.37:9000/"
CLOWDER_KEY = "r1ek3rs"
CLOWDER_USER = "******"
CONN = Connector(
None, mounted_paths={"/Users/mburnette/globus": "/Users/mburnette/globus"})
SPACE_ID = "5997333de98a9d4e498532ce"
SENSOR_FOLDER = "/Users/mburnette/globus/Level_1"
LOGFILE = open(os.path.join(SENSOR_FOLDER, "build_log.txt"), "w+")
SENSOR_LIST = ["scanner3DTop"]
TIMESTAMP_FOLDER = True
DRY_RUN = False
def log(string):
print("%s: %s" %
(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), string))
LOGFILE.write(
"%s: %s" %
(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), string))
def upload_to_clowder(dir, type, scan):
conn = Connector(None , mounted_paths={"/home/clowder/sites":"/home/clowder/sites"})
if args.type == "rgb_geotiff":
print("Submission of RGB GeoTIFF would happen now")
return
disp = "RGB GeoTIFFs"
timestamp = dir.split("/")[-2]
target_dsid = build_dataset_hierarchy(host, secret_key, clow_user, clow_pass, clowspace, disp,
timestamp[:4], timestamp[5:7], timestamp[8:10], leaf_ds_name=disp+' - '+timestamp)
output_ids = {}
# First, upload actual files
for targ_file in ["rgb_geotiff_L1_ua-mac_%s_left.tif" % ts,
"rgb_geotiff_L1_ua-mac_%s_right.tif" % ts,
"rgb_geotiff_L1_ua-mac_%s_nrmac_left.tif" % ts,
"rgb_geotiff_L1_ua-mac_%s_nrmac_right.tif" % ts]:
targ_path = os.path.join(dir, targ_file)
if os.path.isfile(targ_path):
file_id = upload_to_dataset(conn, host, clow_user, clow_pass, target_dsid, targ_path)
output_ids[targ_file] = file_id
# Second, upload metadata
ds_md = os.path.join(dir, "clean_metadata.json")
if os.path.isfile(ds_md):
# Dataset metadata
extractor_info = {
"extractor_name": "terra.stereo-rgb.bin2tif",
"extractor_version": "1.1",
"extractor_author": "Max Burnette <*****@*****.**>",
"extractor_description": "Stereo RGB Image Bin to GeoTIFF Converter",
"extractor_repo": "https://github.com/terraref/extractors-stereo-rgb.git"
}
with open(ds_md, 'r') as contents:
jmd = json.load(contents)
upload_dataset_metadata(conn, host, secret_key, clowder_id, jmd)
lemna_md = build_metadata(host, extractor_info, target_dsid, jmd, 'dataset')
upload_metadata(connector, host, secret_key, target_dsid, lemna_md)
nrmac_md = os.path.join(dir, "nrmac_scores.json")
if os.path.isfile(nrmac_md):
# NRMAC file metadata
extractor_info = {
"extractor_name": "terra.stereo-rgb.nrmac",
"extractor_version": "1.0",
"extractor_author": "Sidike Paheding <*****@*****.**>",
"extractor_description": "Stereo RGB No-Reference Multiscale Autocorrelation",
"extractor_repo": "https://github.com/terraref/quality-metrics.git"
}
with open(nrmac_md, 'r') as contents:
jmd = json.load(contents)
fi_id = output_ids["rgb_geotiff_L1_ua-mac_%s_nrmac_left.tif" % ts]
ext_meta = build_metadata(host, extractor_info, fi_id, {
"quality_score": jmd["quality_score"]["left"]
}, 'file')
upload_metadata(connector, host, secret_key, fi_id, ext_meta)
fi_id = output_ids["rgb_geotiff_L1_ua-mac_%s_nrmac_right.tif" % ts]
ext_meta = build_metadata(host, extractor_info, fi_id, {
"quality_score": jmd["quality_score"]["right"]
}, 'file')
upload_metadata(connector, host, secret_key, fi_id, ext_meta)
# Write output_ids.json
with open(os.path.join(dir, "clowder_ids.json"), 'w') as js:
js.write(json.dumps(output_ids))
elif args.type == "fullfield":
print("Submission of Full Field Mosaic would happen now")
return
disp = "Full Field Stitched Mosaics"
timestamp = dir.split("/")[-2]
target_dsid = build_dataset_hierarchy(host, secret_key, clow_user, clow_pass, clowspace, disp,
timestamp[:4], timestamp[5:7], leaf_ds_name=disp+' - '+timestamp)
# TODO: Can each scan be in a separate folder in Clowder?
output_ids = {}
# First, upload NRMAC files
for targ_file in ["fullfield_L1_ua-mac_%s_%s_nrmac.vrt" % (day, scan),
"fullfield_L1_ua-mac_%s_%s_nrmac.tif" % (day, scan)]:
targ_path = os.path.join(dir, targ_file)
if os.path.isfile(targ_path):
file_id = upload_to_dataset(conn, host, clow_user, clow_pass, target_dsid, targ_path)
output_ids[targ_file] = file_id
# Second, upload main stitched files
for targ_file in ["fullfield_L1_ua-mac_%s_%s.vrt" % (day, scan),
"fullfield_L1_ua-mac_%s_%s.tif" % (day, scan),
"fullfield_L1_ua-mac_%s_%s_thumb.tif" % (day, scan),
"fullfield_L1_ua-mac_%s_%s_10pct.tif" % (day, scan),
"fullfield_L1_ua-mac_%s_%s.png" % (day, scan)]:
targ_path = os.path.join(dir, targ_file)
if os.path.isfile(targ_path):
file_id = upload_to_dataset(conn, host, clow_user, clow_pass, target_dsid, targ_path)
output_ids[targ_file] = file_id
# Third, upload trait CSV files
for targ_file in ["fullfield_L1_ua-mac_%s_%s_canopycover_bety.csv" % (day, scan),
"fullfield_L1_ua-mac_%s_%s_canopycover_geo.csv" % (day, scan)]:
targ_path = os.path.join(dir, targ_file)
if os.path.isfile(targ_path):
file_id = upload_to_dataset(conn, host, clow_user, clow_pass, target_dsid, targ_path)
output_ids[targ_file] = file_id
# Write output_ids.json
with open(os.path.join(dir, "clowder_ids.json"), 'w') as js:
js.write(json.dumps(output_ids))
'perspective': perspective,
'rotation_angle': rotation_angle,
'zoom': zoom,
'imager_stage_vertical_position': stage_position,
'camera_gain': camera_gain,
'camera_exposure': camera_exposure,
'image_id': img_id,
'imagedate': snap_details['timestamp'],
'species': species,
'genotype': genotype,
'treatment': treatment,
'sample_id': barcode['unique_id']
}
conn = Connector({},
mounted_paths={"/home/clowder/sites": "/home/clowder/sites"})
experiment_root = sys.argv[1]
experiment_name = os.path.basename(experiment_root)
if os.path.exists(experiment_root):
logger.debug("Searching for index files in %s" % experiment_root)
md_file = os.path.join(experiment_root, experiment_name + "_metadata.json")
csv_file = os.path.join(experiment_root, "SnapshotInfo.csv")
if not os.path.isfile(md_file):
logger.debug("%s not found" % md_file)
sys.exit(1)
if not os.path.isfile(csv_file):
logger.debug("%s not found" % csv_file)
sys.exit(1)
logger.debug("Found index files; loading %s" % md_file)
default="https://terraref.ncsa.illinois.edu/clowder/")
parser.add_argument('-d',
'--daily',
help="only submit one dataset per day",
default=False,
action='store_true')
parser.add_argument('-t',
'--test',
help="only submit one dataset then exit",
default=False,
action='store_true')
args = parser.parse_args()
logging.basicConfig(filename="submit_%s.log" % args.input, level=logging.DEBUG)
CONN = Connector(None, mounted_paths={"/home/clowder/sites": args.sites})
logging.info("attempting to parse %s" % args.input)
sess = requests.Session()
if args.daily:
seen_days = []
with open(args.input, 'r') as csv:
i = 0
for line in csv:
ds_id, ds_name = line.replace("\n", "").split(",")
if len(ds_id) > 0:
if args.daily:
day = ds_name.split(" - ")[1].split("__")[0]
if day in seen_days:
continue