searchPGS = c.get('resource/search', {
'q': 'Phylogenetic signal',
'types': '["item"]'
})
if len(searchPGS['item']) == 0:
uploadPGS = True
# Read our analyses into Python dictionaries and upload them to girder.
if uploadACR:
ACR = {}
with open("%s/ancestral-state/aceArbor.json" % arborWebAppsPath,
"r") as acrFile:
acrStr = acrFile.read()
ACR['analysis'] = json.loads(acrStr)
item = c.createItem(folderId, 'aceArbor', 'Ancestral state reconstruction')
c.addMetadataToItem(item['_id'], ACR)
print "aceArbor successfully uploaded"
else:
print "aceArbor already exists"
if uploadPGS:
PGS = {}
with open(
"%s/phylogenetic-signal/Phylogenetic_signal.json" %
arborWebAppsPath, "r") as pgsFile:
pgsStr = pgsFile.read()
PGS['analysis'] = json.loads(pgsStr)
item = c.createItem(folderId, 'Phylogenetic signal', 'Phylogenetic signal')
c.addMetadataToItem(item['_id'], PGS)
print "Phylogenetic signal successfully uploaded"
else:
def upload_exported_zipped_dataset(
gc: GirderClient,
manager: JobManager,
folderId: str,
working_directory: Path,
create_subfolder='',
):
"""Uploads a folder that is generated from the export of a zip file and sets metadata"""
listOfFileNames = os.listdir(working_directory)
potential_meta_files = list(
filter(constants.metaRegex.match, listOfFileNames))
if len(potential_meta_files) == 0:
manager.write(
"Could not find meta.json or config.json file within the subdirectroy\n"
)
return
print(listOfFileNames)
# load meta.json to get datatype and verify list of files
meta = {}
for meta_name in potential_meta_files:
with open(f"{working_directory}/{meta_name}") as f:
meta = json.load(f)
type = meta[constants.TypeMarker]
if type == constants.ImageSequenceType:
imageData = meta['imageData']
for image in imageData:
if image["filename"] not in listOfFileNames:
manager.write(
"Could not find {item['filename']} file within the list of files\n"
)
return
elif type == constants.VideoType:
video = meta["video"]
if video["filename"] not in listOfFileNames:
manager.write(
"Could not find {item['filename']} file within the list of files\n"
)
return
# remove the auxilary directory so we don't have to tag them all
if constants.AuxiliaryFolderName in listOfFileNames and os.path.isdir(
f'{working_directory}/{constants.AuxiliaryFolderName}'):
shutil.rmtree(f'{working_directory}/{constants.AuxiliaryFolderName}')
root_folderId = folderId
if create_subfolder != '':
sub_folder = gc.createFolder(
folderId,
create_subfolder,
reuseExisting=True,
)
root_folderId = str(sub_folder['_id'])
manager.updateStatus(JobStatus.PUSHING_OUTPUT)
# create a source folder to place the zipFile inside of
gc.upload(f'{working_directory}/*', root_folderId)
# Now we set all the metadata for the folders and items
all_files = list(gc.listItem(root_folderId))
root_meta = {
"type": type,
"attributes": meta.get("attributes", None),
"customTypeStyling": meta.get("customTypeStyling", None),
"confidenceFilters": meta.get("confidenceFilters", None),
"fps": meta["fps"],
"version": meta["version"],
}
if type == constants.VideoType:
# set transcoded and non-transcoded versions
transcoded_video = list(
gc.listItem(root_folderId, name=video["filename"]))
if len(transcoded_video) == 1:
ffprobe = meta["ffprobe_info"]
avgFpsString = ffprobe["avg_frame_rate"]
dividend, divisor = [int(v) for v in avgFpsString.split('/')]
originalFps = dividend / divisor
transcoded_metadata = {
"codec": "h264",
"originalFps": originalFps,
"originalFpsString": avgFpsString,
"source_video": False,
"transcoder": "ffmpeg",
}
gc.addMetadataToItem(str(transcoded_video[0]['_id']),
transcoded_metadata)
# other video is tagged as the source video
for item in all_files:
if (item["name"].endswith(tuple(constants.validVideoFormats))
and item["name"] != video["filename"]):
source_metadata = {
"codec": ffprobe["codec_name"],
"originalFps": originalFps,
"originalFpsString": avgFpsString,
"source_video": False,
}
gc.addMetadataToItem(str(item['_id']), source_metadata)
root_meta["originalFps"] = originalFps
root_meta["originalFpsString"] = avgFpsString
# Need to tag folder Level data (annotate, and others)
root_meta[constants.DatasetMarker] = True
gc.addMetadataToFolder(root_folderId, root_meta)
gc.post(f'dive_rpc/postprocess/{root_folderId}', data={"skipJobs": True})
searchPGS = c.get('resource/search', {
'q': 'Phylogenetic signal',
'types': '["item"]'
})
if len(searchPGS['item']) == 0:
uploadPGS = True
# Read our analyses into Python dictionaries and upload them to girder.
if uploadACR:
ACR = {}
with open ("%s/ancestral-state/aceArbor.json" % args.path, "r") as acrFile:
acrStr = acrFile.read()
ACR['analysis'] = json.loads(acrStr)
item = c.createItem(folderId, 'aceArbor', 'Ancestral state reconstruction')
c.addMetadataToItem(item['_id'], ACR)
print "aceArbor successfully uploaded"
else:
print "aceArbor already exists"
if uploadPGS:
PGS = {}
with open ("%s/phylogenetic-signal/Phylogenetic_signal.json" % args.path, "r") as pgsFile:
pgsStr = pgsFile.read()
PGS['analysis'] = json.loads(pgsStr)
item = c.createItem(folderId, 'Phylogenetic signal', 'Phylogenetic signal')
c.addMetadataToItem(item['_id'], PGS)
print "Phylogenetic signal successfully uploaded"
else:
print "Phylogenetic signal already exists"
'name': 'Data',
'description': 'Data Folder',
'public': 'true'
})
c.post('folder', parameters={
'parentType': 'collection',
'parentId': collection['_id'],
'name': 'Analyses',
'description': 'Analysis folder',
'public': 'true'
})
else:
collection = collection_search['collection'][0]
# Get the 'Analyses' folder for this collection.
analysis_folder = c.load_or_create_folder('Analyses', collection['_id'], 'collection')
folder_id = analysis_folder['_id']
# Read this analysis into a Python dictionary and upload it to Girder.
analysis = {}
with open (fullpath, "r") as analysis_file:
analysis_str = analysis_file.read()
try:
analysis['analysis'] = json.loads(analysis_str)
except ValueError:
print "Could not read valid JSON from %s" % analysis_filename
continue
item = c.createItem(folder_id, analysis_name, analysis_name)
c.addMetadataToItem(item['_id'], analysis)
print "%s successfully uploaded to %s" % (analysis_filename, collection_name)
