searchPGS = c.get('resource/search', { 'q': 'Phylogenetic signal', 'types': '["item"]' }) if len(searchPGS['item']) == 0: uploadPGS = True # Read our analyses into Python dictionaries and upload them to girder. if uploadACR: ACR = {} with open("%s/ancestral-state/aceArbor.json" % arborWebAppsPath, "r") as acrFile: acrStr = acrFile.read() ACR['analysis'] = json.loads(acrStr) item = c.createItem(folderId, 'aceArbor', 'Ancestral state reconstruction') c.addMetadataToItem(item['_id'], ACR) print "aceArbor successfully uploaded" else: print "aceArbor already exists" if uploadPGS: PGS = {} with open( "%s/phylogenetic-signal/Phylogenetic_signal.json" % arborWebAppsPath, "r") as pgsFile: pgsStr = pgsFile.read() PGS['analysis'] = json.loads(pgsStr) item = c.createItem(folderId, 'Phylogenetic signal', 'Phylogenetic signal') c.addMetadataToItem(item['_id'], PGS) print "Phylogenetic signal successfully uploaded" else:
def upload_exported_zipped_dataset( gc: GirderClient, manager: JobManager, folderId: str, working_directory: Path, create_subfolder='', ): """Uploads a folder that is generated from the export of a zip file and sets metadata""" listOfFileNames = os.listdir(working_directory) potential_meta_files = list( filter(constants.metaRegex.match, listOfFileNames)) if len(potential_meta_files) == 0: manager.write( "Could not find meta.json or config.json file within the subdirectroy\n" ) return print(listOfFileNames) # load meta.json to get datatype and verify list of files meta = {} for meta_name in potential_meta_files: with open(f"{working_directory}/{meta_name}") as f: meta = json.load(f) type = meta[constants.TypeMarker] if type == constants.ImageSequenceType: imageData = meta['imageData'] for image in imageData: if image["filename"] not in listOfFileNames: manager.write( "Could not find {item['filename']} file within the list of files\n" ) return elif type == constants.VideoType: video = meta["video"] if video["filename"] not in listOfFileNames: manager.write( "Could not find {item['filename']} file within the list of files\n" ) return # remove the auxilary directory so we don't have to tag them all if constants.AuxiliaryFolderName in listOfFileNames and os.path.isdir( f'{working_directory}/{constants.AuxiliaryFolderName}'): shutil.rmtree(f'{working_directory}/{constants.AuxiliaryFolderName}') root_folderId = folderId if create_subfolder != '': sub_folder = gc.createFolder( folderId, create_subfolder, reuseExisting=True, ) root_folderId = str(sub_folder['_id']) manager.updateStatus(JobStatus.PUSHING_OUTPUT) # create a source folder to place the zipFile inside of gc.upload(f'{working_directory}/*', root_folderId) # Now we set all the metadata for the folders and items all_files = list(gc.listItem(root_folderId)) root_meta = { "type": type, "attributes": meta.get("attributes", None), "customTypeStyling": meta.get("customTypeStyling", None), "confidenceFilters": meta.get("confidenceFilters", None), "fps": meta["fps"], "version": meta["version"], } if type == constants.VideoType: # set transcoded and non-transcoded versions transcoded_video = list( gc.listItem(root_folderId, name=video["filename"])) if len(transcoded_video) == 1: ffprobe = meta["ffprobe_info"] avgFpsString = ffprobe["avg_frame_rate"] dividend, divisor = [int(v) for v in avgFpsString.split('/')] originalFps = dividend / divisor transcoded_metadata = { "codec": "h264", "originalFps": originalFps, "originalFpsString": avgFpsString, "source_video": False, "transcoder": "ffmpeg", } gc.addMetadataToItem(str(transcoded_video[0]['_id']), transcoded_metadata) # other video is tagged as the source video for item in all_files: if (item["name"].endswith(tuple(constants.validVideoFormats)) and item["name"] != video["filename"]): source_metadata = { "codec": ffprobe["codec_name"], "originalFps": originalFps, "originalFpsString": avgFpsString, "source_video": False, } gc.addMetadataToItem(str(item['_id']), source_metadata) root_meta["originalFps"] = originalFps root_meta["originalFpsString"] = avgFpsString # Need to tag folder Level data (annotate, and others) root_meta[constants.DatasetMarker] = True gc.addMetadataToFolder(root_folderId, root_meta) gc.post(f'dive_rpc/postprocess/{root_folderId}', data={"skipJobs": True})
searchPGS = c.get('resource/search', { 'q': 'Phylogenetic signal', 'types': '["item"]' }) if len(searchPGS['item']) == 0: uploadPGS = True # Read our analyses into Python dictionaries and upload them to girder. if uploadACR: ACR = {} with open ("%s/ancestral-state/aceArbor.json" % args.path, "r") as acrFile: acrStr = acrFile.read() ACR['analysis'] = json.loads(acrStr) item = c.createItem(folderId, 'aceArbor', 'Ancestral state reconstruction') c.addMetadataToItem(item['_id'], ACR) print "aceArbor successfully uploaded" else: print "aceArbor already exists" if uploadPGS: PGS = {} with open ("%s/phylogenetic-signal/Phylogenetic_signal.json" % args.path, "r") as pgsFile: pgsStr = pgsFile.read() PGS['analysis'] = json.loads(pgsStr) item = c.createItem(folderId, 'Phylogenetic signal', 'Phylogenetic signal') c.addMetadataToItem(item['_id'], PGS) print "Phylogenetic signal successfully uploaded" else: print "Phylogenetic signal already exists"
'name': 'Data', 'description': 'Data Folder', 'public': 'true' }) c.post('folder', parameters={ 'parentType': 'collection', 'parentId': collection['_id'], 'name': 'Analyses', 'description': 'Analysis folder', 'public': 'true' }) else: collection = collection_search['collection'][0] # Get the 'Analyses' folder for this collection. analysis_folder = c.load_or_create_folder('Analyses', collection['_id'], 'collection') folder_id = analysis_folder['_id'] # Read this analysis into a Python dictionary and upload it to Girder. analysis = {} with open (fullpath, "r") as analysis_file: analysis_str = analysis_file.read() try: analysis['analysis'] = json.loads(analysis_str) except ValueError: print "Could not read valid JSON from %s" % analysis_filename continue item = c.createItem(folder_id, analysis_name, analysis_name) c.addMetadataToItem(item['_id'], analysis) print "%s successfully uploaded to %s" % (analysis_filename, collection_name)