def __init__(self, env): log.info('GETTING CONFIG FOR: {}'.format(env)) if env == "dev": log.info('SETTING UP CONFIG FOR: {}'.format(env)) self.env = "dev" self.last_updated = None blackfynn_host = "https://api.blackfynn.net" blackfynn_api_token = os.environ.get("BLACKFYNN_API_TOKEN") blackfynn_api_secret = os.environ.get("BLACKFYNN_API_SECRET") self.bf = Blackfynn(api_token=blackfynn_api_token, api_secret=blackfynn_api_secret, host=blackfynn_host) elif env == "prod": log.info('SETTING UP CONFIG FOR: {}'.format(env)) self.env = "prod" self.last_updated = None blackfynn_host = "https://api.blackfynn.io" blackfynn_api_token = os.environ.get("BLACKFYNN_API_TOKEN") blackfynn_api_secret = os.environ.get("BLACKFYNN_API_SECRET") self.bf = Blackfynn(api_token=blackfynn_api_token, api_secret=blackfynn_api_secret, host=blackfynn_host) else: raise (Exception('Incorrect input argument'))
def connect(self): ''' Prompts for csv file with info we need to upload. Checks if files to upload exist. Connects to Blackfynn API and activates top level dataset destination. Returns csv file name, destination dataset. ''' try: b_fynn = Blackfynn(self.working_profile) except Exception as ex: sys.exit('Error Connecting to ' + 'Blackfynn. ' + str(ex)) try: print("Trying to connect to dataset.", flush=True) dataset = b_fynn.get_dataset(self.dataset_name) except Exception as ex: sys.exit('Unable to connect to the dataset.' + str(ex)) print() print('Dataset: {}\nProfile: {}'.format(data_set, self.working_profile)) prompt = input('Continue with upload (y/n)? ') if prompt != 'y': sys.exit('Aborting upload.') return dataset
def push_to_blackfynn(event, context): bucket = event['Records'][0]['s3']['bucket']['name'] key = urllib.unquote_plus( event['Records'][0]['s3']['object']['key'].encode('utf8')) try: bf = Blackfynn() bf.set_context(organization_id) root_collection = bf.get(collection_id) package_name = key.split("/")[-2] package_collection = next( (c for c in root_collection.items if c.name == package_name), None) if package_collection is None: package_collection = Collection(package_name) root_collection.add(package_collection) local_object = '/tmp/{}'.format(os.path.basename(key)) s3.download_file(Bucket=bucket, Key=key, Filename=local_object) package_collection.upload_files(local_object) return 'Successfully pushed {} from bucket {} to Blackfynn.'.format( key, bucket) except Exception as e: print(e) print('Error pushing {} from bucket {} to Blackfynn.'.format( key, bucket)) raise e
def test_client_host_overrides(): host = "http://localhost" # fails authentication in Blackfynn.__init__ with pytest.raises(requests.exceptions.RequestException): bf = Blackfynn(host=host) bf = Blackfynn(model_service_host=host) assert bf.settings.model_service_host == host
def send_digest(): report = [] report.append("<h2>Blackfynn: BioRC Failed Extract Digest</h2>") bf = Blackfynn() root = bf.get('N:collection:4fec4882-925c-4328-bbfd-d1de953ba225') for bucket in root.items: include_count = True if bucket.id == "N:collection:0e476218-ccb9-4c4d-bdb4-e72d0a0f88fd" else False bucket_digest(report, bucket, include_count) send_report(report)
def __init__(self, api_token=None, api_secret=None, host=None, streaming_host=None): self.client = Blackfynn(profile=None, api_token=api_token, api_secret=api_secret, host=host, streaming_host=streaming_host)
def client2(use_dev): bf = Blackfynn('*****@*****.**', 'password') # get organizations orgs = bf.organizations() assert len(orgs) > 0 # explicitly set context to Blackfyn org bf.set_context('Test Organization') assert bf.context is not None return bf
def get_test_client(profile=None, api_token=None, api_secret=None, **overrides): """ Utility function to get a Blackfynn client object """ bf = Blackfynn(profile=profile, api_token=api_token, api_secret=api_secret, **overrides) assert bf.context is not None orgs = bf.organizations() assert len(orgs) > 0 # explicitly set context to Blackfyn org assert bf.context is not None return bf
def lambda_handler(event, context): DATASET = os.environ['DASHBOARD_DATASET_NAME'] bf = Blackfynn() ds = bf.get_dataset(DATASET) update(bf, ds) return { 'statusCode': 200, 'body': json.dumps('Successfully updated the SPARC dashboard') }
def __init__(self, api_token=None, api_secret=None, host=None, streaming_host=None): # WARNING: contruction raise exception if service is not available. Use datacore_wrapper for safe calls self.client = Blackfynn( profile=None, api_token=api_token, api_secret=api_secret, host=host, streaming_host=streaming_host, )
def client2(): api_token = os.environ.get('BLACKFYNN_API_TOKEN2') api_secret = os.environ.get('BLACKFYNN_API_SECRET2') assert api_token != "", "Must define BLACKFYNN_API_TOKEN2" assert api_secret != "", "Must define BLACKFYNN_API_SECRET2" bf = Blackfynn() # get organizations orgs = bf.organizations() assert len(orgs) > 0 # explicitly set context to Blackfyn org assert bf.context is not None return bf
def client(): """ Login via API, return client. Login information, by default, will be taken from environment variables, so ensure those are set properly before testing. Alternatively, to force a particular user, adjust input arguments as necessary. """ bf = Blackfynn() # get organizations orgs = bf.organizations() print 'organizations =', orgs assert len(orgs) > 0 # explicitly set context to Blackfyn org assert bf.context is not None return bf
def __init__(self, api_token=None, api_secret=None, host=None, streaming_host=None): # WARNING: contruction raise exception if service is not available. # Use datacore_wrapper for safe calls # TODO: can use https://developer.blackfynn.io/python/latest/configuration.html#environment-variables self._bf = Blackfynn( profile=None, api_token=api_token, api_secret=api_secret, host=host, streaming_host=streaming_host, )
def connect_to_blackfynn(): global bf bf = Blackfynn( api_token=Config.BLACKFYNN_API_TOKEN, api_secret=Config.BLACKFYNN_API_SECRET, env_override=False, )
def blackfynn_cli(): args = docopt(__doc__, version='bf version {}'.format(blackfynn.__version__), options_first=True) # Test for these two commands first as they # do not require a Blackfynn client if args['<command>'] in ['help', None]: print(__doc__.strip('\n')) return if args['<command>'] == 'profile': import bf_profile bf_profile.main() return # Display warning message if config.ini is not found settings = Settings( ) # create a dummy settings object to load environment variables and defaults only if not os.path.exists(settings.config_file): print( "\033[31m* Warning: No config file found, run 'bf profile' to start the setup assistant\033[0m" ) # Try to use profile specified by --profile, exit if invalid try: bf = Blackfynn(args['--profile']) except Exception, e: exit(e)
def _can_access_blackfynn(self, api_key, api_secret): can_access = False bf = Blackfynn(api_token=api_key, api_secret=api_secret) if bf.context.exists: can_access = True return can_access
def connect_to_blackfynn(): global bf bf = Blackfynn(api_token=Config.BLACKFYNN_API_TOKEN, api_secret=Config.BLACKFYNN_API_SECRET, env_override=False, host=Config.BLACKFYNN_API_HOST, concepts_api_host=Config.BLACKFYNN_CONCEPTS_API_HOST)
def superuser_client(use_dev): """ Client using super-admin permissions """ bf = Blackfynn('*****@*****.**', 'password') assert bf.profile.is_super_admin return bf
def blackfynn_get(dataset='Timeseries Dataset',collection=0,channels='sine 50 Hz'): #establish connection with the API bf = Blackfynn(api_token='**********************************',api_secret='**********************************',) ds = bf.get_dataset(dataset) print(ds) # get all timeseries collections ts = [] for ds_temp in ds: ts.append(ds_temp) # NOTE: THIS NEEDS TO BE MADE MODULAR data = ts[collection].get_data(length='1s') # take the data from the channel and process it to be read by javascript #process y values temp = np.asarray(data[channels]) java_y = [] for el in temp: java_y.append(float(str(el))) #generate x values #NOTE: CURRENTLY NOT MODULAR temp2 = np.linspace(0,1,len(temp)) java_x =[] for el in temp2: java_x.append(el) #write to file f = open('data.js', 'w') f.write('arbData = ') f.write(str(java_y)) f.write('\ndate = ') f.write(repr(java_x)) f.close() #update gitub g = git_hub() g.update() print('update ran successfully')
def connect_blackfynn(): "Connect to Blackfynn" print("Connecting to Blackfynn ...", end=" ") bf = Blackfynn( api_token=Config.BLACKFYNN_API_TOKEN, api_secret=Config.BLACKFYNN_API_SECRET, env_override=False, host=Config.BLACKFYNN_API_HOST, ) print("done") return bf
def connect_blackfynn(): 'Connect to Blackfynn' print('Connecting to Blackfynn ...', end=' ') bf = Blackfynn( api_token=Config.BLACKFYNN_API_TOKEN, api_secret=Config.BLACKFYNN_API_SECRET, env_override=False, host=Config.BLACKFYNN_API_HOST, ) print('done') return bf
def test_client_global_headers(): global_headers = { "X-Custom-Header1": "Custom Value", "X-Custom-Header2": "Custom Value2", } bf = Blackfynn(headers=global_headers) assert bf.settings.headers == global_headers for header, header_value in global_headers.items(): assert header in bf._api._session.headers assert bf._api._session.headers[header] == header_value
def sessionp(): print(request.method) print(request.headers) print(request.data) print('its a post!') data = json.loads(request.data.decode("utf-8")) from blackfynn import Blackfynn global bf bf = Blackfynn(api_token=data['tokenId'], api_secret=data['secret']) data_sets = bf.datasets() global time_series_items time_series_items = [] time_series_names = [] for data_set in data_sets: for item in data_set.items: if item.type is 'TimeSeries': time_series_items.append(item) time_series_names.append(item.name) return json.dumps({'names': time_series_names})
def instantiateDatCore(): ''' instantiate a dat core object if not already done :return: handle to the dat core object ''' # we are going to store the instance as a global # so we can access it from everywhere global dcInstance # check if we already have the dat core instance or not if not isinstance(dcInstance,Blackfynn): dcInstance = Blackfynn() # return the dcInstance return dcInstance
def get_timeseries_dataset_names(): data = json.loads(request.data.decode("utf-8")) global bf bf = Blackfynn(api_token=data['tokenId'], api_secret=data['secret']) data_sets = bf.datasets() global time_series_items, csv_items, csv_names time_series_items = [] time_series_names = [] csv_items = [] csv_names = [] for data_set in data_sets: for item in data_set.items: if item.type is 'TimeSeries': time_series_items.append(item) time_series_names.append(item.name) if item.type is 'Tabular': csv_items.append(item) csv_names.append(item.name) global user_ip user_ip = request.remote_addr return json.dumps({'names': time_series_names + csv_names})
def main(): args = docopt(__doc__, version=blackfynn.__version__) # Test for these two commands first as they # do not require a Blackfynn client or reading the settings file if args["help"]: print(__doc__.strip("\n")) return elif args["version"]: print(blackfynn.__version__) return settings = Settings(args["--profile"]) if not os.path.exists(settings.config_file): setup_assistant(settings) elif args["create"]: create_profile(settings, args["<name>"]) elif args["show"]: show_profile(settings, args["<name>"]) elif args["delete"]: delete_profile(settings, args["<name>"], args["--force"]) elif args["list"]: list_profiles(settings, args["--contents"]) elif args["set-default"]: set_default(settings, args["<name>"]) elif args["unset-default"]: unset_default(settings, args["--force"]) elif args["set"]: set_key( settings, args["<key>"], args["<value>"], args["--profile"], args["--force"] ) elif args["unset"]: unset_key(settings, args["<key>"], args["--profile"], args["--force"]) elif args["keys"]: list_keys(settings, args["--profile"]) elif args["status"]: bf = Blackfynn(args["--profile"]) show_status(bf) else: invalid_usage() with io.open(settings.config_file, "w") as configfile: settings.config.write(configfile)
def main(): args = docopt(__doc__, version=blackfynn.__version__) # Test for these two commands first as they # do not require a Blackfynn client or reading the settings file if args['help']: print(__doc__.strip('\n')) return elif args['version']: print(blackfynn.__version__) return settings = Settings(args['--profile']) if not os.path.exists(settings.config_file): setup_assistant(settings) elif args['create']: create_profile(settings, args['<name>']) elif args['show']: show_profile(settings, args['<name>']) elif args['delete']: delete_profile(settings, args['<name>'], args['--force']) elif args['list']: list_profiles(settings, args['--contents']) elif args['set-default']: set_default(settings, args['<name>']) elif args['unset-default']: unset_default(settings, args['--force']) elif args['set']: set_key(settings, args['<key>'], args['<value>'], args['--profile'], args['--force']) elif args['unset']: unset_key(settings, args['<key>'], args['--profile'], args['--force']) elif args['keys']: list_keys(settings, args['--profile']) elif args['status']: bf = Blackfynn(args['--profile']) show_status(bf) else: invalid_usage() with io.open(settings.config_file, 'w') as configfile: settings.config.write(configfile)
def blackfynn_get(): # make a blackfynn API call according to threeWrapper parameters fpipe = FilePipe() params = fpipe.receive() api_key = params['api_key'] api_secret = params['api_secret'] dataset = params['dataset'] collection = params['collection'] channels = params['channels'] window_from_start = params['window_from_start'] start = params['start'] end = params['end'] error = 0 # Process function input: if start == -1 or end == -1: has_time_window = False else: has_time_window = True #establish connection with the API try: bf = Blackfynn(api_token=api_key, api_secret=api_secret) except: fpipe.send({ 'error': 'Could not connect to the Blackfynn API. Check your API key and internet connection' }) return # get all timeseries collections try: ds = bf.get_dataset(dataset) tstemp = ds.get_items_by_name(collection) time_series = tstemp[0] except: fpipe.send({ 'error': 'Could not find the requested Dataset and Collection. Please check your names if you have not already' }) return # Get data for all channels according to the length set by parameters if has_time_window == False: data = time_series.get_data(length=(str(window_from_start) + 's')) else: data = time_series.get_data(start=start, end=end) # take the data from the channel and process it to be passed into binary (in filePipe) # process y values list_y = data[channels].values.tolist() # generate x values if has_time_window: time = np.linspace(start, end, len(list_y)) else: time = np.linspace(0, window_from_start, len(list_y)) list_x = time.tolist() cache_dict = create_file_cache(data) output = {'x': list_x, 'y': list_y, 'cache': cache_dict, 'error': False} fpipe.send(output) print('update ran successfully')
class DatcoreClient(object): def __init__(self, api_token=None, api_secret=None, host=None, streaming_host=None): # WARNING: contruction raise exception if service is not available. # Use datacore_wrapper for safe calls # TODO: can use https://developer.blackfynn.io/python/latest/configuration.html#environment-variables self._bf = Blackfynn( profile=None, api_token=api_token, api_secret=api_secret, host=host, streaming_host=streaming_host, ) def profile(self): """ Returns profile of current User """ return self._bf.profile def _collection_from_destination(self, destination: str): destination_path = Path(destination) parts = destination_path.parts dataset_name = parts[0] dataset = self.get_dataset(dataset_name) if dataset is None: return None, None collection_id = dataset.id collection = dataset collections = [] if len(parts) > 1: object_path = Path(*parts[1:]) collections = list(object_path.parts) collection_id = "" collection_id = _get_collection_id(dataset, collections, collection_id) collection = self._bf.get(collection_id) return collection, collection_id def _destination_from_id(self, destination_id: str): # NOTE: .get(*) logs # INFO:blackfynn.client.Blackfynn:Unable to retrieve object # if destination_id refers to a Dataset destination: Union[DataPackage, Collection] = self._bf.get(destination_id) if destination is None: destination: Dataset = self._bf.get_dataset(destination_id) return destination def list_files_recursively(self, dataset_filter: str = ""): files = [] for dataset in self._bf.datasets(): if not dataset_filter or dataset_filter in dataset.name: self.list_dataset_files_recursively(files, dataset, Path(dataset.name)) return files def list_files_raw_dataset(self, dataset_id: str) -> List[FileMetaDataEx]: files = [] # raw packages _files = [] # fmds data = {} # map to keep track of parents-child cursor = "" page_size = 1000 api = self._bf._api.datasets dataset = self._bf.get_dataset(dataset_id) if dataset is not None: while True: resp = api._get( api._uri( "/{id}/packages?cursor={cursor}&pageSize={pageSize}&includeSourceFiles={includeSourceFiles}", id=dataset_id, cursor=cursor, pageSize=page_size, includeSourceFiles=False, )) for package in resp.get("packages", list()): id = package["content"]["id"] data[id] = package files.append(package) cursor = resp.get("cursor") if cursor is None: break for f in files: if f["content"]["packageType"] != "Collection": filename = f["content"]["name"] file_path = "" file_id = f["content"]["nodeId"] _f = f while "parentId" in _f["content"].keys(): parentid = _f["content"]["parentId"] _f = data[parentid] file_path = _f["content"]["name"] + "/" + file_path bucket_name = dataset.name file_name = filename file_size = 0 object_name = str(Path(file_path) / file_name) file_uuid = str(Path(bucket_name) / object_name) created_at = f["content"]["createdAt"] last_modified = f["content"]["updatedAt"] parent_id = dataset_id if "parentId" in f["content"]: parentId = f["content"]["parentId"] parent_id = data[parentId]["content"]["nodeId"] fmd = FileMetaData( bucket_name=bucket_name, file_name=file_name, object_name=object_name, location=DATCORE_STR, location_id=DATCORE_ID, file_uuid=file_uuid, file_id=file_id, raw_file_path=file_uuid, display_file_path=file_uuid, created_at=created_at, last_modified=last_modified, file_size=file_size, ) fmdx = FileMetaDataEx(fmd=fmd, parent_id=parent_id) _files.append(fmdx) return _files def list_files_raw(self, dataset_filter: str = "") -> List[FileMetaDataEx]: _files = [] for dataset in self._bf.datasets(): _files = _files + self.list_files_raw_dataset(dataset.id) return _files def list_dataset_files_recursively(self, files: List[FileMetaData], base: BaseCollection, current_root: Path): for item in base: if isinstance(item, Collection): _current_root = current_root / Path(item.name) self.list_dataset_files_recursively(files, item, _current_root) else: parts = current_root.parts bucket_name = parts[0] file_name = item.name file_size = 0 # lets assume we have only one file if item.files: file_name = Path( item.files[0].as_dict()["content"]["s3key"]).name file_size = item.files[0].as_dict()["content"]["size"] # if this is in the root directory, the object_name is the filename only if len(parts) > 1: object_name = str(Path(*list(parts)[1:]) / Path(file_name)) else: object_name = str(Path(file_name)) file_uuid = str(Path(bucket_name) / Path(object_name)) file_id = item.id created_at = item.created_at last_modified = item.updated_at fmd = FileMetaData( bucket_name=bucket_name, file_name=file_name, object_name=object_name, location=DATCORE_STR, location_id=DATCORE_ID, file_uuid=file_uuid, file_id=file_id, raw_file_path=file_uuid, display_file_path=file_uuid, created_at=created_at, last_modified=last_modified, file_size=file_size, ) files.append(fmd) def create_dataset(self, ds_name, *, force_delete=False): """ Creates a new dataset for the current user and returns it. Returns existing one if there is already a dataset with the given name. Args: ds_name (str): Name for the dataset (_,-,' ' and capitalization are ignored) force_delete (bool, optional): Delete first if dataset already exists """ ds = None with suppress(Exception): ds = self._bf.get_dataset(ds_name) if force_delete: ds.delete() ds = None if ds is None: ds = self._bf.create_dataset(ds_name) return ds def get_dataset(self, ds_name, create_if_not_exists=False): """ Returns dataset with the given name. Creates it if required. Args: ds_name (str): Name for the dataset create_if_not_exists (bool, optional): Create first if dataset already exists """ ds = None with suppress(Exception): ds = self._bf.get_dataset(ds_name) if ds is None and create_if_not_exists: ds = self._bf.create_dataset(ds_name) return ds def delete_dataset(self, ds_name): """ Deletes dataset with the given name. Args: ds_name (str): Name for the dataset """ # this is not supported ds = self.get_dataset(ds_name) if ds is not None: self._bf.delete(ds.id) def exists_dataset(self, ds_name): """ Returns True if dataset with the given name exists. Args: ds_name (str): Name for the dataset """ ds = self.get_dataset(ds_name) return ds is not None def upload_file(self, destination: str, filepath: str, meta_data=None) -> bool: """ Uploads a file to a given dataset/collection given its filepath on the host. Optionally adds some meta data Args: dataset (dataset): The dataset into whioch the file shall be uploaded filepath (path): Full path to the file meta_data (dict, optional): Dictionary of meta data Note: Blackfynn postprocesses data based on filendings. If it can do that the filenames on the server change. """ # parse the destination and try to find the package_id to upload to collection, collection_id = self._collection_from_destination( destination) if collection is None: return False files = [ filepath, ] self._bf._api.io.upload_files(collection, files, display_progress=True, use_agent=False) collection.update() if meta_data is not None: for f in files: filename = os.path.basename(f) package = self.get_package(collection, filename) if package is not None: self._update_meta_data(package, meta_data) return True def _update_meta_data(self, package, meta_data): """ Updates or replaces metadata for a package Args: package (package): The package for which the meta data needs update meta_data (dict): Dictionary of meta data """ for key in meta_data.keys(): package.set_property(key, meta_data[key], category="simcore") package.update() def download_file(self, source, filename, destination_path): """ Downloads a frile from a source dataset/collection given its filename. Stores it under destination_path Args: source (dataset/collection): The dataset or collection to donwload from filename (str): Name of the file destination__apth (str): Path on host for storing file """ url = self.download_link(source, filename) if url: _file = urllib.URLopener() # nosec _file.retrieve(url, destination_path) return True return False def download_link(self, destination, filename): """ returns presigned url for download, destination is a dataset or collection """ collection, collection_id = self._collection_from_destination( destination) for item in collection: if isinstance(item, DataPackage): if Path(item.files[0].as_dict()["content"] ["s3key"]).name == filename: file_desc = self._bf._api.packages.get_sources(item.id)[0] url = self._bf._api.packages.get_presigned_url_for_file( item.id, file_desc.id) return url return "" def download_link_by_id(self, file_id): """ returns presigned url for download of a file given its file_id """ url = "" filename = "" package = self._bf.get(file_id) if package is not None: filename = Path( package.files[0].as_dict()["content"]["s3key"]).name file_desc = self._bf._api.packages.get_sources(file_id)[0] url = self._bf._api.packages.get_presigned_url_for_file( file_id, file_desc.id) return url, filename def get_package(self, source, filename): """ Returns package from source by name if exists Args: source (dataset/collection): The dataset or collection to donwload from filename (str): Name of the file """ source.update() for item in source: if item.name == filename: return item return None def delete_file(self, destination, filename): """ Deletes file by name from destination by name Args: destination (dataset/collection): The dataset or collection to delete from filename (str): Name of the file """ collection, collection_id = self._collection_from_destination( destination) if collection is None: return False collection.update() for item in collection: if isinstance(item, DataPackage): if Path(item.files[0].as_dict()["content"] ["s3key"]).name == filename: self._bf.delete(item) return True return False def delete_file_by_id(self, id: str) -> bool: """ Deletes file by id Args: datcore id for the file """ package: DataPackage = self._bf.get(id) package.delete() return not package.exists def delete_files(self, destination): """ Deletes all files in destination Args: destination (dataset/collection): The dataset or collection to delete """ collection, collection_id = self._collection_from_destination( destination) if collection is None: return False collection.update() for item in collection: self._bf.delete(item) def update_meta_data(self, dataset, filename, meta_data): """ Updates metadata for a file Args: dataset (package): Which dataset filename (str): Which file meta_data (dict): Dictionary of meta data """ filename = os.path.basename(filename) package = self.get_package(dataset, filename) if package is not None: self._update_meta_data(package, meta_data) def get_meta_data(self, dataset, filename): """ Returns metadata for a file Args: dataset (package): Which dataset filename (str): Which file """ meta_data = {} filename = os.path.basename(filename) package = self.get_package(dataset, filename) if package is not None: meta_list = package.properties for m in meta_list: meta_data[m.key] = m.value return meta_data def delete_meta_data(self, dataset, filename, keys=None): """ Deletes specified keys in meta data for source/filename. Args: dataset (package): Which dataset filename (str): Which file keys (list of str, optional): Deletes specified keys, deletes all meta data if None """ filename = os.path.basename(filename) package = self.get_package(dataset, filename) if package is not None: if keys is None: for p in package.properties: package.remove_property(p.key, category="simcore") else: for k in keys: package.remove_property(k, category="simcore") def search(self, what, max_count): """ Seraches a thing in the database. Returns max_count results Args: what (str): query max_count (int): Max number of results to return """ return self._bf.search(what, max_count) def upload_file_to_id(self, destination_id: str, filepath: str): """ Uploads file to a given dataset/collection by id given its filepath on the host adds some meta data. Returns the id for the newly created resource Note: filepath could be an array Args: destination_id : The dataset/collection id into which the file shall be uploaded filepath (path): Full path to the file """ _id = "" destination = self._destination_from_id(destination_id) if destination is None: return _id files = [ filepath, ] try: # TODO: PC->MAG: should protected API # TODO: add new agent SEE https://developer.blackfynn.io/python/latest/CHANGELOG.html#id31 result = self._bf._api.io.upload_files(destination, files, display_progress=True, use_agent=False) if result and result[0] and "package" in result[0][0]: _id = result[0][0]["package"]["content"]["id"] except Exception: logger.exception("Error uploading file to datcore") return _id def create_collection(self, destination_id: str, collection_name: str): """ Create a empty collection within destination Args: destination_id : The dataset/collection id into which the file shall be uploaded filepath (path): Full path to the file """ destination = self._destination_from_id(destination_id) _id = "" if destination is None: return _id new_collection = Collection(collection_name) destination.add(new_collection) new_collection.update() destination.update() _id = new_collection.id return _id def list_datasets(self) -> DatasetMetaDataVec: data = [] for dataset in self._bf.datasets(): dmd = DatasetMetaData(dataset_id=dataset.id, display_name=dataset.name) data.append(dmd) return data
# REQUIREMENTS: python2, blackfynn python library, blackfynn key # UPDATES: 180215: Added -f option # AUTHOR: Pete Schmitt (discovery), [email protected] # COMPANY: University of Pennsylvania # VERSION: 0.2.0 # CREATED: Wed Feb 14 13:27:32 EST 2018 # REVISION: Thu Feb 15 11:49:51 EST 2018 #=============================================================================== from blackfynn import Blackfynn from blackfynn.models import BaseCollection from blackfynn.models import Collection import sys import getopt import os bf = Blackfynn() # use 'default' profile ############################################################################### def syntax(): SYNTAX = "\nbfmove -d <dataset> \n" SYNTAX += " --all (loop on all HPAP datasets)\n" SYNTAX += " -f <file containing datasets>\n" SYNTAX += " -S <source path>\n" SYNTAX += " -D <destination path> (MUST be directory)\n\n" SYNTAX += " -h (help)\n" SYNTAX += " -l (list datasets)\n\n" SYNTAX += "Note: -d, -f and --all are mutually exlusive\n" return SYNTAX