def __init__(self, env):
log.info('GETTING CONFIG FOR: {}'.format(env))
if env == "dev":
log.info('SETTING UP CONFIG FOR: {}'.format(env))
self.env = "dev"
self.last_updated = None
blackfynn_host = "https://api.blackfynn.net"
blackfynn_api_token = os.environ.get("BLACKFYNN_API_TOKEN")
blackfynn_api_secret = os.environ.get("BLACKFYNN_API_SECRET")
self.bf = Blackfynn(api_token=blackfynn_api_token,
api_secret=blackfynn_api_secret,
host=blackfynn_host)
elif env == "prod":
log.info('SETTING UP CONFIG FOR: {}'.format(env))
self.env = "prod"
self.last_updated = None
blackfynn_host = "https://api.blackfynn.io"
blackfynn_api_token = os.environ.get("BLACKFYNN_API_TOKEN")
blackfynn_api_secret = os.environ.get("BLACKFYNN_API_SECRET")
self.bf = Blackfynn(api_token=blackfynn_api_token,
api_secret=blackfynn_api_secret,
host=blackfynn_host)
else:
raise (Exception('Incorrect input argument'))
def connect(self):
'''
Prompts for csv file with info we need to upload.
Checks if files to upload exist.
Connects to Blackfynn API and activates top level dataset destination.
Returns csv file name, destination dataset.
'''
try:
b_fynn = Blackfynn(self.working_profile)
except Exception as ex:
sys.exit('Error Connecting to ' + 'Blackfynn. ' + str(ex))
try:
print("Trying to connect to dataset.", flush=True)
dataset = b_fynn.get_dataset(self.dataset_name)
except Exception as ex:
sys.exit('Unable to connect to the dataset.' + str(ex))
print()
print('Dataset: {}\nProfile: {}'.format(data_set,
self.working_profile))
prompt = input('Continue with upload (y/n)? ')
if prompt != 'y':
sys.exit('Aborting upload.')
return dataset
def push_to_blackfynn(event, context):
bucket = event['Records'][0]['s3']['bucket']['name']
key = urllib.unquote_plus(
event['Records'][0]['s3']['object']['key'].encode('utf8'))
try:
bf = Blackfynn()
bf.set_context(organization_id)
root_collection = bf.get(collection_id)
package_name = key.split("/")[-2]
package_collection = next(
(c for c in root_collection.items if c.name == package_name), None)
if package_collection is None:
package_collection = Collection(package_name)
root_collection.add(package_collection)
local_object = '/tmp/{}'.format(os.path.basename(key))
s3.download_file(Bucket=bucket, Key=key, Filename=local_object)
package_collection.upload_files(local_object)
return 'Successfully pushed {} from bucket {} to Blackfynn.'.format(
key, bucket)
except Exception as e:
print(e)
print('Error pushing {} from bucket {} to Blackfynn.'.format(
key, bucket))
raise e
def test_client_host_overrides():
host = "http://localhost"
# fails authentication in Blackfynn.__init__
with pytest.raises(requests.exceptions.RequestException):
bf = Blackfynn(host=host)
bf = Blackfynn(model_service_host=host)
assert bf.settings.model_service_host == host
def send_digest():
report = []
report.append("<h2>Blackfynn: BioRC Failed Extract Digest</h2>")
bf = Blackfynn()
root = bf.get('N:collection:4fec4882-925c-4328-bbfd-d1de953ba225')
for bucket in root.items:
include_count = True if bucket.id == "N:collection:0e476218-ccb9-4c4d-bdb4-e72d0a0f88fd" else False
bucket_digest(report, bucket, include_count)
send_report(report)
def __init__(self,
api_token=None,
api_secret=None,
host=None,
streaming_host=None):
self.client = Blackfynn(profile=None,
api_token=api_token,
api_secret=api_secret,
host=host,
streaming_host=streaming_host)
def client2(use_dev):
bf = Blackfynn('*****@*****.**', 'password')
# get organizations
orgs = bf.organizations()
assert len(orgs) > 0
# explicitly set context to Blackfyn org
bf.set_context('Test Organization')
assert bf.context is not None
return bf
def get_test_client(profile=None, api_token=None, api_secret=None, **overrides):
""" Utility function to get a Blackfynn client object """
bf = Blackfynn(profile=profile, api_token=api_token, api_secret=api_secret, **overrides)
assert bf.context is not None
orgs = bf.organizations()
assert len(orgs) > 0
# explicitly set context to Blackfyn org
assert bf.context is not None
return bf
def lambda_handler(event, context):
DATASET = os.environ['DASHBOARD_DATASET_NAME']
bf = Blackfynn()
ds = bf.get_dataset(DATASET)
update(bf, ds)
return {
'statusCode': 200,
'body': json.dumps('Successfully updated the SPARC dashboard')
}
def __init__(self,
api_token=None,
api_secret=None,
host=None,
streaming_host=None):
# WARNING: contruction raise exception if service is not available. Use datacore_wrapper for safe calls
self.client = Blackfynn(
profile=None,
api_token=api_token,
api_secret=api_secret,
host=host,
streaming_host=streaming_host,
)
def client2():
api_token = os.environ.get('BLACKFYNN_API_TOKEN2')
api_secret = os.environ.get('BLACKFYNN_API_SECRET2')
assert api_token != "", "Must define BLACKFYNN_API_TOKEN2"
assert api_secret != "", "Must define BLACKFYNN_API_SECRET2"
bf = Blackfynn()
# get organizations
orgs = bf.organizations()
assert len(orgs) > 0
# explicitly set context to Blackfyn org
assert bf.context is not None
return bf
def client():
"""
Login via API, return client. Login information, by default, will be taken from
environment variables, so ensure those are set properly before testing. Alternatively,
to force a particular user, adjust input arguments as necessary.
"""
bf = Blackfynn()
# get organizations
orgs = bf.organizations()
print 'organizations =', orgs
assert len(orgs) > 0
# explicitly set context to Blackfyn org
assert bf.context is not None
return bf
def __init__(self,
api_token=None,
api_secret=None,
host=None,
streaming_host=None):
# WARNING: contruction raise exception if service is not available.
# Use datacore_wrapper for safe calls
# TODO: can use https://developer.blackfynn.io/python/latest/configuration.html#environment-variables
self._bf = Blackfynn(
profile=None,
api_token=api_token,
api_secret=api_secret,
host=host,
streaming_host=streaming_host,
)
def connect_to_blackfynn():
global bf
bf = Blackfynn(
api_token=Config.BLACKFYNN_API_TOKEN,
api_secret=Config.BLACKFYNN_API_SECRET,
env_override=False,
)
def blackfynn_cli():
args = docopt(__doc__,
version='bf version {}'.format(blackfynn.__version__),
options_first=True)
# Test for these two commands first as they
# do not require a Blackfynn client
if args['<command>'] in ['help', None]:
print(__doc__.strip('\n'))
return
if args['<command>'] == 'profile':
import bf_profile
bf_profile.main()
return
# Display warning message if config.ini is not found
settings = Settings(
) # create a dummy settings object to load environment variables and defaults only
if not os.path.exists(settings.config_file):
print(
"\033[31m* Warning: No config file found, run 'bf profile' to start the setup assistant\033[0m"
)
# Try to use profile specified by --profile, exit if invalid
try:
bf = Blackfynn(args['--profile'])
except Exception, e:
exit(e)
def _can_access_blackfynn(self, api_key, api_secret):
can_access = False
bf = Blackfynn(api_token=api_key, api_secret=api_secret)
if bf.context.exists:
can_access = True
return can_access
def connect_to_blackfynn():
global bf
bf = Blackfynn(api_token=Config.BLACKFYNN_API_TOKEN,
api_secret=Config.BLACKFYNN_API_SECRET,
env_override=False,
host=Config.BLACKFYNN_API_HOST,
concepts_api_host=Config.BLACKFYNN_CONCEPTS_API_HOST)
def superuser_client(use_dev):
"""
Client using super-admin permissions
"""
bf = Blackfynn('*****@*****.**', 'password')
assert bf.profile.is_super_admin
return bf
def blackfynn_get(dataset='Timeseries Dataset',collection=0,channels='sine 50 Hz'):
#establish connection with the API
bf = Blackfynn(api_token='**********************************',api_secret='**********************************',)
ds = bf.get_dataset(dataset)
print(ds)
# get all timeseries collections
ts = []
for ds_temp in ds:
ts.append(ds_temp)
# NOTE: THIS NEEDS TO BE MADE MODULAR
data = ts[collection].get_data(length='1s')
# take the data from the channel and process it to be read by javascript
#process y values
temp = np.asarray(data[channels])
java_y = []
for el in temp:
java_y.append(float(str(el)))
#generate x values
#NOTE: CURRENTLY NOT MODULAR
temp2 = np.linspace(0,1,len(temp))
java_x =[]
for el in temp2:
java_x.append(el)
#write to file
f = open('data.js', 'w')
f.write('arbData = ')
f.write(str(java_y))
f.write('\ndate = ')
f.write(repr(java_x))
f.close()
#update gitub
g = git_hub()
g.update()
print('update ran successfully')
def connect_blackfynn():
"Connect to Blackfynn"
print("Connecting to Blackfynn ...", end=" ")
bf = Blackfynn(
api_token=Config.BLACKFYNN_API_TOKEN,
api_secret=Config.BLACKFYNN_API_SECRET,
env_override=False,
host=Config.BLACKFYNN_API_HOST,
)
print("done")
return bf
def connect_blackfynn():
'Connect to Blackfynn'
print('Connecting to Blackfynn ...', end=' ')
bf = Blackfynn(
api_token=Config.BLACKFYNN_API_TOKEN,
api_secret=Config.BLACKFYNN_API_SECRET,
env_override=False,
host=Config.BLACKFYNN_API_HOST,
)
print('done')
return bf
def test_client_global_headers():
global_headers = {
"X-Custom-Header1": "Custom Value",
"X-Custom-Header2": "Custom Value2",
}
bf = Blackfynn(headers=global_headers)
assert bf.settings.headers == global_headers
for header, header_value in global_headers.items():
assert header in bf._api._session.headers
assert bf._api._session.headers[header] == header_value
def sessionp():
print(request.method)
print(request.headers)
print(request.data)
print('its a post!')
data = json.loads(request.data.decode("utf-8"))
from blackfynn import Blackfynn
global bf
bf = Blackfynn(api_token=data['tokenId'], api_secret=data['secret'])
data_sets = bf.datasets()
global time_series_items
time_series_items = []
time_series_names = []
for data_set in data_sets:
for item in data_set.items:
if item.type is 'TimeSeries':
time_series_items.append(item)
time_series_names.append(item.name)
return json.dumps({'names': time_series_names})
def instantiateDatCore():
'''
instantiate a dat core object if not already done
:return: handle to the dat core object
'''
# we are going to store the instance as a global
# so we can access it from everywhere
global dcInstance
# check if we already have the dat core instance or not
if not isinstance(dcInstance,Blackfynn):
dcInstance = Blackfynn()
# return the dcInstance
return dcInstance
def get_timeseries_dataset_names():
data = json.loads(request.data.decode("utf-8"))
global bf
bf = Blackfynn(api_token=data['tokenId'], api_secret=data['secret'])
data_sets = bf.datasets()
global time_series_items, csv_items, csv_names
time_series_items = []
time_series_names = []
csv_items = []
csv_names = []
for data_set in data_sets:
for item in data_set.items:
if item.type is 'TimeSeries':
time_series_items.append(item)
time_series_names.append(item.name)
if item.type is 'Tabular':
csv_items.append(item)
csv_names.append(item.name)
global user_ip
user_ip = request.remote_addr
return json.dumps({'names': time_series_names + csv_names})
def main():
args = docopt(__doc__, version=blackfynn.__version__)
# Test for these two commands first as they
# do not require a Blackfynn client or reading the settings file
if args["help"]:
print(__doc__.strip("\n"))
return
elif args["version"]:
print(blackfynn.__version__)
return
settings = Settings(args["--profile"])
if not os.path.exists(settings.config_file):
setup_assistant(settings)
elif args["create"]:
create_profile(settings, args["<name>"])
elif args["show"]:
show_profile(settings, args["<name>"])
elif args["delete"]:
delete_profile(settings, args["<name>"], args["--force"])
elif args["list"]:
list_profiles(settings, args["--contents"])
elif args["set-default"]:
set_default(settings, args["<name>"])
elif args["unset-default"]:
unset_default(settings, args["--force"])
elif args["set"]:
set_key(
settings, args["<key>"], args["<value>"], args["--profile"], args["--force"]
)
elif args["unset"]:
unset_key(settings, args["<key>"], args["--profile"], args["--force"])
elif args["keys"]:
list_keys(settings, args["--profile"])
elif args["status"]:
bf = Blackfynn(args["--profile"])
show_status(bf)
else:
invalid_usage()
with io.open(settings.config_file, "w") as configfile:
settings.config.write(configfile)
def main():
args = docopt(__doc__, version=blackfynn.__version__)
# Test for these two commands first as they
# do not require a Blackfynn client or reading the settings file
if args['help']:
print(__doc__.strip('\n'))
return
elif args['version']:
print(blackfynn.__version__)
return
settings = Settings(args['--profile'])
if not os.path.exists(settings.config_file):
setup_assistant(settings)
elif args['create']:
create_profile(settings, args['<name>'])
elif args['show']:
show_profile(settings, args['<name>'])
elif args['delete']:
delete_profile(settings, args['<name>'], args['--force'])
elif args['list']:
list_profiles(settings, args['--contents'])
elif args['set-default']:
set_default(settings, args['<name>'])
elif args['unset-default']:
unset_default(settings, args['--force'])
elif args['set']:
set_key(settings, args['<key>'], args['<value>'], args['--profile'],
args['--force'])
elif args['unset']:
unset_key(settings, args['<key>'], args['--profile'], args['--force'])
elif args['keys']:
list_keys(settings, args['--profile'])
elif args['status']:
bf = Blackfynn(args['--profile'])
show_status(bf)
else:
invalid_usage()
with io.open(settings.config_file, 'w') as configfile:
settings.config.write(configfile)
def blackfynn_get():
# make a blackfynn API call according to threeWrapper parameters
fpipe = FilePipe()
params = fpipe.receive()
api_key = params['api_key']
api_secret = params['api_secret']
dataset = params['dataset']
collection = params['collection']
channels = params['channels']
window_from_start = params['window_from_start']
start = params['start']
end = params['end']
error = 0
# Process function input:
if start == -1 or end == -1:
has_time_window = False
else:
has_time_window = True
#establish connection with the API
try:
bf = Blackfynn(api_token=api_key, api_secret=api_secret)
except:
fpipe.send({
'error':
'Could not connect to the Blackfynn API. Check your API key and internet connection'
})
return
# get all timeseries collections
try:
ds = bf.get_dataset(dataset)
tstemp = ds.get_items_by_name(collection)
time_series = tstemp[0]
except:
fpipe.send({
'error':
'Could not find the requested Dataset and Collection. Please check your names if you have not already'
})
return
# Get data for all channels according to the length set by parameters
if has_time_window == False:
data = time_series.get_data(length=(str(window_from_start) + 's'))
else:
data = time_series.get_data(start=start, end=end)
# take the data from the channel and process it to be passed into binary (in filePipe)
# process y values
list_y = data[channels].values.tolist()
# generate x values
if has_time_window:
time = np.linspace(start, end, len(list_y))
else:
time = np.linspace(0, window_from_start, len(list_y))
list_x = time.tolist()
cache_dict = create_file_cache(data)
output = {'x': list_x, 'y': list_y, 'cache': cache_dict, 'error': False}
fpipe.send(output)
print('update ran successfully')
class DatcoreClient(object):
def __init__(self,
api_token=None,
api_secret=None,
host=None,
streaming_host=None):
# WARNING: contruction raise exception if service is not available.
# Use datacore_wrapper for safe calls
# TODO: can use https://developer.blackfynn.io/python/latest/configuration.html#environment-variables
self._bf = Blackfynn(
profile=None,
api_token=api_token,
api_secret=api_secret,
host=host,
streaming_host=streaming_host,
)
def profile(self):
"""
Returns profile of current User
"""
return self._bf.profile
def _collection_from_destination(self, destination: str):
destination_path = Path(destination)
parts = destination_path.parts
dataset_name = parts[0]
dataset = self.get_dataset(dataset_name)
if dataset is None:
return None, None
collection_id = dataset.id
collection = dataset
collections = []
if len(parts) > 1:
object_path = Path(*parts[1:])
collections = list(object_path.parts)
collection_id = ""
collection_id = _get_collection_id(dataset, collections,
collection_id)
collection = self._bf.get(collection_id)
return collection, collection_id
def _destination_from_id(self, destination_id: str):
# NOTE: .get(*) logs
# INFO:blackfynn.client.Blackfynn:Unable to retrieve object
# if destination_id refers to a Dataset
destination: Union[DataPackage,
Collection] = self._bf.get(destination_id)
if destination is None:
destination: Dataset = self._bf.get_dataset(destination_id)
return destination
def list_files_recursively(self, dataset_filter: str = ""):
files = []
for dataset in self._bf.datasets():
if not dataset_filter or dataset_filter in dataset.name:
self.list_dataset_files_recursively(files, dataset,
Path(dataset.name))
return files
def list_files_raw_dataset(self, dataset_id: str) -> List[FileMetaDataEx]:
files = [] # raw packages
_files = [] # fmds
data = {} # map to keep track of parents-child
cursor = ""
page_size = 1000
api = self._bf._api.datasets
dataset = self._bf.get_dataset(dataset_id)
if dataset is not None:
while True:
resp = api._get(
api._uri(
"/{id}/packages?cursor={cursor}&pageSize={pageSize}&includeSourceFiles={includeSourceFiles}",
id=dataset_id,
cursor=cursor,
pageSize=page_size,
includeSourceFiles=False,
))
for package in resp.get("packages", list()):
id = package["content"]["id"]
data[id] = package
files.append(package)
cursor = resp.get("cursor")
if cursor is None:
break
for f in files:
if f["content"]["packageType"] != "Collection":
filename = f["content"]["name"]
file_path = ""
file_id = f["content"]["nodeId"]
_f = f
while "parentId" in _f["content"].keys():
parentid = _f["content"]["parentId"]
_f = data[parentid]
file_path = _f["content"]["name"] + "/" + file_path
bucket_name = dataset.name
file_name = filename
file_size = 0
object_name = str(Path(file_path) / file_name)
file_uuid = str(Path(bucket_name) / object_name)
created_at = f["content"]["createdAt"]
last_modified = f["content"]["updatedAt"]
parent_id = dataset_id
if "parentId" in f["content"]:
parentId = f["content"]["parentId"]
parent_id = data[parentId]["content"]["nodeId"]
fmd = FileMetaData(
bucket_name=bucket_name,
file_name=file_name,
object_name=object_name,
location=DATCORE_STR,
location_id=DATCORE_ID,
file_uuid=file_uuid,
file_id=file_id,
raw_file_path=file_uuid,
display_file_path=file_uuid,
created_at=created_at,
last_modified=last_modified,
file_size=file_size,
)
fmdx = FileMetaDataEx(fmd=fmd, parent_id=parent_id)
_files.append(fmdx)
return _files
def list_files_raw(self, dataset_filter: str = "") -> List[FileMetaDataEx]:
_files = []
for dataset in self._bf.datasets():
_files = _files + self.list_files_raw_dataset(dataset.id)
return _files
def list_dataset_files_recursively(self, files: List[FileMetaData],
base: BaseCollection,
current_root: Path):
for item in base:
if isinstance(item, Collection):
_current_root = current_root / Path(item.name)
self.list_dataset_files_recursively(files, item, _current_root)
else:
parts = current_root.parts
bucket_name = parts[0]
file_name = item.name
file_size = 0
# lets assume we have only one file
if item.files:
file_name = Path(
item.files[0].as_dict()["content"]["s3key"]).name
file_size = item.files[0].as_dict()["content"]["size"]
# if this is in the root directory, the object_name is the filename only
if len(parts) > 1:
object_name = str(Path(*list(parts)[1:]) / Path(file_name))
else:
object_name = str(Path(file_name))
file_uuid = str(Path(bucket_name) / Path(object_name))
file_id = item.id
created_at = item.created_at
last_modified = item.updated_at
fmd = FileMetaData(
bucket_name=bucket_name,
file_name=file_name,
object_name=object_name,
location=DATCORE_STR,
location_id=DATCORE_ID,
file_uuid=file_uuid,
file_id=file_id,
raw_file_path=file_uuid,
display_file_path=file_uuid,
created_at=created_at,
last_modified=last_modified,
file_size=file_size,
)
files.append(fmd)
def create_dataset(self, ds_name, *, force_delete=False):
"""
Creates a new dataset for the current user and returns it. Returns existing one
if there is already a dataset with the given name.
Args:
ds_name (str): Name for the dataset (_,-,' ' and capitalization are ignored)
force_delete (bool, optional): Delete first if dataset already exists
"""
ds = None
with suppress(Exception):
ds = self._bf.get_dataset(ds_name)
if force_delete:
ds.delete()
ds = None
if ds is None:
ds = self._bf.create_dataset(ds_name)
return ds
def get_dataset(self, ds_name, create_if_not_exists=False):
"""
Returns dataset with the given name. Creates it if required.
Args:
ds_name (str): Name for the dataset
create_if_not_exists (bool, optional): Create first if dataset already exists
"""
ds = None
with suppress(Exception):
ds = self._bf.get_dataset(ds_name)
if ds is None and create_if_not_exists:
ds = self._bf.create_dataset(ds_name)
return ds
def delete_dataset(self, ds_name):
"""
Deletes dataset with the given name.
Args:
ds_name (str): Name for the dataset
"""
# this is not supported
ds = self.get_dataset(ds_name)
if ds is not None:
self._bf.delete(ds.id)
def exists_dataset(self, ds_name):
"""
Returns True if dataset with the given name exists.
Args:
ds_name (str): Name for the dataset
"""
ds = self.get_dataset(ds_name)
return ds is not None
def upload_file(self,
destination: str,
filepath: str,
meta_data=None) -> bool:
"""
Uploads a file to a given dataset/collection given its filepath on the host. Optionally
adds some meta data
Args:
dataset (dataset): The dataset into whioch the file shall be uploaded
filepath (path): Full path to the file
meta_data (dict, optional): Dictionary of meta data
Note:
Blackfynn postprocesses data based on filendings. If it can do that
the filenames on the server change.
"""
# parse the destination and try to find the package_id to upload to
collection, collection_id = self._collection_from_destination(
destination)
if collection is None:
return False
files = [
filepath,
]
self._bf._api.io.upload_files(collection,
files,
display_progress=True,
use_agent=False)
collection.update()
if meta_data is not None:
for f in files:
filename = os.path.basename(f)
package = self.get_package(collection, filename)
if package is not None:
self._update_meta_data(package, meta_data)
return True
def _update_meta_data(self, package, meta_data):
"""
Updates or replaces metadata for a package
Args:
package (package): The package for which the meta data needs update
meta_data (dict): Dictionary of meta data
"""
for key in meta_data.keys():
package.set_property(key, meta_data[key], category="simcore")
package.update()
def download_file(self, source, filename, destination_path):
"""
Downloads a frile from a source dataset/collection given its filename. Stores
it under destination_path
Args:
source (dataset/collection): The dataset or collection to donwload from
filename (str): Name of the file
destination__apth (str): Path on host for storing file
"""
url = self.download_link(source, filename)
if url:
_file = urllib.URLopener() # nosec
_file.retrieve(url, destination_path)
return True
return False
def download_link(self, destination, filename):
"""
returns presigned url for download, destination is a dataset or collection
"""
collection, collection_id = self._collection_from_destination(
destination)
for item in collection:
if isinstance(item, DataPackage):
if Path(item.files[0].as_dict()["content"]
["s3key"]).name == filename:
file_desc = self._bf._api.packages.get_sources(item.id)[0]
url = self._bf._api.packages.get_presigned_url_for_file(
item.id, file_desc.id)
return url
return ""
def download_link_by_id(self, file_id):
"""
returns presigned url for download of a file given its file_id
"""
url = ""
filename = ""
package = self._bf.get(file_id)
if package is not None:
filename = Path(
package.files[0].as_dict()["content"]["s3key"]).name
file_desc = self._bf._api.packages.get_sources(file_id)[0]
url = self._bf._api.packages.get_presigned_url_for_file(
file_id, file_desc.id)
return url, filename
def get_package(self, source, filename):
"""
Returns package from source by name if exists
Args:
source (dataset/collection): The dataset or collection to donwload from
filename (str): Name of the file
"""
source.update()
for item in source:
if item.name == filename:
return item
return None
def delete_file(self, destination, filename):
"""
Deletes file by name from destination by name
Args:
destination (dataset/collection): The dataset or collection to delete from
filename (str): Name of the file
"""
collection, collection_id = self._collection_from_destination(
destination)
if collection is None:
return False
collection.update()
for item in collection:
if isinstance(item, DataPackage):
if Path(item.files[0].as_dict()["content"]
["s3key"]).name == filename:
self._bf.delete(item)
return True
return False
def delete_file_by_id(self, id: str) -> bool:
"""
Deletes file by id
Args:
datcore id for the file
"""
package: DataPackage = self._bf.get(id)
package.delete()
return not package.exists
def delete_files(self, destination):
"""
Deletes all files in destination
Args:
destination (dataset/collection): The dataset or collection to delete
"""
collection, collection_id = self._collection_from_destination(
destination)
if collection is None:
return False
collection.update()
for item in collection:
self._bf.delete(item)
def update_meta_data(self, dataset, filename, meta_data):
"""
Updates metadata for a file
Args:
dataset (package): Which dataset
filename (str): Which file
meta_data (dict): Dictionary of meta data
"""
filename = os.path.basename(filename)
package = self.get_package(dataset, filename)
if package is not None:
self._update_meta_data(package, meta_data)
def get_meta_data(self, dataset, filename):
"""
Returns metadata for a file
Args:
dataset (package): Which dataset
filename (str): Which file
"""
meta_data = {}
filename = os.path.basename(filename)
package = self.get_package(dataset, filename)
if package is not None:
meta_list = package.properties
for m in meta_list:
meta_data[m.key] = m.value
return meta_data
def delete_meta_data(self, dataset, filename, keys=None):
"""
Deletes specified keys in meta data for source/filename.
Args:
dataset (package): Which dataset
filename (str): Which file
keys (list of str, optional): Deletes specified keys, deletes
all meta data if None
"""
filename = os.path.basename(filename)
package = self.get_package(dataset, filename)
if package is not None:
if keys is None:
for p in package.properties:
package.remove_property(p.key, category="simcore")
else:
for k in keys:
package.remove_property(k, category="simcore")
def search(self, what, max_count):
"""
Seraches a thing in the database. Returns max_count results
Args:
what (str): query
max_count (int): Max number of results to return
"""
return self._bf.search(what, max_count)
def upload_file_to_id(self, destination_id: str, filepath: str):
"""
Uploads file to a given dataset/collection by id given its filepath on the host
adds some meta data.
Returns the id for the newly created resource
Note: filepath could be an array
Args:
destination_id : The dataset/collection id into which the file shall be uploaded
filepath (path): Full path to the file
"""
_id = ""
destination = self._destination_from_id(destination_id)
if destination is None:
return _id
files = [
filepath,
]
try:
# TODO: PC->MAG: should protected API
# TODO: add new agent SEE https://developer.blackfynn.io/python/latest/CHANGELOG.html#id31
result = self._bf._api.io.upload_files(destination,
files,
display_progress=True,
use_agent=False)
if result and result[0] and "package" in result[0][0]:
_id = result[0][0]["package"]["content"]["id"]
except Exception:
logger.exception("Error uploading file to datcore")
return _id
def create_collection(self, destination_id: str, collection_name: str):
"""
Create a empty collection within destination
Args:
destination_id : The dataset/collection id into which the file shall be uploaded
filepath (path): Full path to the file
"""
destination = self._destination_from_id(destination_id)
_id = ""
if destination is None:
return _id
new_collection = Collection(collection_name)
destination.add(new_collection)
new_collection.update()
destination.update()
_id = new_collection.id
return _id
def list_datasets(self) -> DatasetMetaDataVec:
data = []
for dataset in self._bf.datasets():
dmd = DatasetMetaData(dataset_id=dataset.id,
display_name=dataset.name)
data.append(dmd)
return data
# REQUIREMENTS: python2, blackfynn python library, blackfynn key # UPDATES: 180215: Added -f option # AUTHOR: Pete Schmitt (discovery), [email protected] # COMPANY: University of Pennsylvania # VERSION: 0.2.0 # CREATED: Wed Feb 14 13:27:32 EST 2018 # REVISION: Thu Feb 15 11:49:51 EST 2018 #=============================================================================== from blackfynn import Blackfynn from blackfynn.models import BaseCollection from blackfynn.models import Collection import sys import getopt import os bf = Blackfynn() # use 'default' profile ############################################################################### def syntax(): SYNTAX = "\nbfmove -d <dataset> \n" SYNTAX += " --all (loop on all HPAP datasets)\n" SYNTAX += " -f <file containing datasets>\n" SYNTAX += " -S <source path>\n" SYNTAX += " -D <destination path> (MUST be directory)\n\n" SYNTAX += " -h (help)\n" SYNTAX += " -l (list datasets)\n\n" SYNTAX += "Note: -d, -f and --all are mutually exlusive\n" return SYNTAX
