def push_to_blackfynn(event, context):
bucket = event['Records'][0]['s3']['bucket']['name']
key = urllib.unquote_plus(
event['Records'][0]['s3']['object']['key'].encode('utf8'))
try:
bf = Blackfynn()
bf.set_context(organization_id)
root_collection = bf.get(collection_id)
package_name = key.split("/")[-2]
package_collection = next(
(c for c in root_collection.items if c.name == package_name), None)
if package_collection is None:
package_collection = Collection(package_name)
root_collection.add(package_collection)
local_object = '/tmp/{}'.format(os.path.basename(key))
s3.download_file(Bucket=bucket, Key=key, Filename=local_object)
package_collection.upload_files(local_object)
return 'Successfully pushed {} from bucket {} to Blackfynn.'.format(
key, bucket)
except Exception as e:
print(e)
print('Error pushing {} from bucket {} to Blackfynn.'.format(
key, bucket))
raise e
def main(bf):
args = docopt(__doc__)
collection = Collection(args['<name>'])
if args['<destination>']:
parent = get_item(args['<destination>'], bf)
resp = parent.add(collection)
else:
dataset = require_working_dataset(bf)
resp = dataset.add(collection)
print(collection)
def recursively_upload(bf, dest, files):
dirs = [f for f in files if os.path.isdir(f)]
files = [f for f in files if os.path.isfile(f)]
if len(files) > 0:
bf._api.io.upload_files(dest, files, display_progress=True)
for d in dirs:
name = os.path.basename(os.path.normpath(d))
print 'Uploading to {}'.format(name)
c = Collection(name)
parent = bf.get(dest)
parent.add(c)
files = [
os.path.join(d, f) for f in os.listdir(d) if not f.startswith('.')
]
recursively_upload(bf, c.id, files)
def recursively_upload(bf, destination, files):
import os
from blackfynn import Collection
dirs = [f for f in files if os.path.isdir(f)]
files = [f for f in files if os.path.isfile(f)]
if len(files) > 0:
bf._api.io.upload_files(destination, files, display_progress=True)
for d in dirs:
name = os.path.basename(os.path.normpath(d))
print('Uploading to {}'.format(name))
new_collection = Collection(name)
destination.add(new_collection)
files = [os.path.join(d,f) for f in os.listdir(d) if not f.startswith('.')]
recursively_upload(bf, new_collection, files)
def blackfynn_cli():
args = docopt(__doc__)
if args['version']:
print "version: {}".format(blackfynn.__version__)
email = args['--user'] if args['--user'] is not None else settings.api_user
passw = args['--pass'] if args['--pass'] is not None else settings.api_pass
host = args['--host'] if args['--host'] is not None else settings.api_host
org = args['--org']
try:
bf = Blackfynn(email=email, password=passw, host=host)
except:
print "Unable to connect to to Blackfynn using specified user/password."
return
if args['orgs']:
for o in bf.organizations():
print " * {} (id: {})".format(o.name, o.id)
if org is not None:
try:
bf.set_context(org)
except:
print 'Error: Unable to set context to "{}"'.format(org)
return
if args['show']:
item = bf.get(args['<item>'])
print item
if hasattr(item, 'items'):
print "CONTENTS:"
for i in item.items:
print " * {}".format(i)
if hasattr(item, 'channels'):
print "CHANNELS:"
for ch in item.channels:
print " * {} (id: {})".format(ch.name, ch.id)
elif args['search']:
terms = ' '.join(args['<term>'])
results = bf._api.search.query(terms)
if len(results) == 0:
print "No Results."
else:
for r in results:
print " * {}".format(r)
elif args['create']:
if args['collection']:
dest = args['<destination>']
name = args['<name>']
c = Collection(name)
parent = bf.get(dest)
parent.add(c)
print c
elif args['dataset']:
name = args['<name>']
ds = bf.create_dataset(name)
print ds
else:
print "Error: creation for object not supported."
return
elif args['delete']:
item = bf.get(args['<item>'])
if isinstance(item, Dataset):
print "Error: cannot delete dataset"
return
elif not isinstance(item, BaseNode):
print "Error: cannot delete item"
return
bf.delete(item)
elif args['upload']:
files = args['<file>']
dest = args['<destination>']
recursively_upload(bf, dest, files)
elif args['append']:
files = args['<file>']
dest = args['<destination>']
bf._api.io.upload_files(dest,
files,
append=True,
display_progress=True)
elif args['datasets']:
print "Datasets: "
for ds in bf.datasets():
print " - {} (id: {})".format(ds.name, ds.id)
elif args['dataset']:
ds = bf.get(args['<dataset>'])
if args['collaborators']:
if args['<action>'] == 'ls':
resp = ds.collaborators()
print " - Users"
for u in resp['users']:
print " - email:{} id:{}".format(u.email, u.id)
print " - Groups"
for g in resp['groups']:
print " - name:{} id:{}".format(g.name, g.id)
elif args['<action>'] == 'add':
ids = args['<action-args>']
if len(ids) == 0:
print "Error: No ids specified"
sys.exit(1)
resp = ds.add_collaborators(*ids)
print_collaborator_edit_resp(resp)
elif args['<action>'] == 'rm':
ids = args['<action-args>']
if len(ids) == 0:
print "Error: No ids specified"
sys.exit(1)
resp = ds.remove_collaborators(*ids)
print_collaborator_edit_resp(resp)
else:
print "Error: invalid dataset collaborators command. Valid commands are 'ls', 'add' or 'rm'"
else:
print "Error: invalid dataset command. Valid commands are 'collaborators'"
elif args['env']:
print "# Blackfynn environment"
print "API Location: {}".format(host)
print "Streaming API: {}".format(settings.streaming_api_host)
print "User: {}".format(email)
print "Organization: {} (id: {})".format(bf.context.name,
bf.context.id)