def push_to_blackfynn(event, context):
bucket = event['Records'][0]['s3']['bucket']['name']
key = urllib.unquote_plus(
event['Records'][0]['s3']['object']['key'].encode('utf8'))
try:
bf = Blackfynn()
bf.set_context(organization_id)
root_collection = bf.get(collection_id)
package_name = key.split("/")[-2]
package_collection = next(
(c for c in root_collection.items if c.name == package_name), None)
if package_collection is None:
package_collection = Collection(package_name)
root_collection.add(package_collection)
local_object = '/tmp/{}'.format(os.path.basename(key))
s3.download_file(Bucket=bucket, Key=key, Filename=local_object)
package_collection.upload_files(local_object)
return 'Successfully pushed {} from bucket {} to Blackfynn.'.format(
key, bucket)
except Exception as e:
print(e)
print('Error pushing {} from bucket {} to Blackfynn.'.format(
key, bucket))
raise e
def client2(use_dev):
bf = Blackfynn('*****@*****.**', 'password')
# get organizations
orgs = bf.organizations()
assert len(orgs) > 0
# explicitly set context to Blackfyn org
bf.set_context('Test Organization')
assert bf.context is not None
return bf
def client(use_dev):
"""
Login via API, return client. Login information, by default, will be taken from
environment variables, so ensure those are set properly before testing. Alternatively,
to force a particular user, adjust input arguments as necessary.
"""
bf = Blackfynn()
# get organizations
orgs = bf.organizations()
print 'organizations =', orgs
assert len(orgs) > 0
# explicitly set context to Blackfyn org
bf.set_context('Test Organization')
assert bf.context is not None
return bf
def blackfynn_cli():
args = docopt(__doc__)
if args['version']:
print "version: {}".format(blackfynn.__version__)
email = args['--user'] if args['--user'] is not None else settings.api_user
passw = args['--pass'] if args['--pass'] is not None else settings.api_pass
host = args['--host'] if args['--host'] is not None else settings.api_host
org = args['--org']
try:
bf = Blackfynn(email=email, password=passw, host=host)
except:
print "Unable to connect to to Blackfynn using specified user/password."
return
if args['orgs']:
for o in bf.organizations():
print " * {} (id: {})".format(o.name, o.id)
if org is not None:
try:
bf.set_context(org)
except:
print 'Error: Unable to set context to "{}"'.format(org)
return
if args['show']:
item = bf.get(args['<item>'])
print item
if hasattr(item, 'items'):
print "CONTENTS:"
for i in item.items:
print " * {}".format(i)
if hasattr(item, 'channels'):
print "CHANNELS:"
for ch in item.channels:
print " * {} (id: {})".format(ch.name, ch.id)
elif args['search']:
terms = ' '.join(args['<term>'])
results = bf._api.search.query(terms)
if len(results) == 0:
print "No Results."
else:
for r in results:
print " * {}".format(r)
elif args['create']:
if args['collection']:
dest = args['<destination>']
name = args['<name>']
c = Collection(name)
parent = bf.get(dest)
parent.add(c)
print c
elif args['dataset']:
name = args['<name>']
ds = bf.create_dataset(name)
print ds
else:
print "Error: creation for object not supported."
return
elif args['delete']:
item = bf.get(args['<item>'])
if isinstance(item, Dataset):
print "Error: cannot delete dataset"
return
elif not isinstance(item, BaseNode):
print "Error: cannot delete item"
return
bf.delete(item)
elif args['upload']:
files = args['<file>']
dest = args['<destination>']
recursively_upload(bf, dest, files)
elif args['append']:
files = args['<file>']
dest = args['<destination>']
bf._api.io.upload_files(dest,
files,
append=True,
display_progress=True)
elif args['datasets']:
print "Datasets: "
for ds in bf.datasets():
print " - {} (id: {})".format(ds.name, ds.id)
elif args['dataset']:
ds = bf.get(args['<dataset>'])
if args['collaborators']:
if args['<action>'] == 'ls':
resp = ds.collaborators()
print " - Users"
for u in resp['users']:
print " - email:{} id:{}".format(u.email, u.id)
print " - Groups"
for g in resp['groups']:
print " - name:{} id:{}".format(g.name, g.id)
elif args['<action>'] == 'add':
ids = args['<action-args>']
if len(ids) == 0:
print "Error: No ids specified"
sys.exit(1)
resp = ds.add_collaborators(*ids)
print_collaborator_edit_resp(resp)
elif args['<action>'] == 'rm':
ids = args['<action-args>']
if len(ids) == 0:
print "Error: No ids specified"
sys.exit(1)
resp = ds.remove_collaborators(*ids)
print_collaborator_edit_resp(resp)
else:
print "Error: invalid dataset collaborators command. Valid commands are 'ls', 'add' or 'rm'"
else:
print "Error: invalid dataset command. Valid commands are 'collaborators'"
elif args['env']:
print "# Blackfynn environment"
print "API Location: {}".format(host)
print "Streaming API: {}".format(settings.streaming_api_host)
print "User: {}".format(email)
print "Organization: {} (id: {})".format(bf.context.name,
bf.context.id)
for i in range(1, df.shape[0]):
toContinue = False
#print(i)
deltaT = (df.iloc[i].name - df.iloc[i - 1].name).microseconds
if deltaT > 10000:
dfs.append(df.iloc[:i])
df = df.iloc[i:]
toContinue = True
break
dfs.append(df)
return dfs
# Need to fill in personal Blackfynn login information here
bf = Blackfynn(email='you_email_here', password='******')
bf.set_context('Mayo')
ts = bf.datasets()[0].items[
0] # again, this is set up for 951 now. Need to change this line for a different BF dataset
# Here we grab the last 30 minutes of data
tempEnd = datetime.datetime.now()
tempStart = tempEnd - datetime.timedelta(0, 1800)
d = ts.get_data(start=tempStart, end=tempEnd)
d = d.drop_duplicates(
) # had some issues with duplicate data points. Remove them if they exist
if d.shape[0] == 0:
print('no data in time segment...quitting \n')
exit()
print('starting split...')
dfs = split_data(d)
print('done')
