def push_to_blackfynn(event, context):
bucket = event['Records'][0]['s3']['bucket']['name']
key = urllib.unquote_plus(
event['Records'][0]['s3']['object']['key'].encode('utf8'))
try:
bf = Blackfynn()
bf.set_context(organization_id)
root_collection = bf.get(collection_id)
package_name = key.split("/")[-2]
package_collection = next(
(c for c in root_collection.items if c.name == package_name), None)
if package_collection is None:
package_collection = Collection(package_name)
root_collection.add(package_collection)
local_object = '/tmp/{}'.format(os.path.basename(key))
s3.download_file(Bucket=bucket, Key=key, Filename=local_object)
package_collection.upload_files(local_object)
return 'Successfully pushed {} from bucket {} to Blackfynn.'.format(
key, bucket)
except Exception as e:
print(e)
print('Error pushing {} from bucket {} to Blackfynn.'.format(
key, bucket))
raise e
def send_digest():
report = []
report.append("<h2>Blackfynn: BioRC Failed Extract Digest</h2>")
bf = Blackfynn()
root = bf.get('N:collection:4fec4882-925c-4328-bbfd-d1de953ba225')
for bucket in root.items:
include_count = True if bucket.id == "N:collection:0e476218-ccb9-4c4d-bdb4-e72d0a0f88fd" else False
bucket_digest(report, bucket, include_count)
send_report(report)
class DatcoreClient(object):
def __init__(self,
api_token=None,
api_secret=None,
host=None,
streaming_host=None):
# WARNING: contruction raise exception if service is not available.
# Use datacore_wrapper for safe calls
# TODO: can use https://developer.blackfynn.io/python/latest/configuration.html#environment-variables
self._bf = Blackfynn(
profile=None,
api_token=api_token,
api_secret=api_secret,
host=host,
streaming_host=streaming_host,
)
def profile(self):
"""
Returns profile of current User
"""
return self._bf.profile
def _collection_from_destination(self, destination: str):
destination_path = Path(destination)
parts = destination_path.parts
dataset_name = parts[0]
dataset = self.get_dataset(dataset_name)
if dataset is None:
return None, None
collection_id = dataset.id
collection = dataset
collections = []
if len(parts) > 1:
object_path = Path(*parts[1:])
collections = list(object_path.parts)
collection_id = ""
collection_id = _get_collection_id(dataset, collections,
collection_id)
collection = self._bf.get(collection_id)
return collection, collection_id
def _destination_from_id(self, destination_id: str):
# NOTE: .get(*) logs
# INFO:blackfynn.client.Blackfynn:Unable to retrieve object
# if destination_id refers to a Dataset
destination: Union[DataPackage,
Collection] = self._bf.get(destination_id)
if destination is None:
destination: Dataset = self._bf.get_dataset(destination_id)
return destination
def list_files_recursively(self, dataset_filter: str = ""):
files = []
for dataset in self._bf.datasets():
if not dataset_filter or dataset_filter in dataset.name:
self.list_dataset_files_recursively(files, dataset,
Path(dataset.name))
return files
def list_files_raw_dataset(self, dataset_id: str) -> List[FileMetaDataEx]:
files = [] # raw packages
_files = [] # fmds
data = {} # map to keep track of parents-child
cursor = ""
page_size = 1000
api = self._bf._api.datasets
dataset = self._bf.get_dataset(dataset_id)
if dataset is not None:
while True:
resp = api._get(
api._uri(
"/{id}/packages?cursor={cursor}&pageSize={pageSize}&includeSourceFiles={includeSourceFiles}",
id=dataset_id,
cursor=cursor,
pageSize=page_size,
includeSourceFiles=False,
))
for package in resp.get("packages", list()):
id = package["content"]["id"]
data[id] = package
files.append(package)
cursor = resp.get("cursor")
if cursor is None:
break
for f in files:
if f["content"]["packageType"] != "Collection":
filename = f["content"]["name"]
file_path = ""
file_id = f["content"]["nodeId"]
_f = f
while "parentId" in _f["content"].keys():
parentid = _f["content"]["parentId"]
_f = data[parentid]
file_path = _f["content"]["name"] + "/" + file_path
bucket_name = dataset.name
file_name = filename
file_size = 0
object_name = str(Path(file_path) / file_name)
file_uuid = str(Path(bucket_name) / object_name)
created_at = f["content"]["createdAt"]
last_modified = f["content"]["updatedAt"]
parent_id = dataset_id
if "parentId" in f["content"]:
parentId = f["content"]["parentId"]
parent_id = data[parentId]["content"]["nodeId"]
fmd = FileMetaData(
bucket_name=bucket_name,
file_name=file_name,
object_name=object_name,
location=DATCORE_STR,
location_id=DATCORE_ID,
file_uuid=file_uuid,
file_id=file_id,
raw_file_path=file_uuid,
display_file_path=file_uuid,
created_at=created_at,
last_modified=last_modified,
file_size=file_size,
)
fmdx = FileMetaDataEx(fmd=fmd, parent_id=parent_id)
_files.append(fmdx)
return _files
def list_files_raw(self, dataset_filter: str = "") -> List[FileMetaDataEx]:
_files = []
for dataset in self._bf.datasets():
_files = _files + self.list_files_raw_dataset(dataset.id)
return _files
def list_dataset_files_recursively(self, files: List[FileMetaData],
base: BaseCollection,
current_root: Path):
for item in base:
if isinstance(item, Collection):
_current_root = current_root / Path(item.name)
self.list_dataset_files_recursively(files, item, _current_root)
else:
parts = current_root.parts
bucket_name = parts[0]
file_name = item.name
file_size = 0
# lets assume we have only one file
if item.files:
file_name = Path(
item.files[0].as_dict()["content"]["s3key"]).name
file_size = item.files[0].as_dict()["content"]["size"]
# if this is in the root directory, the object_name is the filename only
if len(parts) > 1:
object_name = str(Path(*list(parts)[1:]) / Path(file_name))
else:
object_name = str(Path(file_name))
file_uuid = str(Path(bucket_name) / Path(object_name))
file_id = item.id
created_at = item.created_at
last_modified = item.updated_at
fmd = FileMetaData(
bucket_name=bucket_name,
file_name=file_name,
object_name=object_name,
location=DATCORE_STR,
location_id=DATCORE_ID,
file_uuid=file_uuid,
file_id=file_id,
raw_file_path=file_uuid,
display_file_path=file_uuid,
created_at=created_at,
last_modified=last_modified,
file_size=file_size,
)
files.append(fmd)
def create_dataset(self, ds_name, *, force_delete=False):
"""
Creates a new dataset for the current user and returns it. Returns existing one
if there is already a dataset with the given name.
Args:
ds_name (str): Name for the dataset (_,-,' ' and capitalization are ignored)
force_delete (bool, optional): Delete first if dataset already exists
"""
ds = None
with suppress(Exception):
ds = self._bf.get_dataset(ds_name)
if force_delete:
ds.delete()
ds = None
if ds is None:
ds = self._bf.create_dataset(ds_name)
return ds
def get_dataset(self, ds_name, create_if_not_exists=False):
"""
Returns dataset with the given name. Creates it if required.
Args:
ds_name (str): Name for the dataset
create_if_not_exists (bool, optional): Create first if dataset already exists
"""
ds = None
with suppress(Exception):
ds = self._bf.get_dataset(ds_name)
if ds is None and create_if_not_exists:
ds = self._bf.create_dataset(ds_name)
return ds
def delete_dataset(self, ds_name):
"""
Deletes dataset with the given name.
Args:
ds_name (str): Name for the dataset
"""
# this is not supported
ds = self.get_dataset(ds_name)
if ds is not None:
self._bf.delete(ds.id)
def exists_dataset(self, ds_name):
"""
Returns True if dataset with the given name exists.
Args:
ds_name (str): Name for the dataset
"""
ds = self.get_dataset(ds_name)
return ds is not None
def upload_file(self,
destination: str,
filepath: str,
meta_data=None) -> bool:
"""
Uploads a file to a given dataset/collection given its filepath on the host. Optionally
adds some meta data
Args:
dataset (dataset): The dataset into whioch the file shall be uploaded
filepath (path): Full path to the file
meta_data (dict, optional): Dictionary of meta data
Note:
Blackfynn postprocesses data based on filendings. If it can do that
the filenames on the server change.
"""
# parse the destination and try to find the package_id to upload to
collection, collection_id = self._collection_from_destination(
destination)
if collection is None:
return False
files = [
filepath,
]
self._bf._api.io.upload_files(collection,
files,
display_progress=True,
use_agent=False)
collection.update()
if meta_data is not None:
for f in files:
filename = os.path.basename(f)
package = self.get_package(collection, filename)
if package is not None:
self._update_meta_data(package, meta_data)
return True
def _update_meta_data(self, package, meta_data):
"""
Updates or replaces metadata for a package
Args:
package (package): The package for which the meta data needs update
meta_data (dict): Dictionary of meta data
"""
for key in meta_data.keys():
package.set_property(key, meta_data[key], category="simcore")
package.update()
def download_file(self, source, filename, destination_path):
"""
Downloads a frile from a source dataset/collection given its filename. Stores
it under destination_path
Args:
source (dataset/collection): The dataset or collection to donwload from
filename (str): Name of the file
destination__apth (str): Path on host for storing file
"""
url = self.download_link(source, filename)
if url:
_file = urllib.URLopener() # nosec
_file.retrieve(url, destination_path)
return True
return False
def download_link(self, destination, filename):
"""
returns presigned url for download, destination is a dataset or collection
"""
collection, collection_id = self._collection_from_destination(
destination)
for item in collection:
if isinstance(item, DataPackage):
if Path(item.files[0].as_dict()["content"]
["s3key"]).name == filename:
file_desc = self._bf._api.packages.get_sources(item.id)[0]
url = self._bf._api.packages.get_presigned_url_for_file(
item.id, file_desc.id)
return url
return ""
def download_link_by_id(self, file_id):
"""
returns presigned url for download of a file given its file_id
"""
url = ""
filename = ""
package = self._bf.get(file_id)
if package is not None:
filename = Path(
package.files[0].as_dict()["content"]["s3key"]).name
file_desc = self._bf._api.packages.get_sources(file_id)[0]
url = self._bf._api.packages.get_presigned_url_for_file(
file_id, file_desc.id)
return url, filename
def get_package(self, source, filename):
"""
Returns package from source by name if exists
Args:
source (dataset/collection): The dataset or collection to donwload from
filename (str): Name of the file
"""
source.update()
for item in source:
if item.name == filename:
return item
return None
def delete_file(self, destination, filename):
"""
Deletes file by name from destination by name
Args:
destination (dataset/collection): The dataset or collection to delete from
filename (str): Name of the file
"""
collection, collection_id = self._collection_from_destination(
destination)
if collection is None:
return False
collection.update()
for item in collection:
if isinstance(item, DataPackage):
if Path(item.files[0].as_dict()["content"]
["s3key"]).name == filename:
self._bf.delete(item)
return True
return False
def delete_file_by_id(self, id: str) -> bool:
"""
Deletes file by id
Args:
datcore id for the file
"""
package: DataPackage = self._bf.get(id)
package.delete()
return not package.exists
def delete_files(self, destination):
"""
Deletes all files in destination
Args:
destination (dataset/collection): The dataset or collection to delete
"""
collection, collection_id = self._collection_from_destination(
destination)
if collection is None:
return False
collection.update()
for item in collection:
self._bf.delete(item)
def update_meta_data(self, dataset, filename, meta_data):
"""
Updates metadata for a file
Args:
dataset (package): Which dataset
filename (str): Which file
meta_data (dict): Dictionary of meta data
"""
filename = os.path.basename(filename)
package = self.get_package(dataset, filename)
if package is not None:
self._update_meta_data(package, meta_data)
def get_meta_data(self, dataset, filename):
"""
Returns metadata for a file
Args:
dataset (package): Which dataset
filename (str): Which file
"""
meta_data = {}
filename = os.path.basename(filename)
package = self.get_package(dataset, filename)
if package is not None:
meta_list = package.properties
for m in meta_list:
meta_data[m.key] = m.value
return meta_data
def delete_meta_data(self, dataset, filename, keys=None):
"""
Deletes specified keys in meta data for source/filename.
Args:
dataset (package): Which dataset
filename (str): Which file
keys (list of str, optional): Deletes specified keys, deletes
all meta data if None
"""
filename = os.path.basename(filename)
package = self.get_package(dataset, filename)
if package is not None:
if keys is None:
for p in package.properties:
package.remove_property(p.key, category="simcore")
else:
for k in keys:
package.remove_property(k, category="simcore")
def search(self, what, max_count):
"""
Seraches a thing in the database. Returns max_count results
Args:
what (str): query
max_count (int): Max number of results to return
"""
return self._bf.search(what, max_count)
def upload_file_to_id(self, destination_id: str, filepath: str):
"""
Uploads file to a given dataset/collection by id given its filepath on the host
adds some meta data.
Returns the id for the newly created resource
Note: filepath could be an array
Args:
destination_id : The dataset/collection id into which the file shall be uploaded
filepath (path): Full path to the file
"""
_id = ""
destination = self._destination_from_id(destination_id)
if destination is None:
return _id
files = [
filepath,
]
try:
# TODO: PC->MAG: should protected API
# TODO: add new agent SEE https://developer.blackfynn.io/python/latest/CHANGELOG.html#id31
result = self._bf._api.io.upload_files(destination,
files,
display_progress=True,
use_agent=False)
if result and result[0] and "package" in result[0][0]:
_id = result[0][0]["package"]["content"]["id"]
except Exception:
logger.exception("Error uploading file to datcore")
return _id
def create_collection(self, destination_id: str, collection_name: str):
"""
Create a empty collection within destination
Args:
destination_id : The dataset/collection id into which the file shall be uploaded
filepath (path): Full path to the file
"""
destination = self._destination_from_id(destination_id)
_id = ""
if destination is None:
return _id
new_collection = Collection(collection_name)
destination.add(new_collection)
new_collection.update()
destination.update()
_id = new_collection.id
return _id
def list_datasets(self) -> DatasetMetaDataVec:
data = []
for dataset in self._bf.datasets():
dmd = DatasetMetaData(dataset_id=dataset.id,
display_name=dataset.name)
data.append(dmd)
return data
def lineLength(ptName,
ch,
startTime=None,
endTime=None,
append=False,
layerName=LL_LAYER_NAME):
'''
Runs the line length detector.
ch: channels to annotate
startTime: time (usec) to start from. Default value (None) starts from the beginning of the timeseries
append: Whether to append or overwrite the line length annotation layer
layerName: name of layer to write to
'''
freq = FREQs.get(ptName, DEFAULT_FREQ)
threshold = LL_THRESHOLDS[ptName]
bf = Blackfynn()
ts = bf.get(TS_IDs[ptName])
# Make sure startTime and endTime are valid
if startTime is not None:
if startTime < ts.start:
print(
'Warning: startTime', startTime,
'is earlier than the beginning of the Timeseries. Ignoring startTime argument...'
)
startTime = None
elif startTime > ts.end:
print(
'Warning: startTime', startTime,
'is after the end of the Timeseries. No data will be analyzed.'
)
return
if endTime is not None:
if endTime > ts.end:
print(
'Warning: endTime', endTime,
'is later than the end of the Timeseries. Ignoring endTime argument...'
)
endTime = None
elif endTime < ts.start:
print(
'Warning: endTime', endTime,
'is before the beginning the Timeseries. No data will be analyzed.'
)
return
# Get segments
args = {'start': startTime, 'stop': endTime}
if startTime is None:
args['start'] = ts.start
if endTime is None:
args['stop'] = ts.end
segments = ts.segments(**args)
# edit segments so that it starts at startTime
if startTime is None:
startTime = segments[0][0]
else:
try:
i = next(i for i, (a, b) in enumerate(segments) if b > startTime)
segments[:i] = []
except StopIteration:
pass
startTime = max(segments[0][0], startTime)
print('start time:', startTime)
segments[0] = (startTime, segments[0][1])
# Same thing with end time:
if endTime is None:
endTime = segments[-1][1]
else:
l = len(segments)
try:
i = next(l - 1 - i for i, (a, b) in enumerate(reversed(segments))
if a < endTime)
segments[i + 1:] = []
except StopIteration:
pass
endTime = min(segments[-1][1], endTime)
print('end time:', endTime)
segments[-1] = (segments[-1][0], endTime)
try:
# Get layer if it already exists
layer = ts.get_layer(layerName)
if append:
print("Appending to layer '%s'" % layerName)
else:
print("Overwriting layer '%s'" % layerName)
layer.delete()
layer = ts.add_layer(layerName)
except:
print("Creating layer '%s'" % layerName)
layer = ts.add_layer(layerName)
pos = segments[0][0]
for seg in segments:
pos = max(pos, seg[0])
while pos < seg[1]:
try:
clip = ts.get_data(start=pos,
length=LL_CLIP_LENGTH,
channels=ch,
use_cache=False)
# caching disabled to prevent database disk image errors
# note: actual clip length may be shorter than LL_CLIP_LENGTH
except RequestException as e:
# catch Blackfynn server errors
print('Server error (will retry):', e)
sleep(2)
continue
except Exception as e:
print('Pull failed:', e)
pos += LL_CLIP_LENGTH
continue
if clip.empty or clip.isnull().all().any():
# skip clip if a channel is missing data
pos += LL_CLIP_LENGTH
continue
if clip.shape[0] / freq * 1000000 < LL_CLIP_LENGTH / 2:
# skip clip if it's less than half of max clip length
pos += LL_CLIP_LENGTH
continue
startTime = clip.iloc[
0].name.value / 1000 # convert to Unix epoch time, in usecs
endTime = clip.iloc[-1].name.value / 1000
clip.fillna(0, inplace=True) # replace NaNs with zeros
clip = clip.transpose().values
l = _length(clip)
if l > threshold:
print('+ %f (%d, %d)' % (l, startTime, endTime))
sys.stdout.flush()
layer.insert_annotation('Possible seizure',
start=startTime,
end=endTime)
else:
print('- %f (%d, %d)' % (l, startTime, endTime))
sys.stdout.flush()
pos += LL_CLIP_LENGTH
import sys
import re
from blackfynn import Blackfynn
from settings import VIDEO_DIR_IDs, PL_ROOT
def makeVideoLinkFile(vidList, url_root, filename):
'Write list of video URLs and timestamps to file filename'
#get timestamp-get url
with open(filename, 'w') as f:
for video in vidList:
#Retrieve timestamp string in video name
timestamp = re.search('\d{14}', video.name)
if timestamp != None:
f.write('%s : %s\n' %
(str(timestamp.group(0)), url_root + video.id))
print(('video urls written to %s.' % (filename)))
if __name__ == '__main__':
ptName = sys.argv[1]
vidCollectionID = VIDEO_DIR_IDs[ptName]
bf = Blackfynn()
vids = bf.get(vidCollectionID)
url_root = 'https://app.blackfynn.io/' + bf.context.id + '/datasets/' + vids.dataset + '/viewer/'
makeVideoLinkFile(vids, url_root,
PL_ROOT + '/videoLinks/%s_vidMap.txt' % ptName)
def pipeline(ptName, annotating=True, startTime=None, endTime=None, bf=None):
ch = CHANNELS.get(ptName, None)
freq = FREQs.get(ptName, DEFAULT_FREQ)
### Get patient-specific settings
if bf is None: # WORKAROUND: see cron.py
bf = Blackfynn()
ts = bf.get(TS_IDs[ptName])
while True:
try:
segments = ts.segments()
break
except RequestException:
sleep(2)
continue
layer = ts.get_layer(GOLD_STD_LAYERS[ptName])
layerName = layer.name
### Pull ictal and interictal annotation times
print("Reading annotations from layer '%s'..." % layerName)
annotDir = PL_ROOT + '/annotations'
makeDir(annotDir)
ictals = getIctalAnnots(layer)
makeAnnotFile(ictals, '%s/%s_annotations.txt' % (annotDir, ptName))
print('Generating interictal annotations...')
interictals = getInterictalAnnots(ictals, segments)
makeAnnotFile(interictals, '%s/%s_interictal_annotations.txt' % \
(annotDir, ptName))
print('')
sys.stdout.flush()
### Pull clips
clipDir = PL_ROOT + '/clips/' + ptName
makeDir(clipDir)
print('Pulling ictal clips...')
pullClips('%s/%s_annotations.txt' % (annotDir, ptName),
'ictal',
ts,
clipDir,
ch,
limit=TRAINING_SZ_LIMIT)
print('Pulling interictal clips...')
pullClips('%s/%s_interictal_annotations.txt' % (annotDir, ptName),
'interictal', ts, clipDir, ch)
print('')
sys.stdout.flush()
### Slice and preprocess clips
algoDataDir = PL_ROOT + '/seizure-data/' + ptName
makeDir(algoDataDir)
print('Preparing ictal data for classifier...')
sliceClips(clipDir, 'ictal', freq, ptName)
print('Preparing interictal data for classifier...')
sliceClips(clipDir, 'interictal', freq, ptName)
print('')
sys.stdout.flush()
### Prepare classifier
#with open('liveAlgo/targets.json', 'w') as f:
# json.dump([ptName], f)
### Train classifier
print('Training classifier...')
train('train_model', target=ptName)
print('')
sys.stdout.flush()
### Compute cross-Validation scores
print('Computing cross-validation scores...')
train('cv', target=ptName)
print('')
sys.stdout.flush()
if annotating:
### Make predictions on entire timeseries
print('Testing on entire time series...')
try:
# Delete layer if it already exists
layer = ts.get_layer(PL_LAYER_NAME)
layer.delete()
except:
pass
finally:
layer = ts.add_layer(PL_LAYER_NAME)
testTimeSeries(ts, layer, ptName, startTime, endTime)
# Test segments:
clearDir(PL_ROOT + '/seizure-data/' + ptName)
# Cached classifier data
for fname in os.listdir(PL_ROOT + '/data-cache'):
if (fname.startswith('data_test_' + ptName)
or fname.startswith('predictions_' + ptName)):
os.remove(PL_ROOT + '/data-cache/' + fname)
if __name__ == '__main__':
ptName = sys.argv[1]
startTime = int(sys.argv[2])
kwargs = sys.argv[3:]
annotating = ('annotate' in kwargs)
try:
endTime = int(sys.argv[3])
except (ValueError, IndexError):
endTime = None
bf = Blackfynn()
ts = bf.get(TS_IDs[ptName])
layer = ts.add_layer(PL_LAYER_NAME)
print('Testing on patient:', ptName)
if annotating:
print('Annotating enabled')
else:
print('Annotating disabled')
testTimeSeries(ts, layer, ptName, startTime, endTime, annotating)
def blackfynn_cli():
args = docopt(__doc__)
if args['version']:
print "version: {}".format(blackfynn.__version__)
email = args['--user'] if args['--user'] is not None else settings.api_user
passw = args['--pass'] if args['--pass'] is not None else settings.api_pass
host = args['--host'] if args['--host'] is not None else settings.api_host
org = args['--org']
try:
bf = Blackfynn(email=email, password=passw, host=host)
except:
print "Unable to connect to to Blackfynn using specified user/password."
return
if args['orgs']:
for o in bf.organizations():
print " * {} (id: {})".format(o.name, o.id)
if org is not None:
try:
bf.set_context(org)
except:
print 'Error: Unable to set context to "{}"'.format(org)
return
if args['show']:
item = bf.get(args['<item>'])
print item
if hasattr(item, 'items'):
print "CONTENTS:"
for i in item.items:
print " * {}".format(i)
if hasattr(item, 'channels'):
print "CHANNELS:"
for ch in item.channels:
print " * {} (id: {})".format(ch.name, ch.id)
elif args['search']:
terms = ' '.join(args['<term>'])
results = bf._api.search.query(terms)
if len(results) == 0:
print "No Results."
else:
for r in results:
print " * {}".format(r)
elif args['create']:
if args['collection']:
dest = args['<destination>']
name = args['<name>']
c = Collection(name)
parent = bf.get(dest)
parent.add(c)
print c
elif args['dataset']:
name = args['<name>']
ds = bf.create_dataset(name)
print ds
else:
print "Error: creation for object not supported."
return
elif args['delete']:
item = bf.get(args['<item>'])
if isinstance(item, Dataset):
print "Error: cannot delete dataset"
return
elif not isinstance(item, BaseNode):
print "Error: cannot delete item"
return
bf.delete(item)
elif args['upload']:
files = args['<file>']
dest = args['<destination>']
recursively_upload(bf, dest, files)
elif args['append']:
files = args['<file>']
dest = args['<destination>']
bf._api.io.upload_files(dest,
files,
append=True,
display_progress=True)
elif args['datasets']:
print "Datasets: "
for ds in bf.datasets():
print " - {} (id: {})".format(ds.name, ds.id)
elif args['dataset']:
ds = bf.get(args['<dataset>'])
if args['collaborators']:
if args['<action>'] == 'ls':
resp = ds.collaborators()
print " - Users"
for u in resp['users']:
print " - email:{} id:{}".format(u.email, u.id)
print " - Groups"
for g in resp['groups']:
print " - name:{} id:{}".format(g.name, g.id)
elif args['<action>'] == 'add':
ids = args['<action-args>']
if len(ids) == 0:
print "Error: No ids specified"
sys.exit(1)
resp = ds.add_collaborators(*ids)
print_collaborator_edit_resp(resp)
elif args['<action>'] == 'rm':
ids = args['<action-args>']
if len(ids) == 0:
print "Error: No ids specified"
sys.exit(1)
resp = ds.remove_collaborators(*ids)
print_collaborator_edit_resp(resp)
else:
print "Error: invalid dataset collaborators command. Valid commands are 'ls', 'add' or 'rm'"
else:
print "Error: invalid dataset command. Valid commands are 'collaborators'"
elif args['env']:
print "# Blackfynn environment"
print "API Location: {}".format(host)
print "Streaming API: {}".format(settings.streaming_api_host)
print "User: {}".format(email)
print "Organization: {} (id: {})".format(bf.context.name,
bf.context.id)
dfs.append(df.iloc[i:])
# save clip(s)
for df in dfs:
time = (df.iloc[0].name.value / 1000,
df.iloc[-1].name.value / 1000)
clipTimes.append(time)
array = df.transpose().values
outfile = '%s/%s%d.hkl' % (outDir, outfile_prefix, num_saved + 1)
hickle.dump(array, outfile)
num_saved += 1
print(('%d clips saved.' % num_saved))
return clipTimes
if __name__ == '__main__':
annotFile = sys.argv[1]
clipType = sys.argv[2]
tsID = sys.argv[3]
outDir = sys.argv[4].rstrip('/')
try:
limit = int(sys.argv[5])
except IndexError:
limit = None
bf = Blackfynn()
ts = bf.get(tsID)
pullClips(annotFile, clipType, ts, outDir, limit=limit)
"""
Usage: collection_watcher.py --collection=<collection_id> --threshold=<hours> --notify=<email>
"""
from datetime import datetime
import time
from docopt import docopt
from blackfynn import Blackfynn
from blackfynn import Collection
arguments = docopt(__doc__)
bf = Blackfynn()
collection = bf.get(arguments['--collection'])
current_timestamp = time.time() * 1000
for item in collection.items:
milliseconds_old = current_timestamp - int(
item.get_property('created').value)
hours_old = (milliseconds_old / (1000 * 60 * 60)) % 24
if hours_old > int(arguments['--threshold']):
print "send notification"
def lineLength(ptName,
startTime=None,
endTime=None,
append=False,
layerName=LL_MA_LAYER_NAME):
'''
Runs the line length detector.
ch: channels to annotate
startTime: time (usec) to start from. Default value (None) starts from the beginning
append: Whether to append to (or otherwise overwrite) the line length annotation layer
layerName: name of layer to write to
Returns: endTime (the end of the last full long-term window)
'''
global shortWindow, longWindow, freq, ch
longWindow = LL_LONG_WINDOWS.get(ptName, LL_LONG_WINDOW_DEFAULT)
shortWindow = LL_SHORT_WINDOWS.get(ptName, LL_SHORT_WINDOW_DEFAULT)
freq = FREQs.get(ptName, DEFAULT_FREQ)
ch = CHANNELS.get(ptName, None)
bf = Blackfynn()
ts = bf.get(TS_IDs[ptName])
# Make sure startTime and endTime are valid
if startTime is not None:
if startTime < ts.start:
print(
'Warning: startTime', startTime,
'is before the beginning of the Timeseries. Starting from the beginning...'
)
startTime = None
elif startTime > ts.end:
print('Warning: startTime', startTime,
'is after the end of the Timeseries. Exiting...')
return ts.end
if endTime is not None:
if endTime > ts.end:
print(
'Warning: endTime', endTime,
'is after the end of the Timeseries. Stopping at the end...')
endTime = None
elif endTime < ts.start:
print('Warning: endTime', endTime,
'is before the beginning the Timeseries. Exiting...')
return ts.start
# Get/create annotation layer
try:
layer = ts.get_layer(layerName)
if append:
print("Appending to layer '%s'" % layerName)
else:
print("Overwriting layer '%s'" % layerName)
layer.delete()
layer = ts.add_layer(layerName)
except:
print("Creating layer '%s'" % layerName)
layer = ts.add_layer(layerName)
# Find the long-term windows to start and end from
windowStart = ts.start
windowEnd = windowStart + longWindow
if startTime:
while windowEnd < startTime:
windowStart = windowEnd
windowEnd += longWindow
else:
startTime = ts.start
if not endTime:
endTime = ts.end
# Make sure segments list starts at startTime and ends at endTime
segments = ts.segments(startTime, endTime + 1)
if not segments:
print('No data found between %d and %d.' % (startTime, endTime), \
' Exiting...')
return endTime
startTime = max(segments[0][0], startTime)
print('start time:', startTime)
segments[0] = (startTime, segments[0][1])
endTime = min(segments[-1][1], endTime)
print('end time:', endTime)
segments[-1] = (segments[-1][0], endTime)
# Go through each long-term window
while windowStart < endTime and windowEnd <= ts.end:
# Calculate trend and threshold
try:
trend, shortClips = _trend(ts, windowStart, windowEnd)
except RequestException:
# Workaround in case of server errors
sleep(2)
continue
if trend is None:
print('skipping long window (no clips)...')
sys.stdout.flush()
windowStart = windowEnd
windowEnd += longWindow
continue
# If using a custom start time, trim shortClips
# (This should only ever happen with the 1st long-term window)
if windowStart < startTime:
try:
i = next(i for i, c in enumerate(shortClips)
if c['end'] > startTime)
shortClips[:i] = []
except StopIteration:
pass
shortClips[0]['start'] = max(shortClips[0]['start'], startTime)
# Delete 1st clip if it's too short (less than half of shortWindow):
if shortClips[0]['end'] - shortClips[0]['start'] < shortWindow / 2:
shortClips.pop(0)
# Do the same thing with endTime
# (Should only ever happen to the last long-term window)
if windowEnd > endTime:
l = len(shortClips)
try:
i = next(l - 1 - i for i, c in enumerate(reversed(shortClips))
if c['start'] < endTime)
shortClips[i + 1:] = []
except StopIteration:
pass
# Delete last clip if it's not long enough
lastClip = shortClips.pop(-1)
if lastClip['end'] - lastClip['start'] >= shortWindow / 2:
lastClip['end'] = min(lastClip['end'], endTime)
shortClips.append(lastClip)
else:
pass
# Annotate and/or print predictions
threshold = LL_MA_THRESHOLDS[ptName] * trend
for clip in shortClips:
l = clip['length']
if l > threshold:
print('+ %f (%d, %d)' % (l, clip['start'], clip['end']))
layer.insert_annotation('Possible seizure',
start=clip['start'],
end=clip['end'])
else:
print('- %f (%d, %d)' % (l, clip['start'], clip['end']))
sys.stdout.flush()
# Go to next long term window
windowStart = windowEnd
windowEnd += longWindow
return min(windowStart, endTime)
