def blackfynn_cli():
args = docopt(__doc__,
version='bf version {}'.format(blackfynn.__version__),
options_first=True)
# Test for these two commands first as they
# do not require a Blackfynn client
if args['<command>'] in ['help', None]:
print((__doc__.strip('\n')))
return
if args['<command>'] == 'profile':
from blackfynn.cli import bf_profile
bf_profile.main()
return
# Display warning message if config.ini is not found
settings = Settings(
) # create a dummy settings object to load environment variables and defaults only
if not os.path.exists(settings.config_file):
print(
"\033[31m* Warning: No config file found, run 'bf profile' to start the setup assistant\033[0m"
)
# Try to use profile specified by --profile, exit if invalid
try:
bf = Blackfynn(args['--profile'])
except Exception as e:
exit(e)
#Try to use dataset specified by --dataset, exit if invalid
try:
if args['--dataset'] is not None:
dataset = bf.get_dataset(args['--dataset'])
set_working_dataset(dataset.id)
except Exception as e:
exit(e)
if args['<command>'] == 'status':
from blackfynn.cli import bf_status
bf_status.main(bf)
elif args['<command>'] == 'use':
from . import bf_use
bf_use.main(bf)
elif args['<command>'] == 'init':
from . import bf_init
bf_init.main(bf)
elif args['<command>'] in ['datasets', 'ds']:
from . import bf_datasets
bf_datasets.main(bf)
elif args['<command>'] in ['organizations', 'orgs']:
from . import bf_organizations
bf_organizations.main(bf)
elif args['<command>'] in ['share', 'unshare', 'collaborators']:
from . import bf_share
bf_share.main(bf)
elif args['<command>'] == 'cache':
from . import bf_cache
bf_cache.main(bf)
elif args['<command>'] == 'create':
from . import bf_create
bf_create.main(bf)
elif args['<command>'] == 'delete':
from . import bf_delete
bf_delete.main(bf)
elif args['<command>'] == 'move':
from . import bf_move
bf_move.main(bf)
elif args['<command>'] == 'rename':
from . import bf_rename
bf_rename.main(bf)
elif args['<command>'] == 'props':
from . import bf_props
bf_props.main(bf)
elif args['<command>'] == 'get':
from . import bf_get
bf_get.main(bf)
elif args['<command>'] == 'where':
from . import bf_where
bf_where.main(bf)
elif args['<command>'] == 'upload':
from . import bf_upload
bf_upload.main(bf)
elif args['<command>'] == 'append':
from . import bf_append
bf_append.main(bf)
elif args['<command>'] == 'search':
from . import bf_search
bf_search.main(bf)
else:
exit("Invalid command: '{}'\nSee 'bf help' for available commands".
format(args['<command>']))
def convert_ncs2bfts(dataFolder,
resultFolder=None,
bfFileName=None,
dsName=None,
fs=None):
"""
param dataFolder: folder containing ncs files
param resultFolder: folder to contain results
param bfFileName: name to save generated bfts file
param dsName: name (or id) of dataset
param fs: sampling rate
return: _____
"""
dataFolder = os.path.abspath(dataFolder)
if not bfFileName:
bfFileName = os.path.basename(dataFolder) + '.bfts'
if resultFolder:
resultFolder = os.path.abspath(resultFolder)
resultFile = os.path.join(resultFolder, bfFileName)
else:
resultFile = bfFileName
#
if os.path.isfile(resultFile):
print(bfFileName, 'exists')
else:
print('Converting to', bfFileName, '...')
chls = OrderedDict() # dictionary to store channel values
for chFile in os.listdir(dataFolder):
if chFile.endswith('ncs'):
ncs = ncs2py.load_ncs(
os.path.join(dataFolder, chFile)
) # import neuralynx data. NOTE: import stores information as a dictionary
rawData = ncs['data']
rawData = resample_poly(rawData, 1.0,
ncs['sampling_rate'] / fs)
chls.update({'ch' + chFile.split('.')[0][3:]: rawData})
#
TimeCreated = [
line.strip() for line in ncs['raw_header'].split('\r\n')
if line != '' if 'TimeCreated' == line.strip()[1:12]
][0]
TimeCreated = ncs2py.parse_neuralynx_time_string(TimeCreated)
TimeCreated = (TimeCreated -
datetime(1970, 1, 1)).total_seconds() * (1e6)
timeVec = ncs['timestamp'] + TimeCreated
timeVec = arange(timeVec[0],
timeVec[-1], (1.0 / fs) * (1e6),
dtype=int64)
sampleSize = timeVec.shape[0]
#
df = DataFrame(chls)[0:sampleSize]
df.insert(0, 'timeStamp', timeVec)
df.to_csv(resultFile, index=False)
#
if dsName:
bf = Blackfynn()
ds = bf.get_dataset(dsName)
if os.path.basename(resultFile)[:-5] in ds.get_items_names():
print(bfFileName, 'uploaded')
else:
print('uploading', bfFileName, 'to Blackfynn...')
ds.upload(resultFile)
printf("metadata with key, %s, added to %s\n", key, meta_path)
else:
try:
ds.remove_property(key, CATEGORY)
except:
printf("metadata with key, %s, does not exist in %s.\n", key,
meta_path)
sys.exit()
printf("metadata with key, %s, removed from %s.\n", key, meta_path)
###############################################################################
# program starts HERE
bf = Blackfynn() # use 'default' profile
ALL = False
ADD = True
CATEGORY = 'Blackfynn' # default category
DATASET = False
FILE = False
KEY = False
METAFILE = False
PATH = False
SHOW = False
TYPE = None
VALUE = False
meta_path = ""
if len(sys.argv) < 2:
printf("%s\n", syntax())
from blackfynn import Blackfynn
import glob
import os
import time
import sys
if sys.platform == 'win32':
rootPath = 'R:\\'
else:
rootPath = os.path.join('//', 'media', 'rnelshare')
sys.path.append(
os.path.join(rootPath, 'users', 'amn69', 'Projects', 'cat', 'selectivity',
'surface paper', 'v2018'))
import helperFcns as hf
bf = Blackfynn('lumbar_selectivity')
catDS = bf.get_dataset('N:dataset:1cc6b671-0dea-4aab-ad30-ed3884e17028')
db = hf.db
collection = db.blackfynnUpload
for iSub in ['HA04'
]: #['Electro','Freeze','HA02','HA04','Galactus','Hobgoblin']:
print iSub
if collection.find({'subject': iSub}).count() == 0:
subjFolder = catDS.create_collection(iSub)
else:
subjFolder = catDS.get_items_by_name(iSub)[0]
targetSessions = sorted(hf.PWbySession[iSub].keys())
for iSesh in targetSessions:
#!/usr/bin/env python3
'''
Find line lengths at certain times, in order to figure out threshold values
for the basic line length detector.
Usage: python -m helper_scripts.lineLengthTest ptName [startTime]
'''
import os
import sys
import numpy as np
from blackfynn import Blackfynn
from settings import CHANNELS, LL_CLIP_LENGTH, TS_IDs
ptName = sys.argv[1]
bf = Blackfynn()
ts = bf.get(TS_IDs[ptName])
ch = CHANNELS.get(ptName, None)
try:
startTime = int(sys.argv[2])
segments = ts.segments(start=startTime)
except:
segments = ts.segments()
finally:
startTime = segments[0][0]
print('start time:', startTime)
def lineLength(clip):
lengths = np.zeros(clip.shape[0]).astype('float64')
def _getBlackfynn(self, profile_name):
api_key = self._settings[profile_name]['api_token']
api_secret = self._settings[profile_name]['api_secret']
print('[{0}]:[{1}]'.format(api_key, api_secret))
self._bf = Blackfynn(api_token=api_key, api_secret=api_secret)
return self._bf
def __init__(self, api_token=None, api_secret=None, host=None, streaming_host=None):
self.client = Blackfynn(profile=None, api_token=api_token, api_secret=api_secret,
host=host, streaming_host=streaming_host)
def bf():
return Blackfynn()
from blackfynn import Blackfynn
from datetime import datetime, timedelta
import scipy.io as sio
import sys
username = sys.argv[1]
passwd = sys.argv[2]
tsid = sys.argv[3]
ptname = sys.argv[4]
szfile = sys.argv[5]
#def pullSzClips(username, passwd, tsid, ptname, szfile):
# open up the BF dataset (again, this works for R951 but does not generalize)
# If you were to use a MEF file, you could point to it here
bf = Blackfynn(email=username,password=passwd)
bf.set_context('Mayo')
ts = bf.datasets()[0].items[0]
# get the annotation times in uUTC from file
with open(szfile) as f:
times = f.read().splitlines()
for i in range(0,len(times),2):
tempStart = datetime.utcfromtimestamp(float(times[i])/1e6)
tempEnd = datetime.utcfromtimestamp(float(times[i+1])/1e6)
print(str(tempStart) + "until" + str(tempEnd))
# pull the data for each seizure
try:
print('about to pull')
df = ts.get_data(start=tempStart,end=tempEnd)
while toContinue:
for i in range(1, df.shape[0]):
toContinue = False
#print(i)
deltaT = (df.iloc[i].name - df.iloc[i - 1].name).microseconds
if deltaT > 10000:
dfs.append(df.iloc[:i])
df = df.iloc[i:]
toContinue = True
break
dfs.append(df)
return dfs
# Need to fill in personal Blackfynn login information here
bf = Blackfynn(email='you_email_here', password='******')
bf.set_context('Mayo')
ts = bf.datasets()[0].items[
0] # again, this is set up for 951 now. Need to change this line for a different BF dataset
# Here we grab the last 30 minutes of data
tempEnd = datetime.datetime.now()
tempStart = tempEnd - datetime.timedelta(0, 1800)
d = ts.get_data(start=tempStart, end=tempEnd)
d = d.drop_duplicates(
) # had some issues with duplicate data points. Remove them if they exist
if d.shape[0] == 0:
print('no data in time segment...quitting \n')
exit()
print('starting split...')
dfs = split_data(d)
def connect_to_blackfynn():
global bf
bf = Blackfynn(api_token=Config.BLACKFYNN_API_TOKEN,
api_secret=Config.BLACKFYNN_API_SECRET,
env_override=False,
host=Config.BLACKFYNN_API_HOST)
def test_exception_raise():
bf = Blackfynn()
with pytest.raises(Exception) as excinfo:
bf._api._call("get", "/datasets/plop")
assert "plop not found" in str(excinfo.value)
def test_client_host_overrides():
host = 'http://localhost'
# fails authentication in Blackfynn.__init__
with pytest.raises(requests.exceptions.RequestException):
bf = Blackfynn(host=host)
import click
import sparc_dash
from blackfynn import Blackfynn
bf = Blackfynn('sparc-consortium')
ds = bf.get_dataset('SPARC Datasets')
@click.group()
def cli():
pass
@click.command()
def clear():
print('Clearing')
out = sparc_dash.clearRecords(ds)
@click.command()
def create_models():
"""Example script."""
out = sparc_dash.create_models(ds)
@click.command()
def update():
"""Example script."""
out = sparc_dash.update(bf, ds)
def lineLength(ptName,
startTime=None,
endTime=None,
append=False,
layerName=LL_MA_LAYER_NAME):
'''
Runs the line length detector.
ch: channels to annotate
startTime: time (usec) to start from. Default value (None) starts from the beginning
append: Whether to append to (or otherwise overwrite) the line length annotation layer
layerName: name of layer to write to
Returns: endTime (the end of the last full long-term window)
'''
global shortWindow, longWindow, freq, ch
longWindow = LL_LONG_WINDOWS.get(ptName, LL_LONG_WINDOW_DEFAULT)
shortWindow = LL_SHORT_WINDOWS.get(ptName, LL_SHORT_WINDOW_DEFAULT)
freq = FREQs.get(ptName, DEFAULT_FREQ)
ch = CHANNELS.get(ptName, None)
bf = Blackfynn()
ts = bf.get(TS_IDs[ptName])
# Make sure startTime and endTime are valid
if startTime is not None:
if startTime < ts.start:
print(
'Warning: startTime', startTime,
'is before the beginning of the Timeseries. Starting from the beginning...'
)
startTime = None
elif startTime > ts.end:
print('Warning: startTime', startTime,
'is after the end of the Timeseries. Exiting...')
return ts.end
if endTime is not None:
if endTime > ts.end:
print(
'Warning: endTime', endTime,
'is after the end of the Timeseries. Stopping at the end...')
endTime = None
elif endTime < ts.start:
print('Warning: endTime', endTime,
'is before the beginning the Timeseries. Exiting...')
return ts.start
# Get/create annotation layer
try:
layer = ts.get_layer(layerName)
if append:
print("Appending to layer '%s'" % layerName)
else:
print("Overwriting layer '%s'" % layerName)
layer.delete()
layer = ts.add_layer(layerName)
except:
print("Creating layer '%s'" % layerName)
layer = ts.add_layer(layerName)
# Find the long-term windows to start and end from
windowStart = ts.start
windowEnd = windowStart + longWindow
if startTime:
while windowEnd < startTime:
windowStart = windowEnd
windowEnd += longWindow
else:
startTime = ts.start
if not endTime:
endTime = ts.end
# Make sure segments list starts at startTime and ends at endTime
segments = ts.segments(startTime, endTime + 1)
if not segments:
print('No data found between %d and %d.' % (startTime, endTime), \
' Exiting...')
return endTime
startTime = max(segments[0][0], startTime)
print('start time:', startTime)
segments[0] = (startTime, segments[0][1])
endTime = min(segments[-1][1], endTime)
print('end time:', endTime)
segments[-1] = (segments[-1][0], endTime)
# Go through each long-term window
while windowStart < endTime and windowEnd <= ts.end:
# Calculate trend and threshold
try:
trend, shortClips = _trend(ts, windowStart, windowEnd)
except RequestException:
# Workaround in case of server errors
sleep(2)
continue
if trend is None:
print('skipping long window (no clips)...')
sys.stdout.flush()
windowStart = windowEnd
windowEnd += longWindow
continue
# If using a custom start time, trim shortClips
# (This should only ever happen with the 1st long-term window)
if windowStart < startTime:
try:
i = next(i for i, c in enumerate(shortClips)
if c['end'] > startTime)
shortClips[:i] = []
except StopIteration:
pass
shortClips[0]['start'] = max(shortClips[0]['start'], startTime)
# Delete 1st clip if it's too short (less than half of shortWindow):
if shortClips[0]['end'] - shortClips[0]['start'] < shortWindow / 2:
shortClips.pop(0)
# Do the same thing with endTime
# (Should only ever happen to the last long-term window)
if windowEnd > endTime:
l = len(shortClips)
try:
i = next(l - 1 - i for i, c in enumerate(reversed(shortClips))
if c['start'] < endTime)
shortClips[i + 1:] = []
except StopIteration:
pass
# Delete last clip if it's not long enough
lastClip = shortClips.pop(-1)
if lastClip['end'] - lastClip['start'] >= shortWindow / 2:
lastClip['end'] = min(lastClip['end'], endTime)
shortClips.append(lastClip)
else:
pass
# Annotate and/or print predictions
threshold = LL_MA_THRESHOLDS[ptName] * trend
for clip in shortClips:
l = clip['length']
if l > threshold:
print('+ %f (%d, %d)' % (l, clip['start'], clip['end']))
layer.insert_annotation('Possible seizure',
start=clip['start'],
end=clip['end'])
else:
print('- %f (%d, %d)' % (l, clip['start'], clip['end']))
sys.stdout.flush()
# Go to next long term window
windowStart = windowEnd
windowEnd += longWindow
return min(windowStart, endTime)