def push_to_blackfynn(event, context):

    bucket = event['Records'][0]['s3']['bucket']['name']
    key = urllib.unquote_plus(
        event['Records'][0]['s3']['object']['key'].encode('utf8'))

    try:
        bf = Blackfynn()
        bf.set_context(organization_id)
        root_collection = bf.get(collection_id)

        package_name = key.split("/")[-2]
        package_collection = next(
            (c for c in root_collection.items if c.name == package_name), None)
        if package_collection is None:
            package_collection = Collection(package_name)
            root_collection.add(package_collection)

        local_object = '/tmp/{}'.format(os.path.basename(key))
        s3.download_file(Bucket=bucket, Key=key, Filename=local_object)
        package_collection.upload_files(local_object)
        return 'Successfully pushed {} from bucket {} to Blackfynn.'.format(
            key, bucket)

    except Exception as e:
        print(e)
        print('Error pushing {} from bucket {} to Blackfynn.'.format(
            key, bucket))
        raise e
Exemple #2
0
def send_digest():
    report = []
    report.append("<h2>Blackfynn: BioRC Failed Extract Digest</h2>")
    bf = Blackfynn()
    root = bf.get('N:collection:4fec4882-925c-4328-bbfd-d1de953ba225')
    for bucket in root.items:
        include_count = True if bucket.id == "N:collection:0e476218-ccb9-4c4d-bdb4-e72d0a0f88fd" else False
        bucket_digest(report, bucket, include_count)
    send_report(report)
Exemple #3
0
class DatcoreClient(object):
    def __init__(self,
                 api_token=None,
                 api_secret=None,
                 host=None,
                 streaming_host=None):
        # WARNING: contruction raise exception if service is not available.
        # Use datacore_wrapper for safe calls
        # TODO: can use https://developer.blackfynn.io/python/latest/configuration.html#environment-variables
        self._bf = Blackfynn(
            profile=None,
            api_token=api_token,
            api_secret=api_secret,
            host=host,
            streaming_host=streaming_host,
        )

    def profile(self):
        """
        Returns profile of current User
        """
        return self._bf.profile

    def _collection_from_destination(self, destination: str):
        destination_path = Path(destination)
        parts = destination_path.parts

        dataset_name = parts[0]
        dataset = self.get_dataset(dataset_name)
        if dataset is None:
            return None, None

        collection_id = dataset.id
        collection = dataset
        collections = []
        if len(parts) > 1:
            object_path = Path(*parts[1:])
            collections = list(object_path.parts)
            collection_id = ""
            collection_id = _get_collection_id(dataset, collections,
                                               collection_id)
            collection = self._bf.get(collection_id)

        return collection, collection_id

    def _destination_from_id(self, destination_id: str):
        # NOTE: .get(*) logs
        #  INFO:blackfynn.client.Blackfynn:Unable to retrieve object
        # if destination_id refers to a Dataset

        destination: Union[DataPackage,
                           Collection] = self._bf.get(destination_id)
        if destination is None:
            destination: Dataset = self._bf.get_dataset(destination_id)

        return destination

    def list_files_recursively(self, dataset_filter: str = ""):
        files = []

        for dataset in self._bf.datasets():
            if not dataset_filter or dataset_filter in dataset.name:
                self.list_dataset_files_recursively(files, dataset,
                                                    Path(dataset.name))

        return files

    def list_files_raw_dataset(self, dataset_id: str) -> List[FileMetaDataEx]:
        files = []  # raw packages
        _files = []  # fmds
        data = {}  # map to keep track of parents-child

        cursor = ""
        page_size = 1000
        api = self._bf._api.datasets

        dataset = self._bf.get_dataset(dataset_id)
        if dataset is not None:
            while True:
                resp = api._get(
                    api._uri(
                        "/{id}/packages?cursor={cursor}&pageSize={pageSize}&includeSourceFiles={includeSourceFiles}",
                        id=dataset_id,
                        cursor=cursor,
                        pageSize=page_size,
                        includeSourceFiles=False,
                    ))
                for package in resp.get("packages", list()):
                    id = package["content"]["id"]
                    data[id] = package
                    files.append(package)
                cursor = resp.get("cursor")
                if cursor is None:
                    break

            for f in files:
                if f["content"]["packageType"] != "Collection":
                    filename = f["content"]["name"]
                    file_path = ""
                    file_id = f["content"]["nodeId"]
                    _f = f
                    while "parentId" in _f["content"].keys():
                        parentid = _f["content"]["parentId"]
                        _f = data[parentid]
                        file_path = _f["content"]["name"] + "/" + file_path

                    bucket_name = dataset.name
                    file_name = filename
                    file_size = 0
                    object_name = str(Path(file_path) / file_name)

                    file_uuid = str(Path(bucket_name) / object_name)
                    created_at = f["content"]["createdAt"]
                    last_modified = f["content"]["updatedAt"]
                    parent_id = dataset_id
                    if "parentId" in f["content"]:
                        parentId = f["content"]["parentId"]
                        parent_id = data[parentId]["content"]["nodeId"]

                    fmd = FileMetaData(
                        bucket_name=bucket_name,
                        file_name=file_name,
                        object_name=object_name,
                        location=DATCORE_STR,
                        location_id=DATCORE_ID,
                        file_uuid=file_uuid,
                        file_id=file_id,
                        raw_file_path=file_uuid,
                        display_file_path=file_uuid,
                        created_at=created_at,
                        last_modified=last_modified,
                        file_size=file_size,
                    )
                    fmdx = FileMetaDataEx(fmd=fmd, parent_id=parent_id)
                    _files.append(fmdx)

        return _files

    def list_files_raw(self, dataset_filter: str = "") -> List[FileMetaDataEx]:
        _files = []

        for dataset in self._bf.datasets():
            _files = _files + self.list_files_raw_dataset(dataset.id)

        return _files

    def list_dataset_files_recursively(self, files: List[FileMetaData],
                                       base: BaseCollection,
                                       current_root: Path):
        for item in base:
            if isinstance(item, Collection):
                _current_root = current_root / Path(item.name)
                self.list_dataset_files_recursively(files, item, _current_root)
            else:
                parts = current_root.parts
                bucket_name = parts[0]
                file_name = item.name
                file_size = 0
                # lets assume we have only one file
                if item.files:
                    file_name = Path(
                        item.files[0].as_dict()["content"]["s3key"]).name
                    file_size = item.files[0].as_dict()["content"]["size"]
                # if this is in the root directory, the object_name is the filename only
                if len(parts) > 1:
                    object_name = str(Path(*list(parts)[1:]) / Path(file_name))
                else:
                    object_name = str(Path(file_name))

                file_uuid = str(Path(bucket_name) / Path(object_name))
                file_id = item.id
                created_at = item.created_at
                last_modified = item.updated_at
                fmd = FileMetaData(
                    bucket_name=bucket_name,
                    file_name=file_name,
                    object_name=object_name,
                    location=DATCORE_STR,
                    location_id=DATCORE_ID,
                    file_uuid=file_uuid,
                    file_id=file_id,
                    raw_file_path=file_uuid,
                    display_file_path=file_uuid,
                    created_at=created_at,
                    last_modified=last_modified,
                    file_size=file_size,
                )
                files.append(fmd)

    def create_dataset(self, ds_name, *, force_delete=False):
        """
        Creates a new dataset for the current user and returns it. Returns existing one
        if there is already a dataset with the given name.

        Args:
            ds_name (str): Name for the dataset (_,-,' ' and capitalization are ignored)
            force_delete (bool, optional): Delete first if dataset already exists
        """
        ds = None
        with suppress(Exception):
            ds = self._bf.get_dataset(ds_name)
            if force_delete:
                ds.delete()
                ds = None

        if ds is None:
            ds = self._bf.create_dataset(ds_name)

        return ds

    def get_dataset(self, ds_name, create_if_not_exists=False):
        """
        Returns dataset with the given name. Creates it if required.

        Args:
            ds_name (str): Name for the dataset
            create_if_not_exists (bool, optional): Create first if dataset already exists
        """

        ds = None
        with suppress(Exception):
            ds = self._bf.get_dataset(ds_name)

        if ds is None and create_if_not_exists:
            ds = self._bf.create_dataset(ds_name)

        return ds

    def delete_dataset(self, ds_name):
        """
        Deletes dataset with the given name.

        Args:
            ds_name (str): Name for the dataset
        """

        # this is not supported
        ds = self.get_dataset(ds_name)
        if ds is not None:
            self._bf.delete(ds.id)

    def exists_dataset(self, ds_name):
        """
        Returns True if dataset with the given name exists.

        Args:
            ds_name (str): Name for the dataset
        """

        ds = self.get_dataset(ds_name)
        return ds is not None

    def upload_file(self,
                    destination: str,
                    filepath: str,
                    meta_data=None) -> bool:
        """
        Uploads a file to a given dataset/collection given its filepath on the host. Optionally
        adds some meta data

        Args:
            dataset (dataset): The dataset into whioch the file shall be uploaded
            filepath (path): Full path to the file
            meta_data (dict, optional): Dictionary of meta data

        Note:
            Blackfynn postprocesses data based on filendings. If it can do that
            the filenames on the server change.
        """
        # parse the destination and try to find the package_id to upload to
        collection, collection_id = self._collection_from_destination(
            destination)

        if collection is None:
            return False

        files = [
            filepath,
        ]
        self._bf._api.io.upload_files(collection,
                                      files,
                                      display_progress=True,
                                      use_agent=False)
        collection.update()

        if meta_data is not None:
            for f in files:
                filename = os.path.basename(f)
                package = self.get_package(collection, filename)
                if package is not None:
                    self._update_meta_data(package, meta_data)

        return True

    def _update_meta_data(self, package, meta_data):
        """
        Updates or replaces metadata for a package

        Args:
            package (package): The package for which the meta data needs update
            meta_data (dict): Dictionary of meta data
        """

        for key in meta_data.keys():
            package.set_property(key, meta_data[key], category="simcore")

        package.update()

    def download_file(self, source, filename, destination_path):
        """
        Downloads a frile from a source dataset/collection given its filename. Stores
        it under destination_path

        Args:
            source (dataset/collection): The dataset or collection to donwload from
            filename (str): Name of the file
            destination__apth (str): Path on host for storing file
        """

        url = self.download_link(source, filename)
        if url:
            _file = urllib.URLopener()  # nosec
            _file.retrieve(url, destination_path)
            return True
        return False

    def download_link(self, destination, filename):
        """
            returns presigned url for download, destination is a dataset or collection
        """
        collection, collection_id = self._collection_from_destination(
            destination)

        for item in collection:
            if isinstance(item, DataPackage):
                if Path(item.files[0].as_dict()["content"]
                        ["s3key"]).name == filename:
                    file_desc = self._bf._api.packages.get_sources(item.id)[0]
                    url = self._bf._api.packages.get_presigned_url_for_file(
                        item.id, file_desc.id)
                    return url

        return ""

    def download_link_by_id(self, file_id):
        """
            returns presigned url for download of a file given its file_id
        """
        url = ""
        filename = ""
        package = self._bf.get(file_id)
        if package is not None:
            filename = Path(
                package.files[0].as_dict()["content"]["s3key"]).name

        file_desc = self._bf._api.packages.get_sources(file_id)[0]
        url = self._bf._api.packages.get_presigned_url_for_file(
            file_id, file_desc.id)

        return url, filename

    def get_package(self, source, filename):
        """
        Returns package from source by name if exists

        Args:
            source (dataset/collection): The dataset or collection to donwload from
            filename (str): Name of the file
        """

        source.update()
        for item in source:
            if item.name == filename:
                return item

        return None

    def delete_file(self, destination, filename):
        """
        Deletes file by name from destination by name

        Args:
            destination (dataset/collection): The dataset or collection to delete from
            filename (str): Name of the file
        """
        collection, collection_id = self._collection_from_destination(
            destination)

        if collection is None:
            return False

        collection.update()
        for item in collection:
            if isinstance(item, DataPackage):
                if Path(item.files[0].as_dict()["content"]
                        ["s3key"]).name == filename:
                    self._bf.delete(item)
                    return True

        return False

    def delete_file_by_id(self, id: str) -> bool:
        """
        Deletes file by id

        Args:
            datcore id for the file
        """
        package: DataPackage = self._bf.get(id)
        package.delete()
        return not package.exists

    def delete_files(self, destination):
        """
        Deletes all files in destination

        Args:
            destination (dataset/collection): The dataset or collection to delete
        """

        collection, collection_id = self._collection_from_destination(
            destination)

        if collection is None:
            return False

        collection.update()
        for item in collection:
            self._bf.delete(item)

    def update_meta_data(self, dataset, filename, meta_data):
        """
        Updates metadata for a file

        Args:
            dataset (package): Which dataset
            filename (str): Which file
            meta_data (dict): Dictionary of meta data
        """

        filename = os.path.basename(filename)
        package = self.get_package(dataset, filename)
        if package is not None:
            self._update_meta_data(package, meta_data)

    def get_meta_data(self, dataset, filename):
        """
        Returns metadata for a file

        Args:
            dataset (package): Which dataset
            filename (str): Which file
        """

        meta_data = {}
        filename = os.path.basename(filename)
        package = self.get_package(dataset, filename)
        if package is not None:
            meta_list = package.properties
            for m in meta_list:
                meta_data[m.key] = m.value

        return meta_data

    def delete_meta_data(self, dataset, filename, keys=None):
        """
        Deletes specified keys in meta data for source/filename.

        Args:
            dataset (package): Which dataset
            filename (str): Which file
            keys (list of str, optional): Deletes specified keys, deletes
            all meta data if None
        """

        filename = os.path.basename(filename)
        package = self.get_package(dataset, filename)
        if package is not None:
            if keys is None:
                for p in package.properties:
                    package.remove_property(p.key, category="simcore")
            else:
                for k in keys:
                    package.remove_property(k, category="simcore")

    def search(self, what, max_count):
        """
        Seraches a thing in the database. Returns max_count results

        Args:
            what (str): query
            max_count (int): Max number of results to return
        """
        return self._bf.search(what, max_count)

    def upload_file_to_id(self, destination_id: str, filepath: str):
        """
        Uploads file to a given dataset/collection by id given its filepath on the host
        adds some meta data.

        Returns the id for the newly created resource

        Note: filepath could be an array

        Args:
            destination_id : The dataset/collection id into which the file shall be uploaded
            filepath (path): Full path to the file
        """
        _id = ""
        destination = self._destination_from_id(destination_id)
        if destination is None:
            return _id

        files = [
            filepath,
        ]

        try:
            # TODO: PC->MAG: should protected API
            # TODO: add new agent SEE https://developer.blackfynn.io/python/latest/CHANGELOG.html#id31
            result = self._bf._api.io.upload_files(destination,
                                                   files,
                                                   display_progress=True,
                                                   use_agent=False)
            if result and result[0] and "package" in result[0][0]:
                _id = result[0][0]["package"]["content"]["id"]

        except Exception:
            logger.exception("Error uploading file to datcore")

        return _id

    def create_collection(self, destination_id: str, collection_name: str):
        """
        Create a empty collection within destination
        Args:
            destination_id : The dataset/collection id into which the file shall be uploaded
            filepath (path): Full path to the file
        """
        destination = self._destination_from_id(destination_id)
        _id = ""

        if destination is None:
            return _id

        new_collection = Collection(collection_name)
        destination.add(new_collection)
        new_collection.update()
        destination.update()
        _id = new_collection.id

        return _id

    def list_datasets(self) -> DatasetMetaDataVec:
        data = []
        for dataset in self._bf.datasets():
            dmd = DatasetMetaData(dataset_id=dataset.id,
                                  display_name=dataset.name)
            data.append(dmd)

        return data
Exemple #4
0
def lineLength(ptName,
               ch,
               startTime=None,
               endTime=None,
               append=False,
               layerName=LL_LAYER_NAME):
    '''
    Runs the line length detector.

    ch: channels to annotate
    startTime: time (usec) to start from. Default value (None) starts from the beginning of the timeseries
    append: Whether to append or overwrite the line length annotation layer
    layerName: name of layer to write to
    '''

    freq = FREQs.get(ptName, DEFAULT_FREQ)
    threshold = LL_THRESHOLDS[ptName]

    bf = Blackfynn()
    ts = bf.get(TS_IDs[ptName])

    # Make sure startTime and endTime are valid
    if startTime is not None:
        if startTime < ts.start:
            print(
                'Warning: startTime', startTime,
                'is earlier than the beginning of the Timeseries. Ignoring startTime argument...'
            )
            startTime = None
        elif startTime > ts.end:
            print(
                'Warning: startTime', startTime,
                'is after the end of the Timeseries. No data will be analyzed.'
            )
            return

    if endTime is not None:
        if endTime > ts.end:
            print(
                'Warning: endTime', endTime,
                'is later than the end of the Timeseries. Ignoring endTime argument...'
            )
            endTime = None
        elif endTime < ts.start:
            print(
                'Warning: endTime', endTime,
                'is before the beginning the Timeseries. No data will be analyzed.'
            )
            return

    # Get segments
    args = {'start': startTime, 'stop': endTime}
    if startTime is None:
        args['start'] = ts.start
    if endTime is None:
        args['stop'] = ts.end
    segments = ts.segments(**args)

    # edit segments so that it starts at startTime
    if startTime is None:
        startTime = segments[0][0]
    else:
        try:
            i = next(i for i, (a, b) in enumerate(segments) if b > startTime)
            segments[:i] = []
        except StopIteration:
            pass
        startTime = max(segments[0][0], startTime)
        print('start time:', startTime)
        segments[0] = (startTime, segments[0][1])

    # Same thing with end time:
    if endTime is None:
        endTime = segments[-1][1]
    else:
        l = len(segments)
        try:
            i = next(l - 1 - i for i, (a, b) in enumerate(reversed(segments))
                     if a < endTime)
            segments[i + 1:] = []
        except StopIteration:
            pass
        endTime = min(segments[-1][1], endTime)
        print('end time:', endTime)
        segments[-1] = (segments[-1][0], endTime)

    try:
        # Get layer if it already exists
        layer = ts.get_layer(layerName)
        if append:
            print("Appending to layer '%s'" % layerName)
        else:
            print("Overwriting layer '%s'" % layerName)
            layer.delete()
            layer = ts.add_layer(layerName)
    except:
        print("Creating layer '%s'" % layerName)
        layer = ts.add_layer(layerName)

    pos = segments[0][0]
    for seg in segments:
        pos = max(pos, seg[0])
        while pos < seg[1]:
            try:
                clip = ts.get_data(start=pos,
                                   length=LL_CLIP_LENGTH,
                                   channels=ch,
                                   use_cache=False)
                # caching disabled to prevent database disk image errors
                # note: actual clip length may be shorter than LL_CLIP_LENGTH
            except RequestException as e:
                # catch Blackfynn server errors
                print('Server error (will retry):', e)
                sleep(2)
                continue
            except Exception as e:
                print('Pull failed:', e)
                pos += LL_CLIP_LENGTH
                continue
            if clip.empty or clip.isnull().all().any():
                # skip clip if a channel is missing data
                pos += LL_CLIP_LENGTH
                continue
            if clip.shape[0] / freq * 1000000 < LL_CLIP_LENGTH / 2:
                # skip clip if it's less than half of max clip length
                pos += LL_CLIP_LENGTH
                continue

            startTime = clip.iloc[
                0].name.value / 1000  # convert to Unix epoch time, in usecs
            endTime = clip.iloc[-1].name.value / 1000
            clip.fillna(0, inplace=True)  # replace NaNs with zeros

            clip = clip.transpose().values
            l = _length(clip)

            if l > threshold:
                print('+ %f (%d, %d)' % (l, startTime, endTime))
                sys.stdout.flush()
                layer.insert_annotation('Possible seizure',
                                        start=startTime,
                                        end=endTime)
            else:
                print('- %f (%d, %d)' % (l, startTime, endTime))
                sys.stdout.flush()

            pos += LL_CLIP_LENGTH
Exemple #5
0
import sys
import re

from blackfynn import Blackfynn
from settings import VIDEO_DIR_IDs, PL_ROOT


def makeVideoLinkFile(vidList, url_root, filename):
    'Write list of video URLs and timestamps to file filename'
    #get timestamp-get url

    with open(filename, 'w') as f:
        for video in vidList:
            #Retrieve timestamp string in video name
            timestamp = re.search('\d{14}', video.name)
            if timestamp != None:
                f.write('%s : %s\n' %
                        (str(timestamp.group(0)), url_root + video.id))

        print(('video urls written to %s.' % (filename)))


if __name__ == '__main__':
    ptName = sys.argv[1]
    vidCollectionID = VIDEO_DIR_IDs[ptName]

    bf = Blackfynn()
    vids = bf.get(vidCollectionID)
    url_root = 'https://app.blackfynn.io/' + bf.context.id + '/datasets/' + vids.dataset + '/viewer/'
    makeVideoLinkFile(vids, url_root,
                      PL_ROOT + '/videoLinks/%s_vidMap.txt' % ptName)
Exemple #6
0
def pipeline(ptName, annotating=True, startTime=None, endTime=None, bf=None):
    ch = CHANNELS.get(ptName, None)
    freq = FREQs.get(ptName, DEFAULT_FREQ)

    ### Get patient-specific settings
    if bf is None:  # WORKAROUND: see cron.py
        bf = Blackfynn()
    ts = bf.get(TS_IDs[ptName])

    while True:
        try:
            segments = ts.segments()
            break
        except RequestException:
            sleep(2)
            continue

    layer = ts.get_layer(GOLD_STD_LAYERS[ptName])
    layerName = layer.name

    ### Pull ictal and interictal annotation times
    print("Reading annotations from layer '%s'..." % layerName)
    annotDir = PL_ROOT + '/annotations'
    makeDir(annotDir)
    ictals = getIctalAnnots(layer)
    makeAnnotFile(ictals, '%s/%s_annotations.txt' % (annotDir, ptName))

    print('Generating interictal annotations...')
    interictals = getInterictalAnnots(ictals, segments)
    makeAnnotFile(interictals, '%s/%s_interictal_annotations.txt' % \
                               (annotDir, ptName))
    print('')
    sys.stdout.flush()

    ### Pull clips
    clipDir = PL_ROOT + '/clips/' + ptName
    makeDir(clipDir)
    print('Pulling ictal clips...')
    pullClips('%s/%s_annotations.txt' % (annotDir, ptName),
              'ictal',
              ts,
              clipDir,
              ch,
              limit=TRAINING_SZ_LIMIT)
    print('Pulling interictal clips...')
    pullClips('%s/%s_interictal_annotations.txt' % (annotDir, ptName),
              'interictal', ts, clipDir, ch)
    print('')
    sys.stdout.flush()

    ### Slice and preprocess clips
    algoDataDir = PL_ROOT + '/seizure-data/' + ptName
    makeDir(algoDataDir)
    print('Preparing ictal data for classifier...')
    sliceClips(clipDir, 'ictal', freq, ptName)
    print('Preparing interictal data for classifier...')
    sliceClips(clipDir, 'interictal', freq, ptName)
    print('')
    sys.stdout.flush()

    ### Prepare classifier
    #with open('liveAlgo/targets.json', 'w') as f:
    #    json.dump([ptName], f)

    ### Train classifier
    print('Training classifier...')
    train('train_model', target=ptName)
    print('')
    sys.stdout.flush()

    ### Compute cross-Validation scores
    print('Computing cross-validation scores...')
    train('cv', target=ptName)
    print('')
    sys.stdout.flush()

    if annotating:
        ### Make predictions on entire timeseries
        print('Testing on entire time series...')

        try:
            # Delete layer if it already exists
            layer = ts.get_layer(PL_LAYER_NAME)
            layer.delete()
        except:
            pass
        finally:
            layer = ts.add_layer(PL_LAYER_NAME)

        testTimeSeries(ts, layer, ptName, startTime, endTime)
Exemple #7
0
            # Test segments:
            clearDir(PL_ROOT + '/seizure-data/' + ptName)
            # Cached classifier data
            for fname in os.listdir(PL_ROOT + '/data-cache'):
                if (fname.startswith('data_test_' + ptName)
                        or fname.startswith('predictions_' + ptName)):
                    os.remove(PL_ROOT + '/data-cache/' + fname)


if __name__ == '__main__':
    ptName = sys.argv[1]
    startTime = int(sys.argv[2])
    kwargs = sys.argv[3:]
    annotating = ('annotate' in kwargs)

    try:
        endTime = int(sys.argv[3])
    except (ValueError, IndexError):
        endTime = None

    bf = Blackfynn()
    ts = bf.get(TS_IDs[ptName])
    layer = ts.add_layer(PL_LAYER_NAME)

    print('Testing on patient:', ptName)
    if annotating:
        print('Annotating enabled')
    else:
        print('Annotating disabled')
    testTimeSeries(ts, layer, ptName, startTime, endTime, annotating)
Exemple #8
0
def blackfynn_cli():
    args = docopt(__doc__)

    if args['version']:
        print "version: {}".format(blackfynn.__version__)

    email = args['--user'] if args['--user'] is not None else settings.api_user
    passw = args['--pass'] if args['--pass'] is not None else settings.api_pass
    host = args['--host'] if args['--host'] is not None else settings.api_host
    org = args['--org']

    try:
        bf = Blackfynn(email=email, password=passw, host=host)
    except:
        print "Unable to connect to to Blackfynn using specified user/password."
        return

    if args['orgs']:
        for o in bf.organizations():
            print " * {} (id: {})".format(o.name, o.id)

    if org is not None:
        try:
            bf.set_context(org)
        except:
            print 'Error: Unable to set context to "{}"'.format(org)
            return

    if args['show']:
        item = bf.get(args['<item>'])
        print item
        if hasattr(item, 'items'):
            print "CONTENTS:"
            for i in item.items:
                print " * {}".format(i)
        if hasattr(item, 'channels'):
            print "CHANNELS:"
            for ch in item.channels:
                print " * {} (id: {})".format(ch.name, ch.id)

    elif args['search']:
        terms = ' '.join(args['<term>'])
        results = bf._api.search.query(terms)
        if len(results) == 0:
            print "No Results."
        else:
            for r in results:
                print " * {}".format(r)

    elif args['create']:
        if args['collection']:
            dest = args['<destination>']
            name = args['<name>']
            c = Collection(name)
            parent = bf.get(dest)
            parent.add(c)
            print c
        elif args['dataset']:
            name = args['<name>']
            ds = bf.create_dataset(name)
            print ds
        else:
            print "Error: creation for object not supported."
            return

    elif args['delete']:
        item = bf.get(args['<item>'])
        if isinstance(item, Dataset):
            print "Error: cannot delete dataset"
            return
        elif not isinstance(item, BaseNode):
            print "Error: cannot delete item"
            return
        bf.delete(item)

    elif args['upload']:
        files = args['<file>']
        dest = args['<destination>']
        recursively_upload(bf, dest, files)

    elif args['append']:
        files = args['<file>']
        dest = args['<destination>']
        bf._api.io.upload_files(dest,
                                files,
                                append=True,
                                display_progress=True)

    elif args['datasets']:
        print "Datasets: "
        for ds in bf.datasets():
            print " - {} (id: {})".format(ds.name, ds.id)

    elif args['dataset']:
        ds = bf.get(args['<dataset>'])
        if args['collaborators']:
            if args['<action>'] == 'ls':
                resp = ds.collaborators()
                print " - Users"
                for u in resp['users']:
                    print "   - email:{} id:{}".format(u.email, u.id)
                print " - Groups"
                for g in resp['groups']:
                    print "   - name:{} id:{}".format(g.name, g.id)
            elif args['<action>'] == 'add':
                ids = args['<action-args>']
                if len(ids) == 0:
                    print "Error: No ids specified"
                    sys.exit(1)
                resp = ds.add_collaborators(*ids)
                print_collaborator_edit_resp(resp)
            elif args['<action>'] == 'rm':
                ids = args['<action-args>']
                if len(ids) == 0:
                    print "Error: No ids specified"
                    sys.exit(1)
                resp = ds.remove_collaborators(*ids)
                print_collaborator_edit_resp(resp)
            else:
                print "Error: invalid dataset collaborators command. Valid commands are 'ls', 'add' or 'rm'"
        else:
            print "Error: invalid dataset command. Valid commands are 'collaborators'"

    elif args['env']:
        print "# Blackfynn environment"
        print "API Location:  {}".format(host)
        print "Streaming API: {}".format(settings.streaming_api_host)
        print "User:          {}".format(email)
        print "Organization:  {} (id: {})".format(bf.context.name,
                                                  bf.context.id)
Exemple #9
0
        dfs.append(df.iloc[i:])

        # save clip(s)
        for df in dfs:
            time = (df.iloc[0].name.value / 1000,
                    df.iloc[-1].name.value / 1000)
            clipTimes.append(time)
            array = df.transpose().values
            outfile = '%s/%s%d.hkl' % (outDir, outfile_prefix, num_saved + 1)
            hickle.dump(array, outfile)
            num_saved += 1

    print(('%d clips saved.' % num_saved))
    return clipTimes


if __name__ == '__main__':
    annotFile = sys.argv[1]
    clipType = sys.argv[2]
    tsID = sys.argv[3]
    outDir = sys.argv[4].rstrip('/')

    try:
        limit = int(sys.argv[5])
    except IndexError:
        limit = None

    bf = Blackfynn()
    ts = bf.get(tsID)
    pullClips(annotFile, clipType, ts, outDir, limit=limit)
Exemple #10
0
"""
Usage: collection_watcher.py --collection=<collection_id> --threshold=<hours> --notify=<email>
"""

from datetime import datetime
import time
from docopt import docopt

from blackfynn import Blackfynn
from blackfynn import Collection

arguments = docopt(__doc__)

bf = Blackfynn()
collection = bf.get(arguments['--collection'])
current_timestamp = time.time() * 1000

for item in collection.items:
    milliseconds_old = current_timestamp - int(
        item.get_property('created').value)
    hours_old = (milliseconds_old / (1000 * 60 * 60)) % 24
    if hours_old > int(arguments['--threshold']):
        print "send notification"
Exemple #11
0
def lineLength(ptName,
               startTime=None,
               endTime=None,
               append=False,
               layerName=LL_MA_LAYER_NAME):
    '''
    Runs the line length detector.

    ch: channels to annotate
    startTime: time (usec) to start from. Default value (None) starts from the beginning
    append: Whether to append to (or otherwise overwrite) the line length annotation layer
    layerName: name of layer to write to

    Returns: endTime (the end of the last full long-term window)
    '''
    global shortWindow, longWindow, freq, ch
    longWindow = LL_LONG_WINDOWS.get(ptName, LL_LONG_WINDOW_DEFAULT)
    shortWindow = LL_SHORT_WINDOWS.get(ptName, LL_SHORT_WINDOW_DEFAULT)
    freq = FREQs.get(ptName, DEFAULT_FREQ)
    ch = CHANNELS.get(ptName, None)

    bf = Blackfynn()
    ts = bf.get(TS_IDs[ptName])

    # Make sure startTime and endTime are valid
    if startTime is not None:
        if startTime < ts.start:
            print(
                'Warning: startTime', startTime,
                'is before the beginning of the Timeseries. Starting from the beginning...'
            )
            startTime = None
        elif startTime > ts.end:
            print('Warning: startTime', startTime,
                  'is after the end of the Timeseries. Exiting...')
            return ts.end

    if endTime is not None:
        if endTime > ts.end:
            print(
                'Warning: endTime', endTime,
                'is after the end of the Timeseries. Stopping at the end...')
            endTime = None
        elif endTime < ts.start:
            print('Warning: endTime', endTime,
                  'is before the beginning the Timeseries. Exiting...')
            return ts.start

    # Get/create annotation layer
    try:
        layer = ts.get_layer(layerName)
        if append:
            print("Appending to layer '%s'" % layerName)
        else:
            print("Overwriting layer '%s'" % layerName)
            layer.delete()
            layer = ts.add_layer(layerName)
    except:
        print("Creating layer '%s'" % layerName)
        layer = ts.add_layer(layerName)

    # Find the long-term windows to start and end from
    windowStart = ts.start
    windowEnd = windowStart + longWindow
    if startTime:
        while windowEnd < startTime:
            windowStart = windowEnd
            windowEnd += longWindow
    else:
        startTime = ts.start
    if not endTime:
        endTime = ts.end

    # Make sure segments list starts at startTime and ends at endTime
    segments = ts.segments(startTime, endTime + 1)
    if not segments:
        print('No data found between %d and %d.' % (startTime, endTime), \
              ' Exiting...')
        return endTime
    startTime = max(segments[0][0], startTime)
    print('start time:', startTime)
    segments[0] = (startTime, segments[0][1])

    endTime = min(segments[-1][1], endTime)
    print('end time:', endTime)
    segments[-1] = (segments[-1][0], endTime)

    # Go through each long-term window
    while windowStart < endTime and windowEnd <= ts.end:
        # Calculate trend and threshold
        try:
            trend, shortClips = _trend(ts, windowStart, windowEnd)
        except RequestException:
            # Workaround in case of server errors
            sleep(2)
            continue
        if trend is None:
            print('skipping long window (no clips)...')
            sys.stdout.flush()
            windowStart = windowEnd
            windowEnd += longWindow
            continue

        # If using a custom start time, trim shortClips
        # (This should only ever happen with the 1st long-term window)
        if windowStart < startTime:
            try:
                i = next(i for i, c in enumerate(shortClips)
                         if c['end'] > startTime)
                shortClips[:i] = []
            except StopIteration:
                pass
            shortClips[0]['start'] = max(shortClips[0]['start'], startTime)
            # Delete 1st clip if it's too short (less than half of shortWindow):
            if shortClips[0]['end'] - shortClips[0]['start'] < shortWindow / 2:
                shortClips.pop(0)

        # Do the same thing with endTime
        # (Should only ever happen to the last long-term window)
        if windowEnd > endTime:
            l = len(shortClips)
            try:
                i = next(l - 1 - i for i, c in enumerate(reversed(shortClips))
                         if c['start'] < endTime)
                shortClips[i + 1:] = []
            except StopIteration:
                pass
            # Delete last clip if it's not long enough
            lastClip = shortClips.pop(-1)
            if lastClip['end'] - lastClip['start'] >= shortWindow / 2:
                lastClip['end'] = min(lastClip['end'], endTime)
                shortClips.append(lastClip)
            else:
                pass

        # Annotate and/or print predictions
        threshold = LL_MA_THRESHOLDS[ptName] * trend
        for clip in shortClips:
            l = clip['length']
            if l > threshold:
                print('+ %f (%d, %d)' % (l, clip['start'], clip['end']))
                layer.insert_annotation('Possible seizure',
                                        start=clip['start'],
                                        end=clip['end'])
            else:
                print('- %f (%d, %d)' % (l, clip['start'], clip['end']))
            sys.stdout.flush()

        # Go to next long term window
        windowStart = windowEnd
        windowEnd += longWindow
    return min(windowStart, endTime)