def __init__(self, session):
"""
Base class to be used by all API components.
"""
# api session
self.session = session
self._logger = log.get_logger('blackfynn.api')
def test_log_level_based_on_env():
import blackfynn.log as log
# setting env var should change logging level
os.environ['BLACKFYNN_LOG_LEVEL'] = 'WARN'
warn_log = log.get_logger('warn_log')
assert warn_log.getEffectiveLevel() == logging.WARN
def test_log_level_based_on_env():
from blackfynn import log
# setting env var should change logging level
os.environ["BLACKFYNN_LOG_LEVEL"] = "WARN"
warn_log = log.get_logger("warn_log")
assert warn_log.getEffectiveLevel() == logging.WARN
def __init__(self, func, uri, *args, **kwargs):
self._func = func
self._uri = uri
self._args = args
self._kwargs = kwargs
self._response = None
self._logger = log.get_logger('blackfynn.base.BlackfynnRequest')
def __init__(self, ts, name, max_segment_size, aws_region):
self.name = name
self.max_segment_size = max_segment_size
self.conn = boto3.client('kinesis', region_name=aws_region)
# reference time-series
self.ts = ts
# cache channels
self._channels = ts.channels
self.registered = False
self._logger = log.get_logger('blackfynn.streaming.TimeSeriesStream')
def __init__(self,
profile=None,
api_token=None,
api_secret=None,
host=None,
streaming_host=None,
concepts_host=None,
env_override=True,
**overrides):
self._logger = log.get_logger("blackfynn.client.Blackfynn")
overrides.update({
k: v
for k, v in {
'api_token': api_token,
'api_secret': api_secret,
'api_host': host,
'api_streaming_host': streaming_host,
'api_concepts_host': concepts_host,
}.items() if v != None
})
self.settings = Settings(profile, overrides, env_override)
if self.settings.api_token is None:
raise Exception(
'Error: No API token found. Cannot connect to Blackfynn.')
if self.settings.api_secret is None:
raise Exception(
'Error: No API secret found. Cannot connect to Blackfynn.')
# direct interface to REST API.
self._api = ClientSession(self.settings)
# account
try:
self._api.authenticate()
except Exception as e:
raise e
self._api.register(CoreAPI, OrganizationsAPI, DatasetsAPI, FilesAPI,
DataAPI, PackagesAPI, TimeSeriesAPI, TabularAPI,
SecurityAPI, ComputeAPI, SearchAPI, IOAPI,
LedgerAPI, UserAPI, ModelsAPI, RecordsAPI,
ModelRelationshipsAPI,
ModelRelationshipInstancesAPI)
self._api._context = self._api.organizations.get(
self._api._organization)
def __init__(self, settings):
self._host = settings.api_host
self._api_token = settings.api_token
self._api_secret = settings.api_secret
self._jwt = settings.jwt
self._logger = log.get_logger('blackfynn.base.ClientSession')
self._session = None
self._token = None
self._secret = None
self._context = None
self._organization = None
self.profile = None
self.settings = settings
import errno
import json
import os
import socket
import subprocess
import sys
from collections import OrderedDict
from time import sleep
import semver
from blackfynn.log import get_logger
from blackfynn.models import Collection, DataPackage, Dataset
logger = get_logger("blackfynn.agent")
try:
from websocket import create_connection
except ModuleNotFoundError:
logger.warn(
"websocket-client is not installed - uploading with the Agent will not work"
)
MINIMUM_AGENT_VERSION = semver.parse_version_info("0.2.2")
DEFAULT_LISTEN_PORT = 11235
class AgentError(Exception):
pass
import boto3
import botocore
import requests
#from requests.exceptions import HTTPError, ConnectionError
from joblib import Parallel, delayed
if 'BLACKFYNN_LOG_LEVEL' not in os.environ:
# silence agent import warning
os.environ['BLACKFYNN_LOG_LEVEL'] = 'CRITICAL'
from blackfynn import log as _bflog
# blackfynn.log sets logging.basicConfig which pollutes logs from
# other programs that are sane and do not use the root logger
# so we have to undo the damage done by basic config here
# we add the sparcur local handlers back in later
from sparcur.utils import log, silence_loggers
for __bflog in (_bflog.get_logger(), _bflog.get_logger("blackfynn.agent")):
silence_loggers(__bflog)
__bflog.addHandler(log.handlers[0])
from blackfynn import Blackfynn, Collection, DataPackage, Organization, File
from blackfynn import Dataset, BaseNode
from blackfynn import base as bfb
from blackfynn.api import agent
from blackfynn.api import transfers
from blackfynn.api.data import PackagesAPI
from pyontutils.utils import Async, deferred
from pyontutils.iterio import IterIO
from sparcur import monkey
from sparcur import exceptions as exc
from sparcur.utils import BlackfynnId, ApiWrapper, make_bf_cache_as_classes
import os
import sys
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor, as_completed
from blackfynn import log
from blackfynn.api.agent import agent_upload, validate_agent_installation
from blackfynn.api.base import APIBase
from blackfynn.models import Collection, DataPackage, Dataset, TimeSeries
# GLOBAL
UPLOADS = {}
logger = log.get_logger("blackfynn.api.transfers")
def check_files(files):
for f in files:
if not os.path.exists(f):
raise Exception("File {} not found.".format(f))
class IOAPI(APIBase):
"""
Input/Output interface.
"""
name = "io"
bf move [options] <item> [<destination>]
global options:
-h --help Show help
--dataset=<dataset> Use specified dataset (instead of your current working dataset)
--profile=<name> Use specified profile (instead of default)
'''
from docopt import docopt
from blackfynn import Dataset
from blackfynn.models import Collection
import blackfynn.log as log
from cli_utils import get_item
logger = log.get_logger('blackfynn.cli.bf_move')
def main(bf):
args = docopt(__doc__)
item = get_item(args['<item>'], bf)
if args['<destination>']:
destination = get_item(args['<destination>'], bf)
else:
# item will be moved to to the top of its containing dataset
destination = None
if destination is None or isinstance(destination, Collection):
try:
import sqlite3
import time
from datetime import datetime
from glob import glob
from itertools import groupby
from blackfynn import log
from blackfynn.extensions import numpy as np
from blackfynn.extensions import pandas as pd
from blackfynn.extensions import require_extension
from blackfynn.models import DataPackage, TimeSeriesChannel
from blackfynn.utils import usecs_since_epoch, usecs_to_datetime
from .cache_segment_pb2 import CacheSegment
logger = log.get_logger("blackfynn.cache")
def filter_id(some_id):
return some_id.replace(":", "_").replace("-", "_")
def remove_old_pages(cache, mbdiff):
# taste the rainbow!
n = int(1.5 * ((mbdiff * 1024 * 1024) / 100) / cache.page_size) + 5
# 2. Delete some pages from cache
with cache.index_con as con:
logger.debug("Cache - removing {} pages...".format(n))
# find the oldest/least accessed pages
q = """
def test_default_log_level():
import blackfynn.log as log
# default log level should be INFO
base_log = log.get_logger('base_log')
assert base_log.getEffectiveLevel() == logging.INFO
def test_default_log_level():
from blackfynn import log
# default log level should be INFO
base_log = log.get_logger("base_log")
assert base_log.getEffectiveLevel() == logging.INFO
import uuid
from concurrent.futures import ThreadPoolExecutor, as_completed
import boto3
from boto3.s3.transfer import S3Transfer
from botocore.client import Config
import blackfynn.log as log
from blackfynn.api.agent import validate_agent_installation, agent_upload
from blackfynn.api.base import APIBase
from blackfynn.models import Collection, DataPackage, Dataset, TimeSeries
# GLOBAL
UPLOADS = {}
logger = log.get_logger('blackfynn.api.transfers')
def check_files(files):
for f in files:
if not os.path.exists(f):
raise Exception("File {} not found.".format(f))
class ProgressPercentage(object):
def __init__(self, filename, upload_session_id):
self._filename = filename
self._size = float(os.path.getsize(filename))
self._seen_so_far = 0
self._lock = threading.Lock()
self._done = (self.progress == 1)
import json
import os
import socket
import subprocess
import sys
from collections import OrderedDict
from time import sleep
import semver
from future.utils import raise_from
from websocket import create_connection
from blackfynn.log import get_logger
from blackfynn.models import Collection, Dataset, DataPackage
logger = get_logger('blackfynn.agent')
MINIMUM_AGENT_VERSION = semver.parse_version_info("0.2.2")
DEFAULT_LISTEN_PORT = 11235
class AgentError(Exception):
pass
def agent_cmd():
if sys.platform == 'darwin':
return '/usr/local/opt/blackfynn/bin/blackfynn_agent'
elif sys.platform.startswith('linux'):
return '/opt/blackfynn/bin/blackfynn_agent'
import sqlite3
import platform
import numpy as np
import pandas as pd
from glob import glob
import multiprocessing as mp
from itertools import groupby
from datetime import datetime
# blackfynn-specific
from blackfynn.utils import usecs_to_datetime, usecs_since_epoch
from blackfynn.models import DataPackage, TimeSeriesChannel
import blackfynn.log as log
from .cache_segment_pb2 import CacheSegment
logger = log.get_logger('blackfynn.cache')
def filter_id(some_id):
return some_id.replace(':', '_').replace('-', '_')
def remove_old_pages(cache, mbdiff):
# taste the rainbow!
n = int(1.5 * ((mbdiff * 1024 * 1024) / 100) / cache.page_size) + 5
# 2. Delete some pages from cache
with cache.index_con as con:
logger.debug("Cache - removing {} pages...".format(n))
# find the oldest/least accessed pages
q = """
