def test_token(self):
api = Client(auth={"token": self.token},
address=self.uri,
config=ClientConfig(stream=False, response="json"))
result = api.query(query=self.query)
self.assertIsNotNone(result)
self.assertTrue(len(json.loads(result)['object']) > 0)
def test_query_id(self):
api = Client(key=self.key, secret=self.secret, url=self.uri)
result = api.query(query_id=self.query_id,
stream=False, response="json/compact")
self.assertIsNotNone(result)
self.assertNotEqual(result, {})
self.assertEqual(type(len(json.loads(result)['object'])), type(1))
def test_query_from_seven_days(self):
api = Client(key=self.key, secret=self.secret, url=self.uri)
result = api.query(query=self.query,
dates={'from': 'now()-7*day()', 'to': 'now()'},
stream=False, response="json")
self.assertIsNotNone(result)
self.assertEqual(len(json.loads(result)['object']), 1)
def _query_stream(self, response_type, result,
processor):
client = Client(retries=0, config={'address': "URI", "stream": True,
"response": response_type,
"processor": processor})
client._make_request = MagicMock(return_value=result)
return client.query()
def _query_stream(self, response_type, result,
keepAliveToken=DEFAULT_KEEPALIVE_TOKEN):
client = Client(retries=0, config={'address': "URI", "stream": True,
"response": response_type,
"keepAliveToken": keepAliveToken})
client._make_request = MagicMock(return_value=result)
return client.query()
def test_query_id(self):
api = Client(auth={"key": self.key, "secret": self.secret},
address=self.uri,
config=ClientConfig(stream=False, response="json"))
result = api.query(query_id=self.query_id)
self.assertIsNotNone(result)
self.assertNotEqual(result, {})
self.assertEqual(type(len(json.loads(result)['object'])), type(1))
def test_query_from_seven_days(self):
api = Client(auth={"key": self.key, "secret": self.secret},
address=self.uri,
config=ClientConfig(stream=False, response="json"))
result = api.query(query=self.query,
dates={'from': 'now()-7*day()', 'to': 'now()'})
self.assertIsNotNone(result)
self.assertEqual(len(json.loads(result)['object']), 1)
def test_stream_query(self):
api = Client(auth={"key": self.key, "secret": self.secret},
address=self.uri,
config=ClientConfig(response="json/simple"))
result = api.query(query=self.query)
self.assertTrue(isinstance(result, types.GeneratorType))
result = list(result)
self.assertEqual(len(result), 1)
def test_query_from_fixed_dates(self):
api = Client(self.key, self.secret, self.uri)
result = api.query(query=self.query,
dates={'from': strftime("%Y-%m-%d", gmtime()),
'to': strftime("%Y-%m-%d %H:%M:%S", gmtime())},
stream=False, response="json")
self.assertIsNotNone(result)
self.assertEqual(len(json.loads(result)['object']), 1)
def test_pragmas_not_comment_free(self):
"""Test the api when the pragma comment.free is not used"""
api = Client(key=self.key, secret=self.secret, url=self.uri,
user=self.user, app_name=self.app_name, stream=False)
result = api.query(
query=self.query,
response="json")
self.assertIsNotNone(result)
self.assertEqual(len(json.loads(result)['object']), 1)
def __init__(self,
profile='default',
api_key=None,
api_secret=None,
end_point=None,
oauth_token=None,
jwt=None,
credential_path=None,
timeout=None,
retries=1,
verify=True,
user=None,
app_name=None,
**kwargs):
self.profile = profile
self.api_key = api_key
self.api_secret = api_secret
self.end_point = end_point
self.oauth_token = oauth_token
self.jwt = jwt
if credential_path is None:
self.credential_path = Path.home() / '.devo_credentials'
else:
self.credential_path = Path(credential_path).expanduser().resolve()
if not (self.end_point and (self.oauth_token or self.jwt or
(self.api_key and self.api_secret))):
self._read_profile()
if not (self.end_point and (self.oauth_token or self.jwt or
(self.api_key and self.api_secret))):
raise Exception(
'End point and either API keys or OAuth Token must be specified or in ~/.devo_credentials'
)
config = kwargs
config.update({
'auth': {
'key': self.api_key,
'secret': self.api_secret,
'token': self.oauth_token,
'jwt': self.jwt
},
'address': self.end_point
})
self.client = Client(config=config)
self.client.timeout = timeout
self.client.retries = retries
self.client.verify = verify
if user:
self.client.config.set_user(user)
if app_name:
self.client.config.set_app_name(app_name)
def test_query(self):
config = ClientConfig(stream=False, response="json")
api = Client(auth={"key": self.key, "secret": self.secret},
address=self.uri,
config=config)
result = api.query(query=self.query)
self.assertIsNotNone(result)
self.assertTrue(len(json.loads(result)['object']) > 0)
def test_query_from_fixed_dates(self):
api = Client(auth={"key": self.key, "secret": self.secret},
address=self.uri,
config=ClientConfig(stream=False, response="json"))
result = api.query(query=self.query,
dates={'from': strftime("%Y-%m-%d", gmtime()),
'to': strftime(
"%Y-%m-%d %H:%M:%S",
gmtime())})
self.assertIsNotNone(result)
self.assertEqual(len(json.loads(result)['object']), 1)
def test_query_yesterday_to_today(self):
api = Client(self.key, self.secret, self.uri)
result = api.query(query=self.query,
dates={
'from': 'yesterday()',
'to': 'today()'
},
stream=False,
response="json")
self.assertIsNotNone(result)
self.assertEqual(len(json.loads(result)['object']), 1)
def test_pragmas(self):
"""Test the api when the pragma comment.free is used"""
api = Client(auth={"key": self.key, "secret": self.secret},
address=self.uri,
config=ClientConfig(response="json",
stream=False))
api.config.set_user(user=self.user)
api.config.set_app_name(app_name=self.app_name)
result = api.query(query=self.query, comment=self.comment)
self.assertIsNotNone(result)
self.assertEqual(len(json.loads(result)['object']), 1)
def _query_no_stream(self, response_type, result,
keepAliveToken=DEFAULT_KEEPALIVE_TOKEN):
client = Client(retries=0, config={'address': "URI", "stream": False,
"response": response_type,
"keepAliveToken": keepAliveToken})
with mock.patch(
'devo.api.Client._make_request') as patched_make_request:
if isinstance(result, str):
patched_make_request.return_value.text = result
else:
patched_make_request.return_value.content = result
return client.query()
class TestApi(unittest.TestCase):
def setUp(self):
self.client = Client(
config={
'key':
os.getenv('DEVO_API_KEY', None),
'secret':
os.getenv('DEVO_API_SECRET', None),
'address':
os.getenv('DEVO_API_ADDRESS', 'https://apiv2-us.devo.com/'),
"stream":
False,
"destination": {
"type": "donothing",
"params": {
"friendlyName": "devo-sdk-api-test"
}
}
})
@unittest.skip("temporarily disabled due to Query API bug")
def test_jobs_cycle(self):
self.client.query(query="from demo.ecommerce.data select *",
dates={'from': '2018-01-01 00:00:00'})
# Get all jobs
result = self.client.get_jobs()
self.assertTrue(result['object'])
# Get job by name
result = self.client.get_jobs(name="devo-sdk-api-test")
self.assertTrue(result['object'])
# Get job by type
result = self.client.get_jobs(job_type="donothing")
self.assertTrue(result['object'])
# Get job by name and type
result = self.client.get_jobs(name="devo-sdk-api-test",
job_type="donothing")
self.assertTrue(result['object'])
job_id = result['object'][0]['id']
# Stop job by id
result = self.client.stop_job(job_id)
self.assertEqual(result['object']['status'], "STOPPED")
# Start job by id
result = self.client.start_job(job_id)
self.assertEqual(result['object']['status'], "RUNNING")
# Delete job by id
result = self.client.remove_job(job_id)
self.assertEqual(result['object']['status'], "REMOVED")
def configure(args):
"""
Load CLI configuration
:param args: args from files, launch vars, etc
:return: Client API Object and Config values in array
"""
config = Configuration()
if args.get('config') != "~/.devo.json":
config.load_json(args.get('config'), 'api')
config.mix(dict(args))
if "key" not in args.keys() and "api" not in args.keys() \
and "token" not in args.keys():
config.set("key", os.environ.get('DEVO_API_KEY', None))
config.set("secret", os.environ.get('DEVO_API_SECRET', None))
if "url" not in args.keys():
config.set("url", os.environ.get('DEVO_API_URL', None))
if not config.keys("key") and not config.keys("api") \
and not config.keys("token") \
and os.path.exists("~/.devo.json"):
config.load_default_json('api')
config.keys('from')
config.keys('to')
# Try to compose the api
api = None
try:
api = Client.from_config(config.get())
except DevoClientException as error:
print_error(str(error), show_help=True)
return api, config.get()
def test_from_config(self):
api = Client.from_config(
{'key': self.key, 'secret': self.secret, 'url': self.uri,
'user': self.user, 'app_name': self.app_name}
)
self.assertTrue(isinstance(api, Client))
def configure(args):
"""
Load CLI configuration
:param args: args from files, launch vars, etc
:return: Client API Object and Config values in array
"""
config = Configuration()
try:
if args.get('config'):
config.load_config(args.get('config'), 'api')
if args.get('env'):
config.set("key", os.environ.get('DEVO_API_KEY', None))
config.set("secret", os.environ.get('DEVO_API_SECRET', None))
config.set("url", os.environ.get('DEVO_API_URL', None))
config.set("user", os.environ.get('DEVO_API_USER', None))
config.set("comment", os.environ.get('DEVO_API_COMMENT', None))
if args.get('default'):
config.load_default_config(section="api")
finally:
config.mix(dict(args))
conf = config.get()
# Try to compose the api
api = None
try:
api = Client.from_config(conf)
except DevoClientException as error:
print_error(str(error), show_help=True)
return api, conf
def test_stream_query_no_results_bounded_dates(self):
api = Client(auth={
"key": self.key,
"secret": self.secret
},
address=self.uri,
config=ClientConfig(response="json/simple"),
retries=3)
result = api.query(query=self.query_no_results,
dates={
'from': '1h',
'to': 'now()'
})
self.assertTrue(isinstance(result, types.GeneratorType))
result = list(result)
self.assertEqual(len(result), 0)
def test_from_dict(self):
api = Client(config=
{'key': self.key, 'secret': self.secret, 'address': self.uri,
'user': self.user, 'app_name': self.app_name}
)
self.assertTrue(isinstance(api, Client))
def setUp(self):
self.client = Client(
config={
'key':
os.getenv('DEVO_API_KEY', None),
'secret':
os.getenv('DEVO_API_SECRET', None),
'address':
os.getenv('DEVO_API_ADDRESS', 'https://apiv2-us.devo.com/'),
"stream":
False,
"destination": {
"type": "donothing",
"params": {
"friendlyName": "devo-sdk-api-test"
}
}
})
def test_stream_query_no_results_unbounded_dates(self):
api = Client(auth={
"key": self.key,
"secret": self.secret
},
address=self.uri,
config=ClientConfig(response="json/simple"),
retries=3)
result = api.query(query=self.query_no_results)
self.assertTrue(isinstance(result, types.GeneratorType))
try:
with stopit.ThreadingTimeout(3) as to_ctx_mgr:
result = list(result)
except DevoClientException:
# This exception is sent because
# devo.api.client.Client._make_request catches the
# stopit.TimeoutException, but the latter is not
# wrapped, so we cannot obtain it from here.
self.assertEqual(to_ctx_mgr.state, to_ctx_mgr.TIMED_OUT)
def setUp(self):
self.uri = os.getenv('DEVO_API_URL', 'https://api-us.logtrust.com/')
self.client = Client.from_config({
'key':
os.getenv('DEVO_API_KEY', None),
'secret':
os.getenv('DEVO_API_SECRET', None),
'url':
os.getenv('DEVO_API_URL', 'https://api-us.logtrust.com/'),
})
def test_unsecure_http_query(self):
"""
This test is intended for checking unsecure HTTP requests. Devo will NEVER provide an unsecure HTTP endpoint
for API REST services. Therefore, you are not going to need to use or test this functionality.
In order to enable UNSECURE_HTTP environment var should be TRUE.
The endpoint is served by https://httpbin.org/. You can run with `docker run -p 80:80 kennethreitz/httpbin`. It
will expose an HTTP service at port 80. The URL `http://localhost:80/anything` will answer with the content of
the request.
"""
os.environ["UNSECURE_HTTP"] = "TRUE"
config = ClientConfig(stream=False, response="json")
api = Client(auth={
"key": self.key,
"secret": self.secret
},
address="localhost:80/anything",
config=config,
retries=3)
result = api.query(query=self.query)
self.assertIsNotNone(result)
self.assertIn('json', json.loads(result))
self.assertIn('query', json.loads(result)['json'])
def __init__(self,
profile='default',
api_key=None,
api_secret=None,
end_point=None,
oauth_token=None,
jwt=None,
credential_path=None):
self.profile = profile
self.api_key = api_key
self.api_secret = api_secret
self.end_point = end_point
self.oauth_token = oauth_token
self.jwt = jwt
if credential_path is None:
self.credential_path = Path.home() / '.devo_credentials'
else:
self.credential_path = Path(credential_path).resolve().expanduser()
if not (self.end_point and (self.oauth_token or self.jwt or
(self.api_key and self.api_secret))):
self._read_profile()
if not (self.end_point and (self.oauth_token or self.jwt or
(self.api_key and self.api_secret))):
raise Exception(
'End point and either API keys or OAuth Token must be specified or in ~/.devo_credentials'
)
self.client = Client(auth=dict(key=self.api_key,
secret=self.api_secret,
token=self.oauth_token,
jwt=self.jwt),
address=self.end_point)
def process_response(response, config):
"""
process responses from Client API
:param response: data received from Devo API
:param config: array with launch options
:return: None
"""
try:
file_printer = open(config['output'], 'w')\
if 'output' in config.keys() else None
except (OSError, IOError) as error:
print_error("Error: (%s)" % error)
if not Client.stream_available(config['response']):
config['stream'] = False
printer = line_printer(file_printer)
if config['stream']:
for item in response:
printer(item)
else:
printer(response)
import os
from devo.api import Client, ClientConfig, JSON
key = os.getenv('DEVO_API_KEY', None)
secret = os.getenv('DEVO_API_SECRET', None)
api = Client(auth={
"key": key,
"secret": secret
},
address="https://apiv2-eu.devo.com/search/query",
config=ClientConfig(response="json", processor=JSON))
response = api.query(query="from demo.ecommerce.data select * limit 20",
dates={
'from': "today()-1*day()",
'to': "today()"
})
print(response)
class Reader(object):
def __init__(self,
profile='default',
api_key=None,
api_secret=None,
end_point=None,
oauth_token=None,
jwt=None,
credential_path=None,
timeout=None,
retries=1,
verify=True,
user=None,
app_name=None,
**kwargs):
self.profile = profile
self.api_key = api_key
self.api_secret = api_secret
self.end_point = end_point
self.oauth_token = oauth_token
self.jwt = jwt
if credential_path is None:
self.credential_path = Path.home() / '.devo_credentials'
else:
self.credential_path = Path(credential_path).expanduser().resolve()
if not (self.end_point and (self.oauth_token or self.jwt or
(self.api_key and self.api_secret))):
self._read_profile()
if not (self.end_point and (self.oauth_token or self.jwt or
(self.api_key and self.api_secret))):
raise Exception(
'End point and either API keys or OAuth Token must be specified or in ~/.devo_credentials'
)
config = kwargs
config.update({
'auth': {
'key': self.api_key,
'secret': self.api_secret,
'token': self.oauth_token,
'jwt': self.jwt
},
'address': self.end_point
})
self.client = Client(config=config)
self.client.timeout = timeout
self.client.retries = retries
self.client.verify = verify
if user:
self.client.config.set_user(user)
if app_name:
self.client.config.set_app_name(app_name)
def _read_profile(self):
"""
Read Devo API keys from a credentials file located
at ~/.devo_credentials if credentials are not provided
Use profile to specify which set of credentials to use
"""
config = configparser.ConfigParser()
config.read(self.credential_path)
if self.profile in config:
profile_config = config[self.profile]
self.api_key = profile_config.get('api_key')
self.api_secret = profile_config.get('api_secret')
self.end_point = profile_config.get('end_point')
self.oauth_token = profile_config.get('oauth_token')
if self.end_point == 'USA':
self.end_point = 'https://apiv2-us.devo.com/search/query'
elif self.end_point == 'EU':
self.end_point = 'https://apiv2-eu.devo.com/search/query'
def query(self,
linq_query,
start,
stop=None,
output='dict',
ts_format='datetime',
comment=None):
valid_outputs = ('dict', 'list', 'namedtuple', 'dataframe')
if output not in valid_outputs:
raise Exception(f"Output must be one of {valid_outputs}")
if output == 'dataframe' and stop is None:
raise Exception("DataFrame can't be build from continuous query")
type_dict = self._get_types(linq_query, start, ts_format)
res = self._query(linq_query,
start,
stop,
mode='csv',
stream=True,
comment=comment)
results = self._stream(res, type_dict)
cols = next(results)
results = getattr(self, f'_to_{output}')(results, cols)
if output == 'dataframe':
return results
else:
return DSResults(res, results)
def _stream(self, res, type_dict):
result = self._decode_results(res)
result = csv.reader(result)
try:
cols = next(result)
type_list = [type_dict[c] for c in cols]
if len(cols) != len(type_dict):
raise Exception(
"Duplicate column names encountered, custom columns must be named"
)
yield cols
for row in result:
yield [t(v) for t, v in zip(type_list, row)]
except Exception as e:
res.close()
raise (e)
def _query(self,
linq_query,
start,
stop=None,
mode='csv',
stream=False,
limit=None,
comment=None):
if (getattr(start, 'tzinfo', 1) is None) or (getattr(
stop, 'tzinfo', 1) is None):
warnings.warn('Naive date interpreted as UTC')
start = self._to_unix(start)
stop = self._to_unix(stop)
dates = {'from': start, 'to': stop}
self.client.config.response = mode
self.client.config.stream = stream
response = self.client.query(query=linq_query,
dates=dates,
limit=limit,
comment=comment)
return response
@staticmethod
def _null_decorator(f):
def null_f(v):
if v == '':
return None
else:
return f(v)
return null_f
@staticmethod
def make_ts_func(ts_format):
if ts_format not in ('datetime', 'iso', 'timestamp'):
raise Exception(
'ts_format must be one of: datetime, iso, or timestamp ')
def ts_func(t):
dt = datetime.datetime.strptime(t.strip(), '%Y-%m-%d %H:%M:%S.%f')
dt = dt.replace(tzinfo=timezone.utc)
if ts_format == 'datetime':
return dt
elif ts_format == 'iso':
return dt.isoformat()
elif ts_format == 'timestamp':
return dt.timestamp()
return ts_func
def _make_type_map(self, ts_format):
funcs = {
'timestamp': self.make_ts_func(ts_format),
'str': str,
'int8': int,
'int4': int,
'float8': float,
'float4': float,
'bool': lambda b: b == 'true'
}
decorated_funcs = {
t: self._null_decorator(f)
for t, f in funcs.items()
}
decorated_str = self._null_decorator(str)
return defaultdict(lambda: decorated_str, decorated_funcs)
def _get_types(self, linq_query, start, ts_format):
"""
Gets types of each column of submitted
"""
type_map = self._make_type_map(ts_format)
# so we don't have stop ts in future as required by API V2
stop = self._to_unix(start)
start = stop - 1
response = self._query(linq_query,
start=start,
stop=stop,
mode='json/compact',
limit=1)
try:
data = json.loads(response)
check_status(data)
except ValueError:
raise Exception('API V2 response error')
col_data = data['object']['m']
type_dict = {c: type_map[v['type']] for c, v in col_data.items()}
return type_dict
@staticmethod
def _to_unix(date, milliseconds=False):
"""
Convert date to a unix timestamp
date: A unix timestamp in second, a datetime object,
pandas.Timestamp object, or string to be parsed
by pandas.to_datetime
"""
if date is None:
return None
elif date == 'now':
epoch = datetime.datetime.now().timestamp()
elif type(date) == str:
epoch = pd.to_datetime(date).timestamp()
elif isinstance(date, (pd.Timestamp, datetime.datetime)):
if date.tzinfo is None:
epoch = date.replace(tzinfo=timezone.utc).timestamp()
else:
epoch = date.timestamp()
elif isinstance(date, (int, float)):
epoch = date
else:
raise Exception('Invalid Date')
if milliseconds:
epoch *= 1000
return int(epoch)
@staticmethod
def _decode_results(r):
r = iter(r)
# catch error not reported for json/compact
first = next(r)
try:
data = json.loads(first)
check_status(data)
except ValueError:
pass
yield first.decode(
'utf-8').strip() # APIV2 adding space to first line of aggregates
for l in r:
yield l.decode('utf-8')
@staticmethod
def _to_list(results, cols):
yield from results
@staticmethod
def _to_dict(results, cols):
for row in results:
yield {c: v for c, v in zip(cols, row)}
@staticmethod
def _to_namedtuple(results, cols):
Row = namedtuple('Row', cols)
for row in results:
yield Row(*row)
@staticmethod
def _to_dataframe(results, cols):
return pd.DataFrame(results, columns=cols).fillna(np.nan)
def randomSample(self, linq_query, start, stop, sample_size):
if (sample_size < 1) or (not isinstance(sample_size, int)):
raise Exception('Sample size must be a positive int')
size_query = f'{linq_query} group select count() as count'
r = self.query(size_query, start, stop, output='list')
table_size = next(r)[0]
if sample_size >= table_size:
warning_msg = 'Sample size greater than or equal to total table size. Returning full table'
warnings.warn(warning_msg)
return self.query(linq_query, start, stop, output='dataframe')
p = self._find_optimal_p(n=table_size, k=sample_size, threshold=0.99)
sample_query = f'{linq_query} where simplify(float8(rand())) < {p}'
while True:
df = self.query(sample_query, start, stop, output='dataframe')
if df.shape[0] >= sample_size:
return df.sample(sample_size).sort_index().reset_index(
drop=True)
else:
pass
@staticmethod
def _loc_scale(n, p):
"""
Takes parameters of a binomial
distribution and finds the mean
and std for a normal approximation
:param n: number of trials
:param p: probability of success
:return: mean, std
"""
loc = n * p
scale = np.sqrt(n * p * (1 - p))
return loc, scale
def _find_optimal_p(self, n, k, threshold):
"""
Use a normal approximation to the
binomial distribution. Starts with
p such that mean of B(n,p) = k
and iterates.
:param n: number of trials
:param k: desired number of successes
:param threshold: desired probability to achieve k successes
:return: probability that a single trial that will yield
at least k success with n trials with probability of threshold
"""
p = k / n
while True:
loc, scale = self._loc_scale(n, p)
# sf = 1 - cdf, but can be more accurate according to scipy docs
if norm.sf(x=k - 0.5, loc=loc, scale=scale) > threshold:
break
else:
p = min(1.001 * p, 1)
return p
def population_sample(self, query, start, stop, column, sample_size):
population_query = f'{query} group by {column}'
df = self.randomSample(population_query, start, stop, sample_size)
population = df[column]
sample_set = ','.join(f'"{x}"' for x in population)
sample_query = f'{query} where str({column}) in {{{sample_set}}}'
return self.query(sample_query, start, stop, output='dataframe')
