def _run(self, data):
""" Procedure to run when data received from trigger thread.
Args:
data: Pass to the registered event handlers.
"""
data_source = data["source"]
at = data["at"]
upload_items = []
for label, datum in data["data"].items():
upload_item = datum
upload_item["label"] = label
upload_item["source"] = data_source
upload_item["keen"] = {"timestamp": "{}Z".format(at.isoformat())}
upload_items.append(upload_item)
try:
self.client_.add_events({"offgrid": upload_items})
except Exception as e:
logger.error("{} failed to send data to keenio at {} by {}".format(
type(self).__name__, data["at"],
type(e).__name__))
logger.error("Details: {}".format(str(e)))
del self.client_
self.client_ = KeenClient(project_id=self.project_id_,
write_key=self.write_key_)
# TODO: skip retry to avoid exception in this scope.
# self.client_.add_events({"offgrid": upload_items})
else:
logger.info("{} sent data to keenio at {}".format(
type(self).__name__, at))
def main():
call(['./get_sleep.sh'])
with open('data_json.txt') as raw:
print '[Info] Loading raw sleep data...'
raw_sessions = json.load(raw)
timestamp = (datetime.today() - timedelta(days=TIME_DELTA)).isoformat()
chk_sessions = [s for s in raw_sessions if s['start'] > timestamp]
if len(chk_sessions) > 0:
print '[Info] Found {0} sleep sessions to check...'.format(len(chk_sessions))
client = KeenClient(
project_id = KEEN_PROJECT_ID,
write_key = KEEN_WRITE_KEY,
read_key = KEEN_READ_KEY
)
old_sessions = client.extraction('sessions', timeframe=TIME_FRAME)
new_sessions = [s for s in chk_sessions if 'tyin' + s['start'] not in [s2['id'] for s2 in old_sessions]]
if len(new_sessions) > 0:
print '[Info] Found {0} sleep sessions to add...'.format(len(new_sessions))
sessions = []
events = []
sessions, events = transform(new_sessions, 'tyin')
client.add_events({'sessions': sessions, 'events': events})
print '[Info] Added {0} sleep sessions.'.format(len(new_sessions))
print '[Info] Done.'
def index(request):
hits = cache.get('keen_hit_count')
if not hits:
keen = KeenClient(project_id=settings.KEEN_PROJECT_ID,
write_key=settings.KEEN_WRITE_KEY,
read_key=settings.KEEN_READ_KEY,
get_timeout=3)
try:
detail_count = keen.count('detail_hit')
list_count = keen.count('list_hit')
schema_count = keen.count('schema_hit')
hits = detail_count + list_count + schema_count
except Exception:
print('exception!')
hits = DEFAULT_HITS
cache.set('keen_hit_count', hits, DEFAULT_HITS)
stripe_key = settings.STRIPE_KEYS['publishable']
return render_to_response('index.html', {
"hits": hits,
"stripe_key": stripe_key
})
def index(request):
hits = cache.get('keen_hit_count')
if not hits:
keen = KeenClient(
project_id=settings.KEEN_PROJECT_ID,
write_key=settings.KEEN_WRITE_KEY,
read_key=settings.KEEN_READ_KEY,
get_timeout=3
)
try:
detail_count = keen.count('detail_hit')
list_count = keen.count('list_hit')
schema_count = keen.count('schema_hit')
hits = detail_count + list_count + schema_count
except Exception:
print('exception!')
hits = DEFAULT_HITS
cache.set('keen_hit_count', hits, DEFAULT_HITS)
stripe_key = settings.STRIPE_KEYS['publishable']
return render_to_response('index.html',
{
"hits": hits,
"stripe_key": stripe_key
}
)
def test_post_timeout_batch(self):
keen.project_id = "5004ded1163d66114f000000"
api_key = "2e79c6ec1d0145be8891bf668599c79a"
keen.write_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["write"]})
client = KeenClient(keen.project_id, write_key=keen.write_key, read_key=None,
post_timeout=0.0001)
client.add_events({"python_test": [{"hello": "goodbye"}]})
def test_generate_image_beacon(self):
keen.project_id = "5004ded1163d66114f000000"
api_key = "2e79c6ec1d0145be8891bf668599c79a"
keen.write_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["write"]})
event_collection = "python_test hello!?"
event_data = {"a": "b"}
data = self.base64_encode(json.dumps(event_data))
# module level should work
url = keen.generate_image_beacon(event_collection, event_data)
expected = "https://api.keen.io/3.0/projects/{0}/events/{1}?api_key={2}&data={3}".format(
keen.project_id, self.url_escape(event_collection), keen.write_key.decode(sys.getdefaultencoding()), data
)
self.assert_equal(expected, url)
# so should instance level
client = KeenClient(keen.project_id, write_key=keen.write_key, read_key=None)
url = client.generate_image_beacon(event_collection, event_data)
self.assert_equal(expected, url)
# make sure URL works
response = requests.get(url)
self.assert_equal(200, response.status_code)
self.assert_equal(b"GIF89a\x01\x00\x01\x00\x80\x01\x00\xff\xff\xff\x00\x00\x00!\xf9\x04\x01\n\x00\x01\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02L\x01\x00;",
response.content)
def run(self):
keen_client = KeenClient(
project_id=self.project_id,
write_key=self.write_key,
)
timezone = pytz.timezone(self.data['user']['timezone'])
self.events = []
with click.progressbar(self.data['days'],
label='Preparing keen.io events',
fill_char=click.style('#', fg='blue')) as days:
for day in days:
dt = self._parse_date(day['date'], timezone)
self.append_event(dt, 'total', {
'seconds': day['grand_total']['total_seconds'],
})
categories = [
'editor',
'language',
'operating_system',
'project',
]
for category in categories:
for item in day.get(category + 's', []):
self.append_event(dt, category, {
'seconds': item['total_seconds'],
'name': item['name'],
})
files = {}
for project in day.get('projects', []):
for f in project.get('dump', {}).get('files', []):
if f['name'] not in files:
files[f['name']] = 0
files[f['name']] += f['total_seconds']
for name, seconds in files.items():
self.append_event(dt, 'file', {
'seconds': seconds,
'name': name,
})
if len(self.events) == 0:
click.echo('No events available for exporting to keen.io')
return
click.echo('Uploading events to keen.io...')
start = datetime.utcfromtimestamp(self.data['range']['start'])
end = datetime.utcfromtimestamp(self.data['range']['end'])
collection = 'WakaTime Data Dump from {start} until {end}'.format(
start=start.strftime('%Y-%m-%d'),
end=end.strftime('%Y-%m-%d'),
)
keen_client.add_events({
collection: self.events,
})
def __keen_client(self):
if KEEN_CLIENT_ENABLED and self.keen_client == None:
self.keen_client = KeenClient(project_id=KEEN_PROJECT_ID,
write_key=KEEN_WRITE_KEY,
read_key=KEEN_READ_KEY,
base_url=KEEN_API_URL)
return self.keen_client
def test_generate_image_beacon(self):
keen.project_id = "5004ded1163d66114f000000"
api_key = "2e79c6ec1d0145be8891bf668599c79a"
keen.write_key = scoped_keys.encrypt(api_key,
{"allowed_operations": ["write"]})
event_collection = "python_test hello!?"
event_data = {"a": "b"}
data = self.base64_encode(json.dumps(event_data))
# module level should work
url = keen.generate_image_beacon(event_collection, event_data)
expected = "https://api.keen.io/3.0/projects/{0}/events/{1}?api_key={2}&data={3}".format(
keen.project_id, self.url_escape(event_collection),
keen.write_key.decode(sys.getdefaultencoding()), data)
self.assert_equal(expected, url)
# so should instance level
client = KeenClient(keen.project_id,
write_key=keen.write_key,
read_key=None)
url = client.generate_image_beacon(event_collection, event_data)
self.assert_equal(expected, url)
# make sure URL works
response = requests.get(url)
self.assert_equal(200, response.status_code)
self.assert_equal(
b"GIF89a\x01\x00\x01\x00\x80\x01\x00\xff\xff\xff\x00\x00\x00!\xf9\x04\x01\n\x00\x01\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02L\x01\x00;",
response.content)
def __init__(self, project_id, write_key, q_max=5):
IEventHandler.__init__(self, q_max=q_max)
# for retry
self.project_id_ = project_id
self.write_key_ = write_key
self.client_ = KeenClient(project_id=project_id, write_key=write_key)
class KeenIoEventHandler(IEventHandler):
""" The instance should be registered to event trigger for data update.
This uploads any data to keenio cloud service.
Args:
project_id: Project ID provided by keenio.
write_key: Write key ID provided by keenio.
q_max: internal queue size to be used from another thread.
Returns:
Instance of this class.
"""
def __init__(self, project_id, write_key, q_max=5):
IEventHandler.__init__(self, q_max=q_max)
# for retry
self.project_id_ = project_id
self.write_key_ = write_key
self.client_ = KeenClient(project_id=project_id, write_key=write_key)
def _run(self, data):
""" Procedure to run when data received from trigger thread.
Args:
data: Pass to the registered event handlers.
"""
data_source = data["source"]
at = data["at"]
upload_items = []
for label, datum in data["data"].items():
upload_item = datum
upload_item["label"] = label
upload_item["source"] = data_source
upload_item["keen"] = {"timestamp": "{}Z".format(at.isoformat())}
upload_items.append(upload_item)
try:
self.client_.add_events({"offgrid": upload_items})
except Exception as e:
logger.error("{} failed to send data to keenio at {} by {}".format(
type(self).__name__, data["at"],
type(e).__name__))
logger.error("Details: {}".format(str(e)))
del self.client_
self.client_ = KeenClient(project_id=self.project_id_,
write_key=self.write_key_)
# TODO: skip retry to avoid exception in this scope.
# self.client_.add_events({"offgrid": upload_items})
else:
logger.info("{} sent data to keenio at {}".format(
type(self).__name__, at))
def main():
counts_by_institutions = get_count_by_institutions()
keen_project = keen_settings['private']['project_id']
write_key = keen_settings['private']['write_key']
if keen_project and write_key:
client = KeenClient(
project_id=keen_project,
write_key=write_key,
)
client.add_events(counts_by_institutions)
else:
print(counts_by_institutions)
def send_events(self, events):
keen_project = keen_settings['private']['project_id']
write_key = keen_settings['private']['write_key']
if keen_project and write_key:
client = KeenClient(
project_id=keen_project,
write_key=write_key,
)
logger.info('Adding {} events to the {} collection'.format(len(events), self.collection_name))
client.add_events({self.collection_name: events})
else:
logger.info('Keen not enabled - would otherwise be adding the following {} events to the {} collection'.format(len(events), self.collection_name))
print(events)
def send_events(self, events):
keen_project = keen_settings['private']['project_id']
write_key = keen_settings['private']['write_key']
if keen_project and write_key:
client = KeenClient(
project_id=keen_project,
write_key=write_key,
)
logger.info('Adding {} events to the {} collection'.format(len(events), self.collection_name))
client.add_events({self.collection_name: events})
else:
logger.info('Keen not enabled - would otherwise be adding the following {} events to the {} collection'.format(len(events), self.collection_name))
print(events)
def test_configure_through_code(self, post):
client = KeenClient(project_id="123456", read_key=None, write_key=None)
self.assert_raises(exceptions.InvalidEnvironmentError,
client.add_event, "python_test", {"hello": "goodbye"})
# force client to reinitialize
client = KeenClient(project_id="123456", read_key=None, write_key="abcdef")
with patch("requests.Session.post") as post:
post.return_value = MockedFailedResponse(
status_code=401,
json_response={"message": "authorization error", "error_code": "AdminOnlyEndpointError"},
)
self.assert_raises(exceptions.KeenApiError,
client.add_event, "python_test", {"hello": "goodbye"})
def negative_helper(expected_exception, project_id,
**kwargs):
try:
KeenClient(project_id, **kwargs)
except expected_exception as e:
self.assert_true(str(e))
return e
def test_generate_image_beacon(self, post):
event_collection = "python_test hello!?"
event_data = {"a": "b"}
data = self.base64_encode(json.dumps(event_data))
# module level should work
url = keen.generate_image_beacon(event_collection, event_data)
expected = "https://api.keen.io/3.0/projects/{0}/events/{1}?api_key={2}&data={3}".format(
keen.project_id, self.url_escape(event_collection), keen.write_key.decode(sys.getdefaultencoding()), data
)
self.assert_equal(expected, url)
# so should instance level
client = KeenClient(keen.project_id, write_key=keen.write_key, read_key=None)
url = client.generate_image_beacon(event_collection, event_data)
self.assert_equal(expected, url)
def init_app(self, app):
# self.app = app
self.client = KeenClient(
project_id=app.config.get('KEEN_PROJECT_ID'),
read_key=app.config.get('KEEN_READ_KEY'),
write_key=app.config.get('KEEN_WRITE_KEY'),
)
def get_keen_client(credentials_key):
""" Get KeenCLient
Parameters
----------
credentials_key : str
either a path to a json file containing 'project_id' and 'read_key'
or a service_name for keyring entries containing 'project_id' and
'read_key'
Returns
-------
KeenClient
"""
if credentials_key.endswith('.json'):
credentials = open(credentials_key, 'r').read()
credentials = json.loads(credentials)
project_id = credentials['project_id']
read_key = credentials['read_key']
else:
project_id=keyring.get_password(credentials_key, 'project_id')
read_key=keyring.get_password(credentials_key, 'read_key')
client = KeenClient(project_id=project_id, read_key=read_key)
return client
def setUp(self):
super(SavedQueryTests, self).setUp()
self.exp_project_id = "xxxx1234"
exp_master_key = "abcd3456"
self.client = KeenClient(project_id=self.exp_project_id,
read_key="efgh7890",
master_key=exp_master_key)
def test_passing_invalid_custom_api_client(self, get):
class CustomApiClient(object):
api_version = "3.0"
def __init__(self,
project_id,
write_key=None,
read_key=None,
base_url=None,
api_version=None,
**kwargs):
super(CustomApiClient, self).__init__()
self.project_id = project_id
self.write_key = write_key
self.read_key = read_key
if base_url:
self.base_url = base_url
if api_version:
self.api_version = api_version
api_key = "2e79c6ec1d0145be8891bf668599c79a"
client = KeenClient(
"5004ded1163d66114f000000",
write_key=scoped_keys.encrypt(api_key,
{"allowed_operations": ["write"]}),
read_key=scoped_keys.encrypt(api_key,
{"allowed_operations": ["read"]}),
api_class=CustomApiClient,
base_url="keen.base.url")
# Should raise an error, we never added this method on our class
# But it shows it is actually using our class
self.assertRaises(TypeError, client.add_event)
def setUp(self):
super(CustomJSONEncoderTests, self).setUp()
api_key = "2e79c6ec1d0145be8891bf668599c79a"
self.client = KeenClient(project_id='5004ded1163d66114f000000',
write_key=scoped_keys.encrypt(api_key, {"allowed_operations": ["write"]}),
read_key=scoped_keys.encrypt(api_key, {"allowed_operations": ["read"]}),
json_encoder=CustomEncoder)
def test_generate_image_beacon(self, post):
event_collection = "python_test hello!?"
event_data = {"a": "b"}
data = self.base64_encode(json.dumps(event_data))
# module level should work
url = keen.generate_image_beacon(event_collection, event_data)
expected = "https://api.keen.io/3.0/projects/{0}/events/{1}?api_key={2}&data={3}".format(
keen.project_id, self.url_escape(event_collection), keen.write_key.decode(sys.getdefaultencoding()), data
)
self.assert_equal(expected, url)
# so should instance level
client = KeenClient(keen.project_id, write_key=keen.write_key, read_key=None)
url = client.generate_image_beacon(event_collection, event_data)
self.assert_equal(expected, url)
def __init__(self):
self.client = KeenClient(
project_id=ENV["keen"]["project_id"],
write_key=ENV["keen"]["write_key"],
read_key=ENV["keen"]["read_key"],
master_key=ENV["keen"]["master_key"],
)
class KeenEventService:
def __init__(self):
self.client = KeenClient(
project_id=ENV["keen"]["project_id"],
write_key=ENV["keen"]["write_key"],
read_key=ENV["keen"]["read_key"],
master_key=ENV["keen"]["master_key"],
)
#
# insert an event log
#
def insert_event(self, event_collection, event_log):
try:
self.client.add_event(event_collection, event_log)
except Exception, e:
log.exception(e)
def __keen_client(self):
if KEEN_CLIENT_ENABLED and self.keen_client == None:
self.keen_client = KeenClient(
project_id=KEEN_PROJECT_ID,
write_key=KEEN_WRITE_KEY,
read_key=KEEN_READ_KEY,
base_url=KEEN_API_URL
)
return self.keen_client
def send_events(self, events):
keen_project = keen_settings["private"]["project_id"]
write_key = keen_settings["private"]["write_key"]
if keen_project and write_key:
client = KeenClient(project_id=keen_project, write_key=write_key)
logger.info("Adding {} events to the {} collection".format(len(events), self.collection_name))
for chunk in self.yield_chunked_events(events):
client.add_events({self.collection_name: chunk})
time.sleep(1)
else:
logger.info(
"Keen not enabled - would otherwise be adding the following {} events to the {} collection".format(
len(events), self.collection_name
)
)
print(events)
def main():
print "[INFO] Getting raw spire data..."
date = (datetime.today() - timedelta(days=DAYS_AGO)).strftime("%Y-%m-%d")
# Breath
breath_client = KeenClient(
project_id=keen["breath"]["project_id"],
read_key=keen["breath"]["read_key"],
write_key=keen["breath"]["write_key"],
)
if len(breath_client.extraction("sessions", timeframe="this_" + str(DAYS_AGO) + "_days")) > 0:
print "[INFO] Already uploaded breath data for " + date + "."
else:
breaths = get_spire(spire["breath"], date)
if len(breaths["data"]) > 0:
print "[INFO] Uploading breath data..."
keen_breaths = {}
keen_breaths["breaths"] = [{"timestamp": b["timestamp"], "value": b["value"]} for b in breaths["data"]]
keen_breaths["metadata"] = breaths["metadata"]
keen_breaths["metadata"]["min"] = min([b["value"] for b in breaths["data"]])
keen_breaths["metadata"]["max"] = max([b["value"] for b in breaths["data"]])
keen_breaths["id"] = USER + date
breath_client.add_event("sessions", keen_breaths)
print "[INFO] Uploaded breath data for " + date + "."
else:
print "[INFO] No breath data found for " + date + "."
# Steps
steps_client = KeenClient(
project_id=keen["steps"]["project_id"], read_key=keen["steps"]["read_key"], write_key=keen["steps"]["write_key"]
)
if len(steps_client.extraction("sessions", timeframe="this_" + str(DAYS_AGO) + "_days")) > 0:
print "[INFO] Already uploaded steps data for " + date + "."
else:
steps = get_spire(spire["step"], date)
if len(steps["data"]) > 0:
print "[INFO] Uploading steps data..."
keen_steps = {}
keen_steps["steps"] = [{"timestamp": s["timestamp"], "value": s["value"]} for s in steps["data"]]
keen_steps["metadata"] = steps["metadata"]
keen_steps["id"] = USER + date
steps_client.add_event("sessions", keen_steps)
print "[INFO] Uploaded steps data for " + date + "."
else:
print "[INFO] No steps data found for " + date + "."
class AsyncKeenClient(object):
def __init__(self):
self._client = KeenClient(
project_id=os.environ['KEEN_PROJECT_ID'],
write_key=os.environ['KEEN_WRITE_KEY'],
post_timeout=100
)
def add_event(self, collection, body):
push = lambda: self._client.add_event(collection, body)
IOLoop.current().spawn_callback(push)
class CustomJSONEncoderTests(BaseTestCase):
def setUp(self):
super(CustomJSONEncoderTests, self).setUp()
api_key = "2e79c6ec1d0145be8891bf668599c79a"
self.client = KeenClient(project_id='5004ded1163d66114f000000',
write_key=scoped_keys.encrypt(api_key, {"allowed_operations": ["write"]}),
read_key=scoped_keys.encrypt(api_key, {"allowed_operations": ["read"]}),
json_encoder=CustomEncoder)
def tearDown(self):
self.client = None
super(CustomJSONEncoderTests, self).tearDown()
def test_custom_encoder_with_datetime_type(self):
self.client.add_event("sign_ups", {
"username": "******",
"referred_by": "harry",
"confirmed_at": datetime.datetime.utcnow()
})
def test_direct_persistence_strategy(self):
project_id = "5004ded1163d66114f000000"
api_key = "2e79c6ec1d0145be8891bf668599c79a"
write_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["write"]})
read_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["read"]})
client = KeenClient(project_id, write_key=write_key, read_key=read_key)
client.add_event("python_test", {"hello": "goodbye"})
client.add_event("python_test", {"hello": "goodbye"})
client.add_events(
{
"sign_ups": [{
"username": "******",
"referred_by": "steve",
"son_of": "my_mom"
}],
"purchases": [
{"price": 5},
{"price": 6},
{"price": 7}
]}
)
def test_timeout_count(self, get):
get.side_effect = requests.Timeout
client = KeenClient(keen.project_id,
write_key=None,
read_key=keen.read_key,
get_timeout=0.0001)
self.assert_raises(requests.Timeout,
client.count,
"query test",
timeframe="today",
filters=self.get_filter())
# Make sure the requests library was called with `timeout`.
self.assert_equals(get.call_args[1]["timeout"], 0.0001)
def get_keen_client():
keen_project = keen_settings['private'].get('project_id')
read_key = keen_settings['private'].get('read_key')
master_key = keen_settings['private'].get('master_key')
write_key = keen_settings['private'].get('write_key')
if keen_project and read_key and master_key:
client = KeenClient(project_id=keen_project,
read_key=read_key,
master_key=master_key,
write_key=write_key)
else:
raise ValueError(
'Cannot connect to Keen clients - all keys not provided.')
return client
def _initialize_client_from_environment():
global _client, project_id, write_key, read_key
if _client is None:
# check environment for project ID and keys
project_id = project_id or os.environ.get("KEEN_PROJECT_ID")
write_key = write_key or os.environ.get("KEEN_WRITE_KEY")
read_key = read_key or os.environ.get("KEEN_READ_KEY")
if not project_id:
raise InvalidEnvironmentError("Please set the KEEN_PROJECT_ID environment variable or set keen.project_id!")
_client = KeenClient(project_id,
write_key=write_key,
read_key=read_key)
def test_new_client_instance(self, post):
exp_project_id = "xxxx1234"
exp_write_key = "yyyy4567"
exp_read_key = "zzzz8912"
exp_master_key = "abcd3456"
# create Client instance
client = KeenClient(project_id=exp_project_id,
write_key=exp_write_key,
read_key=exp_read_key,
master_key=exp_master_key)
# assert values
self.assertEquals(exp_project_id, client.api.project_id)
self.assertEquals(exp_write_key, client.api.write_key)
self.assertEquals(exp_read_key, client.api.read_key)
self.assertEquals(exp_master_key, client.api.master_key)
def _initialize_client_from_environment():
''' Initialize a KeenClient instance using environment variables. '''
global _client, project_id, write_key, read_key, master_key, base_url
if _client is None:
# check environment for project ID and keys
project_id = project_id or os.environ.get("KEEN_PROJECT_ID")
write_key = write_key or os.environ.get("KEEN_WRITE_KEY")
read_key = read_key or os.environ.get("KEEN_READ_KEY")
master_key = master_key or os.environ.get("KEEN_MASTER_KEY")
base_url = base_url or os.environ.get("KEEN_BASE_URL")
if not project_id:
raise InvalidEnvironmentError("Please set the KEEN_PROJECT_ID environment variable or set keen.project_id!")
_client = KeenClient(project_id,
write_key=write_key,
read_key=read_key,
master_key=master_key,
base_url=base_url)
def test_direct_persistence_strategy(self):
project_id = "5004ded1163d66114f000000"
api_key = "2e79c6ec1d0145be8891bf668599c79a"
write_key = scoped_keys.encrypt(api_key,
{"allowed_operations": ["write"]})
read_key = scoped_keys.encrypt(api_key,
{"allowed_operations": ["read"]})
client = KeenClient(project_id, write_key=write_key, read_key=read_key)
client.add_event("python_test", {"hello": "goodbye"})
client.add_event("python_test", {"hello": "goodbye"})
client.add_events({
"sign_ups": [{
"username": "******",
"referred_by": "steve",
"son_of": "my_mom"
}],
"purchases": [{
"price": 5
}, {
"price": 6
}, {
"price": 7
}]
})
def test_direct_persistence_strategy(self):
project_id = "5004ded1163d66114f000000"
auth_token = "2e79c6ec1d0145be8891bf668599c79a"
client = KeenClient(project_id, auth_token)
client.add_event("python_test", {"hello": "goodbye"})
#!/usr/bin/env python
# -*- coding:utf8 -*-
from keen.client import KeenClient
keen_client = KeenClient(
project_id="55f93f85d2eaaa05a699de39",
write_key="8f77f837b629b8519e8d23c541fedc42d423f8f71dc10e2244b50e364e7034cb753d960ea31d7925d044a109233cdf55e88792d4fa9f16f4e2410a7e09309242565415ef3d7299526c2fbf6860603bd349d186f068f5bd940ff047c2f4a4f0959c1cda4c3c3208729524b233b096e4de",
read_key="2f1b89a70f545caaeff7964b5daa60236844aa0291a20749e15ca83e30c63d3cb939d7eb720666c03164403680d77d8e6086e3f1d60b5a34f696b09fc1bd695a0a0f0a1f947757cd5bd5666d5448b614e9928a544bfcecfe4bb9e52b1c02d77e49e359c04dc2dd901764097364b00573",
master_key="3855C83BD7B0720323E0B1E447149B01")
class Keen(object):
keen = None
def __init__(self):
self.keen = keen_client
def add_girl(self, collection, name, count_dict, **kwargs):
data_array = list()
for i in count_dict.keys():
if self.is_exist_girl(collection, name, i):
pass
else:
data_dict = dict()
data_dict["캐릭터"] = name
data_dict["keen"] = dict()
data_dict["count"] = count_dict[i]
data_dict["keen"]["timestamp"] = i
data_array.append(data_dict)
self.keen.add_events({
collection : data_array})
return True
def is_exist_girl(self, collection, name, timestamp, **kwargs):
# -*- coding: utf-8 -*-
# <nbformat>3.0</nbformat>
# <codecell>
from keen.client import KeenClient
import json
from dateutil.parser import *
client = KeenClient(
project_id="50ba759d897a2c45c2000000",
write_key="72a1ad85456fd2361cfe8b65d240eab16b083caa944c11d9507761e536d34822eca4cefad12e482373ee41278953f98c6ff724b33b56fd5971e8d34cdd284cb6c3f202d9f13329e39c01eb9a35d1124cae81d9a4505992fce03be4030e32b19c72df266c55a18937f001e7158f6eac44",
read_key="f7a829ecc9e237949b0b97245c828661ff0c9f46eaec35a7dc4cfd3b4ef42804f5fc1859ecad05bd7d72b6c3a2e4047ae39cba86b5d51c770be26b0a406cbb44650d20fd05fb7896293f015cec3af3ca2df8a507927bf4cfed9275eb89e8dee855cf45db75b221074bda2f737d2810f2"
)
# <codecell>
import datetime
# <codecell>
current = datetime.datetime.now()
# <codecell>
from dateutil.relativedelta import *
# <codecell>
two_week = relativedelta(weeks=2)
import matplotlib.pyplot as plt
from keen.client import KeenClient
from functools import wraps
from queue import Queue
from threading import Thread
from datetime import datetime, timedelta
#load keen credentials and initialize the class
os.chdir('/users/csaunders/Desktop')
with open('Keen_API_credentials.pickle', 'rb') as f:
Keen_API_credentials = pickle.load(f)
Keen_silo = 'QZ prod'
projectID = Keen_API_credentials[Keen_silo]['projectID']
readKey = Keen_API_credentials[Keen_silo]['readKey']
keen = KeenClient(project_id=projectID, read_key=readKey)
######### TIMEFRAME GENERATOR ##############################################
def timeframe_gen(start, end, hour_interval=24, tz='US/Eastern'):
"""creates timeframe for use in making Keen API calls
args:
start - start date (str - '2017-08-04'); inclusive
end - end date (str - '2017-08-04'); inclusive
kwargs:
hour_interval - interval for breaking up start, end tuple
tz - timezone
returns:
List of tuples, tuple - (start, end)
"""
def __init__(self):
self.keenWriter = KeenClient(
project_id=config.KEEN_PROJECTID,
write_key=config.KEEN_WRITEKEY,
read_key=config.KEEN_READKEY
)
def __init__(self):
self._client = KeenClient(
project_id=os.environ['KEEN_PROJECT_ID'],
write_key=os.environ['KEEN_WRITE_KEY'],
post_timeout=100
)
class Alerter:
def __init__(self):
logging.basicConfig()
self.logger = logging.getLogger()
self.logger.setLevel(logging.INFO)
self.garage_data = anyconfig.load("garage_data.json")['garage_data']
self.sg = sendgrid.SendGridClient(SENDGRID_USERNAME, SENDGRID_PASSWORD)
self.keen_client = None
def __keen_client(self):
if KEEN_CLIENT_ENABLED and self.keen_client == None:
self.keen_client = KeenClient(
project_id=KEEN_PROJECT_ID,
write_key=KEEN_WRITE_KEY,
read_key=KEEN_READ_KEY,
base_url=KEEN_API_URL
)
return self.keen_client
def is_email(self, address):
assert isinstance(address, str)
return "@" in address
def is_phone(self, address):
assert isinstance(address, str)
address = address.replace("-", "")
pattern = re.compile("^[\dA-Z]{3}[\dA-Z]{3}[\dA-Z]{4}$", re.IGNORECASE)
return pattern.match(address) is not None
@newrelic.agent.background_task()
def send_alert(self, address, garage):
if self.is_email(address):
self.send_email(address, garage)
elif self.is_phone(address):
self.send_txt(address, garage)
self.clear_subs_for_user(address)
@newrelic.agent.background_task()
def send_email(self, address, garage):
self.logger.info("send_email(): Sending email about {} to {}".format(garage, address))
message = sendgrid.Mail()
message.add_to(address)
message_text = "{} has a new spot open".format(garage)
message.set_subject(message_text)
message.set_text(message_text) #todo add how many spots are there
message.set_from(SENDGRID_EMAIL_FROM)
self.sg.send(message)
@newrelic.agent.background_task()
def send_txt(self, address, garage):
self.logger.info("send_txt(): Sending txt about {} to {}".format(garage, address))
body = "{} has a new spot open".format(garage),
EasySms.send_sms(to=address, body=body)
@newrelic.agent.background_task()
def find_changes(self):
current = json.loads(self.redis_connection.hget("current", "data"))
previous = json.loads(self.redis_connection.hget("previous", "data"))
cur_stations = {}
prev_stations = {}
for garage in current.itervalues():
cur_stations.update(garage['stations'])
for garage in previous.itervalues():
prev_stations.update(garage['stations'])
stations_with_new_spots = []
for station_name in cur_stations.keys():
if cur_stations[station_name] > prev_stations[station_name]:
stations_with_new_spots.append(station_name)
all_changes = {}
for station_name in cur_stations.keys():
if cur_stations[station_name] != prev_stations[station_name]:
all_changes[station_name] = cur_stations[station_name]
self.logger.info("find_changes(): all chargers changes: {}".format(all_changes))
if KEEN_CLIENT_ENABLED and all_changes > 0:
self.logger.info("find_changes(): sending keen events: {}".format(all_changes))
self.keen_client.add_event("chargers", all_changes)
self.logger.info("find_changes(): stations with new spots {}".format(stations_with_new_spots))
return stations_with_new_spots
@newrelic.agent.background_task()
def clear_subs_for_user(self, target):
self.logger.info("clear_subs_for_user(): Clearing sub for user {}".format(target))
for key in self.redis_connection.keys("SUB-*"):
if self.redis_connection.type(key) == "set":
self.redis_connection.srem(key, target)
self.logger.info("clear_subs_for_user(): Removing user {} from subscription to {}".format(target, key))
def find_garage(self, station):
# station = PG3-STATION 26 for example
# PG1 is Hilltop, PG2 is Central, PG3 is Creekside
garages = ['Hilltop', 'Central', 'Creekside']
garage_array_index = int(station[2]) - 1
return garages[garage_array_index]
@newrelic.agent.background_task()
def main_loop(self):
self.redis_connection = redis.StrictRedis.from_url(REDIS_URL)
pubsub = self.redis_connection.pubsub()
pubsub.subscribe(REDIS_CHANNEL)
# todo add tests
for item in pubsub.listen():
self.logger.info("alerter.main_loop(): Got a new alert. Processing ...")
if item['type'] == "message":
self.logger.info("alerter.main_loop(): received alert of new info with timstamp {}".format(item['data']))
for station in self.find_changes():
self.logger.info("alerter.main_loop(): Found new station {}...checking for subscriptions".format(station))
garage_name = self.find_garage(station)
queue_name = "SUB-{}".format(garage_name)
self.logger.info("alerter.main_loop(): Checking queue {} for members".format(queue_name))
queue_members = self.redis_connection.smembers(queue_name)
if len(queue_members) == 0:
self.logger.info("alerter.main_loop(): No notifications to send for garage {}".format(garage_name))
for target in queue_members:
self.logger.info("alerter.main_loop(): Notifying {} for {} changes.".format(target, garage_name))
self.send_alert(target, garage_name)
self.logger.info("alerter.main_loop(): done processing alert.")
def test_create_saved_query_master_key(self):
client = KeenClient(project_id="123123")
self.assertRaises(
exceptions.InvalidEnvironmentError,
lambda: client.saved_queries.create("saved-query-name", {})
)
def test_get_one_saved_query_keys(self):
client = KeenClient(project_id="123123")
self.assertRaises(
exceptions.InvalidEnvironmentError,
lambda: client.saved_queries.get("saved-query-name")
)
def test_get_all_saved_queries_keys(self):
client = KeenClient(project_id="123123")
self.assertRaises(
exceptions.InvalidEnvironmentError, client.saved_queries.all
)
class HireKeen:
def __init__(self):
self.keenWriter = KeenClient(
project_id=config.KEEN_PROJECTID,
write_key=config.KEEN_WRITEKEY,
read_key=config.KEEN_READKEY
)
def record_keen(self, index, event):
self.keenWriter.add_event(index, event)
def extract_keen(self, index, timeframe, filters):
return self.keenWriter.extraction(index, timeframe, filters=filters)
def pred_identify(self, user_id):
self.identified = True
self.predclient.identify(user_id)
def pred_ranked(self, engine, data):
return self.predclient.get_itemrank_ranked(engine, data)
def pred_top_rec(self, engine, number):
return self.predclient.get_itemrec_topn(engine, number)
def pred_create(self, user, userType):
self.predclient.create_user(user, {'userType': userType})
def event_record_register(self, loggedin_id):
user_details = self.db_user.find_one({'linkedin_id': loggedin_id})
if user_details:
event_data = {
"action": "registration",
"userId": user_details['linkedin_id'],
"userCompanyId": user_details['linkedin_company_id']
}
recordEvent.delay('siteActions', event_data)
def event_record_activate(self, loggedin_id):
user_details = self.db_user.find_one({'linkedin_id': loggedin_id})
if user_details:
event_data = {
"action": "activation",
"userId": user_details['linkedin_id'],
"userCompanyId": user_details['linkedin_company_id']
}
recordEvent.delay('siteActions', event_data)
def event_record_create_project(self, project_slug, user_id):
project_details = self.db_project.find_one({'slug': project_slug, 'loggedin_id': user_id})
if project_details:
try:
user_details = self.db_user.find_one({'linkedin_id': project_details['loggedin_id']})
company_details = self.db_linkedincompany.find_one({'id': user_details['linkedin_company_id']})
event_data = {
"action": "project creation",
"userId": project_details['loggedin_id'],
"userCompanyId": user_details['linkedin_company_id'],
"slug": project_slug,
"skills": [],
"addlSkills": [],
"userCompany": company_details['name'],
"forCompanyId": project_details['companyId'],
"forCompanyName": project_details['company'],
"name": project_details['name']
}
for skill in project_details['skills']:
if skill['skill'] not in event_data['skills']:
event_data['skills'].append(skill['skill'])
for addlskill in project_details['addlSkills']:
if addlskill not in event_data['addlSkills']:
event_data['addlSkills'].append(addlskill)
recordEvent.delay('siteActions', event_data)
except:
rollbar.report_exc_info()
def event_record_create_position(self, project_slug, position_slug, user_id):
position_details = self.db_position.find_one({'project': project_slug, 'slug': position_slug, 'loggedin_id': user_id})
if position_details:
try:
user_details = self.db_user.find_one({'linkedin_id': position_details['loggedin_id']})
company_details = self.db_linkedincompany.find_one({'id': user_details['linkedin_company_id']})
project_details = self.db_project.find_one({'slug': project_slug})
event_data = {
"action": "position creation",
"userId": user_details['linkedin_id'],
"companyId": user_details['linkedin_company_id'],
"slug": position_details['slug'],
"projectslug": position_details['project'],
"skills": [],
"addlSkills": [],
"userCompany": company_details['name'],
"forCompanyId": project_details['companyId'],
"forCompanyName": project_details['company'],
"title": position_details['title'],
"projectName": project_details['name']
}
for skill in position_details['skills']:
if skill['skill'] not in event_data['skills']:
event_data['skills'].append(skill['skill'])
for addlskill in position_details['addlSkills']:
if addlskill not in event_data['addlSkills']:
event_data['addlSkills'].append(addlskill)
recordEvent.delay('siteActions', event_data)
except:
rollbar.report_exc_info()
def event_record_resumesearch(self, position_slug, loggedin_id):
resumesearch_details = self.db_searchlog.find_one({'slug': position_slug, 'loggedin_id': loggedin_id})
if resumesearch_details:
try:
user_details = self.db_user.find_one({'linkedin_id': loggedin_id})
company_details = self.db_linkedincompany.find_one({'id': user_details['linkedin_company_id']})
position_details = self.db_position.find_one({'loggedin_id': loggedin_id, 'slug': position_slug})
project_details = self.db_project.find_one({'slug': position_details['project']})
event_data = {
"action": "resume search",
"userId": loggedin_id,
"companyId": user_details['linkedin_company_id'],
"positionslug": position_slug,
"projectslug": position_details['project'],
"userCompany": company_details['name'],
"forCompanyId": project_details['companyId'],
"forCompanyName": project_details['company'],
"title": position_details['title'],
"projectName": project_details['name'],
"results": resumesearch_details['results'],
"site": resumesearch_details['site']
}
recordEvent.delay('siteActions', event_data)
except:
rollbar.report_exc_info()
def event_record_resumedownload(self, position_slug, loggedin_id, doc_id):
download_details = self.db_positionexternalrel.find_one({'position_slug': position_slug, 'loggedin_id': loggedin_id, 'doc_id': doc_id})
if download_details:
try:
user_details = self.db_user.find_one({'linkedin_id': loggedin_id})
company_details = self.db_linkedincompany.find_one({'id': user_details['linkedin_company_id']})
position_details = self.db_position.find_one({'loggedin_id': loggedin_id, 'slug': position_slug})
project_details = self.db_project.find_one({'slug': position_details['project']})
event_data = {
"action": "resume download",
"userId": loggedin_id,
"companyId": user_details['linkedin_company_id'],
"positionslug": position_slug,
"projectslug": position_details['project'],
"userCompany": company_details['name'],
"forCompanyId": project_details['companyId'],
"forCompanyName": project_details['company'],
"title": position_details['title'],
"projectName": project_details['name'],
"site": download_details['source'],
"resumeid": doc_id
}
recordEvent.delay('siteActions', event_data)
except:
rollbar.report_exc_info()
from keen.client import KeenClient
keen = KeenClient(project_id="123", write_key="abc")
@app.before_request
def before_request():
if "/static/" in request.path:
return
g.start_time = time.time()
@app.after_request
def send_to_keen(response):
if request.path.startswith("/static"):
return response
keen.add_event(
"request", {
"keen": {
"addons": [{
"name": "keen:ip_to_geo",
"input": {
"ip": "ip_address"
},
"output": "ip_geo_info"
}, {
"name": "keen:url_parser",
"input": {
"url": "url"
},
def get_keen_sonar(collection, timeframe, filters = [], timezone="US/Pacific"):
client = KeenClient(
project_id=conf['keen_project_id'], read_key=conf['keen_read_key'])
return client.extraction(
collection, timeframe=timeframe, filters=filters, timezone=timezone)
from sseclient import SSEClient
from keen.client import KeenClient
import json
client = KeenClient(
project_id="50ba759d897a2c45c2000000",
write_key="72a1ad85456fd2361cfe8b65d240eab16b083caa944c11d9507761e536d34822eca4cefad12e482373ee41278953f98c6ff724b33b56fd5971e8d34cdd284cb6c3f202d9f13329e39c01eb9a35d1124cae81d9a4505992fce03be4030e32b19c72df266c55a18937f001e7158f6eac44",
read_key="f7a829ecc9e237949b0b97245c828661ff0c9f46eaec35a7dc4cfd3b4ef42804f5fc1859ecad05bd7d72b6c3a2e4047ae39cba86b5d51c770be26b0a406cbb44650d20fd05fb7896293f015cec3af3ca2df8a507927bf4cfed9275eb89e8dee855cf45db75b221074bda2f737d2810f2"
)
headers = {"Accept": "text/event-stream"}
messages = SSEClient('https://flickering-fire-322.firebaseio.com/.json', headers=headers)
first = True
for msg in messages:
print msg.event
print msg.data
if not first and 'put' in msg.event:
data = json.loads(msg.data)
path = data['path'].split("/")
device = path[1]
timestamp = path[2]
data = int(data['data'])
event = {
"device": device,
"volume": data,
"timestamp": timestamp
}
client.add_event("flow", event)
def test_direct_persistence_strategy(self):
project_id = "5004ded1163d66114f000000"
api_key = "2e79c6ec1d0145be8891bf668599c79a"
write_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["write"]})
client = KeenClient(project_id, write_key=write_key)
client.add_event("python_test", {"hello": "goodbye"})
def positive_helper(project_id, **kwargs):
client = KeenClient(project_id, **kwargs)
self.assert_not_equal(client, None)
self.assert_equal(project_id, client.project_id)
return client
from sseclient import SSEClient
from keen.client import KeenClient
import json
client = KeenClient(
project_id="50ba759d897a2c45c2000000",
write_key=
"72a1ad85456fd2361cfe8b65d240eab16b083caa944c11d9507761e536d34822eca4cefad12e482373ee41278953f98c6ff724b33b56fd5971e8d34cdd284cb6c3f202d9f13329e39c01eb9a35d1124cae81d9a4505992fce03be4030e32b19c72df266c55a18937f001e7158f6eac44",
read_key=
"f7a829ecc9e237949b0b97245c828661ff0c9f46eaec35a7dc4cfd3b4ef42804f5fc1859ecad05bd7d72b6c3a2e4047ae39cba86b5d51c770be26b0a406cbb44650d20fd05fb7896293f015cec3af3ca2df8a507927bf4cfed9275eb89e8dee855cf45db75b221074bda2f737d2810f2"
)
headers = {"Accept": "text/event-stream"}
messages = SSEClient('https://flickering-fire-322.firebaseio.com/.json',
headers=headers)
first = True
for msg in messages:
print msg.event
print msg.data
if not first and 'put' in msg.event:
data = json.loads(msg.data)
path = data['path'].split("/")
device = path[1]
timestamp = path[2]
data = int(data['data'])
event = {"device": device, "volume": data, "timestamp": timestamp}
client.add_event("flow", event)
first = False
def test_timeout_count(self):
keen.project_id = "5004ded1163d66114f000000"
api_key = "2e79c6ec1d0145be8891bf668599c79a"
keen.read_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["read"]})
client = KeenClient(keen.project_id, write_key=None, read_key=keen.read_key, get_timeout=0.0001)
resp = client.count("query test", timeframe="today", filters=self.get_filter())
