def test_flushes_after_first_event_if_first_flush_true(self, sync_flush):
self.consumer = AsyncBufferedConsumer(max_size=self.MAX_SIZE)
self.send_event()
self.wait_for_threads()
sync_flush.assert_called_once_with()
def test_endpoint_events_get_flushed_instantly_with_max_size_1(
self, sync_flush):
self.consumer = AsyncBufferedConsumer(max_size=1, flush_first=False)
self.send_event()
self.wait_for_threads()
sync_flush.assert_called_once_with(endpoint=self.ENDPOINT)
def test_does_not_drop_events(self):
self.consumer = AsyncBufferedConsumer(flush_first=True)
send_patch = patch.object(self.consumer._consumer, 'send').start()
self.send_event()
self.send_event()
self.wait_for_threads()
send_patch.assert_called_once_with(self.ENDPOINT, '[{"test": true}]')
self.assertEqual(self.consumer._async_buffers[self.ENDPOINT],
[self.JSON])
def test_flushes_after_first_event_if_first_flush_true(self, sync_flush):
self.consumer = AsyncBufferedConsumer(max_size=self.MAX_SIZE)
self.send_event()
self.wait_for_threads()
sync_flush.assert_called_once_with()
def test_endpoint_events_get_flushed_instantly_with_max_size_1(self, sync_flush):
self.consumer = AsyncBufferedConsumer(max_size=1, flush_first=False)
self.send_event()
self.wait_for_threads()
sync_flush.assert_called_once_with(endpoint=self.ENDPOINT)
def init_mixpannel_clients(mxp_tokens: List[str]) -> List[Mixpanel]:
"""
Return a list of mixpannel clients.
"""
projects: List[Mixpanel] = []
logger.info('Found %s Mixpannel tokens.', len(mxp_tokens))
for project_token in mxp_tokens:
mp = Mixpanel(project_token, consumer=AsyncBufferedConsumer())
projects.append(mp)
logger.info('%s Mixpannel projects ready to go.', len(projects))
return projects
def test_does_not_drop_events(self):
self.consumer = AsyncBufferedConsumer(flush_first=True)
send_patch = patch.object(self.consumer._consumer, 'send').start()
self.send_event()
self.send_event()
self.wait_for_threads()
send_patch.assert_called_once_with(self.ENDPOINT, '[{"test": true}]')
self.assertEqual(self.consumer._async_buffers[self.ENDPOINT], [self.JSON])
def mixpanel_event(name, username=None, properties=None):
"""
Takes an event name and a dict of args and registers it with Mixpanel.
If the username is None, it will assumed that it can be found in a
.siphon file in the directory.
"""
# Use AsyncBufferedConsumer to avoid blocking the main thread
mp = Mixpanel(MIXPANEL_TOKEN, consumer=AsyncBufferedConsumer())
if not username:
auth = Auth()
username = auth.username
props = {'user_platform': get_platform_name()}
if properties:
props.update(properties)
mp.track(username, name, props)
def _trace_mixpanel(action_name, auth_user, client_user, client_token, kwargs):
from mixpanel import Mixpanel
from mixpanel_async import AsyncBufferedConsumer
some_user_id = auth_user or client_user or client_token
try:
if action_name in ["get_server_info", "publish"]:
mp = Mixpanel(BII_MIXPANEL_TOKEN, consumer=AsyncBufferedConsumer())
properties = {
'action': action_name,
'anonymous': (some_user_id == client_token)
}
if action_name == "get_server_info":
properties["os"] = kwargs["bson_data"]["data"][0]["family"]
properties["biicode_version"] = kwargs["bson_data"]["data"][1]
mp.track(some_user_id, BII_API_MIXPANEL_EVENT_NAME, properties)
except Exception as e:
logger.warning("Error sending action to mixpanel: %s" % e)
def setUp(self):
self.consumer = AsyncBufferedConsumer(max_size=self.MAX_SIZE,
flush_first=False)
class AsyncBufferedConsumerTestCase(unittest.TestCase):
MAX_SIZE = 5
ENDPOINT = 'people'
JSON = json.dumps(dict(test=True))
def setUp(self):
self.consumer = AsyncBufferedConsumer(max_size=self.MAX_SIZE,
flush_first=False)
@patch.object(mixpanel.BufferedConsumer, '_flush_endpoint')
def test_sync_flush_endpoint_calls_buffered_consumer_flush_endpoint(
self, _flush_endpoint):
self.send_event()
self.assertFalse(_flush_endpoint.called)
self.consumer._sync_flush(endpoint=self.ENDPOINT)
_flush_endpoint.assert_called_once_with(self.ENDPOINT)
@patch.object(mixpanel.BufferedConsumer, '_flush_endpoint')
def test_sync_flush_calls_buffered_consumer_flush_endpoint(
self, flush_endpoint):
self.send_event()
self.assertFalse(flush_endpoint.called)
self.consumer._sync_flush()
flush_endpoint.assert_any_call(self.ENDPOINT)
@patch.object(AsyncBufferedConsumer, '_sync_flush')
def test_flush_gets_called_in_different_thread_if_async(self, sync_flush):
main_thread_id = thread.get_ident()
flush_thread_id = None
def side_effect(endpoint=None):
self.assertNotEqual(main_thread_id, thread.get_ident())
return DEFAULT
sync_flush.side_effect = side_effect
self.send_max_events()
self.wait_for_threads()
@patch.object(AsyncBufferedConsumer, '_sync_flush')
def test_flush_gets_called_in_same_thread_if_not_async(self, sync_flush):
main_thread_id = thread.get_ident()
flush_thread_id = None
def side_effect(endpoint=None):
self.assertEqual(main_thread_id, thread.get_ident())
return DEFAULT
sync_flush.side_effect = side_effect
self.send_event()
self.consumer.flush(use_async=False)
@patch.object(AsyncBufferedConsumer, '_sync_flush')
def test_flushes_after_first_event_if_first_flush_true(self, sync_flush):
self.consumer = AsyncBufferedConsumer(max_size=self.MAX_SIZE)
self.send_event()
self.wait_for_threads()
sync_flush.assert_called_once_with()
@patch.object(AsyncBufferedConsumer, '_sync_flush')
def test_does_not_flush_after_first_event_if_first_flush_false(
self, sync_flush):
self.send_event()
self.wait_for_threads()
self.assertFalse(sync_flush.called)
@patch.object(AsyncBufferedConsumer, '_sync_flush')
def test_endpoints_get_flushed_when_they_hit_max_size(self, sync_flush):
self.send_event("events")
self.send_max_events()
self.wait_for_threads()
sync_flush.assert_called_once_with(endpoint=self.ENDPOINT)
@patch.object(AsyncBufferedConsumer, '_sync_flush')
def test_all_events_get_flushed_after_flush_after(self, sync_flush):
self.consumer.flush_after = datetime.timedelta(0, .5)
self.send_event()
self.assertFalse(sync_flush.called)
time.sleep(.6)
self.send_event()
self.wait_for_threads()
sync_flush.assert_called_once_with()
@patch.object(AsyncBufferedConsumer, '_sync_flush')
def test_endpoint_events_get_flushed_instantly_with_max_size_1(
self, sync_flush):
self.consumer = AsyncBufferedConsumer(max_size=1, flush_first=False)
self.send_event()
self.wait_for_threads()
sync_flush.assert_called_once_with(endpoint=self.ENDPOINT)
def test_does_not_drop_events(self):
self.consumer = AsyncBufferedConsumer(flush_first=True)
send_patch = patch.object(self.consumer._consumer, 'send').start()
self.send_event()
self.send_event()
self.wait_for_threads()
send_patch.assert_called_once_with(self.ENDPOINT, '[{"test": true}]')
self.assertEqual(self.consumer._async_buffers[self.ENDPOINT],
[self.JSON])
def test_raises_exception_with_bad_endpoint(self):
with self.assertRaises(mixpanel.MixpanelException):
self.consumer.send('badendpoint', True)
def send_event(self, endpoint=None):
endpoint = endpoint or self.ENDPOINT
self.consumer.send(endpoint, self.JSON)
def send_max_events(self):
for i in range(self.MAX_SIZE):
self.send_event()
def wait_for_threads(self):
if not self.consumer._flush_thread_is_free():
self.consumer.flushing_thread.join()
def setUp(self):
self.consumer = AsyncBufferedConsumer(
max_size=self.MAX_SIZE,
flush_first=False
)
class AsyncBufferedConsumerTestCase(unittest.TestCase):
MAX_SIZE = 5
ENDPOINT = 'people'
JSON = json.dumps(dict(test=True))
def setUp(self):
self.consumer = AsyncBufferedConsumer(
max_size=self.MAX_SIZE,
flush_first=False
)
@patch.object(mixpanel.BufferedConsumer, '_flush_endpoint')
def test_sync_flush_endpoint_calls_buffered_consumer_flush_endpoint(self, _flush_endpoint):
self.send_event()
self.assertFalse(_flush_endpoint.called)
self.consumer._sync_flush(endpoint=self.ENDPOINT)
_flush_endpoint.assert_called_once_with(self.ENDPOINT)
@patch.object(mixpanel.BufferedConsumer, '_flush_endpoint')
def test_sync_flush_calls_buffered_consumer_flush_endpoint(self, flush_endpoint):
self.send_event()
self.assertFalse(flush_endpoint.called)
self.consumer._sync_flush()
flush_endpoint.assert_called_with(self.ENDPOINT)
@patch.object(AsyncBufferedConsumer, '_sync_flush')
def test_flush_gets_called_in_different_thread_if_async(self, sync_flush):
main_thread_id = thread.get_ident()
flush_thread_id = None
def side_effect(endpoint=None):
self.assertNotEqual(main_thread_id, thread.get_ident())
return DEFAULT
sync_flush.side_effect = side_effect
self.send_max_events()
self.wait_for_threads()
@patch.object(AsyncBufferedConsumer, '_sync_flush')
def test_flush_gets_called_in_same_thread_if_not_async(self, sync_flush):
main_thread_id = thread.get_ident()
flush_thread_id = None
def side_effect(endpoint=None):
self.assertEqual(main_thread_id, thread.get_ident())
return DEFAULT
sync_flush.side_effect = side_effect
self.send_event()
self.consumer.flush(async=False)
@patch.object(AsyncBufferedConsumer, '_sync_flush')
def test_flushes_after_first_event_if_first_flush_true(self, sync_flush):
self.consumer = AsyncBufferedConsumer(max_size=self.MAX_SIZE)
self.send_event()
self.wait_for_threads()
sync_flush.assert_called_once_with()
@patch.object(AsyncBufferedConsumer, '_sync_flush')
def test_does_not_flush_after_first_event_if_first_flush_false(self, sync_flush):
self.send_event()
self.wait_for_threads()
self.assertFalse(sync_flush.called)
@patch.object(AsyncBufferedConsumer, '_sync_flush')
def test_endpoints_get_flushed_when_they_hit_max_size(self, sync_flush):
self.send_event("events")
self.send_max_events()
self.wait_for_threads()
sync_flush.assert_called_once_with(endpoint=self.ENDPOINT)
@patch.object(AsyncBufferedConsumer, '_sync_flush')
def test_all_events_get_flushed_after_flush_after(self, sync_flush):
self.consumer.flush_after = datetime.timedelta(0, .5)
self.send_event()
self.assertFalse(sync_flush.called)
time.sleep(.6)
self.send_event()
self.wait_for_threads()
sync_flush.assert_called_once_with()
@patch.object(AsyncBufferedConsumer, '_sync_flush')
def test_endpoint_events_get_flushed_instantly_with_max_size_1(self, sync_flush):
self.consumer = AsyncBufferedConsumer(max_size=1, flush_first=False)
self.send_event()
self.wait_for_threads()
sync_flush.assert_called_once_with(endpoint=self.ENDPOINT)
def send_event(self, endpoint=None):
endpoint = endpoint or self.ENDPOINT
self.consumer.send(endpoint, self.JSON)
def send_max_events(self):
for i in range(self.MAX_SIZE):
self.send_event()
def wait_for_threads(self):
if not self.consumer._flush_thread_is_free():
self.consumer.flushing_thread.join()
def create_app(config_name):
app = Flask(__name__, static_url_path='')
app.config.from_object(config[config_name])
app.config.from_envvar("OPENTARGETS_API_LOCAL_SETTINGS", silent=True)
config[config_name].init_app(app)
api_version = app.config['API_VERSION']
api_version_minor = app.config['API_VERSION_MINOR']
# log_level = logging.INFO
# if app.config['DEBUG']:
# log_level = logging.DEBUG
# Flask has a default logger which works well and pushes to stderr
# if you want to add different handlers (to file, or logstash, or whatever)
# you can use code similar to the one below and set the error level accordingly.
# logHandler = logging.StreamHandler()
# formatter = jsonlogger.JsonFormatter()
# logHandler.setFormatter(formatter)
# loghandler.setLevel(logging.INFO)
# app.logger.addHandler(logHandler)
# or for LOGSTASH
# app.logger.addHandler(logstash.LogstashHandler(app.config['LOGSTASH_HOST'], app.config['LOGSTASH_PORT'], version=1))
app.logger.info('looking for elasticsearch at: %s' %
app.config['ELASTICSEARCH_URL'])
print('looking for elasticsearch at: %s' % app.config['ELASTICSEARCH_URL'])
app.extensions['redis-core'] = Redis(app.config['REDIS_SERVER_PATH'],
db=0) #served data
app.extensions['redis-service'] = Redis(
app.config['REDIS_SERVER_PATH'],
db=1) #cache, rate limit and internal things
app.extensions['redis-user'] = Redis(app.config['REDIS_SERVER_PATH'],
db=2) # user info
'''setup cache'''
app.extensions['redis-service'].config_set('save', '')
app.extensions['redis-service'].config_set('appendonly', 'no')
icache = InternalCache(app.extensions['redis-service'],
str(api_version_minor))
ip2org = IP2Org(icache)
es = Elasticsearch(
app.config['ELASTICSEARCH_URL'],
# # sniff before doing anything
# sniff_on_start=True,
# # refresh nodes after a node fails to respond
# sniff_on_connection_fail=True,
# # and also every 60 seconds
# sniffer_timeout=60
timeout=60 * 20,
maxsize=100,
)
'''elasticsearch handlers'''
app.extensions['esquery'] = esQuery(
es,
DataTypes(app),
DataSourceScoring(app),
index_data=app.config['ELASTICSEARCH_DATA_INDEX_NAME'],
index_efo=app.config['ELASTICSEARCH_EFO_LABEL_INDEX_NAME'],
index_eco=app.config['ELASTICSEARCH_ECO_INDEX_NAME'],
index_genename=app.config['ELASTICSEARCH_GENE_NAME_INDEX_NAME'],
index_expression=app.config['ELASTICSEARCH_EXPRESSION_INDEX_NAME'],
index_reactome=app.config['ELASTICSEARCH_REACTOME_INDEX_NAME'],
index_association=app.
config['ELASTICSEARCH_DATA_ASSOCIATION_INDEX_NAME'],
index_search=app.config['ELASTICSEARCH_DATA_SEARCH_INDEX_NAME'],
index_relation=app.config['ELASTICSEARCH_DATA_RELATION_INDEX_NAME'],
docname_data=app.config['ELASTICSEARCH_DATA_DOC_NAME'],
docname_efo=app.config['ELASTICSEARCH_EFO_LABEL_DOC_NAME'],
docname_eco=app.config['ELASTICSEARCH_ECO_DOC_NAME'],
docname_genename=app.config['ELASTICSEARCH_GENE_NAME_DOC_NAME'],
docname_expression=app.config['ELASTICSEARCH_EXPRESSION_DOC_NAME'],
docname_reactome=app.
config['ELASTICSEARCH_REACTOME_REACTION_DOC_NAME'],
docname_association=app.
config['ELASTICSEARCH_DATA_ASSOCIATION_DOC_NAME'],
docname_search=app.config['ELASTICSEARCH_DATA_SEARCH_DOC_NAME'],
# docname_search_target=app.config['ELASTICSEARCH_DATA_SEARCH_TARGET_DOC_NAME'],
# docname_search_disease=app.config['ELASTICSEARCH_DATA_SEARCH_DISEASE_DOC_NAME'],
docname_relation=app.config['ELASTICSEARCH_DATA_RELATION_DOC_NAME'],
log_level=app.logger.getEffectiveLevel(),
cache=icache)
app.extensions['es_access_store'] = esStore(
es,
eventlog_index=app.config['ELASTICSEARCH_LOG_EVENT_INDEX_NAME'],
ip2org=ip2org,
)
'''mixpanel handlers'''
if Config.MIXPANEL_TOKEN:
mp = Mixpanel(Config.MIXPANEL_TOKEN, consumer=AsyncBufferedConsumer())
app.extensions['mixpanel'] = mp
app.extensions['mp_access_store'] = MixPanelStore(
mp,
ip2org=ip2org,
)
app.extensions['proxy'] = ProxyHandler(
allowed_targets=app.config['PROXY_SETTINGS']['allowed_targets'],
allowed_domains=app.config['PROXY_SETTINGS']['allowed_domains'],
allowed_request_domains=app.config['PROXY_SETTINGS']
['allowed_request_domains'])
basepath = app.config['PUBLIC_API_BASE_PATH'] + api_version
# cors = CORS(app, resources=r'/api/*', allow_headers='Content-Type,Auth-Token')
''' define cache'''
# cache = Cache(config={'CACHE_TYPE': 'simple'})
# cache.init_app(latest_blueprint)
# latest_blueprint.cache = cache
# latest_blueprint.extensions['cache'] = cache
# app.cache = SimpleCache()
app.cache = FileSystemCache('/tmp/cttv-rest-api-cache',
threshold=100000,
default_timeout=60 * 60,
mode=777)
'''Set usage limiter '''
# limiter = Limiter(global_limits=["2000 per hour", "20 per second"])
# limiter.init_app(app)# use redis to store limits
'''Load api keys in redis'''
rate_limit_file = app.config['USAGE_LIMIT_PATH']
if not os.path.exists(rate_limit_file):
rate_limit_file = '../' + rate_limit_file
if os.path.exists(rate_limit_file):
with open(rate_limit_file) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
auth_key = AuthKey(**row)
app.extensions['redis-user'].hmset(auth_key.get_key(),
auth_key.__dict__)
print('INFO - succesfully loaded rate limit file')
else:
print('ERROR - cannot find rate limit file')
app.logger.error(
'cannot find rate limit file: %s. RATE LIMIT QUOTA LOAD SKIPPED!' %
rate_limit_file)
'''load ip name resolution'''
ip_resolver = defaultdict(lambda: "PUBLIC")
ip_list_file = app.config['IP_RESOLVER_LIST_PATH']
if not os.path.exists(ip_list_file):
ip_list_file = '../' + ip_list_file
if os.path.exists(ip_list_file):
with open(ip_list_file) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
net = IPNetwork(row['ip'])
ip_resolver[net] = row['org']
else:
app.logger.warning(
'cannot find IP list for IP resolver. All traffic will be logged as PUBLIC'
)
app.config['IP_RESOLVER'] = ip_resolver
'''compress http response'''
compress = Compress()
compress.init_app(app)
latest_blueprint = Blueprint('latest', __name__)
current_version_blueprint = Blueprint(str(api_version), __name__)
current_minor_version_blueprint = Blueprint(str(api_version_minor),
__name__)
specpath = '/cttv'
if app.config['PROFILE'] == True:
from werkzeug.contrib.profiler import ProfilerMiddleware
app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[30])
create_api(latest_blueprint, api_version, specpath)
create_api(current_version_blueprint, api_version, specpath)
create_api(current_minor_version_blueprint, api_version_minor, specpath)
app.register_blueprint(latest_blueprint, url_prefix='/api/latest')
app.register_blueprint(current_version_blueprint,
url_prefix='/api/' + str(api_version))
app.register_blueprint(current_minor_version_blueprint,
url_prefix='/api/' + str(api_version_minor))
@app.route('/api-docs/%s' % str(api_version_minor))
def docs_current_minor_version():
return redirect('/api/swagger/index.html')
@app.route('/api-docs/%s' % str(api_version))
def docs_current_version():
return redirect('/api/swagger/index.html')
@app.route('/api-docs')
def docs():
return redirect('/api/swagger/index.html')
def serve_swagger():
return app.send_static_file('docs/swagger/swagger.yaml')
@app.route('/api/docs/swagger.yaml')
def send_swagger():
return serve_swagger()
@app.route('/api/latest/docs/swagger.yaml')
def send_swagger_latest():
return serve_swagger()
@app.route('/api/' + str(api_version) + '/docs/swagger.yaml')
def send_swagger_current_cersion():
return serve_swagger()
@app.before_request
def before_request():
g.request_start = datetime.now()
@app.after_request
def after(resp):
rate_limiter = RateLimiter()
now = datetime.now()
took = (now - g.request_start).total_seconds() * 1000
if took > 500:
cache_time = str(
int(3600 * took)
) # set cache to last one our for each second spent in the request
resp.headers.add('X-Accel-Expires', cache_time)
took = int(round(took))
LogApiCallWeight(took)
# if took < RateLimiter.DEFAULT_CALL_WEIGHT:
# took = RateLimiter.DEFAULT_CALL_WEIGHT
current_values = increment_call_rate(took, rate_limiter)
now = datetime.now()
ceil10s = round(ceil_dt_to_future_time(now, 10), 2)
ceil1h = round(ceil_dt_to_future_time(now, 3600), 2)
usage_left_10s = rate_limiter.short_window_rate - current_values[
'short']
usage_left_1h = rate_limiter.long_window_rate - current_values['long']
min_ceil = ceil10s
if usage_left_1h < 0:
min_ceil = ceil1h
if (usage_left_10s < 0) or (usage_left_1h < 0):
resp.headers.add('Retry-After', min_ceil)
resp.headers.add('X-API-Took', took)
resp.headers.add('X-Usage-Limit-10s', rate_limiter.short_window_rate)
resp.headers.add('X-Usage-Limit-1h', rate_limiter.long_window_rate)
resp.headers.add('X-Usage-Remaining-10s', usage_left_10s)
resp.headers.add('X-Usage-Remaining-1h', usage_left_1h)
# resp.headers.add('X-Usage-Limit-Reset-10s', ceil10s)
# resp.headers.add('X-Usage-Limit-Reset-1h', ceil1h)
resp.headers.add('Access-Control-Allow-Origin', '*')
resp.headers.add('Access-Control-Allow-Headers',
'Content-Type,Auth-Token')
if do_not_cache(request): # do not cache in the browser
resp.headers.add('Cache-Control',
"no-cache, must-revalidate, max-age=0")
else:
resp.headers.add(
'Cache-Control',
"no-transform, public, max-age=%i, s-maxage=%i" %
(took * 1800 / 1000, took * 9000 / 1000))
return resp
return app
class LampiApp(App):
_updated = False
_updatingUI = False
_hue = NumericProperty()
_saturation = NumericProperty()
_brightness = NumericProperty()
_preset = NumericProperty()
lamp_is_on = BooleanProperty()
_preset_color = NumericProperty()
_preset_temp = NumericProperty()
remote_connection = StringProperty("[b]Connected:[/b] No")
trusted_remotes = StringProperty("[b]Trusted Remotes:[/b] None")
mp = Mixpanel(MIXPANEL_TOKEN, consumer=AsyncBufferedConsumer())
def _get_hue(self):
return self._hue
def _set_hue(self, value):
self._hue = value
def _get_saturation(self):
return self._saturation
def _set_saturation(self, value):
self._saturation = value
def _get_brightness(self):
return self._brightness
def _set_brightness(self, value):
self._brightness = value
def _get_preset(self):
return self._preset
def _set_preset(self, value):
self._preset = value
def _get_preset_color(self):
return self._preset_color
def _set_preset_color(self, value):
self._preset_color = value
def _get_preset_temp(self):
return self._preset_temp
def _set_preset_temp(self, value):
self._preset_temp = value
hue = AliasProperty(_get_hue, _set_hue, bind=['_hue'])
saturation = AliasProperty(_get_saturation,
_set_saturation,
bind=['_saturation'])
brightness = AliasProperty(_get_brightness,
_set_brightness,
bind=['_brightness'])
preset = AliasProperty(_get_preset, _set_preset, bind=['_preset'])
preset_color = AliasProperty(_get_preset_color,
_set_preset_color,
bind=['_preset_color'])
preset_temp = AliasProperty(_get_preset_temp,
_set_preset_temp,
bind=['_preset_temp'])
gpio17_pressed = BooleanProperty(False)
device_associated = BooleanProperty(True)
def on_start(self):
self._publish_clock = None
self.mqtt_broker_bridged = False
self._associated = True
self.association_code = None
self.mqtt = Client(client_id=MQTT_CLIENT_ID)
self.mqtt.enable_logger()
self.mqtt.will_set(client_state_topic(MQTT_CLIENT_ID),
"0",
qos=2,
retain=True)
self.mqtt.on_connect = self.on_connect
self.mqtt.connect(MQTT_BROKER_HOST,
port=MQTT_BROKER_PORT,
keepalive=MQTT_BROKER_KEEP_ALIVE_SECS)
self.mqtt.loop_start()
self.set_up_GPIO_and_device_status_popup()
self.associated_status_popup = self._build_associated_status_popup()
self.associated_status_popup.bind(on_open=self.update_popup_associated)
self._remote = None
self._popup_remote = None
self.pairing_popup = self._build_pairing_popup()
self._update_remotes_ui()
self.discoverswitch = self.root.ids.discoverswitch
self.discoverswitch.bind(active=self.toggle_discovery)
Clock.schedule_interval(self._poll_associated, 0.1)
def _build_associated_status_popup(self):
return Popup(title='Associate your Lamp',
content=Label(text='Msg here', font_size='30sp'),
size_hint=(1, 1),
auto_dismiss=False)
def _build_pairing_popup(self):
layout = StackLayout()
label = Label(
text=
'A new remote is attempting\nto connect to your lamp.\n\nWould you like to\nallow it?',
size_hint=(1, None),
padding=(4, 4))
label.bind(size=self._update_textsize)
deny = Button(text='Deny', size_hint=(0.49, None), height=40)
allow = Button(text='Trust', size_hint=(0.49, None), height=40)
allow.on_release = self._allow_remote
deny.on_release = self._decline_remote
layout.add_widget(label)
layout.add_widget(Label(size_hint=(1, None), height=15))
layout.add_widget(deny)
layout.add_widget(Label(size_hint=(0.02, None), height=1))
layout.add_widget(allow)
return Popup(title='Remote Pairing Request',
content=layout,
size_hint=(1, 0.68),
auto_dismiss=False)
def _update_textsize(self, instance, value):
instance.text_size = (value[0], value[1])
def on_new_remote(self, client, userdata, message):
isEmpty = message.payload == b''
if isEmpty:
self._remote = None
else:
remote = json.loads(message.payload.decode('utf-8'))
self._remote = remote
self._popup_remote = remote
if (not remote['allowed']):
self.pairing_popup.open()
self._update_remotes_ui()
def _allow_remote(self):
print("Pairing allowed for {}".format(self._popup_remote['address']))
remotes.saveAddress(self._popup_remote['address'])
self._remote = None
self._popup_remote = None
self.pairing_popup.dismiss()
self._update_remotes_ui()
# Display confirmation
conf = Popup(
title='Remote Trusted',
content=Label(
text=
'You have successfully trusted\nyour remote. Pair it again to\nuse it'
),
size_hint=(1, 0.5),
auto_dismiss=False)
conf.open()
Clock.schedule_once(lambda dt: conf.dismiss(), 3)
def _decline_remote(self):
print("Pairing denied for {}".format(self._popup_remote['address']))
self._popup_remote = None
self._remote = None
self.pairing_popup.dismiss()
self._update_remotes_ui()
def clear_remotes(self):
remotes.clear()
self.mqtt.publish(DISCONNECT_TOPIC, b'')
self._update_remotes_ui()
def toggle_discovery(self, instance, value):
# Send message accordingly
self.mqtt.publish(DISCOVERY_TOPIC,
("true" if value else "false").encode('utf8'),
retain=True)
def _update_remotes_ui(self):
savedremotes = remotes._read()
statustext = "[b]Connected:[/b] False\n\n"
if (self._remote is not None):
self.remote_connection = "[b]Connected:[/b] [color=32ff32]{}[/color]".format(
self._remote['address'])
else:
self.remote_connection = "[b]Connected:[/b] [color=ff3232]Not connected[/color]"
if (len(savedremotes) == 0):
self.trusted_remotes = "[b]Trusted Remotes:[/b] None"
else:
self.trusted_remotes = "[b]Trusted Remotes:[/b]\n" + "\n".join(
[" • {}".format(addr) for addr in savedremotes])
def on_hue(self, instance, value):
if self._updatingUI:
return
self._track_ui_event('Slider Change', {
'slider': 'hue-slider',
'value': value
})
if self._publish_clock is None:
self._publish_clock = Clock.schedule_once(
lambda dt: self._update_leds(), 0.01)
def on_saturation(self, instance, value):
if self._updatingUI:
return
self._track_ui_event('Slider Change', {
'slider': 'saturation-slider',
'value': value
})
if self._publish_clock is None:
self._publish_clock = Clock.schedule_once(
lambda dt: self._update_leds(), 0.01)
def on_brightness(self, instance, value):
if self._updatingUI:
return
self._track_ui_event('Slider Change', {
'slider': 'brightness-slider',
'value': value
})
if self._publish_clock is None:
self._publish_clock = Clock.schedule_once(
lambda dt: self._update_leds(), 0.01)
def on_lamp_is_on(self, instance, value):
if self._updatingUI:
return
self._track_ui_event('Toggle Power', {'isOn': value})
if self._publish_clock is None:
self._publish_clock = Clock.schedule_once(
lambda dt: self._update_leds(), 0.01)
def on_preset_temp(self, instance, value):
if self._updatingUI:
return
self._track_ui_event('Slider Change', {
'slider': 'preset_hue_slider',
'value': value
})
def _track_ui_event(self, event_name, additional_props={}):
device_id = lampi.lampi_util.get_device_id()
event_props = {
'event_type': 'ui',
'interface': 'lampi',
'device_id': device_id
}
event_props.update(additional_props)
self.mp.track(device_id, event_name, event_props)
def on_connect(self, client, userdata, flags, rc):
self.mqtt.publish(client_state_topic(MQTT_CLIENT_ID),
b"1",
qos=2,
retain=True)
self.mqtt.message_callback_add(TOPIC_LAMP_CHANGE_NOTIFICATION,
self.receive_new_lamp_state)
self.mqtt.message_callback_add(broker_bridge_connection_topic(),
self.receive_bridge_connection_status)
self.mqtt.message_callback_add(TOPIC_LAMP_ASSOCIATED,
self.receive_associated)
self.mqtt.message_callback_add(NEW_REMOTE_TOPIC, self.on_new_remote)
self.mqtt.subscribe(broker_bridge_connection_topic(), qos=1)
self.mqtt.subscribe(TOPIC_LAMP_CHANGE_NOTIFICATION, qos=1)
self.mqtt.subscribe(TOPIC_LAMP_ASSOCIATED, qos=2)
self.mqtt.subscribe(NEW_REMOTE_TOPIC, qos=2)
def _poll_associated(self, dt):
# this polling loop allows us to synchronize changes from the
# MQTT callbacks (which happen in a different thread) to the
# Kivy UI
self.device_associated = self._associated
def receive_associated(self, client, userdata, message):
# this is called in MQTT event loop thread
new_associated = json.loads(message.payload.decode('utf-8'))
if self._associated != new_associated['associated']:
if not new_associated['associated']:
self.association_code = new_associated['code']
else:
self.association_code = None
self._associated = new_associated['associated']
def on_device_associated(self, instance, value):
if value:
self.associated_status_popup.dismiss()
else:
self.associated_status_popup.open()
def update_popup_associated(self, instance):
code = self.association_code[0:6]
instance.content.text = ("Please use the\n"
"following code\n"
"to associate\n"
"your device\n"
"on the Web\n{}".format(code))
def receive_bridge_connection_status(self, client, userdata, message):
# monitor if the MQTT bridge to our cloud broker is up
if message.payload == b"1":
self.mqtt_broker_bridged = True
else:
self.mqtt_broker_bridged = False
def receive_new_lamp_state(self, client, userdata, message):
new_state = json.loads(message.payload.decode('utf-8'))
Clock.schedule_once(lambda dt: self._update_ui(new_state), 0.01)
def _update_ui(self, new_state):
if self._updated and new_state['client'] == MQTT_CLIENT_ID:
# ignore updates generated by this client, except the first to
# make sure the UI is syncrhonized with the lamp_service
return
self._updatingUI = True
try:
if 'color' in new_state:
self.hue = new_state['color']['h']
self.saturation = new_state['color']['s']
if 'brightness' in new_state:
self.brightness = new_state['brightness']
if 'on' in new_state:
self.lamp_is_on = new_state['on']
finally:
self._updatingUI = False
self._updated = True
def _update_leds(self):
msg = {
'color': {
'h': self._hue,
's': self._saturation
},
'brightness': self._brightness,
'on': self.lamp_is_on,
'client': MQTT_CLIENT_ID
}
self.mqtt.publish(TOPIC_SET_LAMP_CONFIG,
json.dumps(msg).encode('utf-8'),
qos=1)
self._publish_clock = None
def set_up_GPIO_and_device_status_popup(self):
self.pi = pigpio.pi()
self.pi.set_mode(17, pigpio.INPUT)
self.pi.set_pull_up_down(17, pigpio.PUD_UP)
Clock.schedule_interval(self._poll_GPIO, 0.05)
self.network_status_popup = self._build_network_status_popup()
self.network_status_popup.bind(on_open=self.update_device_status_popup)
def _build_network_status_popup(self):
return Popup(title='Device Status',
content=Label(text='IP ADDRESS WILL GO HERE'),
size_hint=(1, 1),
auto_dismiss=False)
def update_device_status_popup(self, instance):
interface = "wlan0"
ipaddr = lampi.lampi_util.get_ip_address(interface)
deviceid = lampi.lampi_util.get_device_id()
msg = ("Version: {}\n"
"{}: {}\n"
"DeviceID: {}\n"
"Broker Bridged: {}\n"
"Async Analytics").format(LAMPI_APP_VERSION, interface, ipaddr,
deviceid, self.mqtt_broker_bridged)
instance.content.text = msg
def on_gpio17_pressed(self, instance, value):
if value:
self.network_status_popup.open()
else:
self.network_status_popup.dismiss()
def _poll_GPIO(self, dt):
# GPIO17 is the rightmost button when looking front of LAMPI
self.gpio17_pressed = not self.pi.read(17)
def write_preset(self, num):
filewrite = {
"stateList": [
{
"state": {
"h": self._preset_color,
"s": 1.0,
"b": 1.0
},
"smooth": False,
"waitTime": 0,
"transitionTime": 0
},
],
'loop':
False
}
with open(PRESETS[num - 1] + ".json", "w") as f:
json.dump(filewrite, f)
def create_app(config_name):
app = Flask(__name__, static_url_path='')
# This first loads the configuration from eg. config['development'] which corresponds to the DevelopmentConfig class in the config.py
app.config.from_object(config[config_name])
# Then you can override the values with the contents of the file the OPENTARGETS_API_LOCAL_SETTINGS environment variable points to.
# For eg:
# $ export OPENTARGETS_API_LOCAL_SETTINGS=/path/to/settings.cfg
#
# where settings.cfg looks like:
#
# DEBUG = False
# SECRET_KEY = 'foo'
#
app.config.from_envvar("OPENTARGETS_API_LOCAL_SETTINGS", silent=True)
config[config_name].init_app(app)
api_version = app.config['API_VERSION']
api_version_minor = app.config['API_VERSION_MINOR']
app.logger.info('looking for elasticsearch at: %s' % app.config['ELASTICSEARCH_URL'])
app.extensions['redis-core'] = Redis(app.config['REDIS_SERVER_PATH'], db=0) #served data
app.extensions['redis-service'] = Redis(app.config['REDIS_SERVER_PATH'], db=1) #cache, rate limit and internal things
app.extensions['redis-user'] = Redis(app.config['REDIS_SERVER_PATH'], db=2)# user info
'''setup cache'''
app.extensions['redis-service'].config_set('save','')
app.extensions['redis-service'].config_set('appendonly', 'no')
icache = InternalCache(app.extensions['redis-service'],
str(api_version_minor))
ip2org = IP2Org(icache)
if app.config['ELASTICSEARCH_URL']:
es = Elasticsearch(app.config['ELASTICSEARCH_URL'],
# # sniff before doing anything
# sniff_on_start=True,
# # refresh nodes after a node fails to respond
# sniff_on_connection_fail=True,
# # and also every 60 seconds
# sniffer_timeout=60
timeout=60 * 20,
maxsize=32,
)
else:
es = None
'''elasticsearch handlers'''
app.extensions['esquery'] = esQuery(es,
DataTypes(app),
DataSourceScoring(app),
index_data=app.config['ELASTICSEARCH_DATA_INDEX_NAME'],
index_efo=app.config['ELASTICSEARCH_EFO_LABEL_INDEX_NAME'],
index_eco=app.config['ELASTICSEARCH_ECO_INDEX_NAME'],
index_genename=app.config['ELASTICSEARCH_GENE_NAME_INDEX_NAME'],
index_expression=app.config['ELASTICSEARCH_EXPRESSION_INDEX_NAME'],
index_reactome=app.config['ELASTICSEARCH_REACTOME_INDEX_NAME'],
index_association=app.config['ELASTICSEARCH_DATA_ASSOCIATION_INDEX_NAME'],
index_search=app.config['ELASTICSEARCH_DATA_SEARCH_INDEX_NAME'],
index_relation=app.config['ELASTICSEARCH_DATA_RELATION_INDEX_NAME'],
docname_data=app.config['ELASTICSEARCH_DATA_DOC_NAME'],
docname_efo=app.config['ELASTICSEARCH_EFO_LABEL_DOC_NAME'],
docname_eco=app.config['ELASTICSEARCH_ECO_DOC_NAME'],
docname_genename=app.config['ELASTICSEARCH_GENE_NAME_DOC_NAME'],
docname_expression=app.config['ELASTICSEARCH_EXPRESSION_DOC_NAME'],
docname_reactome=app.config['ELASTICSEARCH_REACTOME_REACTION_DOC_NAME'],
docname_association=app.config['ELASTICSEARCH_DATA_ASSOCIATION_DOC_NAME'],
docname_search=app.config['ELASTICSEARCH_DATA_SEARCH_DOC_NAME'],
# docname_search_target=app.config['ELASTICSEARCH_DATA_SEARCH_TARGET_DOC_NAME'],
# docname_search_disease=app.config['ELASTICSEARCH_DATA_SEARCH_DISEASE_DOC_NAME'],
docname_relation=app.config['ELASTICSEARCH_DATA_RELATION_DOC_NAME'],
log_level=app.logger.getEffectiveLevel(),
cache=icache
)
app.extensions['es_access_store'] = esStore(es,
eventlog_index=app.config['ELASTICSEARCH_LOG_EVENT_INDEX_NAME'],
ip2org=ip2org,
)
'''mixpanel handlers'''
if Config.MIXPANEL_TOKEN:
mp = Mixpanel(Config.MIXPANEL_TOKEN, consumer=AsyncBufferedConsumer())
app.extensions['mixpanel']= mp
app.extensions['mp_access_store'] = MixPanelStore(mp,
ip2org=ip2org,
)
app.extensions['proxy'] = ProxyHandler(allowed_targets=app.config['PROXY_SETTINGS']['allowed_targets'],
allowed_domains=app.config['PROXY_SETTINGS']['allowed_domains'],
allowed_request_domains=app.config['PROXY_SETTINGS']['allowed_request_domains'])
# basepath = app.config['PUBLIC_API_BASE_PATH']+api_version
# cors = CORS(app, resources=r'/api/*', allow_headers='Content-Type,Auth-Token')
''' define cache'''
# cache = Cache(config={'CACHE_TYPE': 'simple'})
# cache.init_app(latest_blueprint)
# latest_blueprint.cache = cache
# latest_blueprint.extensions['cache'] = cache
# app.cache = SimpleCache()
app.cache = FileSystemCache('/tmp/cttv-rest-api-cache', threshold=100000, default_timeout=60*60, mode=777)
'''Set usage limiter '''
# limiter = Limiter(global_limits=["2000 per hour", "20 per second"])
# limiter.init_app(app)# use redis to store limits
'''Load api keys in redis'''
rate_limit_file = app.config['USAGE_LIMIT_PATH']
if not os.path.exists(rate_limit_file):
rate_limit_file = '../'+rate_limit_file
csvfile = None
if Config.GITHUB_AUTH_TOKEN:
r = requests.get('https://api.github.com/repos/opentargets/rest_api_auth/contents/rate_limit.csv',
headers = {'Authorization': 'token %s'%Config.GITHUB_AUTH_TOKEN,
'Accept': 'application/vnd.github.v3.raw'})
if r.ok:
csvfile = r.text.split('\n')
app.logger.info('Retrieved rate limit file from github remote')
else:
app.logger.warning('Cannot retrieve rate limit file from remote, SKIPPED!')
elif os.path.exists(rate_limit_file):
csvfile = open(rate_limit_file)
app.logger.info('Using dummy rate limit file')
if csvfile is None:
app.logger.error('cannot find rate limit file: %s. RATE LIMIT QUOTA LOAD SKIPPED!'%rate_limit_file)
else:
reader = csv.DictReader(csvfile)
for row in reader:
auth_key = AuthKey(**row)
app.extensions['redis-user'].hmset(auth_key.get_key(), auth_key.__dict__)
try:
csvfile.close()
except:
pass
app.logger.info('succesfully loaded rate limit file')
'''load ip name resolution'''
ip_resolver = defaultdict(lambda: "PUBLIC")
ip_list_file = app.config['IP_RESOLVER_LIST_PATH']
if not os.path.exists(ip_list_file):
ip_list_file = '../' + ip_list_file
if os.path.exists(ip_list_file):
with open(ip_list_file) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
net = IPNetwork(row['ip'])
ip_resolver[net] = row['org']
else:
app.logger.warning('cannot find IP list for IP resolver. All traffic will be logged as PUBLIC')
app.config['IP_RESOLVER'] = ip_resolver
'''compress http response'''
compress = Compress()
compress.init_app(app)
latest_blueprint = Blueprint('latest', __name__)
current_version_blueprint = Blueprint(str(api_version), __name__)
current_minor_version_blueprint = Blueprint(str(api_version_minor), __name__)
specpath = '/cttv'
if app.config['PROFILE'] == True:
from werkzeug.contrib.profiler import ProfilerMiddleware
app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[30])
'''set the right prefixes'''
create_api(latest_blueprint, api_version, specpath)
create_api(current_version_blueprint, api_version, specpath)
create_api(current_minor_version_blueprint, api_version_minor, specpath)
# app.register_blueprint(latest_blueprint, url_prefix='/latest/platform')
app.register_blueprint(current_version_blueprint, url_prefix='/v'+str(api_version) + '/platform')
app.register_blueprint(current_minor_version_blueprint, url_prefix='/v'+str(api_version_minor) + '/platform')
'''serve the static docs'''
try:
'''
NOTE: this file gets created only at deployment time
'''
openapi_def = yaml.load(file('app/static/openapi.yaml', 'r'))
app.logger.info('parsing swagger from static/openapi.yaml')
except IOError:
'''if we are not deployed, then simply use the template'''
openapi_def = yaml.load(file('openapi.template.yaml', 'r'))
app.logger.error('parsing swagger from openapi.template.yaml')
with open("api-description.md", "r") as f:
desc = f.read()
openapi_def['info']['description'] = desc
openapi_def['basePath'] = '/v%s' % str(api_version)
@app.route('/v%s/platform/swagger' % str(api_version))
def serve_swagger(apiversion=api_version):
return jsonify(openapi_def)
@app.route('/v%s/platform/docs' % str(api_version))
def render_redoc(apiversion=api_version):
return render_template('docs.html',api_version=apiversion)
'''pre and post-request'''
@app.before_request
def before_request():
g.request_start = datetime.now()
@app.after_request
def after(resp):
try:
rate_limiter = RateLimiter()
now = datetime.now()
took = (now - g.request_start).total_seconds()*1000
if took > 500:
cache_time = str(int(3600*took))# set cache to last one our for each second spent in the request
resp.headers.add('X-Accel-Expires', cache_time)
took = int(round(took))
LogApiCallWeight(took)
# if took < RateLimiter.DEFAULT_CALL_WEIGHT:
# took = RateLimiter.DEFAULT_CALL_WEIGHT
current_values = increment_call_rate(took,rate_limiter)
now = datetime.now()
ceil10s=round(ceil_dt_to_future_time(now, 10),2)
ceil1h=round(ceil_dt_to_future_time(now, 3600),2)
usage_left_10s = rate_limiter.short_window_rate-current_values['short']
usage_left_1h = rate_limiter.long_window_rate - current_values['long']
min_ceil = ceil10s
if usage_left_1h <0:
min_ceil = ceil1h
if (usage_left_10s < 0) or (usage_left_1h <0):
resp.headers.add('Retry-After', min_ceil)
resp.headers.add('X-API-Took', took)
resp.headers.add('X-Usage-Limit-10s', rate_limiter.short_window_rate)
resp.headers.add('X-Usage-Limit-1h', rate_limiter.long_window_rate)
resp.headers.add('X-Usage-Remaining-10s', usage_left_10s)
resp.headers.add('X-Usage-Remaining-1h', usage_left_1h)
# resp.headers.add('X-Usage-Limit-Reset-10s', ceil10s)
# resp.headers.add('X-Usage-Limit-Reset-1h', ceil1h)
resp.headers.add('Access-Control-Allow-Origin', '*')
resp.headers.add('Access-Control-Allow-Headers','Content-Type,Auth-Token')
resp.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS')
if do_not_cache(request):# do not cache in the browser
resp.headers.add('Cache-Control', "no-cache, must-revalidate, max-age=0")
else:
resp.headers.add('Cache-Control', "no-transform, public, max-age=%i, s-maxage=%i"%(took*1800/1000, took*9000/1000))
return resp
except Exception as e:
app.logger.exception('failed request teardown function', str(e))
return resp
# Override the HTTP exception handler.
app.handle_http_exception = get_http_exception_handler(app)
return app
def create_app(config_name):
app = Flask(__name__, static_url_path='')
# This first loads the configuration from eg. config['development'] which corresponds to the DevelopmentConfig class in the config.py
app.config.from_object(config[config_name])
# Then you can override the values with the contents of the file the OPENTARGETS_API_LOCAL_SETTINGS environment variable points to.
# For eg:
# $ export OPENTARGETS_API_LOCAL_SETTINGS=/path/to/settings.cfg
#
# where settings.cfg looks like:
#
# DEBUG = False
# SECRET_KEY = 'foo'
#
app.config.from_envvar("OPENTARGETS_API_LOCAL_SETTINGS", silent=True)
config[config_name].init_app(app)
api_version = app.config['API_VERSION']
api_version_minor = app.config['API_VERSION_MINOR']
app.logger.info('looking for elasticsearch at: %s' % app.config['ELASTICSEARCH_URL'])
app.extensions['redis-core'] = Redis(app.config['REDIS_SERVER_PATH'], db=0) #served data
app.extensions['redis-service'] = Redis(app.config['REDIS_SERVER_PATH'], db=1) #cache, rate limit and internal things
app.extensions['redis-user'] = Redis(app.config['REDIS_SERVER_PATH'], db=2)# user info
'''setup cache'''
app.extensions['redis-service'].config_set('save','')
app.extensions['redis-service'].config_set('appendonly', 'no')
icache = InternalCache(app.extensions['redis-service'],
str(api_version_minor))
ip2org = IP2Org(icache)
if app.config['ELASTICSEARCH_URL']:
es = Elasticsearch(app.config['ELASTICSEARCH_URL'],
# # sniff before doing anything
# sniff_on_start=True,
# # refresh nodes after a node fails to respond
# sniff_on_connection_fail=True,
# # and also every 60 seconds
# sniffer_timeout=60
timeout=60 * 20,
maxsize=32,
)
else:
es = None
'''elasticsearch handlers'''
app.extensions['esquery'] = esQuery(
es,
DataTypes(app),
DataSourceScoring(app),
index_data=app.config['ELASTICSEARCH_DATA_INDEX_NAME'],
index_drug=app.config['ELASTICSEARCH_DRUG_INDEX_NAME'],
index_efo=app.config['ELASTICSEARCH_EFO_LABEL_INDEX_NAME'],
index_eco=app.config['ELASTICSEARCH_ECO_INDEX_NAME'],
index_genename=app.config['ELASTICSEARCH_GENE_NAME_INDEX_NAME'],
index_expression=app.config['ELASTICSEARCH_EXPRESSION_INDEX_NAME'],
index_reactome=app.config['ELASTICSEARCH_REACTOME_INDEX_NAME'],
index_association=app.config['ELASTICSEARCH_DATA_ASSOCIATION_INDEX_NAME'],
index_search=app.config['ELASTICSEARCH_DATA_SEARCH_INDEX_NAME'],
index_relation=app.config['ELASTICSEARCH_DATA_RELATION_INDEX_NAME'],
docname_data=app.config['ELASTICSEARCH_DATA_DOC_NAME'],
docname_drug=app.config['ELASTICSEARCH_DRUG_DOC_NAME'],
docname_efo=app.config['ELASTICSEARCH_EFO_LABEL_DOC_NAME'],
docname_eco=app.config['ELASTICSEARCH_ECO_DOC_NAME'],
docname_genename=app.config['ELASTICSEARCH_GENE_NAME_DOC_NAME'],
docname_expression=app.config['ELASTICSEARCH_EXPRESSION_DOC_NAME'],
docname_reactome=app.config['ELASTICSEARCH_REACTOME_REACTION_DOC_NAME'],
docname_association=app.config['ELASTICSEARCH_DATA_ASSOCIATION_DOC_NAME'],
docname_search=app.config['ELASTICSEARCH_DATA_SEARCH_DOC_NAME'],
# docname_search_target=app.config['ELASTICSEARCH_DATA_SEARCH_TARGET_DOC_NAME'],
# docname_search_disease=app.config['ELASTICSEARCH_DATA_SEARCH_DISEASE_DOC_NAME'],
docname_relation=app.config['ELASTICSEARCH_DATA_RELATION_DOC_NAME'],
log_level=app.logger.getEffectiveLevel(),
cache=icache
)
app.extensions['es_access_store'] = esStore(es,
eventlog_index=app.config['ELASTICSEARCH_LOG_EVENT_INDEX_NAME'],
ip2org=ip2org,
)
'''mixpanel handlers'''
if Config.MIXPANEL_TOKEN:
mp = Mixpanel(Config.MIXPANEL_TOKEN, consumer=AsyncBufferedConsumer())
app.extensions['mixpanel']= mp
app.extensions['mp_access_store'] = MixPanelStore(
mp,
ip2org=ip2org,
)
app.extensions['proxy'] = ProxyHandler(
allowed_targets=app.config['PROXY_SETTINGS']['allowed_targets'],
allowed_domains=app.config['PROXY_SETTINGS']['allowed_domains'],
allowed_request_domains=app.config['PROXY_SETTINGS']['allowed_request_domains'])
# basepath = app.config['PUBLIC_API_BASE_PATH']+api_version
# cors = CORS(app, resources=r'/api/*', allow_headers='Content-Type,Auth-Token')
''' define cache'''
# cache = Cache(config={'CACHE_TYPE': 'simple'})
# cache.init_app(latest_blueprint)
# latest_blueprint.cache = cache
# latest_blueprint.extensions['cache'] = cache
# app.cache = SimpleCache()
app.cache = FileSystemCache('/tmp/cttv-rest-api-cache', threshold=100000, default_timeout=60*60, mode=777)
'''load ip name resolution'''
ip_resolver = defaultdict(lambda: "PUBLIC")
ip_list_file = app.config['IP_RESOLVER_LIST_PATH']
if not os.path.exists(ip_list_file):
ip_list_file = '../' + ip_list_file
if os.path.exists(ip_list_file):
with open(ip_list_file) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
net = IPNetwork(row['ip'])
ip_resolver[net] = row['org']
else:
app.logger.warning('cannot find IP list for IP resolver. All traffic will be logged as PUBLIC')
app.config['IP_RESOLVER'] = ip_resolver
'''compress http response'''
compress = Compress()
compress.init_app(app)
latest_blueprint = Blueprint('latest', __name__)
current_version_blueprint = Blueprint(str(api_version), __name__)
current_minor_version_blueprint = Blueprint(str(api_version_minor), __name__)
specpath = '/cttv'
if app.config['PROFILE'] == True:
from werkzeug.contrib.profiler import ProfilerMiddleware
app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[30])
'''set the right prefixes'''
create_api(latest_blueprint, api_version, specpath)
create_api(current_version_blueprint, api_version, specpath)
create_api(current_minor_version_blueprint, api_version_minor, specpath)
# app.register_blueprint(latest_blueprint, url_prefix='/latest/platform')
app.register_blueprint(current_version_blueprint, url_prefix='/v'+str(api_version) + '/platform')
app.register_blueprint(current_minor_version_blueprint, url_prefix='/v'+str(api_version_minor) + '/platform')
'''serve the static docs'''
openapi_def = yaml.load(file('app/static/openapi.template.yaml', 'r'))
app.logger.info('parsing swagger from app/static/openapi.template.yaml')
#inject the description into the docs
with open("api-description.md", "r") as f:
desc = f.read()
openapi_def['info']['description'] = desc
openapi_def['basePath'] = '/v%s' % str(api_version)
@app.route('/v%s/platform/swagger' % str(api_version))
def serve_swagger(apiversion=api_version):
return jsonify(openapi_def)
@app.route('/v%s/platform/docs/swagger-ui' % str(api_version))
def render_swaggerui(apiversion=api_version):
return render_template('swaggerui.html',api_version=apiversion)
'''pre and post-request'''
@app.before_request
def before_request():
g.request_start = datetime.now()
@app.after_request
def after(resp):
try:
now = datetime.now()
took = int(round((now - g.request_start).total_seconds()))
resp.headers.add('Access-Control-Allow-Origin', '*')
resp.headers.add('Access-Control-Allow-Headers','Content-Type,Auth-Token')
resp.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS')
if do_not_cache(request):# do not cache in the browser
resp.headers.add('Cache-Control', "no-cache, must-revalidate, max-age=0")
else:
cache = 30 * 24 * 60 * 60 #cache for seven days
resp.headers.add('Cache-Control', "no-transform, max-age=%i"%(cache))
return resp
except Exception as e:
app.logger.exception('failed request teardown function', str(e))
return resp
# Override the HTTP exception handler.
app.handle_http_exception = get_http_exception_handler(app)
return app
def __init__(self):
self.mp = Mixpanel(self.PROJECT_TOKEN,
consumer=AsyncBufferedConsumer())