def config(static_folder, cache_folder):
"""Test configuration."""
config = Config()
config.video_folder = ""
config.database.override_uri = 'sqlite:///:memory:'
config.thumbnail_cache_folder = cache_folder
config.static_folder = static_folder
yield config
def init_client():
global es
if es is None:
conf = Config()
# TODO: Read config props here
es_url = conf.es_url
es = Elasticsearch([es_url])
def get_config(config_path):
lookup_paths = [os.curdir,
expanduser('~'),
'/etc/',
dirname(__file__)]
conf = Config.load(config_path, conf_name='application.conf', lookup_paths=lookup_paths)
logger.debug(json.dumps(conf.items, indent=4))
return conf
def geocode_google_place(address):
google_url = 'https://maps.googleapis.com/maps/api/geocode/json?address={}&sensor=false&key={}'.format(
address,
Config.get_env_var('GOOGLE_API_KEY'),
)
r = requests.get(url=google_url)
if r.status_code != 200:
raise ValueError('Unknown/Not Finished Error Occured: {}'.format(
r.text))
return r.json()
def serve(host=None,
port=None,
db_host=None,
db_port=None,
db_name=None,
db_user=None,
db_secret=None,
db_dialect=None,
db_uri=None,
static=None,
videos=None):
"""Start Deduplication API Server."""
# Read configuration
config = Config()
config.port = port or config.port
config.host = host or config.host
config.video_folder = videos or config.video_folder
config.static_folder = static or config.static_folder
config.database.port = db_port or config.database.port
config.database.host = db_host or config.database.host
config.database.name = db_name or config.database.name
config.database.user = db_user or config.database.user
config.database.secret = db_secret or config.database.secret
config.database.dialect = db_dialect or config.database.dialect
config.database.override_uri = db_uri or config.database.override_uri
# Create application
application = create_application(config)
# Initialize database
database.init_app(application)
# Serve REST API
application.run(host=config.host, port=config.port)
def search_google_places(type, text):
google_url = 'https://maps.googleapis.com/maps/api/place/autocomplete/json?&input={}&key={}'.format(
text,
Config.get_env_var('GOOGLE_API_KEY'),
)
# google_url = 'https://maps.googleapis.com/maps/api/place/autoComplete/json?types=establishment&input={}&key={}'.format(
# text,
# Config.get_env_var('GOOGLE_API_KEY'),
# )
r = requests.get(url=google_url)
if r.status_code != 200:
raise ValueError('Unknown/Not Finished Error Occured: {}'.format(
r.text))
return r.json()
def configure_logging(log_level=None):
"""
Configure structlog to play nice with the stdlib according to these directions:
http://www.structlog.org/en/stable/standard-library.html#rendering-using-structlog-based-formatters-within-logging
"""
config = Config()
shared_processors = [
create_pennsieve_log_context(config),
structlog.processors.format_exc_info,
structlog.processors.TimeStamper(fmt="iso"),
]
# Prepare structlog events for the stdlib formatter
structlog.configure(
processors=shared_processors +
[structlog.stdlib.ProcessorFormatter.wrap_for_formatter],
logger_factory=structlog.stdlib.LoggerFactory(),
cache_logger_on_first_use=True,
)
# Aggregate stdlib and structlog and render to JSON
formatter = structlog.stdlib.ProcessorFormatter(
processor=structlog.processors.JSONRenderer(),
foreign_pre_chain=shared_processors,
)
handler = logging.StreamHandler()
handler.setFormatter(formatter)
root_logger = logging.getLogger()
root_logger.handlers = [handler]
if log_level is not None:
env_log_level = LOG_LEVELS.get(log_level.lower().strip(), log_level)
elif config.log_level is not None:
env_log_level = LOG_LEVELS.get(config.log_level.lower().strip(),
config.log_level)
else:
env_log_level = root_logger.getEffectiveLevel()
root_logger.setLevel(env_log_level)
logging.getLogger("connexion.operations.abstract").addFilter(
IgnoreApiKeyQueryParameter())
return root_logger
def get_google_directions(origin, destination, waypoints):
new_waypoints = ''
for waypoint in waypoints:
new_waypoints += '{}'.format(waypoint) + '|'
print(new_waypoints)
google_url = 'https://maps.googleapis.com/maps/api/directions/json?origin={}&destination={}&key={}'.format(
# google_url = 'https://maps.googleapis.com/maps/api/directions/json?origin={}&destination={}&waypoints=optimize:true|{}&key={}'.format(
origin,
destination,
# new_waypoints,
Config.get_env_var('GOOGLE_API_KEY'),
)
r = requests.get(url=google_url)
if r.status_code == 200:
print(r.json())
return r.json()
raise ValueError('Unknown Google Directions Error Occured: {}'.format(
r.text))
def wrapper(db: PartitionedDatabase, **kwargs):
result = func(db, **kwargs)
config = Config.from_app()
jwt = service_claim(db.organization_id, db.dataset_id,
config.jwt_config)
try:
PennsieveApiClient.get().touch_dataset(
db.organization_id,
db.dataset_id,
headers={"Authorization": f"Bearer {jwt}"},
)
except Exception as e:
VictorOpsClient.get().warning(
f"organization/{db.organization_id}/dataset/{db.dataset_id}",
f"Couldn't touch dataset {db.dataset_id} for organization={db.organization_id}",
)
logger.warn(
f"couldn't touch timestamp for organization/{db.organization_id}/dataset/{db.dataset_id}: {e}"
)
return result
def load_default_config(self):
self.cfg = Config(self.usage, prog=self.prog)
import pickle
import json
import pkg_resources
import numpy as np
import os
from flask import Blueprint, request, jsonify
from server.prediction import predict
from server.config import Config
from server.logging import get_logger
from server.util import APIError
from collections import Iterable
LOGGER = get_logger(__name__)
config = Config()
LOGGER.info("Started server with config")
LOGGER.info(json.dumps(config.config_data))
models = {}
endpoints_config = {}
for endpoint in config.endpoints:
# We remove the leading /
route_key = endpoint.route[1:]
models[route_key] = pickle.loads(
pkg_resources.resource_string(__name__,
f'resources/{endpoint.model_name}'))
endpoints_config[route_key] = endpoint
app_blueprint = Blueprint('app', __name__)
def config():
return Config()
DEFAULT_CONFIG = './config.conf'
def create_socket(server_address: typing.Tuple[str, int], request_queue_size: int):
address_family = socket.AF_INET
socket_type = socket.SOCK_STREAM
socket_listen = socket.socket(
address_family,
socket_type
)
socket_listen.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
socket_listen.bind(server_address)
socket_listen.listen(request_queue_size)
return socket_listen
if __name__ == '__main__':
config_path = DEFAULT_CONFIG if len(sys.argv) < 2 else sys.argv[1]
config = Config(config_path)
if config.path_project != './':
sys.path.insert(0, config.path_project)
wsgi = importlib.import_module(config.app_module)
application = getattr(wsgi, config.app_name)
server_address = (config.host, config.port)
socket_listen = create_socket(server_address, config.request_queue_size)
logging.info(f'WSGIServer: Serving HTTP on port {config.port}')
start_workers(config, socket_listen, application)
import pickle
import numpy as np
import json
from server.scoring import predict
from server.config import Config
import os.path as osp
def validate_model(clf, endpoint):
sample = {i['name']: 0.0 for i in endpoint.inputs}
x = predict(clf, sample, endpoint)
json.dumps(x)
if __name__ == "__main__":
with open("./server/resources/config.json") as f:
config = Config(json.load(f))
for endpoint in config.endpoints:
with open(osp.join("./server/resources", endpoint.model_name),
'rb') as f:
clf = pickle.load(f)
validate_model(clf, endpoint)
import cherrypy
import sys
sys.path.append("..")
from server.config import Config
from server import alerts
from os import path
from jinja2 import Environment, FileSystemLoader
import json, httplib, urllib, urllib2
from auth import require
import hashlib
current_dir = path.dirname(path.abspath(__file__))
env = Environment(loader=FileSystemLoader(path.join(current_dir,'templates')))
conf = Config()
web_app_server = conf.get_web_app_server()
port = int(web_app_server['port'])
cherrypy.config.update({'server.socket_host': '0.0.0.0',
'server.socket_port': port,
})
config = {
'/': {
'tools.staticdir.root': current_dir,
},
'/static': {
'tools.staticdir.on': True,
'tools.staticdir.dir': "static",
},
}