def serve(self) -> None:
'''
todo: docs
'''
conf = dotdict(instance.config['api'])
wapp = Flask(conf.name)
CORS(wapp)
wapp.config['PROPAGATE_EXCEPTIONS'] = True
for elex in instance.elexes.values():
for echo in instance.echoes:
echo(elex).deploy(wapp)
class process(Process):
def run(self) -> None:
logger().info('Deploying web server at %s:%s', conf.host,
conf.port)
wapp.run(host=conf.host, port=conf.port)
try:
instance.server.terminate()
instance.server.join()
collect()
except:
pass
instance.server = process(daemon=True, name='server')
instance.server.start()
def watch(self) -> None:
'''
todo: docs
'''
conf = dotdict(instance.config['data'])
tick = Queue()
def lexer(elex):
instance.elexes[elex.uid] = elex
index.update(elex)
self.serve()
class thread(Thread):
def run(self) -> None:
logger().info('Starting data sync in %s', conf.root)
for call in index.notify(conf.root, conf.spec):
tick.put(call)
thread(daemon=True, name='update').start()
while True:
try:
[lexer(i) for i in tick.get(False)()]
except Empty:
sleep(1)
def __init__(self, elex: Dict[str, Any]) -> None:
'''
todo: docs
'''
self.conf = dotdict(instance.config['api'])
self.elex = elex
self.emap = dotdict({
'id': {
'type': 'keyword'
},
**self.elex.schema.mappings.properties, 'created': {
'type': 'date'
},
'xml': {
'type': 'text'
}
})
self.path = '{}/{}/{}'.format(self.conf.root, self.elex.uid,
self.__class__.__name__.split('.')[-1])
def ids(cls, elex: Dict[str, Any], ids: List[str]) -> List[Dict[str, str]]:
'''
todo: docs
'''
find = entry(elex).schema()
try:
return [
dotdict({
**item.to_dict(), 'id': item.meta.id
}) for item in find.mget(ids)
]
except:
return []
def __parser(cls, file: str) -> List[Dict[str, Any]]:
'''
todo: docs
'''
conf = ConfigParser()
root = path.dirname(file)
conf.read_file(open(file))
return [
dotdict(elex) for elex
in [[('uid', uid),
('files',
['{}/{}'.format(root, i)
for i in loads(conf[uid]['files'])]),
('schema',
load(open('{}/{}'.format(root, conf[uid]['schema']))))]
for uid in conf.sections()]
]
def main(self) -> None:
'''
todo: docs
'''
try:
instance.config = ConfigParser()
instance.config.read_dict(defaultconfig())
logger().info('Started kosh with pid %s', getpid())
root = '{}/{}'.format(path.dirname(__file__), 'api')
mods = [i for _, i, _ in iter_modules([root]) if not i[0] is ('_')]
instance.echoes = [
mod('kosh.api.{}'.format(i)).__dict__[i] for i in mods
]
logger().info('Loaded API endpoint modules %s', mods)
for i in [i for i in argv if i.startswith('--')]:
try:
mod('kosh.param.{}'.format(i[2:])).__dict__[i[2:]](argv)
except:
exit('Invalid parameter or argument to {}'.format(i[2:]))
conf = dotdict(instance.config['data'])
connections.create_connection(hosts=[conf.host])
instance.elexes = {
i.uid: i
for i in index.lookup(conf.root, conf.spec)
}
for elex in instance.elexes.values():
index.update(elex)
self.serve()
self.watch() if strtobool(conf.sync) else pause()
except KeyboardInterrupt:
print('\N{bomb}')
except Exception as exception:
logger().exception(exception)
except SystemExit as exception:
logger().critical(str(exception))
finally:
logger().info('Stopped kosh with pid %s', getpid())
def entries(cls, elex: Dict[str, Any], field: str, query: str,
query_type: str, size: int) -> List[Dict[str, str]]:
'''
todo: docs
'''
find = Search(index=elex.uid).query(
query_type, **{field if field != 'id' else '_id': query})
try:
return [
dotdict({
**item.to_dict(), 'id':
item.meta.id,
'created':
datetime(*map(int, split(r'\D', item.created)))
}) for item in find[:size].execute()
]
except:
return []
def __schema(cls, elex: Dict[str, Any]) -> Dict[str, Any]:
'''
todo: docs
'''
emap = elex.schema.mappings.properties
emap.created = {'type': 'date'}
emap.xml = {'analyzer': 'strip_tags', 'type': 'text'}
elex.schema.mappings.properties = dotdict(emap)
elex.schema.settings = {
'analysis': {
'analyzer': {
'strip_tags': {
'type': 'custom',
'tokenizer': 'standard',
'char_filter': ['html_strip'],
'filter': ['lowercase']
}
}
}
}
return elex.schema