def start_service(api_class, service_class, log,
services=None, host='localhost', port=9100, killprocgrp=False):
"""Start a Data API service.
Args:
api_class (BaseService): The custom API service class, e.g.,
`doekbase.data_api.taxonomy.taxon.api.TaxonService`
service_class (type): The Thrift auto-generated service class
log (logging.Logger): Logging object
services (dict): Service configuration dictionary, passed to
constructor of the `api_class`.
host (str): Service host (will default to 'localhost')
port (int): Service port, e.g. 9101
killprocgrp (bool): if True, kill process group on exit
"""
assert issubclass(api_class, BaseService), \
'Invalid "api_class": must be a subclass of ' \
'doekbase.data_api.service_core.BaseService'
assert hasattr(service_class, 'Processor'), 'Invalid "service_class": ' \
'missing "Processor" attribute'
assert isinstance(port, int), 'The "port" must be an integer'
svc_t0 = util.log_start(log, 'start_service',
kvp=dict(host=host, port=port))
# Create server
services = services or SERVICES_DICT
handler = api_class(services)
processor = service_class.Processor(handler)
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
resource = TTwisted.ThriftResource(processor, pfactory, pfactory)
site = twisted.web.server.Site(resource=resource)
twisted.internet.reactor.listenTCP(port, site, interface=host)
# Kill entire process group on shutdown
if killprocgrp:
twisted.internet.reactor.addSystemEventTrigger('before', 'shutdown',
functools.partial(
kill_process_group,
log=log))
# Run server
sname = api_class.__name__
shost = host or 'localhost'
util.log_start(log, 'server', kvp=dict(name=sname, host=shost, port=port))
t0 = util.log_start(log, 'twisted.internet.reactor.run',
level=logging.DEBUG)
try:
twisted.internet.reactor.run()
except Exception as err:
log.error('msg="Abort {} server on error"'.format(sname))
util.log_end(log, t0, 'twisted.internet.reactor.run',
status_code=1, level=logging.ERROR, kvp=dict(msg=err))
raise
finally:
util.log_end(log, t0, 'twisted.internet.reactor.run')
util.log_end(log, svc_t0, 'start_service',
kvp=dict(host=host, port=port))
return 0
def __init__(self, client, ref):
"""Create API instance with an object reference and client
Args:
client (object): Implementation of thrift_service.Iface
ref (str): Object reference
"""
assert isinstance(client, thrift_service.Iface)
if not REF_PATTERN.match(ref):
raise ValueError('Format error for "{}"'.format(ref))
self._client, self._ref = client, ref
# Set up client
t0 = log_start(_log, 'client.init')
auth_info = ttypes.AuthInfo(token=get_auth_token())
self._client.init(auth_info)
log_end(_log, t0, 'client.init')
# Create internal state
t0 = log_start(_log, 'client.get_info', kvp=dict(ref=ref))
self._info = self._client.get_info(ref)
log_end(_log, t0, 'client.get_info')
self._id = self._info.object_id
self._name = self._info.object_name
self._typestring = self._info.type_string
self._version = self._info.version
def __init__(self, client, ref):
"""Create API instance with an object reference and client
Args:
client (object): Implementation of thrift_service.Iface
ref (str): Object reference
"""
assert isinstance(client, thrift_service.Iface)
if not REF_PATTERN.match(ref):
raise ValueError('Format error for "{}"'.format(ref))
self._client, self._ref = client, ref
# Set up client
t0 = log_start(_log, 'client.init')
auth_info = ttypes.AuthInfo(token=get_auth_token())
self._client.init(auth_info)
log_end(_log, t0, 'client.init')
# Create internal state
t0 = log_start(_log, 'client.get_info', kvp=dict(ref=ref))
self._info = self._client.get_info(ref)
log_end(_log, t0, 'client.get_info')
self._id = self._info.object_id
self._name = self._info.object_name
self._typestring = self._info.type_string
self._version = self._info.version
def get_data(self):
t0 = log_start(_log, 'get_data')
s = ''
try:
t1 = log_start(_log, 'get_data.query')
data_dict = self.ws_client.get_objects([
{"ref": self.ref}])[0]["data"]
log_end(_log, t1, 'get_data.query')
t1 = log_start(_log, 'get_data.dump')
s = json.dumps(data_dict)
log_end(_log, t1, 'get_data.dump')
except Exception as err:
print("@@ died in .dumps: {}".format(err))
log_end(_log, t0, 'get_data')
return s
def _init_ws_from_files(self, path):
ext = '.msgpack'
extlen = len(ext)
WorkspaceFile.use_msgpack = True
client = WorkspaceFile(path)
num_loaded = 0
for name in os.listdir(path):
if name.endswith(ext):
ref = name[:-extlen]
t0 = log_start(_log,
'load',
level=logging.DEBUG,
kvp={'ref': ref})
client.load(ref)
log_end(_log,
t0,
'client.load',
level=logging.DEBUG,
kvp={'ref': ref})
num_loaded += 1
if num_loaded == 0:
raise ValueError(
'No files with extension "{e}" found in path {p}'.format(
e=ext, p=path))
return client
def _get_annotation(ga):
t0 = log_start(_log, "get_annotation")
try:
feature_types = ga.get_feature_types()
except Exception as err:
raise RuntimeError("Cannot get feature_types: {}".format(err))
ann = {
"feature_" + k: getattr(ga, "get_feature_" + k)(feature_types) for k in ("type_descriptions", "type_counts")
}
ann["feature_types"] = feature_types
log_end(_log, t0, "get_annotation")
return ann
def _get_annotation(ga):
t0 = log_start(_log, 'get_annotation')
try:
feature_types = ga.get_feature_types()
except Exception as err:
raise RuntimeError('Cannot get feature_types: {}'.format(err))
ann = {
'feature_' + k: getattr(ga, 'get_feature_' + k)(feature_types)
for k in ('type_descriptions', 'type_counts')
}
ann['feature_types'] = feature_types
log_end(_log, t0, 'get_annotation')
return ann
def _get_taxon(ga):
t0 = log_start(_log, 'get_taxon')
try:
taxon = ga.get_taxon()
except Exception as err:
raise RuntimeError('Cannot get taxon: {}'.format(err))
txn = {
k: getattr(taxon, 'get_' + k)()
for k in ('taxonomic_id', 'kingdom', 'domain', 'genetic_code',
'scientific_name', 'aliases', 'scientific_lineage')
}
txn['lineage_list'] = txn['scientific_lineage'].split(';')
log_end(_log, t0, 'get_taxon')
return txn
def _get_assembly(ga):
t0 = log_start(_log, 'get_assembly')
try:
assembly = ga.get_assembly()
except Exception as err:
raise RuntimeError('Cannot get assembly: {}'.format(err))
asy = {
k1: getattr(assembly, 'get_' + k2)()
for k1, k2 in (('number_of_contigs',
'number_contigs'), ('total_length', 'dna_size'),
('total_gc_content',
'gc_content'), ('contig_length', 'contig_lengths'),
('contig_gc_content', 'contig_gc_content'))
}
log_end(_log, t0, 'get_assembly')
return asy
def kill_process_group(log):
"""Kill entire process group on Twisted shutdown.
Args:
log (logging.Logger): Logger
"""
pid = os.getpid() # my pid
grpid = -os.getpgid(pid) # my process group
signo = signal.SIGINT # the signal to send
t = util.log_start(log, 'kill_process_group', level=logging.WARN,
kvp=dict(pid=pid, group_pid=grpid, signal=signo))
# ignore signal in this process (Twisted is already shutting down)
signal.signal(signo, signal.SIG_IGN)
# send the signal to my process group
os.kill(grpid, signo)
util.log_end(log, t, 'kill_process_group', level=logging.WARN,
kvp=dict(pid=pid, group_pid=grpid, signal=signo))
def _get_assembly(ga):
t0 = log_start(_log, "get_assembly")
try:
assembly = ga.get_assembly()
except Exception as err:
raise RuntimeError("Cannot get assembly: {}".format(err))
asy = {
k1: getattr(assembly, "get_" + k2)()
for k1, k2 in (
("number_of_contigs", "number_contigs"),
("total_length", "dna_size"),
("total_gc_content", "gc_content"),
("contig_length", "contig_lengths"),
("contig_gc_content", "contig_gc_content"),
)
}
log_end(_log, t0, "get_assembly")
return asy
def _init_ws_from_files(self, path):
ext = '.msgpack'
extlen = len(ext)
WorkspaceFile.use_msgpack = True
client = WorkspaceFile(path)
num_loaded = 0
for name in os.listdir(path):
if name.endswith(ext):
ref = name[:-extlen]
t0 = log_start(_log, 'load', level=logging.DEBUG,
kvp={'ref': ref})
client.load(ref)
log_end(_log, t0, 'client.load', level=logging.DEBUG,
kvp={'ref': ref})
num_loaded += 1
if num_loaded == 0:
raise ValueError('No files with extension "{e}" found in path {p}'
.format(e=ext, p=path))
return client
def _get_taxon(ga):
t0 = log_start(_log, "get_taxon")
try:
taxon = ga.get_taxon()
except Exception as err:
raise RuntimeError("Cannot get taxon: {}".format(err))
txn = {
k: getattr(taxon, "get_" + k)()
for k in (
"taxonomic_id",
"kingdom",
"domain",
"genetic_code",
"scientific_name",
"aliases",
"scientific_lineage",
)
}
txn["lineage_list"] = txn["scientific_lineage"].split(";")
log_end(_log, t0, "get_taxon")
return txn
def kill_process_group(log):
"""Kill entire process group on Twisted shutdown.
Args:
log (logging.Logger): Logger
"""
pid = os.getpid() # my pid
grpid = -os.getpgid(pid) # my process group
signo = signal.SIGINT # the signal to send
t = util.log_start(log,
'kill_process_group',
level=logging.WARN,
kvp=dict(pid=pid, group_pid=grpid, signal=signo))
# ignore signal in this process (Twisted is already shutting down)
signal.signal(signo, signal.SIG_IGN)
# send the signal to my process group
os.kill(grpid, signo)
util.log_end(log,
t,
'kill_process_group',
level=logging.WARN,
kvp=dict(pid=pid, group_pid=grpid, signal=signo))
def load(self, ref):
"""Load data from a given reference.
The reference will be translated into a file to load,
using the following formula:
``<working_directory> + '/' + <ref> + <ext>`,
where ``<working_directory>`` is the path given to the class
constructor, ``<ref>`` is the reference given to this
function, and
``<ext>`` is a file extension '.msgpack' if
`use_msgpack` is True and '.json' otherwise.
Thus, for ``WorkspaceFile('/tmp/data').load('foo_bar')``,
the path loaded would be '/tmp/data/foo_bar.msgpack'.
See class documentation on format of input data.
Args:
ref (str): The reference
Notes:
* Post-condition: Object is loaded if and only if that reference was
not loaded previously. Modification timestamp of the underlying
file is NOT checked, you must manually invalidate modified
data with :meth:`unload(ref)`.
Raises:
IOError: file not found or not readable.
ValueError: parsing failed.
"""
# log start
t0 = log_start(_log, 'WorkspaceFile.load', level=logging.INFO,
kvp=dict(ref=ref))
# stop if already loaded in the past
if ref in self._loaded:
# log done and return
log_end(_log, t0, 'WorkspaceFile.load', level=logging.INFO,
kvp=dict(ref=ref, cached='yes'))
return
# create the full path from the reference
ext = 'msgpack' if self.use_msgpack else 'json'
full_path = '{}.{}'.format(os.path.join(self._wd, ref), ext)
# open the file; raises IOError on failure
f = open(full_path)
# parse the file
try:
record = msgpack.load(f) if self.use_msgpack else json.load(f)
except Exception as err:
raise ValueError('Loading {}: {}'.format(full_path, err))
finally:
f.close()
# cache the parsed data, both by reference and by 'name'
# (if name is not the same as reference)
#print("@@ REF={r} RECORD[ref]={rr} RECORD[name]={n}"
# .format(r=ref, rr=record['ref'], n=record['name']))
self._loaded[ref] = record
self._loaded[record['ref']] = record
self._loaded[record['name']] = record
#print('@@ STORE RECORD BY ({},{})'.format(record['ref'], record['name']))
# insert the parsed data into mongomock
self.collection.insert(record)
# log done
log_end(_log, t0, 'WorkspaceFile.load', level=logging.INFO,
kvp=dict(ref=ref, cached='no'))
def load(self, ref):
"""Load data from a given reference.
The reference will be translated into a file to load,
using the following formula:
``<working_directory> + '/' + <ref> + <ext>`,
where ``<working_directory>`` is the path given to the class
constructor, ``<ref>`` is the reference given to this
function, and
``<ext>`` is a file extension '.msgpack' if
`use_msgpack` is True and '.json' otherwise.
Thus, for ``WorkspaceFile('/tmp/data').load('foo_bar')``,
the path loaded would be '/tmp/data/foo_bar.msgpack'.
See class documentation on format of input data.
Args:
ref (str): The reference
Notes:
* Post-condition: Object is loaded if and only if that reference was
not loaded previously. Modification timestamp of the underlying
file is NOT checked, you must manually invalidate modified
data with :meth:`unload(ref)`.
Raises:
IOError: file not found or not readable.
ValueError: parsing failed.
"""
# log start
t0 = log_start(_log,
'WorkspaceFile.load',
level=logging.INFO,
kvp=dict(ref=ref))
# stop if already loaded in the past
if ref in self._loaded:
# log done and return
log_end(_log,
t0,
'WorkspaceFile.load',
level=logging.INFO,
kvp=dict(ref=ref, cached='yes'))
return
# create the full path from the reference
ext = 'msgpack' if self.use_msgpack else 'json'
full_path = '{}.{}'.format(os.path.join(self._wd, ref), ext)
# open the file; raises IOError on failure
f = open(full_path)
# parse the file
try:
record = msgpack.load(f) if self.use_msgpack else json.load(f)
except Exception as err:
raise ValueError('Loading {}: {}'.format(full_path, err))
finally:
f.close()
# cache the parsed data, both by reference and by 'name'
# (if name is not the same as reference)
#print("@@ REF={r} RECORD[ref]={rr} RECORD[name]={n}"
# .format(r=ref, rr=record['ref'], n=record['name']))
self._loaded[ref] = record
self._loaded[record['ref']] = record
self._loaded[record['name']] = record
#print('@@ STORE RECORD BY ({},{})'.format(record['ref'], record['name']))
# insert the parsed data into mongomock
self.collection.insert(record)
# log done
log_end(_log,
t0,
'WorkspaceFile.load',
level=logging.INFO,
kvp=dict(ref=ref, cached='no'))
def start_service(api_class,
service_class,
log,
services=None,
host='localhost',
port=9100,
killprocgrp=False):
"""Start a Data API service.
Args:
api_class (BaseService): The custom API service class, e.g.,
`doekbase.data_api.taxonomy.taxon.api.TaxonService`
service_class (type): The Thrift auto-generated service class
log (logging.Logger): Logging object
services (dict): Service configuration dictionary, passed to
constructor of the `api_class`.
host (str): Service host (will default to 'localhost')
port (int): Service port, e.g. 9101
killprocgrp (bool): if True, kill process group on exit
"""
assert issubclass(api_class, BaseService), \
'Invalid "api_class": must be a subclass of ' \
'doekbase.data_api.service_core.BaseService'
assert hasattr(service_class, 'Processor'), 'Invalid "service_class": ' \
'missing "Processor" attribute'
assert isinstance(port, int), 'The "port" must be an integer'
svc_t0 = util.log_start(log,
'start_service',
kvp=dict(host=host, port=port))
# Create server
services = services or SERVICES_DICT
handler = api_class(services)
processor = service_class.Processor(handler)
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
resource = TTwisted.ThriftResource(processor, pfactory, pfactory)
site = twisted.web.server.Site(resource=resource)
twisted.internet.reactor.listenTCP(port, site, interface=host)
# Kill entire process group on shutdown
if killprocgrp:
twisted.internet.reactor.addSystemEventTrigger(
'before', 'shutdown', functools.partial(kill_process_group,
log=log))
# Run server
sname = api_class.__name__
shost = host or 'localhost'
util.log_start(log, 'server', kvp=dict(name=sname, host=shost, port=port))
t0 = util.log_start(log,
'twisted.internet.reactor.run',
level=logging.DEBUG)
try:
twisted.internet.reactor.run()
except Exception as err:
log.error('msg="Abort {} server on error"'.format(sname))
util.log_end(log,
t0,
'twisted.internet.reactor.run',
status_code=1,
level=logging.ERROR,
kvp=dict(msg=err))
raise
finally:
util.log_end(log, t0, 'twisted.internet.reactor.run')
util.log_end(log, svc_t0, 'start_service', kvp=dict(host=host, port=port))
return 0