def __tabularprint(self,
data,
headers=None,
other_headers=[],
fields=None,
maxsize=20):
if not isinstance(data, list):
values = [data]
else:
values = data
if headers is None:
headers = [u'id', u'name']
headers.extend(other_headers)
table = []
if fields is None:
fields = headers
for item in values:
raw = []
if isinstance(item, dict):
for key in fields:
val = self.__multi_get(item, key)
raw.append(truncate(val, maxsize))
else:
raw.append(truncate(item, maxsize))
table.append(raw)
print(tabulate(table, headers=headers, tablefmt=u'fancy_grid'))
print(u'')
def add_client(self,
name,
authorization_code=None,
redirect_uri=u'https://localhost',
scopes=u'beehive',
response_type=u'code',
expiry_date=None):
if expiry_date is None:
expiry_date = datetime.today() + timedelta(days=365)
expiry_date = expiry_date.strftime(u'%d-%m-%Y')
if authorization_code is None:
authorization_code = GrantType.JWT_BEARER
data = {
u'client': {
u'name': name,
u'grant-type': authorization_code,
u'redirect-uri': redirect_uri,
u'description': u'Client %s' % name,
u'response-type': u'code',
u'scopes': scopes,
u'expiry-date': expiry_date
}
}
uri = u'%s/clients/' % (self.baseuri)
res = self._call(uri, u'POST', data=data)
self.logger.info(u'Add client: %s' % truncate(res))
#self.result(res)
print(u'Add client: %s' % res)
def get_identity(self, uid):
"""Get identity
:return: {u'uid':..., u'user':..., u'timestamp':..., u'pubkey':...,
'seckey':...}
:rtype: dict
"""
#self.check_authorization(Token.objtype, Token.objdef, u'*', u'view')
try:
identity = self.module.redis_manager.get(self.prefix + uid)
except Exception as ex:
self.logger.error(u'Identity %s retrieve error: %s' % (uid, ex))
raise ApiManagerError(u'Identity %s retrieve error' % uid, code=404)
if identity is not None:
data = pickle.loads(identity)
data[u'ttl'] = self.module.redis_manager.ttl(self.prefix + uid)
#User(self).send_event(u'identity.get', params={u'uid':uid})
self.logger.debug(u'Get identity %s from redis: %s' %
(uid, truncate(data)))
return data
else:
self.logger.error(u'Identity %s does not exist or is expired' % uid)
raise ApiManagerError(u'Identity %s does not exist or is '\
u'expired' % uid, code=404)
def add_scope(self, name, desc):
data = {u'scope': {u'name': name, u'desc': desc}}
uri = u'%s/scopes/' % (self.baseuri)
res = self._call(uri, u'POST', data=data)
self.logger.info(u'Add scope: %s' % truncate(res))
#self.result(res)
print(u'Add scope: %s' % res)
def add(self, data):
data = self.parent.load_config(data)
uri = u'%s/%ss/' % (self.parent.baseuri, self.name)
res = self.parent._call(uri, u'POST', data=data)
self.parent.logger.info(u'Add %s: %s' % (self.name,
truncate(res)))
self.parent.result(res)
def add(self, eventid, etype, objid, objdef, objtype, creation, data,
source, dest):
"""Add new event.
:param eventid: event id
:param etype: event type
:param objid: event object id
:param objdef: event object definition
:param objtype: event object objtype
:param creation: creation time
:param data: operation data
:param source: event source
:param dest: event destionation
:raise TransactionError: if transaction return error
"""
res = None
# add event
if objid is not None:
data = truncate(json.dumps(data), size=4000)
res = self.add_entity(DbEvent, eventid, etype, objid, objdef,
objtype, creation, data, json.dumps(source),
json.dumps(dest))
# add permtag
ids = self.get_all_valid_objids(objid.split('//'))
for i in ids:
perm = '%s-%s' % (objdef.lower(), i)
tag = self.hash_from_permission(objdef.lower(), i)
self.add_perm_tag(tag, perm, res.id, 'event')
return res
def info(self, extended=False):
"""Get uwsgi instance infos
memory:
rss: aka "Resident Set Size", this is the non-swapped physical memory a process has used. On UNIX it matches "top"'s RES column (see doc). On Windows this is an alias for wset field and it matches "Mem Usage" column of taskmgr.exe.
vms: aka "Virtual Memory Size", this is the total amount of virtual memory used by the process. On UNIX it matches "top"'s VIRT column (see doc). On Windows this is an alias for pagefile field and it matches "Mem Usage" "VM Size" column of taskmgr.exe.
shared: (Linux) memory that could be potentially shared with other processes. This matches "top"'s SHR column (see doc).
text (Linux, BSD): aka TRS (text resident set) the amount of memory devoted to executable code. This matches "top"'s CODE column (see doc).
data (Linux, BSD): aka DRS (data resident set) the amount of physical memory devoted to other than executable code. It matches "top"'s DATA column (see doc).
lib (Linux): the memory used by shared libraries.
dirty (Linux): the number of dirty pages.
open files:
path: the absolute file name.
fd: the file descriptor number; on Windows this is always -1.
position (Linux): the file (offset) position.
mode (Linux): a string indicating how the file was opened, similarly open's mode argument. Possible values are 'r', 'w', 'a', 'r+' and 'a+'. There's no distinction between files opened in bynary or text mode ("b" or "t").
flags (Linux): the flags which were passed to the underlying os.open C call when the file was opened (e.g. os.O_RDONLY, os.O_TRUNC, etc).
connections:
fd: the socket file descriptor. This can be passed to socket.fromfd() to obtain a usable socket object. This is only available on UNIX; on Windows -1 is always returned.
family: the address family, either AF_INET, AF_INET6 or AF_UNIX.
type: the address type, either SOCK_STREAM or SOCK_DGRAM.
laddr: the local address as a (ip, port) tuple or a path in case of AF_UNIX sockets.
raddr: the remote address as a (ip, port) tuple or an absolute path in case of UNIX sockets. When the remote endpoint is not connected you'll get an empty tuple (AF_INET) or None (AF_UNIX). On Linux AF_UNIX sockets will always have this set to None.
status: represents the status of a TCP connection. The return value is one of the psutil.CONN_* constants. For UDP and UNIX sockets this is always going to be psutil.CONN_NONE.
:param extended: if True print processes memory maps
:raise UwsgiManagerError:
"""
master_proc = psutil.Process(int(uwsgi_util.masterpid()))
resp = self._get_proc_infos(master_proc, extended=extended)
self.logger.debug('Get uwsgi processes: %s' % truncate(resp))
return resp
def callback(self, event, message):
"""Consume event relative to api where new access token is requested
:param event: event received
:param message: message received
:return:
"""
event_type = event.get('type')
if event_type == 'API':
data = event.get('data')
route = data.get('op')
source = event.get('source')
tmpl = '%(ip)s - %(user)s - %(identity)s [%(timestamp)s] "%(id)s %(method)s %(path)s" %(params)s ' \
'%(response)s %(elapsed)s'
log = {
'id': data.get('opid'),
'timestamp': format_date(datetime.fromtimestamp(event.get('creation'))),
'ip': source.get('ip'),
'user': source.get('user'),
'identity': source.get('identity'),
'response': data.get('response'),
'elapsed': data.get('elapsed'),
'method': route.get('method'),
'path': route.get('path'),
'params': truncate(json.dumps(data.get('params')))
}
self.logger2.info(tmpl % log)
def run(self, tags, *args, **kvargs):
"""Make query
:param list tags: list of permission tags
"""
if tags is None or len(tags) == 0:
tags = [u'']
# count all records
stmp = self.base_stmp(count=True)
total = self.session.query(u'count').\
from_statement(stmp).\
params(tags=tags, **kvargs).first()[0]
# make query
stmp = self.base_stmp()
query = self.session.query(self.entity).\
from_statement(stmp).\
params(tags=tags, **kvargs)
#self.logger.warn(u'stmp: %s' % query.statement.compile(dialect=mysql.dialect()))
query = query.all()
self.logger.debug(u'Get %ss (total:%s): %s' %
(self.entity, total, truncate(query)))
return query, total
def add_endpoint(self, name, desc, service, uri, active=True):
"""Add endpoint.
:param name: endpoint name
:param desc: endpoint desc
:param service: endpoint service
:param uri: endpoint uri
:param active: endpoint status
:return: endpoint id
:rtype: bool
:raises ApiManagerError: raise :class:`ApiManagerError`
:raises ApiAuthorizationError: raise :class:`ApiAuthorizationError`
"""
# check authorization
self.controller.check_authorization(CatalogEndpoint.objtype,
CatalogEndpoint.objdef, self.objid,
u'insert')
try:
# create catalog endpoint reference
objid = u'%s//%s' % (self.objid, id_gen())
res = self.manager.add_endpoint(objid, name, service, desc,
self.oid, uri, active)
# create object and permission
CatalogEndpoint(self.controller).register_object(
objid.split(u'//'), desc=desc)
self.logger.debug(u'Add catalog endpoint: %s' % truncate(res))
return res.id
except (QueryError, TransactionError, ModelError) as ex:
self.logger.error(ex, exc_info=1)
raise ApiManagerError(ex, code=ex.code)
def get_scope(self, scope_id):
uri = u'%s/scopes/%s/' % (self.baseuri, scope_id)
res = self._call(uri, u'GET', data=u'')
self.logger.info(u'Get scope: %s' % truncate(res))
self.result(res,
key=u'scope',
headers=self.scope_headers,
details=True)
def run2(self, tags, *args, **kvargs):
"""Make query. Use base_smtp2
:param list tags: list of permission tags
:param args: custom args
:param kvargs: custom kvargs
"""
start = time()
if self.with_perm_tag is True:
self.logger.debug2('Authorization with permission tags ENABLED')
else:
self.logger.debug2('Authorization with permission tags DISABLED')
if tags is None or len(tags) == 0:
tags = ['']
# make query
if self.size > 0:
# count all records
stmp = self.base_stmp2(count=True)
total = self.session.query('count').from_statement(stmp).params(
tags=tags, **kvargs).first()[0]
stmp = self.base_stmp2()
# set query entities
entities = [self.entity]
entities.extend(self.other_entities)
query = self.session.query(*entities).from_statement(stmp).params(
tags=tags, **kvargs)
self.logger.debug2('stmp: %s' %
query.statement.compile(dialect=mysql.dialect()))
self.logger.debug2('kvargs: %s' % truncate(kvargs))
self.logger.debug2('tags: %s' % truncate(tags))
res = query.all()
if self.size == 0 or self.size == -1:
total = len(res)
elapsed = round(time() - start, 3)
self.logger.debug2(
'Get %ss (total:%s): %s [%s]' %
(self.entity.__tablename__, total, truncate(res), elapsed))
return res, total
def get_configs(self, app=None):
if app is None:
uri = u'%s/%s/' % (self.baseuri, app)
else:
uri = u'%s/' % (self.baseuri)
res = self._call(uri, u'GET')
self.logger.info(u'Get configs: %s' % truncate(res))
self.result(res, key=u'configs', headers=self.headers)
def get_client(self, client_id):
uri = u'%s/clients/%s/' % (self.baseuri, client_id)
res = self._call(uri, u'GET', data=u'')
self.logger.info(u'Get client: %s' % truncate(res))
self.result(res,
key=u'client',
headers=self.client_headers,
details=True)
def push_stack_data(self, task_id, data):
"""Set data to shared memory stack. Use this to pass data from different
tasks that must ensure synchronization.
"""
val = json.dumps(data)
self._redis.lpush(self.prefix_stack + task_id, val)
logger.debug('Push stack data for job %s: %s' %
(task_id, truncate(data)))
return True
def list(self, *args):
data = self.parent.format_http_get_query_params(*args)
uri = u'%s/%ss/' % (self.parent.baseuri, self.name)
res = self.parent._call(uri, u'GET', data=data)
self.parent.logger.info(u'Get %s: %s' % (self.name, truncate(res)))
self.parent.result(res,
other_headers=self.headers,
key=self.name + u's',
maxsize=30)
def log_event(self, event, message):
"""Log received event
:param event: event received
:param message: message received
:raise EventConsumerError:
"""
message.ack()
self.logger.info('Consume event : %s' % truncate(event))
def ping_endpoints(self, catalog_id):
services = []
catalog = self.client.get_catalog(catalog_id)
for k, v in catalog.get(u'services', {}).items():
for v1 in v:
res = self.client.ping(endpoint=v1)
services.append({u'service': k, u'endpoint': v1, u'ping': res})
self.logger.info(u'Ping endpoint %s: %s' % (v1, truncate(res)))
self.result(services, headers=[u'service', u'endpoint', u'ping'])
def get_task_trace(self, task_id):
uri = u'/v1.0/worker/tasks/%s/' % task_id
res = self._call(uri, u'GET').get(u'task-instance').get(u'trace')
self.logger.info(res)
resp = []
for i in res:
resp.append({u'timestamp':i[0], u'task':i[1], u'task id':i[2],
u'msg':truncate(i[3], 150)})
self.result(resp, headers=[u'timestamp', u'msg'], maxsize=200)
def list(self, *args):
data = self.parent.format_http_get_query_params(*args)
uri = u'%s/%ss/' % (self.parent.baseuri, self.name)
res = self.parent._call(uri, u'GET', data=data)
self.parent.logger.info(u'Get %s: %s' % (self.name, truncate(res)))
if self.parent.format == u'text':
res.pop(u'count')
def print_dict(data):
for k,v in data.items():
yield (Token.Name, u' %s' % k)
yield (Token.Text.Whitespace, u' : ')
yield (Token.Literal.String, u'%s' % v)
def create_data():
for item in res.values()[0]:
for key in [u'id', u'uuid', u'name']:
yield (Token.Name, u' %-10s' % key)
yield (Token.Text.Whitespace, u' : ')
yield (Token.Literal.String, u'%s\n' % item.get(key))
yield (Token.Name, u' %-10s' % u'parent')
yield (Token.Text.Whitespace, u' : ')
yield (Token.Literal.String, u'%s\n' % item.get(u'parent_name'))
configs = item.get(u'attributes').pop(u'configs')
yield (Token.Name, u' %-10s' % u'configs')
if len(configs.keys()) == 0:
yield (Token.Text.Whitespace, u' : ')
yield (Token.Literal.Number, u'Empty\n')
else:
yield (Token.Text.Whitespace, u' :\n')
for k,v in configs.items():
yield (Token.Name, u' - %-12s' % k)
yield (Token.Text.Whitespace, u' : ')
yield (Token.Literal.String, u'%s\n' % v)
attribs = item.get(u'attributes')
yield (Token.Name, u' %-10s' % u'attribs')
if len(attribs.keys()) == 0:
yield (Token.Text.Whitespace, u' : ')
yield (Token.Literal.Number, u'Empty\n')
else:
yield (Token.Text.Whitespace, u' :\n')
for k,v in attribs.items():
yield (Token.Name, u' - %-12s' % k)
yield (Token.Text.Whitespace, u' : ')
yield (Token.Literal.String, u'%s\n' % v)
yield (Token.Text.Whitespace, u' ===========================\n')
data = format(create_data(), Terminal256Formatter(style=ListStyle))
print(data)
else:
self.parent.result(res, other_headers=self.headers, key=self.name+u's')
def list(self, *args):
data = self.parent.format_http_get_query_params(*args)
uri = u'%s/%ss/' % (self.parent.baseuri, self.name)
res = self.parent._call(uri, u'GET', data=data)
self.parent.logger.info(u'Get %s: %s' % (self.name, truncate(res)))
if self.format == u'table':
if data is not None:
if isinstance(data, dict) or isinstance(data, list):
self.__tabularprint(data)
self.parent.result(res)
def get_catalog(self, catalog_id):
res = self.client.get_catalog(catalog_id)
self.logger.info(u'Get catalog: %s' % truncate(res))
services = []
for k, v in res.get(u'services', {}).items():
for v1 in v:
services.append({u'service': k, u'endpoint': v1})
self.result(res, headers=self.cat_headers)
if self.format == u'table':
print(u'Services: ')
self.result(services, headers=[u'service', u'endpoint'])
def ping_endpoint(self, endpoint_id):
endpoint = self.client.get_endpoint(endpoint_id).get(u'endpoint')\
.get(u'endpoint')
res = self.client.ping(endpoint=endpoint)
self.logger.info(u'Ping endpoint %s: %s' % (endpoint, truncate(res)))
self.result({
u'endpoint': endpoint,
u'ping': res
},
headers=[u'endpoint', u'ping'])
def get_user_perms(self, name):
"""Get user permissions
:raise BeehiveApiClientError:
"""
data = urlencode({u'user': name, u'size': 1000})
uri = u'/v1.0/auth/objects/perms/'
res = self.invoke(u'auth', uri, u'GET', data, parse=True)
self.logger.debug(u'Get user %s permission : %s' %
(name, truncate(res)))
return res.get(u'perms', [])
def logout_user(self, token, seckey):
uri = u'%s/logout/' % (self.baseuri)
res = self.client.send_signed_request(u'auth',
uri,
u'DELETE',
data=u'',
uid=token,
seckey=seckey)
res = res[u'response']
self.logger.info(u'Logout %s: %s' % (token, truncate(res)))
self.result(res)
def pop_stack_data(self, task_id):
"""Pop item from shared memory stack. Use this to pass data from different
tasks that must ensure synchronization.
"""
data = None
val = self._redis.lpop(self.prefix_stack + task_id)
if val is not None:
data = json.loads(val)
logger.debug('Pop stack data for job %s: %s' %
(task_id, truncate(data)))
return data
def get_events(self, *args):
data = self.format_http_get_query_params(*args)
params = self.get_query_params(*args)
uri = u'%s/' % (self.baseuri)
res = self._call(uri, u'GET', data=data)
self.logger.info(u'Get events: %s' % truncate(res))
print(u'Page: %s' % res[u'page'])
print(u'Count: %s' % res[u'count'])
print(u'Total: %s' % res[u'total'])
print(u'Order: %s %s' % (params.get(u'field', u'id'),
params.get(u'order', u'DESC')))
self.result(res, key=u'events', headers=self.headers)
def get_steps(self, task_id):
"""Get task steps
:param task_id: task id
:return: SchedulerStep instance list
"""
session = self.get_session()
query = session.query(SchedulerStep).filter_by(task_id=task_id).order_by(asc(SchedulerStep.start_time))
steps = query.all()
self.logger.debug('Get task %s steps: %s' % (task_id, truncate(steps)))
return steps
def get_trace(self, task_id):
"""Get task trace
:param task_id: task id
:return: SchedulerTrace instance list
"""
session = self.get_session()
query = session.query(SchedulerTrace).filter_by(task_id=task_id).order_by(asc(SchedulerTrace.date))
steps = query.all()
self.logger.debug('Get task %s trace: %s' % (task_id, truncate(steps)))
return steps
def get_all_tasks(self, details=False):
"""Get all task of type TASK and JOB. Inner job task are not returned.
:return:
:rtype: dict
:raises ApiManagerError: raise :class:`.ApiManagerError`
"""
self.verify_permisssions(u'view')
try:
res = []
manager = self.controller.redis_taskmanager
keys = manager.inspect(pattern=self.prefix + u'*', debug=False)
if details is False:
for key in keys:
key = key[0].lstrip(self.prefix + '-')
res.append(key)
else:
data = manager.query(keys, ttl=True)
for key, item in data.iteritems():
val = json.loads(item[0])
ttl = item[1]
tasktype = val[u'type']
val.pop(u'trace')
# add time to live
val[u'ttl'] = ttl
# add elapsed
val[u'elapsed'] = val[u'stop_time'] - val[u'start_time']
val[u'stop_time'] = self.__convert_timestamp(
val[u'stop_time'])
val[u'start_time'] = self.__convert_timestamp(
val[u'start_time'])
# task status
#if tasktype == 'JOB':
# status = self.query_chain_status(tid)[2]
if tasktype in [u'JOB', u'TASK']:
res.append(val)
#res = AsyncResult(key, app=task_manager).get()
# sort task by date
res = sorted(res, key=lambda task: task[u'start_time'])
self.logger.debug(u'Get all tasks: %s' % truncate(res))
return res
except Exception as ex:
self.logger.error(ex, exc_info=1)
raise ApiManagerError(ex, code=404)
