def get_service_status_without_restart(self, ctx, service):
"""
:param service: instance of type "Service" (module_name - the name of
the service module, case-insensitive version - specify the
service version, which can be either: (1) full git commit hash of
the module version (2) semantic version or semantic version
specification Note: semantic version lookup will only work for
released versions of the module. (3) release tag, which is one of:
dev | beta | release This information is always fetched from the
Catalog, so for more details on specifying the version, see the
Catalog documentation for the get_module_version method.) ->
structure: parameter "module_name" of String, parameter "version"
of String
:returns: instance of type "ServiceStatus" (module_name - name of
the service module version - semantic version number of
the service module git_commit_hash - git commit hash of the
service module release_tags - list of release tags currently
for this service module (dev/beta/release) url - the
url of the service up - 1 if the service is up, 0
otherwise status - status of the service as reported by
rancher health - health of the service as reported by
Rancher TODO: add something to return: string
last_request_timestamp;) -> structure: parameter "module_name" of
String, parameter "version" of String, parameter "git_commit_hash"
of String, parameter "release_tags" of list of String, parameter
"hash" of String, parameter "url" of String, parameter "up" of
type "boolean", parameter "status" of String, parameter "health"
of String
"""
# ctx is the context object
# return variables are: status
#BEGIN get_service_status_without_restart
cc = Catalog(self.CATALOG_URL, token=ctx['token'])
mv = cc.get_module_version({
'module_name': service['module_name'],
'version': service['version']
})
if 'dynamic_service' not in mv:
raise ValueError(
'Specified module is not marked as a dynamic service. (' +
mv['module_name'] + '-' + mv['git_commit_hash'] + ')')
if mv['dynamic_service'] != 1:
raise ValueError(
'Specified module is not marked as a dynamic service. (' +
mv['module_name'] + '-' + mv['git_commit_hash'] + ')')
status = self.get_single_service_status(mv)
#END get_service_status_without_restart
# At some point might do deeper type checking...
if not isinstance(status, dict):
raise ValueError(
'Method get_service_status_without_restart return value ' +
'status is not type dict as required.')
# return the results
return [status]
def create_app_dictionary():
#Create App Dictionary: Main function
requests.packages.urllib3.disable_warnings()
catalog = Catalog(url=os.environ['CATALOG_URL'])
nms = NarrativeMethodStore(url=os.environ['NARRATIVE_METHOD_STORE'])
apps = nms.list_methods({"tag": "release"})
apps_datastruc = pd.DataFrame.from_dict(apps)
ModDfApps = data_configure(apps_datastruc)
ModDfApps.drop([
'app_type', 'authors', 'git_commit_hash', 'icon', 'input_types',
'module_name', 'name', 'namespace', 'output_types', 'subtitle',
'tooltip', 'ver'
],
axis=1,
inplace=True)
keys = list(
set([
item for sublist in list(ModDfApps.categories) for item in sublist
]))
app_dict = {k: [] for k in keys}
for i in ModDfApps.index.values:
app_category_lst = ModDfApps["categories"][i]
for category in app_category_lst:
if category in app_dict.keys():
app_dict[category].append(ModDfApps["id"][i])
app_dict[category] = list(set(app_dict[category]))
else:
raise KeyError("{} not a KBase app category".format(category))
return app_dict
def __init_client(client_name):
if client_name == 'workspace':
c = Workspace(URLS.workspace)
elif client_name == 'job_service':
c = NarrativeJobService(URLS.job_service)
elif client_name == 'narrative_method_store':
c = NarrativeMethodStore(URLS.narrative_method_store)
elif client_name == 'user_and_job_state':
c = UserAndJobState(URLS.user_and_job_state)
elif client_name == 'catalog':
c = Catalog(URLS.catalog)
else:
raise ValueError('Unknown client name "%s"' % client_name)
__clients[client_name] = c
return c
def __init_client(client_name, token=None):
if client_name == 'workspace':
c = Workspace(URLS.workspace, token=token)
elif client_name == 'narrative_method_store':
c = NarrativeMethodStore(URLS.narrative_method_store, token=token)
elif client_name == 'user_and_job_state':
c = UserAndJobState(URLS.user_and_job_state, token=token)
elif client_name == 'catalog':
c = Catalog(URLS.catalog, token=token)
elif client_name == 'service' or client_name == 'service_wizard':
c = ServiceClient(URLS.service_wizard,
use_url_lookup=True,
token=token)
elif client_name == 'execution_engine2' or client_name == 'execution_engine' or client_name == 'job_service':
c = execution_engine2(URLS.execution_engine2, token=token)
elif client_name == 'job_service_mock':
c = JobServiceMock()
else:
raise ValueError('Unknown client name "%s"' % client_name)
return c
def __init_client(client_name, token=None):
if client_name == "workspace":
c = Workspace(URLS.workspace, token=token)
elif client_name == "execution_engine2":
c = execution_engine2(URLS.execution_engine2, token=token)
elif client_name == "narrative_method_store":
c = NarrativeMethodStore(URLS.narrative_method_store, token=token)
elif client_name == "service":
c = ServiceClient(URLS.service_wizard,
use_url_lookup=True,
token=token)
elif client_name == "catalog":
c = Catalog(URLS.catalog, token=token)
else:
raise ValueError('Unknown client name "%s"\n' % client_name +
"The following client names are recognised:\n" +
'Catalog: "catalog"\n' +
'Execution Engine 2: "execution_engine2"\n' +
'NMS: "narrative_method_store"\n' +
'Service Wizard: "service"\n' +
'Workspace: "workspace"')
return c
#Create App Dictionary: Main function
import requests
requests.packages.urllib3.disable_warnings()
from biokbase.catalog.Client import Catalog
from biokbase.narrative_method_store.client import NarrativeMethodStore
catalog = Catalog(url="https://kbase.us/services/catalog")
nms = NarrativeMethodStore(
url="https://kbase.us/services/narrative_method_store/rpc")
from data_configure import data_configure
import pandas as pd
def create_app_dictionary():
apps = nms.list_methods({"tag": "release"})
apps_datastruc = pd.DataFrame.from_dict(apps)
ModDfApps = data_configure(apps_datastruc)
ModDfApps.drop([
'app_type', 'authors', 'git_commit_hash', 'icon', 'input_types',
'module_name', 'name', 'namespace', 'output_types', 'subtitle',
'tooltip', 'ver'
],
axis=1,
inplace=True)
keys = list(
set([
item for sublist in list(ModDfApps.categories) for item in sublist
]))
app_dict = {k: [] for k in keys}
internal_users = dict()
import urllib2
response = urllib2.urlopen('https://raw.githubusercontent.com/kbase/metrics/master/kbase-staff.lst')
for user in response.read().split("\n"):
internal_users[user] = True
###############################################################################
# NEW CELL
###############################################################################
###########################################################################################
# https://github.com/dcchivian/kb_sdk_catalog_metrics/blob/master/src/time_based_plots.py #
###########################################################################################
from biokbase.catalog.Client import Catalog
catalog = Catalog(url="https://kbase.us/services/catalog")
import pandas as pd
import numpy as np
import math as math
import matplotlib.pyplot as plt
%matplotlib inline
#pd.__version__
import datetime, time
secs_per_week = 7*24*60*60
# Init data containers
#
total_users = []
total_regular_users = []
import sys
from biokbase.catalog.Client import Catalog
from pprint import pprint
catalog = Catalog('http://localhost:5000',user_id='wstester1',password='******')
catalogAdmin = Catalog('http://localhost:5000',user_id='wstester2',password='******')
print(catalog.version())
#catalog.push_dev_to_beta({'module_name':'onerepotest'})
#sys.exit()
print(catalog.register_repo({'git_url':'https://github.com/kbaseIncubator/onerepotest'}))
while True:
state = catalog.get_module_state({'git_url':'https://github.com/kbaseIncubator/onerepotest'})
pprint(state)
if state['registration'] in ['complete','error']:
break
sys.exit()
sys.exit()
selection = {
'git_url':'https://github.com/kbaseIncubator/contigcount'
import sys
from biokbase.catalog.Client import Catalog
from pprint import pprint
catalog = Catalog('http://localhost:5000',
user_id='wstester1',
password='******')
catalogAdmin = Catalog('http://localhost:5000',
user_id='wstester2',
password='******')
print(catalog.version())
#catalog.push_dev_to_beta({'module_name':'onerepotest'})
#sys.exit()
print(
catalog.register_repo(
{'git_url': 'https://github.com/kbaseIncubator/onerepotest'}))
while True:
state = catalog.get_module_state(
{'git_url': 'https://github.com/kbaseIncubator/onerepotest'})
pprint(state)
if state['registration'] in ['complete', 'error']:
break
sys.exit()
sys.exit()
def get_service_log_web_socket(self, ctx, params):
"""
returns connection info for a websocket connection to get realtime service logs
:param params: instance of type "GetServiceLogParams" (optional
instance_id to get logs for a specific instance. Otherwise logs
from all instances are returned, TODO: add line number
constraints.) -> structure: parameter "service" of type "Service"
(module_name - the name of the service module, case-insensitive
version - specify the service version, which can be either:
(1) full git commit hash of the module version (2) semantic
version or semantic version specification Note: semantic version
lookup will only work for released versions of the module. (3)
release tag, which is one of: dev | beta | release This
information is always fetched from the Catalog, so for more
details on specifying the version, see the Catalog documentation
for the get_module_version method.) -> structure: parameter
"module_name" of String, parameter "version" of String, parameter
"instance_id" of String
:returns: instance of list of type "ServiceLogWebSocket" ->
structure: parameter "instance_id" of String, parameter
"socket_url" of String
"""
# ctx is the context object
# return variables are: sockets
#BEGIN get_service_log_web_socket
service = params['service']
user_id = ctx['user_id']
cc = Catalog(self.CATALOG_URL, token=ctx['token'])
module = cc.get_module_info({'module_name' : service['module_name']})
has_access = False
for o in module['owners']:
if o == user_id:
has_access = True
if not has_access:
if cc.is_admin(user_id)==1:
has_access = True
if not has_access:
raise ValueError('Only module owners and catalog admins can view service logs.')
mv = cc.get_module_version({'module_name' : service['module_name'], 'version' : service['version']})
if 'dynamic_service' not in mv:
raise ValueError('Specified module is not marked as a dynamic service. ('+mv['module_name']+'-' + mv['git_commit_hash']+')')
if mv['dynamic_service'] != 1:
raise ValueError('Specified module is not marked as a dynamic service. ('+mv['module_name']+'-' + mv['git_commit_hash']+')')
rancher = gdapi.Client(url=self.RANCHER_URL,
access_key=self.RANCHER_ACCESS_KEY,
secret_key=self.RANCHER_SECRET_KEY)
#service_info = rancher.list_servicess(name=self.get_service_name(mv))
GET_SERVICE_URL = self.RANCHER_URL + '/v1/services?name=' + self.get_service_name(mv) + '&include=instances'
service_info = requests.get(GET_SERVICE_URL, auth=(self.RANCHER_ACCESS_KEY,self.RANCHER_SECRET_KEY),verify=False).json()
if len(service_info['data'])==0:
raise ValueError('Unable to fetch service information. That service version may not be available.')
if len(service_info['data'][0]['instances'])==0:
raise ValueError('The service version specified has no available container instances.')
instances = service_info['data'][0]['instances']
#pprint(instances)
match_instance_id = False
if 'instance_id' in params:
match_instance_id = True
sockets = []
for i in instances:
if match_instance_id and params['instance_id']!=i['id']:
continue;
LOG_URL = i['actions']['logs']
payload = { 'follow':True }
log_ws = requests.post(LOG_URL, data = json.dumps(payload), auth=(self.RANCHER_ACCESS_KEY,self.RANCHER_SECRET_KEY),verify=False).json()
sockets.append({
'instance_id':i['id'],
'socket_url':log_ws['url'] + '?token=' + log_ws['token']
})
#END get_service_log_web_socket
# At some point might do deeper type checking...
if not isinstance(sockets, list):
raise ValueError('Method get_service_log_web_socket return value ' +
'sockets is not type list as required.')
# return the results
return [sockets]
def get_service_status(self, ctx, service):
"""
For a given service, check on the status. If the service is down or
not running, this function will attempt to start or restart the
service once, then return the status.
This function will throw an error if the specified service cannot be
found or encountered errors on startup.
:param service: instance of type "Service" (module_name - the name of
the service module, case-insensitive version - specify the
service version, which can be either: (1) full git commit hash of
the module version (2) semantic version or semantic version
specification Note: semantic version lookup will only work for
released versions of the module. (3) release tag, which is one of:
dev | beta | release This information is always fetched from the
Catalog, so for more details on specifying the version, see the
Catalog documentation for the get_module_version method.) ->
structure: parameter "module_name" of String, parameter "version"
of String
:returns: instance of type "ServiceStatus" (module_name - name of
the service module version - semantic version number of
the service module git_commit_hash - git commit hash of the
service module release_tags - list of release tags currently
for this service module (dev/beta/release) url - the
url of the service up - 1 if the service is up, 0
otherwise status - status of the service as reported by
rancher health - health of the service as reported by
Rancher TODO: add something to return: string
last_request_timestamp;) -> structure: parameter "module_name" of
String, parameter "version" of String, parameter "git_commit_hash"
of String, parameter "release_tags" of list of String, parameter
"hash" of String, parameter "url" of String, parameter "up" of
type "boolean", parameter "status" of String, parameter "health"
of String
"""
# ctx is the context object
# return variables are: status
#BEGIN get_service_status
# TODO: handle case where version is not registered in the catalog- this may be the case for core services
# that were not registered in the usual way.
# first get infor from the catalog- it must be a dynamic service
cc = Catalog(self.CATALOG_URL, token=ctx['token'])
mv = cc.get_module_version({'module_name' : service['module_name'], 'version' : service['version']})
if 'dynamic_service' not in mv:
raise ValueError('Specified module is not marked as a dynamic service. ('+mv['module_name']+'-' + mv['git_commit_hash']+')')
if mv['dynamic_service'] != 1:
raise ValueError('Specified module is not marked as a dynamic service. ('+mv['module_name']+'-' + mv['git_commit_hash']+')')
status = self.get_single_service_status(mv)
# if we cannot get the status, or it is not up, then try to start it
if status is None or status['up']!=1:
self.start(ctx, service)
# try to get status
status = self.get_single_service_status(mv)
if status is None:
raise ValueError('Unable to get service status, or service was unable to start properly');
#END get_service_status
# At some point might do deeper type checking...
if not isinstance(status, dict):
raise ValueError('Method get_service_status return value ' +
'status is not type dict as required.')
# return the results
return [status]
def list_service_status(self, ctx, params):
"""
:param params: instance of type "ListServiceStatusParams" (not yet
implemented funcdef pause(Service service) returns (ServiceStatus
status);) -> structure: parameter "is_up" of type "boolean",
parameter "module_names" of list of String
:returns: instance of list of type "ServiceStatus" (module_name -
name of the service module version - semantic version
number of the service module git_commit_hash - git commit hash of
the service module release_tags - list of release tags
currently for this service module (dev/beta/release) url
- the url of the service up - 1 if the service is up,
0 otherwise status - status of the service as reported by
rancher health - health of the service as reported by
Rancher TODO: add something to return: string
last_request_timestamp;) -> structure: parameter "module_name" of
String, parameter "version" of String, parameter "git_commit_hash"
of String, parameter "release_tags" of list of String, parameter
"hash" of String, parameter "url" of String, parameter "up" of
type "boolean", parameter "status" of String, parameter "health"
of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN list_service_status
rancher = gdapi.Client(url=self.RANCHER_URL,
access_key=self.RANCHER_ACCESS_KEY,
secret_key=self.RANCHER_SECRET_KEY)
cc = Catalog(self.CATALOG_URL, token=ctx['token'])
# first create simple module_name lookup based on hash (TODO: in catalog, allow us to only fetch dynamic service modules)
modules = cc.list_basic_module_info({'include_released':1, 'include_unreleased':1})
module_hash_lookup = {} # hash => module_name
for m in modules:
if 'dynamic_service' not in m or m['dynamic_service']!=1:
continue
module_hash_lookup[self.get_module_name_hash(m['module_name'])] = m['module_name']
# next get environment id (could be a config parameter in the future rather than looping over everything)
result = []
# not sure if this is supposed to pass False, or the string 'false'
slists = rancher.list_environment(system='false')
if len(slists) == 0: return [] # I shouldn't return
for slist in slists:
eid = slist['id']
# get service info
entries = rancher.list_service(environmentId = eid)
if len(entries) == 0: continue
for entry in entries:
rs = entry['name'].split('-')
if len(rs) != 2: continue
es = {'status' : entry['state'], 'health' : entry['healthState'], 'hash' : rs[1]}
#if es['health'] == 'healthy' and es['status'] == 'active':
if es['status'] == 'active':
es['up'] = 1
else:
es['up'] = 0
try:
mv = cc.get_module_version({'module_name': module_hash_lookup[rs[0]],'version':rs[1]})
es['url'] = self.get_service_url(mv)
es['version'] = mv['version']
es['module_name'] = mv['module_name']
es['release_tags'] = mv['release_tags']
es['git_commit_hash'] = mv['git_commit_hash']
except:
# this will occur if the module version is not registered with the catalog, or if the module
# was not marked as a service, or if something was started in Rancher directly and pulled
# from somewhere else, or an old version of the catalog was used to start this service
es['url'] = "https://{0}:{1}/dynserv/{3}.{2}".format(self.SVC_HOSTNAME, self.NGINX_PORT, rs[0], rs[1])
es['version'] = ''
es['release_tags'] = []
es['git_commit_hash'] = ''
es['module_name'] = '!'+rs[0]+''
result.append(es)
returnVal = result
#END list_service_status
# At some point might do deeper type checking...
if not isinstance(returnVal, list):
raise ValueError('Method list_service_status return value ' +
'returnVal is not type list as required.')
# return the results
return [returnVal]
def stop(self, ctx, service):
"""
Try to stop the specified service; this will generate an error if the
specified service cannot be stopped. If the stop did not give any
errors, then the status of the stopped service is provided.
:param service: instance of type "Service" (module_name - the name of
the service module, case-insensitive version - specify the
service version, which can be either: (1) full git commit hash of
the module version (2) semantic version or semantic version
specification Note: semantic version lookup will only work for
released versions of the module. (3) release tag, which is one of:
dev | beta | release This information is always fetched from the
Catalog, so for more details on specifying the version, see the
Catalog documentation for the get_module_version method.) ->
structure: parameter "module_name" of String, parameter "version"
of String
:returns: instance of type "ServiceStatus" (module_name - name of
the service module version - semantic version number of
the service module git_commit_hash - git commit hash of the
service module release_tags - list of release tags currently
for this service module (dev/beta/release) url - the
url of the service up - 1 if the service is up, 0
otherwise status - status of the service as reported by
rancher health - health of the service as reported by
Rancher TODO: add something to return: string
last_request_timestamp;) -> structure: parameter "module_name" of
String, parameter "version" of String, parameter "git_commit_hash"
of String, parameter "release_tags" of list of String, parameter
"hash" of String, parameter "url" of String, parameter "up" of
type "boolean", parameter "status" of String, parameter "health"
of String
"""
# ctx is the context object
# return variables are: status
#BEGIN stop
print('STOP REQUEST: ' + str(service))
# lookup the module info from the catalog
cc = Catalog(self.CATALOG_URL, token=self.CATALOG_ADMIN_TOKEN)
mv = cc.get_module_version({'module_name' : service['module_name'], 'version' : service['version']})
if 'dynamic_service' not in mv:
raise ValueError('Specified module is not marked as a dynamic service. ('+mv['module_name']+'-' + mv['git_commit_hash']+')')
if mv['dynamic_service'] != 1:
raise ValueError('Specified module is not marked as a dynamic service. ('+mv['module_name']+'-' + mv['git_commit_hash']+')')
secure_param_list = cc.get_secure_config_params({'module_name' : mv['module_name'], 'version': mv['git_commit_hash']})
docker_compose, rancher_compose, is_new_stack = self.create_compose_files(mv, secure_param_list)
ymlpath = self.SCRATCH_DIR + '/' + mv['module_name'] + '/' + str(int(time.time()*1000))
os.makedirs(ymlpath)
docker_compose_path=ymlpath + '/docker-compose.yml'
rancher_compose_path=ymlpath + '/rancher-compose.yml'
with open(docker_compose_path, 'w') as outfile:
outfile.write( yaml.safe_dump(docker_compose, default_flow_style=False) )
with open(rancher_compose_path, 'w') as outfile:
outfile.write( yaml.safe_dump(rancher_compose, default_flow_style=False) )
eenv = os.environ.copy()
eenv['RANCHER_URL'] = self.RANCHER_URL
eenv['RANCHER_ACCESS_KEY'] = self.RANCHER_ACCESS_KEY
eenv['RANCHER_SECRET_KEY'] = self.RANCHER_SECRET_KEY
stack_name = self.get_stack_name(mv)
print('STOPPING STACK: ' + stack_name)
cmd_list = [self.RANCHER_COMPOSE_BIN, '-p', stack_name, 'stop']
try:
p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE, env=eenv, cwd=ymlpath)
stdout, stderr = p.communicate()
except:
pprint(traceback.format_exc())
raise ValueError('Unable to stop service: Error calling rancher-compose: '+traceback.format_exc())
print('STDOUT:')
print(stdout)
print('STDERR:')
print(stderr)
status = self.get_single_service_status(mv)
#END stop
# At some point might do deeper type checking...
if not isinstance(status, dict):
raise ValueError('Method stop return value ' +
'status is not type dict as required.')
# return the results
return [status]
# GetAppStats
#
import requests
import os
import datetime, time
import mysql.connector as mysql
from biokbase.catalog.Client import Catalog
from biokbase.narrative_method_store.client import NarrativeMethodStore
requests.packages.urllib3.disable_warnings()
catalog = Catalog(url=os.environ['CATALOG_URL'],
token=os.environ['METRICS_USER_TOKEN'])
nms = NarrativeMethodStore(url=os.environ['NARRATIVE_METHOD_STORE'])
sql_host = os.environ['SQL_HOST']
query_on = os.environ['QUERY_ON']
#Insures all finish times within last day.
yesterday = (datetime.date.today() - datetime.timedelta(days=1))
def get_user_app_stats(
start_date=datetime.datetime.combine(yesterday,
datetime.datetime.min.time()),
end_date=datetime.datetime.combine(yesterday,
datetime.datetime.max.time())):
"""
Gets a data dump from the app cataloge for a certain date window.
If no statt and end date are entered it will default to the last 15 calendar days (UTC TIME).
It is 15 days because it uses an underlying method that
filters by creation_time and not finish_time
import requests
import os
requests.packages.urllib3.disable_warnings()
from biokbase.catalog.Client import Catalog
from biokbase.narrative_method_store.client import NarrativeMethodStore
from category_to_app_dict import create_app_dictionary
catalog = Catalog(url="https://kbase.us/services/catalog",
token=os.environ['METRICS_USER_TOKEN'])
nms = NarrativeMethodStore(
url="https://kbase.us/services/narrative_method_store/rpc")
import pandas as pd
import datetime, time
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', 500)
from operator import itemgetter
# Get User_App_Stats: Main Function
"""Requires a unique list of username as input and runs it 3-4min on average.
outputs a list of user dictionaries with app statistics for each userm example user : [{app1_usage_info}, {app2_usage_info}]
If you would like to generate user-app stats for all of KBase time please choose: start = "2016-03-04" end = "todays-date"""
# Get User_App_Stats
yesterday = (datetime.date.today() - datetime.timedelta(days=1))
def user_app_stats(
unique_usernames,
start_date=datetime.datetime.combine(yesterday,
internal_users = dict()
import urllib2
response = urllib2.urlopen('https://raw.githubusercontent.com/kbase/metrics/master/kbase-staff.lst')
for user in response.read().split("\n"):
internal_users[user] = True
###############################################################################
# NEW CELL
###############################################################################
####################################################################################
# https://github.com/dcchivian/kb_sdk_catalog_metrics/blob/master/src/bar_plots.py #
####################################################################################
from biokbase.catalog.Client import Catalog
catalog = Catalog(url="https://kbase.us/services/catalog")
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
#pd.__version__
# requires Admin priveleges
#aggregated_by_app = catalog.get_exec_aggr_stats({})
aggregated_by_user = catalog.get_exec_aggr_table({})
total_runs_by_app = dict()
total_runs_by_module = dict()
total_users_by_app = dict()
total_users_by_module = dict()
total_app_runs_by_user = dict()