def create_app_dictionary():
#Create App Dictionary: Main function
requests.packages.urllib3.disable_warnings()
catalog = Catalog(url=os.environ['CATALOG_URL'])
nms = NarrativeMethodStore(url=os.environ['NARRATIVE_METHOD_STORE'])
apps = nms.list_methods({"tag": "release"})
apps_datastruc = pd.DataFrame.from_dict(apps)
ModDfApps = data_configure(apps_datastruc)
ModDfApps.drop([
'app_type', 'authors', 'git_commit_hash', 'icon', 'input_types',
'module_name', 'name', 'namespace', 'output_types', 'subtitle',
'tooltip', 'ver'
],
axis=1,
inplace=True)
keys = list(
set([
item for sublist in list(ModDfApps.categories) for item in sublist
]))
app_dict = {k: [] for k in keys}
for i in ModDfApps.index.values:
app_category_lst = ModDfApps["categories"][i]
for category in app_category_lst:
if category in app_dict.keys():
app_dict[category].append(ModDfApps["id"][i])
app_dict[category] = list(set(app_dict[category]))
else:
raise KeyError("{} not a KBase app category".format(category))
return app_dict
def __init__(self, params, registration_id, timestamp, username, is_admin,token, db, temp_dir, docker_base_url,
docker_registry_host, docker_push_allow_insecure, nms_url, nms_admin_token, module_details,
ref_data_base, kbase_endpoint, prev_dev_version):
self.db = db
self.params = params
# at this point, we assume git_url has been checked
self.git_url = params['git_url']
self.registration_id = registration_id
self.timestamp = timestamp
self.username = username
self.is_admin = is_admin
self.token = token
self.db = db
self.temp_dir = temp_dir
self.docker_base_url = docker_base_url
self.docker_registry_host = docker_registry_host
self.docker_push_allow_insecure = docker_push_allow_insecure
self.nms_url = nms_url
self.nms = NarrativeMethodStore(self.nms_url, token=nms_admin_token)
self.local_function_reader = LocalFunctionReader()
# (most) of the mongo document for this module snapshot before this registration
self.module_details = module_details
self.log_buffer = [];
self.last_log_time = time.time() # in seconds
self.log_interval = 1.0 # save log to mongo every second
self.ref_data_base = ref_data_base
self.kbase_endpoint = kbase_endpoint
self.prev_dev_version = prev_dev_version
def __init_client(client_name):
if client_name == 'workspace':
c = Workspace(URLS.workspace)
elif client_name == 'job_service':
c = NarrativeJobService(URLS.job_service)
elif client_name == 'narrative_method_store':
c = NarrativeMethodStore(URLS.narrative_method_store)
elif client_name == 'user_and_job_state':
c = UserAndJobState(URLS.user_and_job_state)
elif client_name == 'catalog':
c = Catalog(URLS.catalog)
else:
raise ValueError('Unknown client name "%s"' % client_name)
__clients[client_name] = c
return c
def setUpClass(cls):
print('++++++++++++ RUNNING core_registration_test.py +++++++++++')
# hack for testing!! remove when docker and NMS components can be tested
from biokbase.catalog.registrar import Registrar
Registrar._TEST_WITHOUT_DOCKER = True
cls.cUtil = CatalogTestUtil(
'.') # TODO: pass in test directory from outside
cls.cUtil.setUp()
cls.catalog = Catalog(cls.cUtil.getCatalogConfig())
# approve developers we will use
cls.catalog.approve_developer(cls.cUtil.admin_ctx(),
cls.cUtil.admin_ctx()['user_id'])
cls.catalog.approve_developer(cls.cUtil.admin_ctx(),
cls.cUtil.user_ctx()['user_id'])
cls.nms = NarrativeMethodStore(cls.cUtil.getCatalogConfig()['nms-url'])
def __init_client(client_name, token=None):
if client_name == 'workspace':
c = Workspace(URLS.workspace, token=token)
elif client_name == 'narrative_method_store':
c = NarrativeMethodStore(URLS.narrative_method_store, token=token)
elif client_name == 'user_and_job_state':
c = UserAndJobState(URLS.user_and_job_state, token=token)
elif client_name == 'catalog':
c = Catalog(URLS.catalog, token=token)
elif client_name == 'service' or client_name == 'service_wizard':
c = ServiceClient(URLS.service_wizard,
use_url_lookup=True,
token=token)
elif client_name == 'execution_engine2' or client_name == 'execution_engine' or client_name == 'job_service':
c = execution_engine2(URLS.execution_engine2, token=token)
elif client_name == 'job_service_mock':
c = JobServiceMock()
else:
raise ValueError('Unknown client name "%s"' % client_name)
return c
def __init_client(client_name, token=None):
if client_name == "workspace":
c = Workspace(URLS.workspace, token=token)
elif client_name == "execution_engine2":
c = execution_engine2(URLS.execution_engine2, token=token)
elif client_name == "narrative_method_store":
c = NarrativeMethodStore(URLS.narrative_method_store, token=token)
elif client_name == "service":
c = ServiceClient(URLS.service_wizard,
use_url_lookup=True,
token=token)
elif client_name == "catalog":
c = Catalog(URLS.catalog, token=token)
else:
raise ValueError('Unknown client name "%s"\n' % client_name +
"The following client names are recognised:\n" +
'Catalog: "catalog"\n' +
'Execution Engine 2: "execution_engine2"\n' +
'NMS: "narrative_method_store"\n' +
'Service Wizard: "service"\n' +
'Workspace: "workspace"')
return c
#Create App Dictionary: Main function
import requests
requests.packages.urllib3.disable_warnings()
from biokbase.catalog.Client import Catalog
from biokbase.narrative_method_store.client import NarrativeMethodStore
catalog = Catalog(url="https://kbase.us/services/catalog")
nms = NarrativeMethodStore(
url="https://kbase.us/services/narrative_method_store/rpc")
from data_configure import data_configure
import pandas as pd
def create_app_dictionary():
apps = nms.list_methods({"tag": "release"})
apps_datastruc = pd.DataFrame.from_dict(apps)
ModDfApps = data_configure(apps_datastruc)
ModDfApps.drop([
'app_type', 'authors', 'git_commit_hash', 'icon', 'input_types',
'module_name', 'name', 'namespace', 'output_types', 'subtitle',
'tooltip', 'ver'
],
axis=1,
inplace=True)
keys = list(
set([
item for sublist in list(ModDfApps.categories) for item in sublist
]))
app_dict = {k: [] for k in keys}
# GetAppStats
#
import requests
import os
import datetime, time
import mysql.connector as mysql
from biokbase.catalog.Client import Catalog
from biokbase.narrative_method_store.client import NarrativeMethodStore
requests.packages.urllib3.disable_warnings()
catalog = Catalog(url=os.environ['CATALOG_URL'],
token=os.environ['METRICS_USER_TOKEN'])
nms = NarrativeMethodStore(url=os.environ['NARRATIVE_METHOD_STORE'])
sql_host = os.environ['SQL_HOST']
query_on = os.environ['QUERY_ON']
#Insures all finish times within last day.
yesterday = (datetime.date.today() - datetime.timedelta(days=1))
def get_user_app_stats(
start_date=datetime.datetime.combine(yesterday,
datetime.datetime.min.time()),
end_date=datetime.datetime.combine(yesterday,
datetime.datetime.max.time())):
"""
Gets a data dump from the app cataloge for a certain date window.
If no statt and end date are entered it will default to the last 15 calendar days (UTC TIME).
It is 15 days because it uses an underlying method that
filters by creation_time and not finish_time
