def __init__(
self,
url=None,
timeout=30 * 60,
user_id=None,
password=None,
token=None,
ignore_authrc=False,
trust_all_ssl_certificates=False,
auth_svc="https://kbase.us/services/authorization/Sessions/Login",
service_ver="dev",
):
if url is None:
url = "https://kbase.us/services/service_wizard"
self._service_ver = service_ver
self._client = _BaseClient(
url,
timeout=timeout,
user_id=user_id,
password=password,
token=token,
ignore_authrc=ignore_authrc,
trust_all_ssl_certificates=trust_all_ssl_certificates,
auth_svc=auth_svc,
lookup_url=True,
)
def __init__(
self,
url=None,
timeout=30 * 60,
user_id=None,
password=None,
token=None,
ignore_authrc=False,
trust_all_ssl_certificates=False,
auth_svc='https://kbase.us/services/authorization/Sessions/Login',
service_ver='dev',
async_job_check_time_ms=100,
async_job_check_time_scale_percent=150,
async_job_check_max_time_ms=300000):
if url is None:
raise ValueError('A url is required')
self._service_ver = service_ver
self._client = _BaseClient(
url,
timeout=timeout,
user_id=user_id,
password=password,
token=token,
ignore_authrc=ignore_authrc,
trust_all_ssl_certificates=trust_all_ssl_certificates,
auth_svc=auth_svc,
async_job_check_time_ms=async_job_check_time_ms,
async_job_check_time_scale_percent=
async_job_check_time_scale_percent,
async_job_check_max_time_ms=async_job_check_max_time_ms)
def __init__(
self,
url=None,
timeout=30 * 60,
user_id=None,
password=None,
token=None,
ignore_authrc=False,
trust_all_ssl_certificates=False,
auth_svc='https://kbase.us/services/authorization/Sessions/Login',
service_ver='release',
async_job_check_time_ms=5000):
if url is None:
url = 'https://kbase.us/services/njs_wrapper'
self._service_ver = service_ver
self._client = _BaseClient(
url,
timeout=timeout,
user_id=user_id,
password=password,
token=token,
ignore_authrc=ignore_authrc,
trust_all_ssl_certificates=trust_all_ssl_certificates,
auth_svc=auth_svc,
async_job_check_time_ms=async_job_check_time_ms)
def _lookup_url(self):
bc = _BaseClient(url=self.sw_url, lookup_url=False)
self.cached_url = bc.call_method('ServiceWizard.get_service_status',
[{
'module_name': self.module_name,
'version': self.service_ver
}])['url']
self.last_refresh_time = time.time()
def __init__(
self, url=None, timeout=30 * 60, user_id=None,
password=None, token=None, ignore_authrc=False,
trust_all_ssl_certificates=False,
auth_svc='https://kbase.us/services/authorization/Sessions/Login'):
if url is None:
raise ValueError('A url is required')
self._service_ver = None
self._client = _BaseClient(
url, timeout=timeout, user_id=user_id, password=password,
token=token, ignore_authrc=ignore_authrc,
trust_all_ssl_certificates=trust_all_ssl_certificates,
auth_svc=auth_svc)
def __init__(
self, url=None, timeout=30 * 60, user_id=None,
password=None, token=None, ignore_authrc=False,
trust_all_ssl_certificates=False,
auth_svc='https://kbase.us/services/authorization/Sessions/Login'):
if url is None:
raise ValueError('A url is required')
self._service_ver = None
self._client = _BaseClient(
url, timeout=timeout, user_id=user_id, password=password,
token=token, ignore_authrc=ignore_authrc,
trust_all_ssl_certificates=trust_all_ssl_certificates,
auth_svc=auth_svc)
def __init__(
self, url=None, timeout=30 * 60, user_id=None,
password=None, token=None, ignore_authrc=False,
trust_all_ssl_certificates=False,
auth_svc='https://kbase.us/services/authorization/Sessions/Login',
service_ver='dev'):
if url is None:
url = 'https://kbase.us/services/service_wizard'
self._service_ver = service_ver
self._client = _BaseClient(
url, timeout=timeout, user_id=user_id, password=password,
token=token, ignore_authrc=ignore_authrc,
trust_all_ssl_certificates=trust_all_ssl_certificates,
auth_svc=auth_svc,
lookup_url=True)
def __init__(
self, url=None, timeout=30 * 60, user_id=None,
password=None, token=None, ignore_authrc=False,
trust_all_ssl_certificates=False,
auth_svc='https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login',
service_ver='release'):
if url is None:
raise ValueError('A url is required')
self._service_ver = service_ver
self._client = _BaseClient(
url, timeout=timeout, user_id=user_id, password=password,
token=token, ignore_authrc=ignore_authrc,
trust_all_ssl_certificates=trust_all_ssl_certificates,
auth_svc=auth_svc,
lookup_url=True)
def __init__(
self, url=None, timeout=30 * 60, user_id=None,
password=None, token=None, ignore_authrc=False,
trust_all_ssl_certificates=False,
auth_svc='https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login',
service_ver='release'):
if url is None:
url = 'https://kbase.us/services/service_wizard'
self._service_ver = service_ver
self._client = _BaseClient(
url, timeout=timeout, user_id=user_id, password=password,
token=token, ignore_authrc=ignore_authrc,
trust_all_ssl_certificates=trust_all_ssl_certificates,
auth_svc=auth_svc,
lookup_url=True)
def __init__(
self, url=None, timeout=30 * 60, user_id=None,
password=None, token=None, ignore_authrc=False,
trust_all_ssl_certificates=False,
auth_svc='https://kbase.us/services/authorization/Sessions/Login',
service_ver='dev',
async_job_check_time_ms=5000):
if url is None:
url = 'https://kbase.us/services/njs_wrapper'
self._service_ver = service_ver
self._client = _BaseClient(
url, timeout=timeout, user_id=user_id, password=password,
token=token, ignore_authrc=ignore_authrc,
trust_all_ssl_certificates=trust_all_ssl_certificates,
auth_svc=auth_svc,
async_job_check_time_ms=async_job_check_time_ms)
def __init__(
self, url=None, timeout=30 * 60, user_id=None,
password=None, token=None, ignore_authrc=False,
trust_all_ssl_certificates=False,
auth_svc='https://kbase.us/services/authorization/Sessions/Login',
service_ver='release',
async_job_check_time_ms=100, async_job_check_time_scale_percent=150,
async_job_check_max_time_ms=300000):
if url is None:
raise ValueError('A url is required')
self._service_ver = service_ver
self._client = _BaseClient(
url, timeout=timeout, user_id=user_id, password=password,
token=token, ignore_authrc=ignore_authrc,
trust_all_ssl_certificates=trust_all_ssl_certificates,
auth_svc=auth_svc,
async_job_check_time_ms=async_job_check_time_ms,
async_job_check_time_scale_percent=async_job_check_time_scale_percent,
async_job_check_max_time_ms=async_job_check_max_time_ms)
def run(self, ctx, input_params):
"""
:param input_params: instance of type "KBParallelrunInputParams"
(Input parameters for run() method. method - optional method where
_prepare(), _runEach() and _collect() suffixes are applied,
prepare_method - optional method (if defined overrides _prepare
suffix rule), is_local - optional flag defining way of scheduling
sub-job, in case is_local=false sub-jobs are scheduled against
remote execution engine, if is_local=true then sub_jobs are run as
local functions through CALLBACK mechanism, default value is
false, global_input - input data which is supposed to be sent as a
part to <module_name>.<method_name>_prepare() method, time_limit -
time limit in seconds, equals to 5000 by default.) -> structure:
parameter "method" of type "FullMethodQualifier" (module_name -
SDK module name (ie. ManyHellos, RNAseq), method_name - method in
SDK module (TopHatcall, Hiseqcall etc each will have own
_prepare(), _runEach(), _collect() methods defined), service_ver -
optional version of SDK module (may be dev/beta/release, or
symantic version or particular git commit hash), it's release by
default,) -> structure: parameter "module_name" of String,
parameter "method_name" of String, parameter "service_ver" of
String, parameter "prepare_method" of type "FullMethodQualifier"
(module_name - SDK module name (ie. ManyHellos, RNAseq),
method_name - method in SDK module (TopHatcall, Hiseqcall etc each
will have own _prepare(), _runEach(), _collect() methods defined),
service_ver - optional version of SDK module (may be
dev/beta/release, or symantic version or particular git commit
hash), it's release by default,) -> structure: parameter
"module_name" of String, parameter "method_name" of String,
parameter "service_ver" of String, parameter "is_local" of type
"boolean" (A boolean - 0 for false, 1 for true. @range (0, 1)),
parameter "global_input" of unspecified object, parameter
"time_limit" of Long
:returns: instance of unspecified object
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN run
print( "Hi this is KBParallel.run() input_params are")
pprint( input_params )
token = ctx['token']
method = input_params.get('method')
#instantiate ManyHellos client here
print( "about to initiate client .." )
client = _BaseClient(self.callbackURL, token=token)
# issue prepare call
print( "about to invoke prepare()")
prepare_input_params = {'global_input_params': input_params["global_params"],
'global_method': method}
prepare_method_name = None
prepare_service_ver = None
if 'prepare_method' in input_params:
prepare_method = input_params.get('prepare_method')
prepare_method_name = (prepare_method['module_name'] + '.' +
prepare_method['method_name'])
prepare_service_ver = prepare_method.get('service_ver')
else:
if not method:
raise ValueError("Global method wasn't defined but task doesn't provide " +
"local _runEach alternative")
module_name = method['module_name']
method_name = method['method_name']
prepare_method_name = module_name + '.' + method_name + "_prepare"
prepare_service_ver = method.get('service_ver', "release")
schedule = client.call_method(prepare_method_name,
[prepare_input_params],
service_ver = prepare_service_ver,
context=None)
print( "back in run")
tasks = schedule['tasks']
collect_method = schedule.get('collect_method', None)
pprint( tasks )
# initiate NJS wrapper
print( "initiating Execution Engine")
exec_engine_url = None
remote_ee_client = None
is_local = ('is_local' in input_params and input_params['is_local'] == 1)
if is_local:
exec_engine_url = self.callbackURL
else:
exec_engine_url = self.config['njs-wrapper-url']
remote_ee_client = NJS(url=self.config['njs-wrapper-url'], token=token)
exec_engine_client = _BaseClient(exec_engine_url, token=token)
jobid_list = [] # list of jobids when launched
job_running_list = [] # list of booleans which are True if the job is queued or running, False when completed
job_timeout_list = [] # list of job timeout values, None until the job starts
njobs = 0
job_input_result_map = {}
for task in tasks:
pprint( [" launching task", task] )
task_method_module_name = None
task_service_ver = None
if 'method' in task:
task_method = task['method']
task_method_module_name = (task_method['module_name'] + '.' +
task_method['method_name'])
task_service_ver = task_method.get('service_ver', None)
else:
if not method:
raise ValueError("Global method wasn't defined but task doesn't provide " +
"local _runEach alternative")
module_name = method['module_name']
method_name = method['method_name']
task_method_module_name = module_name + '.' + method_name + "_runEach"
task_service_ver = method.get('service_ver', "release")
task_input = task['input_arguments']
jobid = exec_engine_client._submit_job(task_method_module_name,
task_input,
service_ver=task_service_ver)
print( "job_id", jobid )
jobid_list.append( jobid )
job_running_list.append( True )
job_timeout_list.append( None )
job_input_result_map[jobid] = {'input': task}
njobs = njobs + 1
# Polling loop to see when job is finished
print("polling ", njobs, " NJS job status")
pprint(job_running_list)
njobs_remaining = njobs # this will count down to 0 when all jobs completed
try:
while njobs_remaining > 0:
# poll each job
for j in range(0, njobs):
if job_running_list[j]: # only consider those which are still running
jobid = jobid_list[j]
task = job_input_result_map[jobid]['input']
task_module_name = None
if 'method' in task:
task_module_name = task['method']['module_name']
else:
task_module_name = method['module_name']
job_state_desc = exec_engine_client._check_job(task_module_name, jobid )
pprint( job_state_desc )
if job_state_desc["finished"] == 1: # has this job completed
print( "**************job ", j, " is done***********" )
job_running_list[j] = False
njobs_remaining = njobs_remaining - 1
if "error" in job_state_desc: # if it crashed,
print( "************** job ", j, " has crashed*********" )
raise Exception("jobs canceled because of failure")
elif 'result' in job_state_desc:
job_result = job_state_desc['result']
if len(job_result) == 1:
job_result = job_result[0]
job_input_result_map[jobid_list[j]]['result'] = job_result
else:
raise Exception("Unexpected job state (no error/result fields")
else: # if its running, check for timeout
print( "checking timeout for this job" )
# check timeout of job
if job_timeout_list[j] == None :
job_timeout_list[j] = time.time() + 60 * self.config['time_limit'] # start the clock if just started
if time.time() > job_timeout_list[j] :
print "*************TIMEOUT****************"
raise Exception("jobs canceled because of timeout")
time.sleep( self.checkwait )
finally:
if is_local:
# TODO: make sure we stop all local sub-containers in SDK CALLBACK
pass
else:
jobs_to_stop = reduce_list( jobid_list, job_running_list )
if len(jobs_to_stop) > 0:
print("Stopping all sub-jobs...")
for jobid in jobs_to_stop:
try:
remote_ee_client.cancel_job( { 'job_id': jobid } )
except:
print("Error canceling job " + jobid)
# at this point, NJS informed us that job is finished, must now check error status
# Question: best way to determine a successful completion or not from the
# jobs state
print( "about to invoke collect()" )
input_result_pairs = [job_input_result_map[key] for key in job_input_result_map]
collect_input_params = {'global_params': input_params["global_params"],
'input_result_pairs': input_result_pairs}
collect_module_method_name = None
collect_service_ver = None
if collect_method:
collect_module_method_name = (collect_method['module_name'] + '.' +
collect_method['method_name'])
collect_service_ver = collect_method['service_ver']
else:
if not method:
raise ValueError("Global method wasn't defined but schedule doesn't provide " +
"local _collect alternative")
module_name = method['module_name']
method_name = method['method_name']
collect_module_method_name = module_name + '.' + method_name + "_collect"
collect_service_ver = method.get('service_ver', "release")
print("Collect input: " + json.dumps(collect_input_params))
returnVal = client.call_method(collect_module_method_name,
[collect_input_params],
service_ver = collect_service_ver,
context=None)
pprint( returnVal )
#END run
# At some point might do deeper type checking...
if not isinstance(returnVal, object):
raise ValueError('Method run return value ' +
'returnVal is not type object as required.')
# return the results
return [returnVal]
def _call(self, method, params_array, token):
bc = _BaseClient(url=self.cached_url, token=token, lookup_url=False)
return bc.call_method(self.module_name + '.' + method, params_array)
