def __init__(
     self,
     url=None,
     timeout=30 * 60,
     user_id=None,
     password=None,
     token=None,
     ignore_authrc=False,
     trust_all_ssl_certificates=False,
     auth_svc="https://kbase.us/services/authorization/Sessions/Login",
     service_ver="dev",
 ):
     if url is None:
         url = "https://kbase.us/services/service_wizard"
     self._service_ver = service_ver
     self._client = _BaseClient(
         url,
         timeout=timeout,
         user_id=user_id,
         password=password,
         token=token,
         ignore_authrc=ignore_authrc,
         trust_all_ssl_certificates=trust_all_ssl_certificates,
         auth_svc=auth_svc,
         lookup_url=True,
     )
 def __init__(
         self,
         url=None,
         timeout=30 * 60,
         user_id=None,
         password=None,
         token=None,
         ignore_authrc=False,
         trust_all_ssl_certificates=False,
         auth_svc='https://kbase.us/services/authorization/Sessions/Login',
         service_ver='dev',
         async_job_check_time_ms=100,
         async_job_check_time_scale_percent=150,
         async_job_check_max_time_ms=300000):
     if url is None:
         raise ValueError('A url is required')
     self._service_ver = service_ver
     self._client = _BaseClient(
         url,
         timeout=timeout,
         user_id=user_id,
         password=password,
         token=token,
         ignore_authrc=ignore_authrc,
         trust_all_ssl_certificates=trust_all_ssl_certificates,
         auth_svc=auth_svc,
         async_job_check_time_ms=async_job_check_time_ms,
         async_job_check_time_scale_percent=
         async_job_check_time_scale_percent,
         async_job_check_max_time_ms=async_job_check_max_time_ms)
 def __init__(
         self,
         url=None,
         timeout=30 * 60,
         user_id=None,
         password=None,
         token=None,
         ignore_authrc=False,
         trust_all_ssl_certificates=False,
         auth_svc='https://kbase.us/services/authorization/Sessions/Login',
         service_ver='release',
         async_job_check_time_ms=5000):
     if url is None:
         url = 'https://kbase.us/services/njs_wrapper'
     self._service_ver = service_ver
     self._client = _BaseClient(
         url,
         timeout=timeout,
         user_id=user_id,
         password=password,
         token=token,
         ignore_authrc=ignore_authrc,
         trust_all_ssl_certificates=trust_all_ssl_certificates,
         auth_svc=auth_svc,
         async_job_check_time_ms=async_job_check_time_ms)
Пример #4
0
 def _lookup_url(self):
     bc = _BaseClient(url=self.sw_url, lookup_url=False)
     self.cached_url = bc.call_method('ServiceWizard.get_service_status',
                                      [{
                                          'module_name': self.module_name,
                                          'version': self.service_ver
                                      }])['url']
     self.last_refresh_time = time.time()
 def __init__(
         self, url=None, timeout=30 * 60, user_id=None,
         password=None, token=None, ignore_authrc=False,
         trust_all_ssl_certificates=False,
         auth_svc='https://kbase.us/services/authorization/Sessions/Login'):
     if url is None:
         raise ValueError('A url is required')
     self._service_ver = None
     self._client = _BaseClient(
         url, timeout=timeout, user_id=user_id, password=password,
         token=token, ignore_authrc=ignore_authrc,
         trust_all_ssl_certificates=trust_all_ssl_certificates,
         auth_svc=auth_svc)
 def __init__(
         self, url=None, timeout=30 * 60, user_id=None,
         password=None, token=None, ignore_authrc=False,
         trust_all_ssl_certificates=False,
         auth_svc='https://kbase.us/services/authorization/Sessions/Login'):
     if url is None:
         raise ValueError('A url is required')
     self._service_ver = None
     self._client = _BaseClient(
         url, timeout=timeout, user_id=user_id, password=password,
         token=token, ignore_authrc=ignore_authrc,
         trust_all_ssl_certificates=trust_all_ssl_certificates,
         auth_svc=auth_svc)
Пример #7
0
 def __init__(
         self, url=None, timeout=30 * 60, user_id=None,
         password=None, token=None, ignore_authrc=False,
         trust_all_ssl_certificates=False,
         auth_svc='https://kbase.us/services/authorization/Sessions/Login',
         service_ver='dev'):
     if url is None:
         url = 'https://kbase.us/services/service_wizard'
     self._service_ver = service_ver
     self._client = _BaseClient(
         url, timeout=timeout, user_id=user_id, password=password,
         token=token, ignore_authrc=ignore_authrc,
         trust_all_ssl_certificates=trust_all_ssl_certificates,
         auth_svc=auth_svc,
         lookup_url=True)
 def __init__(
         self, url=None, timeout=30 * 60, user_id=None,
         password=None, token=None, ignore_authrc=False,
         trust_all_ssl_certificates=False,
         auth_svc='https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login',
         service_ver='release'):
     if url is None:
         raise ValueError('A url is required')
     self._service_ver = service_ver
     self._client = _BaseClient(
         url, timeout=timeout, user_id=user_id, password=password,
         token=token, ignore_authrc=ignore_authrc,
         trust_all_ssl_certificates=trust_all_ssl_certificates,
         auth_svc=auth_svc,
         lookup_url=True)
 def __init__(
         self, url=None, timeout=30 * 60, user_id=None,
         password=None, token=None, ignore_authrc=False,
         trust_all_ssl_certificates=False,
         auth_svc='https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login',
         service_ver='release'):
     if url is None:
         url = 'https://kbase.us/services/service_wizard'
     self._service_ver = service_ver
     self._client = _BaseClient(
         url, timeout=timeout, user_id=user_id, password=password,
         token=token, ignore_authrc=ignore_authrc,
         trust_all_ssl_certificates=trust_all_ssl_certificates,
         auth_svc=auth_svc,
         lookup_url=True)
 def __init__(
         self, url=None, timeout=30 * 60, user_id=None,
         password=None, token=None, ignore_authrc=False,
         trust_all_ssl_certificates=False,
         auth_svc='https://kbase.us/services/authorization/Sessions/Login',
         service_ver='dev',
         async_job_check_time_ms=5000):
     if url is None:
         url = 'https://kbase.us/services/njs_wrapper'
     self._service_ver = service_ver
     self._client = _BaseClient(
         url, timeout=timeout, user_id=user_id, password=password,
         token=token, ignore_authrc=ignore_authrc,
         trust_all_ssl_certificates=trust_all_ssl_certificates,
         auth_svc=auth_svc,
         async_job_check_time_ms=async_job_check_time_ms)
 def __init__(
         self, url=None, timeout=30 * 60, user_id=None,
         password=None, token=None, ignore_authrc=False,
         trust_all_ssl_certificates=False,
         auth_svc='https://kbase.us/services/authorization/Sessions/Login',
         service_ver='release',
         async_job_check_time_ms=100, async_job_check_time_scale_percent=150, 
         async_job_check_max_time_ms=300000):
     if url is None:
         raise ValueError('A url is required')
     self._service_ver = service_ver
     self._client = _BaseClient(
         url, timeout=timeout, user_id=user_id, password=password,
         token=token, ignore_authrc=ignore_authrc,
         trust_all_ssl_certificates=trust_all_ssl_certificates,
         auth_svc=auth_svc,
         async_job_check_time_ms=async_job_check_time_ms,
         async_job_check_time_scale_percent=async_job_check_time_scale_percent,
         async_job_check_max_time_ms=async_job_check_max_time_ms)
Пример #12
0
    def run(self, ctx, input_params):
        """
        :param input_params: instance of type "KBParallelrunInputParams"
           (Input parameters for run() method. method - optional method where
           _prepare(), _runEach() and _collect() suffixes are applied,
           prepare_method - optional method (if defined overrides _prepare
           suffix rule), is_local - optional flag defining way of scheduling
           sub-job, in case is_local=false sub-jobs are scheduled against
           remote execution engine, if is_local=true then sub_jobs are run as
           local functions through CALLBACK mechanism, default value is
           false, global_input - input data which is supposed to be sent as a
           part to <module_name>.<method_name>_prepare() method, time_limit -
           time limit in seconds, equals to 5000 by default.) -> structure:
           parameter "method" of type "FullMethodQualifier" (module_name -
           SDK module name (ie. ManyHellos, RNAseq), method_name - method in
           SDK module (TopHatcall, Hiseqcall etc each will have own
           _prepare(), _runEach(), _collect() methods defined), service_ver -
           optional version of SDK module (may be dev/beta/release, or
           symantic version or particular git commit hash), it's release by
           default,) -> structure: parameter "module_name" of String,
           parameter "method_name" of String, parameter "service_ver" of
           String, parameter "prepare_method" of type "FullMethodQualifier"
           (module_name - SDK module name (ie. ManyHellos, RNAseq),
           method_name - method in SDK module (TopHatcall, Hiseqcall etc each
           will have own _prepare(), _runEach(), _collect() methods defined),
           service_ver - optional version of SDK module (may be
           dev/beta/release, or symantic version or particular git commit
           hash), it's release by default,) -> structure: parameter
           "module_name" of String, parameter "method_name" of String,
           parameter "service_ver" of String, parameter "is_local" of type
           "boolean" (A boolean - 0 for false, 1 for true. @range (0, 1)),
           parameter "global_input" of unspecified object, parameter
           "time_limit" of Long
        :returns: instance of unspecified object
        """
        # ctx is the context object
        # return variables are: returnVal
        #BEGIN run
        print( "Hi this is KBParallel.run() input_params are")
        pprint( input_params )

        token = ctx['token']
        method = input_params.get('method')


        #instantiate ManyHellos client here
        print( "about to initiate client .." )
        client = _BaseClient(self.callbackURL, token=token)


        # issue prepare call
        print( "about to invoke prepare()")
        prepare_input_params = {'global_input_params': input_params["global_params"], 
                                'global_method': method}
        prepare_method_name = None
        prepare_service_ver = None
        if 'prepare_method' in input_params:
            prepare_method = input_params.get('prepare_method')
            prepare_method_name = (prepare_method['module_name'] + '.' + 
                                   prepare_method['method_name'])
            prepare_service_ver = prepare_method.get('service_ver')
        else:
            if not method:
                raise ValueError("Global method wasn't defined but task doesn't provide " +
                                 "local _runEach alternative")
            module_name = method['module_name']
            method_name = method['method_name']
            prepare_method_name = module_name + '.' + method_name + "_prepare"
            prepare_service_ver = method.get('service_ver', "release")
        schedule = client.call_method(prepare_method_name,
                                      [prepare_input_params], 
                                      service_ver = prepare_service_ver,
                                      context=None)
        print( "back in run")
        tasks = schedule['tasks']
        collect_method = schedule.get('collect_method', None)
        pprint( tasks )

        # initiate NJS wrapper
        print( "initiating Execution Engine")
        exec_engine_url = None
        remote_ee_client = None
        is_local = ('is_local' in input_params and input_params['is_local'] == 1)
        if is_local:
            exec_engine_url = self.callbackURL
        else:
            exec_engine_url = self.config['njs-wrapper-url']
            remote_ee_client = NJS(url=self.config['njs-wrapper-url'], token=token)
        exec_engine_client = _BaseClient(exec_engine_url, token=token)

        jobid_list = []         # list of jobids when launched
        job_running_list = []   # list of booleans which are True if the job is queued or running, False when completed
        job_timeout_list = []   # list of job timeout values, None until the job starts
        njobs = 0
        job_input_result_map = {}
        for task in tasks:
            pprint( ["   launching task", task]  )
            task_method_module_name = None
            task_service_ver = None
            if 'method' in task:
                task_method = task['method']
                task_method_module_name = (task_method['module_name'] + '.' + 
                                           task_method['method_name'])
                task_service_ver = task_method.get('service_ver', None)
            else:
                if not method:
                    raise ValueError("Global method wasn't defined but task doesn't provide " +
                                     "local _runEach alternative")
                module_name = method['module_name']
                method_name = method['method_name']
                task_method_module_name = module_name + '.' + method_name + "_runEach"
                task_service_ver = method.get('service_ver', "release")
            task_input = task['input_arguments']
            
            jobid = exec_engine_client._submit_job(task_method_module_name,
                                                   task_input, 
                                                   service_ver=task_service_ver)
            print( "job_id", jobid )
            jobid_list.append( jobid )
            job_running_list.append( True )
            job_timeout_list.append( None )
            job_input_result_map[jobid] = {'input': task}
            njobs = njobs + 1

        # Polling loop to see when job is finished

        print("polling ", njobs, " NJS job status")
        pprint(job_running_list)
        njobs_remaining = njobs      # this will count down to 0 when all jobs completed
        try:
            while njobs_remaining > 0:
    
                # poll each job
                for j in range(0, njobs):
                    if job_running_list[j]:     # only consider those which are still running
                        jobid = jobid_list[j]
                        task = job_input_result_map[jobid]['input']
                        task_module_name = None
                        if 'method' in task:
                            task_module_name = task['method']['module_name']
                        else:
                            task_module_name = method['module_name']
                        job_state_desc = exec_engine_client._check_job(task_module_name, jobid )
                        pprint( job_state_desc )
    
                        if job_state_desc["finished"] == 1: # has this job completed
                            print( "**************job ", j, " is done***********" )
                            job_running_list[j] = False
                            njobs_remaining = njobs_remaining - 1
    
                            if "error" in job_state_desc:                                # if it crashed, 
                                print( "************** job ", j, " has crashed*********" )
                                raise Exception("jobs canceled because of failure")
                            elif 'result' in job_state_desc:
                                job_result = job_state_desc['result']
                                if len(job_result) == 1:
                                    job_result = job_result[0]
                                job_input_result_map[jobid_list[j]]['result'] = job_result
                            else:
                                raise Exception("Unexpected job state (no error/result fields")
                        else:                                                            # if its running, check for timeout
                            print( "checking timeout for this job" )
                            # check timeout of job
                            if job_timeout_list[j] == None :
                                job_timeout_list[j] = time.time() + 60 * self.config['time_limit']   # start the clock if just started
                            if time.time() > job_timeout_list[j] :
                                print "*************TIMEOUT****************"
                                raise Exception("jobs canceled because of timeout")
    
                time.sleep( self.checkwait )
        finally:
            if is_local:
                # TODO: make sure we stop all local sub-containers in SDK CALLBACK
                pass
            else:
                jobs_to_stop = reduce_list( jobid_list, job_running_list )
                if len(jobs_to_stop) > 0:
                    print("Stopping all sub-jobs...")
                    for jobid in jobs_to_stop:
                        try:
                            remote_ee_client.cancel_job( { 'job_id': jobid } )
                        except:
                            print("Error canceling job " + jobid)

        # at this point, NJS informed us that job is finished, must now check error status

        # Question: best way to determine a successful completion or not from the
        #           jobs state

        print( "about to invoke collect()" )
        input_result_pairs = [job_input_result_map[key] for key in job_input_result_map]
        collect_input_params = {'global_params': input_params["global_params"],
                                'input_result_pairs': input_result_pairs}
        collect_module_method_name = None
        collect_service_ver = None
        if collect_method:
            collect_module_method_name = (collect_method['module_name'] + '.' + 
                                          collect_method['method_name'])
            collect_service_ver = collect_method['service_ver']
        else:
            if not method:
                raise ValueError("Global method wasn't defined but schedule doesn't provide " +
                                 "local _collect alternative")
            module_name = method['module_name']
            method_name = method['method_name']
            collect_module_method_name = module_name + '.' + method_name + "_collect"
            collect_service_ver = method.get('service_ver', "release")
        print("Collect input: " + json.dumps(collect_input_params))
        returnVal = client.call_method(collect_module_method_name,
                                 [collect_input_params], 
                                 service_ver = collect_service_ver,
                                 context=None)
        pprint( returnVal )
        #END run

        # At some point might do deeper type checking...
        if not isinstance(returnVal, object):
            raise ValueError('Method run return value ' +
                             'returnVal is not type object as required.')
        # return the results
        return [returnVal]
Пример #13
0
 def _call(self, method, params_array, token):
     bc = _BaseClient(url=self.cached_url, token=token, lookup_url=False)
     return bc.call_method(self.module_name + '.' + method, params_array)