def setup_class(cls): ''' Sets up the token, service urls, and service clients for use in tests. ''' cls.token = script_utils.get_token() cfg = ConfigObj(TEST_CFG_LOC) cls.runner = TransformTaskRunnerDriver(cfg, PLUGIN_CFG_LOC) mapping = cls.runner.get_service_mapping() # TODO discuss why we would need different names here than what is used # by the transform service, client code, and all scripts, why is this necessary at all? cls.ws_url = mapping["workspace"]["url"] cls.ujs_url = mapping["ujs"]["url"] cls.shock_url = mapping["shock"]["url"] cls.ws = mapping["workspace"]["client"] cls.handle = mapping["handle"]["client"] cls.ujs = mapping["ujs"]["client"] keep_venv = cls._keep_venv if os.environ.get(KEEP_VENV): keep_venv = True tve = TransformVirtualEnv(FILE_LOC, 'venv', TRANSFORM_LOC, keep_current_venv=keep_venv) tve.activate_for_current_py_process() cls.staged = {} cls.stage_data()
help='path to place downloaded files for validation', const=".", default=".") parser.add_argument('--config_file', nargs='?', help='path to config file with parameters', const="", default="") parser.add_argument('--verify', help='verify uploaded files', action="store_true") #parser.add_argument('--create_log', help='create pass/fail log file', action='store_true') args = parser.parse_args() token = script_utils.get_token() plugin = None plugin = biokbase.Transform.handler_utils.PlugIns(args.plugin_directory, logger) inputs = list() if not args.demo: if args.config_file: f = open(args.config_file, 'r') config = simplejson.loads(f.read()) f.close() services = config["services"] inputs = config["upload"] else:
parser.add_argument('--workspace', nargs='?', help='name of the workspace where your objects should be created', const="", default="upload_testing") parser.add_argument('--object_name', nargs='?', help='name of the workspace object to create', const="", default="") parser.add_argument('--download_path', nargs='?', help='path to place downloaded files for validation', const=".", default=".") #parser.add_argument('--handler', help='Client bypass test', dest="handler_mode", action='store_true') #parser.add_argument('--client', help='Client mode test', dest="handler_mode", action='store_false') #parser.set_defaults(handler_mode=False) ## TODO: change the default path to be relative to __FILE__ parser.add_argument('--config_file', nargs='?', help='path to config file with parameters', const="", default="") #parser.add_argument('--verify', help='verify uploaded files', action="store_true") parser.add_argument('--plugin_directory', nargs='?', help='path to the plugin dir', const="", default="/kb/dev_container/modules/transform/plugins/configs") args = parser.parse_args() print args token = script_utils.get_token() plugin = None plugin = biokbase.Transform.handler_utils.PlugIns(args.plugin_directory, logger) inputs = list() if not args.demo: if args.config_file: f = open(args.config_file, 'r') config = simplejson.loads(f.read()) f.close() services = config["services"] inputs = config["download"] else: inputs = {"user":
def download_taskrunner(ujs_service_url = None, workspace_service_url = None, shock_service_url = None, handle_service_url = None, fba_service_url = None, workspace_name = None, object_name = None, object_id = None, external_type = None, kbase_type = None, optional_arguments = None, ujs_job_id = None, job_details = None, working_directory = None, keep_working_directory = False, debug = False, logger = None): """ KBase Download task manager for converting from KBase objects to external data formats. Step 1 - Convert the objects to local files Step 2 - Extract provenance and metadata Step 3 - Package all files into a tarball or zip Step 4 - Upload the compressed file to shock and return the download url Args: workspace_service_url: URL for a KBase Workspace service where KBase objects are stored. ujs_service_url: URL for a User and Job State service to report task progress back to the user. shock_service_url: URL for a KBase SHOCK data store service for storing files and large reference data. handle_service_url: URL for a KBase Handle service that maps permissions from the Workspace to SHOCK for KBase types that specify a Handle reference instead of a SHOCK reference. fba_service_url: URL for the FBA Model service used by Model related types. workspace_name: The name of the source workspace. object_name: The source object name. object_id: A source object id, which can be used instead of object_name. external_type: The external data format being transformed to from a KBase type. E.g., FASTA.DNA.Assembly This is simply a string, but denotes the format of the data and some context about what it actually is, which is used to differentiate between files of the same general format with different fundamental values. kbase_type: The KBase Workspace type string that indicates the module and type of the object being created. optional_arguments: This is a JSON string containing optional parameters that can be passed in to customize behavior based on available exposed options. ujs_job_id: The job id from the User and Job State service that can be used to report status on task progress back to the user. job_details: This is a JSON string that passes in the script specific command line options for a given transformation type. The service pulls these config settings from a script config created by the developer of the transformation script and passes that into the AWE job that calls this script. working_directory: The working directory on disk where files can be created and will be cleaned when the job ends with success or failure. keep_working_directory: A flag to tell the script not to delete the working directory, which is mainly for debugging purposes. debug: Run the taskrunner in debug mode for local execution in a virtualenv. Returns: Literal return value is 0 for success and 1 for failure. Actual data output is a shock url that contains data files that were created based on KBase objects. Authors: Shinjae Yoo, Matt Henderson """ if logger is None: raise Exception("A logger must be provided for status information.") kb_token = None try: kb_token = script_utils.get_token() assert type(kb_token) == type(str()) except Exception, e: logger.debug("Exception getting token!") raise
def upload_taskrunner(ujs_service_url=None, workspace_service_url=None, shock_service_url=None, handle_service_url=None, fba_service_url=None, url_mapping=None, workspace_name=None, object_name=None, object_id=None, external_type=None, kbase_type=None, optional_arguments=None, ujs_job_id=None, job_details=None, working_directory=None, keep_working_directory=None, debug=False, logger=None): """ KBase Upload task manager for converting from external data formats to KBase objects. Step 1 - Download the input data and unpack to local files Step 2 - Validate the input files Step 3 - Transform the input files to KBase Object JSON and save any references Step 4 - Save Workspace objects from the JSON and reference info Steps 2,3,4 may be combined by certain data transformations, in which case only step 3 is executed, with the understanding that it has accepted responsibility for validation of the inputs and saving the workspace data. Args: workspace_service_url: URL for a KBase Workspace service where KBase objects are stored. ujs_service_url: URL for a User and Job State service to report task progress back to the user. shock_service_url: URL for a KBase SHOCK data store service for storing files and large reference data. handle_service_url: URL for a KBase Handle service that maps permissions from the Workspace to SHOCK for KBase types that specify a Handle reference instead of a SHOCK reference. fba_service_url: URL for a KBase FBA Model service that has to handle any model related conversions. url_mapping: A JSON mapping of names to urls that identifies specific data elements as well as special information such as KBase metadata and provenance. workspace_name: The name of the destination workspace. object_name: The destination object name. object_id: A destination object id, which can be used instead of object_name. external_type: The external data format being transformed to a KBase type. E.g., FASTA.DNA.Assembly This is simply a string, but denotes the format of the data and some context about what it actually is, which is used to differentiate between files of the same general format with different fundamental values. kbase_type: The KBase Workspace type string that indicates the modules and type of the object being created. optional_arguments: This is a JSON string containing optional parameters that can be passed in for validation or transformation steps to customize behavior based on available exposed options. ujs_job_id: The job id from the User and Job State service that can be used to report status on task progress back to the user. job_details: This is a JSON string that passes in the script specific command line options for a given transformation type. The service pulls these config settings from a script config created by the developer of the transformation script and passes that into the AWE job that calls this script. working_directory: The working directory on disk where files can be created and will be cleaned when the job ends with success or failure. keep_working_directory: A flag to tell the script not to delete the working directory, which is mainly for debugging purposes. debug: Run the taskrunner in debug mode for local execution in a virtualenv. Returns: One or more Workspace objects saved to a user's workspace based on input files provided by urls. Authors: Shinjae Yoo, Matt Henderson """ # validate inputs and environment if logger is None: raise Exception("A logger must be provided for status information.") kb_token = None try: kb_token = script_utils.get_token() assert type(kb_token) == type(str()) except Exception, e: logger.error("Unable to get token!") logger.exception(e) sys.exit(1)
def upload_taskrunner(ujs_service_url = None, workspace_service_url = None, shock_service_url = None, handle_service_url = None, fba_service_url = None, url_mapping = None, workspace_name = None, object_name = None, object_id = None, external_type = None, kbase_type = None, optional_arguments = None, ujs_job_id = None, job_details = None, working_directory = None, keep_working_directory = None, debug = False, logger = None): """ KBase Upload task manager for converting from external data formats to KBase objects. Step 1 - Download the input data and unpack to local files Step 2 - Validate the input files Step 3 - Transform the input files to KBase Object JSON and save any references Step 4 - Save Workspace objects from the JSON and reference info Steps 2,3,4 may be combined by certain data transformations, in which case only step 3 is executed, with the understanding that it has accepted responsibility for validation of the inputs and saving the workspace data. Args: workspace_service_url: URL for a KBase Workspace service where KBase objects are stored. ujs_service_url: URL for a User and Job State service to report task progress back to the user. shock_service_url: URL for a KBase SHOCK data store service for storing files and large reference data. handle_service_url: URL for a KBase Handle service that maps permissions from the Workspace to SHOCK for KBase types that specify a Handle reference instead of a SHOCK reference. fba_service_url: URL for a KBase FBA Model service that has to handle any model related conversions. url_mapping: A JSON mapping of names to urls that identifies specific data elements as well as special information such as KBase metadata and provenance. workspace_name: The name of the destination workspace. object_name: The destination object name. object_id: A destination object id, which can be used instead of object_name. external_type: The external data format being transformed to a KBase type. E.g., FASTA.DNA.Assembly This is simply a string, but denotes the format of the data and some context about what it actually is, which is used to differentiate between files of the same general format with different fundamental values. kbase_type: The KBase Workspace type string that indicates the modules and type of the object being created. optional_arguments: This is a JSON string containing optional parameters that can be passed in for validation or transformation steps to customize behavior based on available exposed options. ujs_job_id: The job id from the User and Job State service that can be used to report status on task progress back to the user. job_details: This is a JSON string that passes in the script specific command line options for a given transformation type. The service pulls these config settings from a script config created by the developer of the transformation script and passes that into the AWE job that calls this script. working_directory: The working directory on disk where files can be created and will be cleaned when the job ends with success or failure. keep_working_directory: A flag to tell the script not to delete the working directory, which is mainly for debugging purposes. debug: Run the taskrunner in debug mode for local execution in a virtualenv. Returns: One or more Workspace objects saved to a user's workspace based on input files provided by urls. Authors: Shinjae Yoo, Matt Henderson """ # validate inputs and environment if logger is None: raise Exception("A logger must be provided for status information.") kb_token = None try: kb_token = script_utils.get_token() assert type(kb_token) == type(str()) except Exception, e: logger.error("Unable to get token!") logger.exception(e) sys.exit(1)