def setup_class(cls):
'''
Sets up the token, service urls, and service clients for use in
tests.
'''
cls.token = script_utils.get_token()
cfg = ConfigObj(TEST_CFG_LOC)
cls.runner = TransformTaskRunnerDriver(cfg, PLUGIN_CFG_LOC)
mapping = cls.runner.get_service_mapping()
# TODO discuss why we would need different names here than what is used
# by the transform service, client code, and all scripts, why is this necessary at all?
cls.ws_url = mapping["workspace"]["url"]
cls.ujs_url = mapping["ujs"]["url"]
cls.shock_url = mapping["shock"]["url"]
cls.ws = mapping["workspace"]["client"]
cls.handle = mapping["handle"]["client"]
cls.ujs = mapping["ujs"]["client"]
keep_venv = cls._keep_venv
if os.environ.get(KEEP_VENV):
keep_venv = True
tve = TransformVirtualEnv(FILE_LOC, 'venv', TRANSFORM_LOC,
keep_current_venv=keep_venv)
tve.activate_for_current_py_process()
cls.staged = {}
cls.stage_data()
help='path to place downloaded files for validation',
const=".",
default=".")
parser.add_argument('--config_file',
nargs='?',
help='path to config file with parameters',
const="",
default="")
parser.add_argument('--verify',
help='verify uploaded files',
action="store_true")
#parser.add_argument('--create_log', help='create pass/fail log file', action='store_true')
args = parser.parse_args()
token = script_utils.get_token()
plugin = None
plugin = biokbase.Transform.handler_utils.PlugIns(args.plugin_directory,
logger)
inputs = list()
if not args.demo:
if args.config_file:
f = open(args.config_file, 'r')
config = simplejson.loads(f.read())
f.close()
services = config["services"]
inputs = config["upload"]
else:
parser.add_argument('--workspace', nargs='?', help='name of the workspace where your objects should be created', const="", default="upload_testing")
parser.add_argument('--object_name', nargs='?', help='name of the workspace object to create', const="", default="")
parser.add_argument('--download_path', nargs='?', help='path to place downloaded files for validation', const=".", default=".")
#parser.add_argument('--handler', help='Client bypass test', dest="handler_mode", action='store_true')
#parser.add_argument('--client', help='Client mode test', dest="handler_mode", action='store_false')
#parser.set_defaults(handler_mode=False)
## TODO: change the default path to be relative to __FILE__
parser.add_argument('--config_file', nargs='?', help='path to config file with parameters', const="", default="")
#parser.add_argument('--verify', help='verify uploaded files', action="store_true")
parser.add_argument('--plugin_directory', nargs='?', help='path to the plugin dir', const="", default="/kb/dev_container/modules/transform/plugins/configs")
args = parser.parse_args()
print args
token = script_utils.get_token()
plugin = None
plugin = biokbase.Transform.handler_utils.PlugIns(args.plugin_directory, logger)
inputs = list()
if not args.demo:
if args.config_file:
f = open(args.config_file, 'r')
config = simplejson.loads(f.read())
f.close()
services = config["services"]
inputs = config["download"]
else:
inputs = {"user":
def download_taskrunner(ujs_service_url = None, workspace_service_url = None,
shock_service_url = None, handle_service_url = None,
fba_service_url = None, workspace_name = None,
object_name = None, object_id = None,
external_type = None, kbase_type = None,
optional_arguments = None, ujs_job_id = None,
job_details = None, working_directory = None,
keep_working_directory = False, debug = False,
logger = None):
"""
KBase Download task manager for converting from KBase objects to external data formats.
Step 1 - Convert the objects to local files
Step 2 - Extract provenance and metadata
Step 3 - Package all files into a tarball or zip
Step 4 - Upload the compressed file to shock and return the download url
Args:
workspace_service_url: URL for a KBase Workspace service where KBase objects
are stored.
ujs_service_url: URL for a User and Job State service to report task progress
back to the user.
shock_service_url: URL for a KBase SHOCK data store service for storing files
and large reference data.
handle_service_url: URL for a KBase Handle service that maps permissions from
the Workspace to SHOCK for KBase types that specify a Handle
reference instead of a SHOCK reference.
fba_service_url: URL for the FBA Model service used by Model related types.
workspace_name: The name of the source workspace.
object_name: The source object name.
object_id: A source object id, which can be used instead of object_name.
external_type: The external data format being transformed to from a KBase type.
E.g., FASTA.DNA.Assembly
This is simply a string, but denotes the format of the data and
some context about what it actually is, which is used to
differentiate between files of the same general format with
different fundamental values.
kbase_type: The KBase Workspace type string that indicates the module and type
of the object being created.
optional_arguments: This is a JSON string containing optional parameters that can
be passed in to customize behavior based on available
exposed options.
ujs_job_id: The job id from the User and Job State service that can be used to
report status on task progress back to the user.
job_details: This is a JSON string that passes in the script specific command
line options for a given transformation type. The service pulls
these config settings from a script config created by the developer
of the transformation script and passes that into the AWE job that
calls this script.
working_directory: The working directory on disk where files can be created and
will be cleaned when the job ends with success or failure.
keep_working_directory: A flag to tell the script not to delete the working
directory, which is mainly for debugging purposes.
debug: Run the taskrunner in debug mode for local execution in a virtualenv.
Returns:
Literal return value is 0 for success and 1 for failure.
Actual data output is a shock url that contains data files that were created
based on KBase objects.
Authors:
Shinjae Yoo, Matt Henderson
"""
if logger is None:
raise Exception("A logger must be provided for status information.")
kb_token = None
try:
kb_token = script_utils.get_token()
assert type(kb_token) == type(str())
except Exception, e:
logger.debug("Exception getting token!")
raise
def upload_taskrunner(ujs_service_url=None,
workspace_service_url=None,
shock_service_url=None,
handle_service_url=None,
fba_service_url=None,
url_mapping=None,
workspace_name=None,
object_name=None,
object_id=None,
external_type=None,
kbase_type=None,
optional_arguments=None,
ujs_job_id=None,
job_details=None,
working_directory=None,
keep_working_directory=None,
debug=False,
logger=None):
"""
KBase Upload task manager for converting from external data formats to KBase objects.
Step 1 - Download the input data and unpack to local files
Step 2 - Validate the input files
Step 3 - Transform the input files to KBase Object JSON and save any references
Step 4 - Save Workspace objects from the JSON and reference info
Steps 2,3,4 may be combined by certain data transformations, in which case
only step 3 is executed, with the understanding that it has accepted
responsibility for validation of the inputs and saving the workspace data.
Args:
workspace_service_url: URL for a KBase Workspace service where KBase objects
are stored.
ujs_service_url: URL for a User and Job State service to report task progress
back to the user.
shock_service_url: URL for a KBase SHOCK data store service for storing files
and large reference data.
handle_service_url: URL for a KBase Handle service that maps permissions from
the Workspace to SHOCK for KBase types that specify a Handle
reference instead of a SHOCK reference.
fba_service_url: URL for a KBase FBA Model service that has to handle any
model related conversions.
url_mapping: A JSON mapping of names to urls that identifies specific data elements
as well as special information such as KBase metadata and provenance.
workspace_name: The name of the destination workspace.
object_name: The destination object name.
object_id: A destination object id, which can be used instead of object_name.
external_type: The external data format being transformed to a KBase type.
E.g., FASTA.DNA.Assembly
This is simply a string, but denotes the format of the data and
some context about what it actually is, which is used to
differentiate between files of the same general format with
different fundamental values.
kbase_type: The KBase Workspace type string that indicates the modules and type
of the object being created.
optional_arguments: This is a JSON string containing optional parameters that can
be passed in for validation or transformation steps to
customize behavior based on available exposed options.
ujs_job_id: The job id from the User and Job State service that can be used to
report status on task progress back to the user.
job_details: This is a JSON string that passes in the script specific command
line options for a given transformation type. The service pulls
these config settings from a script config created by the developer
of the transformation script and passes that into the AWE job that
calls this script.
working_directory: The working directory on disk where files can be created and
will be cleaned when the job ends with success or failure.
keep_working_directory: A flag to tell the script not to delete the working
directory, which is mainly for debugging purposes.
debug: Run the taskrunner in debug mode for local execution in a virtualenv.
Returns:
One or more Workspace objects saved to a user's workspace
based on input files provided by urls.
Authors:
Shinjae Yoo, Matt Henderson
"""
# validate inputs and environment
if logger is None:
raise Exception("A logger must be provided for status information.")
kb_token = None
try:
kb_token = script_utils.get_token()
assert type(kb_token) == type(str())
except Exception, e:
logger.error("Unable to get token!")
logger.exception(e)
sys.exit(1)
def upload_taskrunner(ujs_service_url = None, workspace_service_url = None,
shock_service_url = None, handle_service_url = None,
fba_service_url = None, url_mapping = None,
workspace_name = None, object_name = None,
object_id = None, external_type = None,
kbase_type = None, optional_arguments = None,
ujs_job_id = None, job_details = None,
working_directory = None, keep_working_directory = None,
debug = False, logger = None):
"""
KBase Upload task manager for converting from external data formats to KBase objects.
Step 1 - Download the input data and unpack to local files
Step 2 - Validate the input files
Step 3 - Transform the input files to KBase Object JSON and save any references
Step 4 - Save Workspace objects from the JSON and reference info
Steps 2,3,4 may be combined by certain data transformations, in which case
only step 3 is executed, with the understanding that it has accepted
responsibility for validation of the inputs and saving the workspace data.
Args:
workspace_service_url: URL for a KBase Workspace service where KBase objects
are stored.
ujs_service_url: URL for a User and Job State service to report task progress
back to the user.
shock_service_url: URL for a KBase SHOCK data store service for storing files
and large reference data.
handle_service_url: URL for a KBase Handle service that maps permissions from
the Workspace to SHOCK for KBase types that specify a Handle
reference instead of a SHOCK reference.
fba_service_url: URL for a KBase FBA Model service that has to handle any
model related conversions.
url_mapping: A JSON mapping of names to urls that identifies specific data elements
as well as special information such as KBase metadata and provenance.
workspace_name: The name of the destination workspace.
object_name: The destination object name.
object_id: A destination object id, which can be used instead of object_name.
external_type: The external data format being transformed to a KBase type.
E.g., FASTA.DNA.Assembly
This is simply a string, but denotes the format of the data and
some context about what it actually is, which is used to
differentiate between files of the same general format with
different fundamental values.
kbase_type: The KBase Workspace type string that indicates the modules and type
of the object being created.
optional_arguments: This is a JSON string containing optional parameters that can
be passed in for validation or transformation steps to
customize behavior based on available exposed options.
ujs_job_id: The job id from the User and Job State service that can be used to
report status on task progress back to the user.
job_details: This is a JSON string that passes in the script specific command
line options for a given transformation type. The service pulls
these config settings from a script config created by the developer
of the transformation script and passes that into the AWE job that
calls this script.
working_directory: The working directory on disk where files can be created and
will be cleaned when the job ends with success or failure.
keep_working_directory: A flag to tell the script not to delete the working
directory, which is mainly for debugging purposes.
debug: Run the taskrunner in debug mode for local execution in a virtualenv.
Returns:
One or more Workspace objects saved to a user's workspace
based on input files provided by urls.
Authors:
Shinjae Yoo, Matt Henderson
"""
# validate inputs and environment
if logger is None:
raise Exception("A logger must be provided for status information.")
kb_token = None
try:
kb_token = script_utils.get_token()
assert type(kb_token) == type(str())
except Exception, e:
logger.error("Unable to get token!")
logger.exception(e)
sys.exit(1)