def cliToMacroModule(executablePath, targetDirectory, defFile = True,
includePanelScreenshots = True, env = None,
unexpandFilename = None):
"""Write .script/.mlab/.mhelp files for the CLI module `executablePath`
to `targetDirectory`[/mhelp]. If `defFile` is set to an MLDFile instance,
the .def file contents are appended to that object, otherwise a
.def file for that single module gets written."""
logger.info("processing %s..." % executablePath)
#ET.dump(elementTree)
m = CLIModule(executablePath, env = env)
m.classifyParameters() # performs additional sanity checks
mdefFile, scriptFile, mlabFile, mhelpFile = mdlDescription(m, includePanelScreenshots)
if defFile is True:
mdefFile.write(os.path.join(targetDirectory, "%s.def" % m.name))
else:
if defFile: # not empty, add separating newline
defFile.append(MDLNewline)
defFile.extend(mdefFile)
if unexpandFilename is not None:
cliExecutablePath = scriptFile.group('Interface').group('Parameters').group('Field', 'cliExecutablePath').tag('value')
cliExecutablePath.tagValue = unexpandFilename(cliExecutablePath.tagValue)
scriptFile.write(os.path.join(targetDirectory, "%s.script" % m.name))
mlabFile.write(os.path.join(targetDirectory, "%s.mlab" % m.name))
mhelpFile.write(os.path.join(targetDirectory, "mhelp", "CLI_%s.mhelp" % m.name))
return mdefFile
def genHandlerToRunDockerCLI(dockerImage, cliRelPath, restResource):
"""Generates a handler to run docker CLI using girder_worker
Parameters
----------
dockerImage : str
Docker image in which the CLI resides
cliRelPath : str
Relative path of the CLI which is needed to run the CLI by running
the command docker run `dockerImage` `cliRelPath`
restResource : girder.api.rest.Resource
The object of a class derived from girder.api.rest.Resource to which
this handler will be attached
Returns
-------
function
Returns a function that runs the CLI using girder_worker
"""
cliName = os.path.normpath(cliRelPath).replace(os.sep, '.')
# get xml spec
str_xml = subprocess.check_output(
['docker', 'run', dockerImage, cliRelPath, '--xml'])
# parse cli xml spec
xmlFile = 'temp.xml'
with open(xmlFile, 'w') as f:
f.write(str_xml)
clim = CLIModule(xmlFile)
os.remove(xmlFile)
# create CLI description string
str_description = ['Description: <br/><br/>' + clim.description]
if clim.version is not None and len(clim.version) > 0:
str_description.append('Version: ' + clim.version)
if clim.license is not None and len(clim.license) > 0:
str_description.append('License: ' + clim.license)
if clim.contributor is not None and len(clim.contributor) > 0:
str_description.append('Author(s): ' + clim.contributor)
if clim.acknowledgements is not None and \
len(clim.acknowledgements) > 0:
str_description.append('Acknowledgements: ' + clim.acknowledgements)
str_description = '<br/><br/>'.join(str_description)
# do stuff needed to create REST endpoint for cLI
handlerDesc = Description(clim.title).notes(str_description)
# get CLI parameters
index_params, opt_params, simple_out_params = _getCLIParameters(clim)
# add indexed input parameters
index_input_params = filter(lambda p: p.channel == 'input', index_params)
_addIndexedInputParamsToHandler(index_input_params, handlerDesc)
# add indexed output parameters
index_output_params = filter(lambda p: p.channel == 'output', index_params)
_addIndexedOutputParamsToHandler(index_output_params, handlerDesc)
# add optional input parameters
opt_input_params = filter(lambda p: p.channel != 'output', opt_params)
_addOptionalInputParamsToHandler(opt_input_params, handlerDesc)
# add optional output parameters
opt_output_params = filter(lambda p: p.channel == 'output', opt_params)
_addOptionalOutputParamsToHandler(opt_output_params, handlerDesc)
# add returnparameterfile if there are simple output params
if len(simple_out_params) > 0:
_addReturnParameterFileParamToHandler(handlerDesc)
# define CLI handler function
@boundHandler(restResource)
@access.user
@describeRoute(handlerDesc)
def cliHandler(self, **hargs):
user = self.getCurrentUser()
token = self.getCurrentToken()['_id']
# create job
jobModel = self.model('job', 'jobs')
jobTitle = '.'.join((restResource.resourceName, cliName))
job = jobModel.createJob(title=jobTitle,
type=jobTitle,
handler='worker_handler',
user=user)
kwargs = {
'validate': False,
'auto_convert': True,
'cleanup': True,
'inputs': dict(),
'outputs': dict()
}
# create job info
jobToken = jobModel.createJobToken(job)
kwargs['jobInfo'] = wutils.jobInfoSpec(job, jobToken)
# initialize task spec
taskSpec = {
'name': cliName,
'mode': 'docker',
'docker_image': dockerImage,
'pull_image': True,
'inputs': [],
'outputs': []
}
_addIndexedInputParamsToTaskSpec(index_input_params, taskSpec)
_addIndexedOutputParamsToTaskSpec(index_output_params, taskSpec, hargs)
_addOptionalInputParamsToTaskSpec(opt_input_params, taskSpec)
_addOptionalOutputParamsToTaskSpec(opt_output_params, taskSpec, hargs)
if len(simple_out_params) > 0:
_addReturnParameterFileParamToTaskSpec(taskSpec, hargs)
kwargs['task'] = taskSpec
# add input/output parameter bindings
_addIndexedInputParamBindings(index_input_params, kwargs['inputs'],
hargs, token)
_addIndexedOutputParamBindings(index_output_params, kwargs['outputs'],
hargs, user, token)
_addOptionalInputParamBindings(opt_input_params, kwargs['inputs'],
hargs, user, token)
_addOptionalOutputParamBindings(opt_output_params, kwargs['outputs'],
hargs, user, token)
if len(simple_out_params) > 0:
_addReturnParameterFileBinding(kwargs['outputs'], hargs, user,
token)
# construct container arguments
containerArgs = [cliRelPath]
_addOptionalInputParamsToContainerArgs(opt_input_params, containerArgs,
hargs)
_addOptionalOutputParamsToContainerArgs(opt_input_params,
containerArgs, kwargs, hargs)
_addReturnParameterFileToContainerArgs(containerArgs, kwargs, hargs)
_addIndexedParamsToContainerArgs(index_params, containerArgs, hargs)
taskSpec['container_args'] = containerArgs
# schedule job
job['kwargs'] = kwargs
job = jobModel.save(job)
jobModel.scheduleJob(job)
# return result
return jobModel.filter(job, user)
handlerFunc = cliHandler
# loadmodel stuff for indexed input params on girder
index_input_params_on_girder = filter(_is_on_girder, index_input_params)
for param in index_input_params_on_girder:
curModel = _SLICER_TYPE_TO_GIRDER_MODEL_MAP[param.typ]
curMap = {param.name + _girderInputFileSuffix: param.name}
handlerFunc = loadmodel(map=curMap,
model=curModel,
level=AccessType.READ)(handlerFunc)
# loadmodel stuff for indexed output params on girder
index_output_params_on_girder = filter(_is_on_girder, index_output_params)
for param in index_output_params_on_girder:
curModel = 'folder'
curMap = {param.name + _girderOutputFolderSuffix: param.name}
handlerFunc = loadmodel(map=curMap,
model=curModel,
level=AccessType.WRITE)(handlerFunc)
return handlerFunc
def genHandlerToRunDockerCLI(dockerImage, cliRelPath, cliXML, restResource): # noqa
"""Generates a handler to run docker CLI using girder_worker
Parameters
----------
dockerImage : str
Docker image in which the CLI resides
cliRelPath : str
Relative path of the CLI which is needed to run the CLI by running
the command docker run `dockerImage` `cliRelPath`
cliXML:str
Cached copy of xml spec for this cli
restResource : girder.api.rest.Resource
The object of a class derived from girder.api.rest.Resource to which
this handler will be attached
Returns
-------
function
Returns a function that runs the CLI using girder_worker
"""
cliName = os.path.normpath(cliRelPath).replace(os.sep, '.')
# get xml spec
str_xml = cliXML
# parse cli xml spec
with tempfile.NamedTemporaryFile(suffix='.xml') as f:
f.write(str_xml)
f.flush()
clim = CLIModule(f.name)
# create CLI description string
str_description = ['Description: <br/><br/>' + clim.description]
if clim.version is not None and len(clim.version) > 0:
str_description.append('Version: ' + clim.version)
if clim.license is not None and len(clim.license) > 0:
str_description.append('License: ' + clim.license)
if clim.contributor is not None and len(clim.contributor) > 0:
str_description.append('Author(s): ' + clim.contributor)
if clim.acknowledgements is not None and \
len(clim.acknowledgements) > 0:
str_description.append(
'Acknowledgements: ' + clim.acknowledgements)
str_description = '<br/><br/>'.join(str_description)
# do stuff needed to create REST endpoint for cLI
handlerDesc = Description(clim.title).notes(str_description)
# print handlerDesc
# get CLI parameters
index_params, opt_params, simple_out_params = _getCLIParameters(clim)
# print index_params [<CLIParameter 'inputMultipleImage' of type directory>,
# <CLIParameter 'outputThresholding' of type file>, <CLIParameter 'tableFile' of type file>]
# add indexed input parameters
index_input_params = filter(lambda p: p.channel != 'output', index_params)
# print index_input_params
# print index_input_params [<CLIParameter 'inputMultipleImage' of type directory>]
_addIndexedInputParamsToHandler(index_input_params, handlerDesc)
# add indexed output parameters
index_output_params = filter(lambda p: p.channel == 'output', index_params)
_addIndexedOutputParamsToHandler(index_output_params, handlerDesc)
# add optional input parameters
opt_input_params = filter(lambda p: p.channel != 'output', opt_params)
_addOptionalInputParamsToHandler(opt_input_params, handlerDesc)
# add optional output parameters
opt_output_params = filter(lambda p: p.channel == 'output', opt_params)
_addOptionalOutputParamsToHandler(opt_output_params, handlerDesc)
# print simple_out_params
# add returnparameterfile if there are simple output params
if len(simple_out_params) > 0:
_addReturnParameterFileParamToHandler(handlerDesc)
# define CLI handler function
@boundHandler(restResource)
@access.user
@describeRoute(handlerDesc)
def cliHandler(self, **hargs):
# print 'in cliHandler hargs is '
# print hargs
user = self.getCurrentUser()
token = self.getCurrentToken()['_id']
# create job
jobModel = self.model('job', 'jobs')
jobTitle = '.'.join((restResource.resourceName, cliName))
# User Group access control,
# register group into particular job so that this user can access this job
groups = list(Group().list(user=user))
groupsAccess = []
for eachGroup in groups:
eachGroupAccess = {'id': eachGroup['_id'], 'level': 0}
groupsAccess.append(eachGroupAccess)
job = jobModel.createJob(title=jobTitle,
type=jobTitle,
handler='worker_handler',
user=user,
otherFields={'access': {'groups': groupsAccess}})
kwargs = {
'validate': False,
'auto_convert': True,
'cleanup': True,
'inputs': dict(),
'outputs': dict()
}
# create job info
jobToken = jobModel.createJobToken(job)
kwargs['jobInfo'] = wutils.jobInfoSpec(job, jobToken)
# initialize task spec
taskSpec = {'name': cliName,
'mode': 'docker',
'docker_image': dockerImage,
'pull_image': False,
'inputs': [],
'outputs': []}
_addIndexedInputParamsToTaskSpec(index_input_params, taskSpec)
_addIndexedOutputParamsToTaskSpec(index_output_params, taskSpec, hargs)
_addOptionalInputParamsToTaskSpec(opt_input_params, taskSpec)
_addOptionalOutputParamsToTaskSpec(opt_output_params, taskSpec, hargs)
if len(simple_out_params) > 0:
_addReturnParameterFileParamToTaskSpec(taskSpec, hargs)
kwargs['task'] = taskSpec
# add input/output parameter bindings
_addIndexedInputParamBindings(index_input_params,
kwargs['inputs'], hargs, token)
_addIndexedOutputParamBindings(index_output_params,
kwargs['outputs'], hargs, user, token)
_addOptionalInputParamBindings(opt_input_params,
kwargs['inputs'], hargs, user, token)
_addOptionalOutputParamBindings(opt_output_params,
kwargs['outputs'], hargs, user, token)
if len(simple_out_params) > 0:
_addReturnParameterFileBinding(kwargs['outputs'],
hargs, user, token)
# construct container arguments
containerArgs = [cliRelPath]
_addOptionalInputParamsToContainerArgs(opt_input_params,
containerArgs, hargs)
_addOptionalOutputParamsToContainerArgs(opt_output_params,
containerArgs, kwargs, hargs)
_addReturnParameterFileToContainerArgs(containerArgs, kwargs, hargs)
# print 'index_params'
# print index_params
_addIndexedParamsToContainerArgs(index_params,
containerArgs, hargs)
taskSpec['container_args'] = containerArgs
# schedule job
job['kwargs'] = kwargs
# print '-------job is-------'
# print job
job = jobModel.save(job)
jobModel.scheduleJob(job)
# return result
return jobModel.filter(job, user)
handlerFunc = cliHandler
# print _is_on_girder
# loadmodel stuff for indexed input params on girder
index_input_params_on_girder = filter(_is_on_girder, index_input_params)
# print '---------'
# print index_input_params_on_girder
for param in index_input_params_on_girder:
if param.flag == '-item':
curModel = 'item'
# print curModel
if curModel != 'url':
suffix = '_girderItemId'
curMap = {param.identifier() + suffix: param.identifier()}
# print curMap
handlerFunc = loadmodel(map=curMap,
model=curModel,
level=AccessType.READ)(handlerFunc)
else:
curModel = _SLICER_TYPE_TO_GIRDER_MODEL_MAP[param.typ]
# print curModel
if curModel != 'url':
suffix = _SLICER_TYPE_TO_GIRDER_INPUT_SUFFIX_MAP[param.typ]
curMap = {param.identifier() + suffix: param.identifier()}
# print curMap
handlerFunc = loadmodel(map=curMap,
model=curModel,
level=AccessType.READ)(handlerFunc)
# loadmodel stuff for indexed output params on girder
index_output_params_on_girder = filter(_is_on_girder, index_output_params)
for param in index_output_params_on_girder:
if param.flag == '-item':
_girderOutputItemSuffix = '_girderItemId'
curModel = 'item'
curMap = {param.identifier() + _girderOutputItemSuffix: param.identifier()}
else:
curModel = 'folder'
curMap = {param.identifier() + _girderOutputFolderSuffix: param.identifier()}
handlerFunc = loadmodel(map=curMap,
model=curModel,
level=AccessType.WRITE)(handlerFunc)
return handlerFunc