def _execute(plugin, action, inputs):
pm = sdk.PluginManager()
action = pm.plugins[plugin.replace('_', '-')].actions[action]
signature = action.signature
kwargs = {}
for name, spec in _iter_provided(signature.inputs, inputs):
if inputs[name] is None:
continue
if spec.qiime_type.name in ('List', 'Set'):
kwargs[name] = [sdk.Artifact.load(v['path']) for v in inputs[name]]
if spec.qiime_type.name == 'Set':
kwargs[name] = set(kwargs[name])
else:
kwargs[name] = sdk.Artifact.load(inputs[name]['path'])
for name, spec in _iter_provided(signature.parameters, inputs):
if inputs[name] is None:
continue
if spec.qiime_type.name == 'Set':
kwargs[name] = set(kwargs[name])
elif spec.qiime_type.name == 'Metadata':
kwargs[name] = _handle_metadata(inputs[name])
elif spec.qiime_type.name == 'MetadataColumn':
md = _handle_metadata(inputs[name])
kwargs[name] = md.get_column(inputs[name + '_column'])
else:
kwargs[name] = inputs[name]
results = action(**kwargs)
for name, result in zip(results._fields, results):
result.save(name)
def run(plugin, action, inputs):
with open(inputs, 'r') as fh:
inputs = json.load(fh)['_'] # junk outer wrapper from templating
pm = sdk.PluginManager()
action = pm.plugins[plugin.replace('_', '-')].actions[action]
signature = action.signature
kwargs = {}
for name, spec in _iter_provided(signature.inputs, inputs):
if spec.qiime_type.name in ('List', 'Set'):
kwargs[name] = [sdk.Artifact.load(v['path']) for v in inputs[name]]
if spec.qiime_type.name == 'Set':
kwargs[name] = set(kwargs[name])
else:
kwargs[name] = sdk.Artifact.load(inputs[name]['path'])
for name, spec in _iter_provided(signature.parameters, inputs):
if spec.qiime_type.name == 'Set':
kwargs[name] = set(kwargs[name])
elif spec.qiime_type.name == 'Metadata':
kwargs[name] = qiime2.Metadata.load(inputs[name]['path'])
elif spec.qiime_type.name == 'MetadataColumn':
kwargs[name] = qiime2.MetadataColumn.load(inputs[name]['path'],
inputs[name + "_column"])
else:
kwargs[name] = inputs[name]
results = action(**kwargs)
for name, result in zip(results._fields, results):
result.save(name)
def __init__(self, view_type):
self._pm = sdk.PluginManager()
self._view_type = view_type
self._view_name = util.get_view_name(self._view_type)
self._record = None
if self._view_name in self._pm.views:
self._record = self._pm.views[self._view_name]
def find_all_actions(plugin=None):
pm = sdk.PluginManager()
if plugin is not None:
plugins = [pm.plugins[plugin]]
else:
plugins = pm.plugins.values()
for plugin in plugins:
for action in plugin.actions.values():
yield plugin, action
def __init__(self, *, plugin_id: str, action_id: str):
if plugin_id == '':
raise ValueError('Must specify a value for plugin_id.')
if action_id == '':
raise ValueError('Must specify a value for action_id.')
self.plugin_id = plugin_id
self.action_id = action_id
self._plugin_manager = sdk.PluginManager()
def plugin(plugin, output):
pm = sdk.PluginManager()
plugin = pm.get_plugin(id=plugin)
for status in template_plugin_iter(plugin, output):
line = json.dumps(status)
if status['status'] == 'error':
click.secho(line, fg='red', err=True)
elif status['status'] == 'created':
click.secho(line, fg='green')
else:
click.secho(line, fg='yellow')
def action_runner(plugin_id, action_id, inputs):
pm = sdk.PluginManager()
plugin = pm.get_plugin(id=plugin_id)
action = plugin.actions[action_id]
inputs = {k: sdk.Artifact.load(v) for k, v in inputs.items()
if v is not None}
results = action(**inputs)
for name, result in zip(results._fields, results):
result.save(name)
def write_tool(directory, plugin_id, action_id):
pm = sdk.PluginManager()
plugin = pm.plugins[plugin_id.replace('_', '-')]
action = plugin.actions[action_id]
filename = os.path.join(directory, get_tool_id(action) + '.xml')
tool = make_tool(plugin_id, action, plugin.version)
with open(filename, 'w') as fh:
xmlstr = dom.parseString(xml.tostring(tool)).toprettyxml(indent=" ")
fh.write(xmlstr)
def get_mystery_stew():
from q2_mystery_stew.plugin_setup import create_plugin
pm = sdk.PluginManager(add_plugins=False)
test_plugin = create_plugin(ints=True,
strings=True,
bools=True,
floats=True,
artifacts=True,
primitive_unions=True,
metadata=True,
collections=True)
pm.add_plugin(test_plugin)
return pm.get_plugin(id='mystery_stew')
def make_requirements(conda_meta, *project_names):
# HACK:
# use a single environment when templating instead of following the full
# trail. An exception to this is q2-mystery-stew
if len(project_names) == 1 and project_names[0] == 'q2-mystery-stew':
pass
else:
pm = sdk.PluginManager()
project_names = [p.project_name for p in pm.plugins.values()]
requirements = XMLNode('requirements')
for dep, version in conda_meta.iter_deps(*project_names,
include_self=True):
r = XMLNode('requirement', dep, type='package', version=version)
requirements.append(r)
requirements.append(XMLNode('requirement', 'q2galaxy',
type='package', version=q2galaxy.__version__))
return requirements
def template_tool_conf(directory, out_path):
toolbox = _util.XMLNode('toolbox')
section = _util.XMLNode('section', id='getext', name='Get Data')
section.append(_util.XMLNode('tool', file='data_source/upload.xml'))
toolbox.append(section)
section = _util.XMLNode('section', id='qiime2_tools', name='QIIME 2 Tools')
suite_name = 'suite_qiime2_tools'
suite_dir = os.path.join(directory, suite_name)
for tool_id in _templaters.BUILTIN_MAKERS:
path = os.path.join(suite_dir, tool_id + '.xml')
section.append(_util.XMLNode('tool', file=path))
toolbox.append(section)
pm = _sdk.PluginManager()
for plugin in sorted(pm.plugins.values(), key=lambda x: x.id):
plugin_name = plugin.id.replace('_', '-')
suite_name = f'suite_qiime2_{plugin_name}'
section = _util.XMLNode('section',
id=suite_name,
name=f'QIIME 2 {plugin_name}')
for action in sorted(plugin.actions.values(), key=lambda x: x.id):
filename = _templaters.make_tool_id(plugin.id, action.id) + '.xml'
path = os.path.join(directory, suite_name, filename)
section.append(_util.XMLNode('tool', file=path))
toolbox.append(section)
with open(out_path, 'wb') as fh:
_xml.indent(toolbox, ' ' * 4)
fh.write(
_xml.tostring(toolbox,
pretty_print=True,
encoding='utf-8',
xml_declaration=True))
def plugin(plugin, output):
pm = sdk.PluginManager()
plugin = pm.get_plugin(id=plugin)
for status in template_plugin_iter(plugin, output):
_echo_status(status)
def make_builtin_import(meta, tool_id):
pm = sdk.PluginManager()
inputs = XMLNode('inputs')
# Not a galaxy_ui_var() because this will be taken by name from the
# Cheetah searchList (see `def _inline_code` below)
conditional = XMLNode('conditional', name='import_root')
inputs.append(conditional)
type_ = XMLNode('param',
name='type',
type='select',
label=('Type of data to import:'))
type_.append(
XMLNode('option', 'Select a QIIME 2 type to import.', value='None'))
conditional.append(type_)
when = XMLNode('when', value="None")
conditional.append(when)
default_formats = _get_default_formats(pm)
plugins = set()
known_formats = set()
for record in sorted(pm.get_semantic_types().values(),
key=lambda x: str(x.semantic_type)):
plugins.add(record.plugin)
type_option = XMLNode('option',
str(record.semantic_type),
value=galaxy_esc(str(record.semantic_type)))
type_.append(type_option)
when = XMLNode('when', value=galaxy_esc(str(record.semantic_type)))
fmt_conditional = XMLNode('conditional',
name=galaxy_ui_var(tag='cond',
name='format'))
select = XMLNode('param',
type='select',
name='format',
label="QIIME 2 file format to import from:")
fmt_conditional.append(select)
when.append(fmt_conditional)
default_format = default_formats[record.semantic_type]
for fmt_rec in sorted(pm.get_formats(
filter=GetFormatFilters.IMPORTABLE,
semantic_type=record.semantic_type).values(),
key=lambda x: x.format.__name__):
if issubclass(fmt_rec.format, model.SingleFileDirectoryFormatBase):
# These are really just noise for the galaxy UI
# an implicit transformation from the backing file format
# is simpler and removes the redundancy
continue
plugins.add(fmt_rec.plugin)
known_formats.add(fmt_rec.format)
option = XMLNode(
'option',
pretty_fmt_name(fmt_rec.format),
value=galaxy_esc(fmt_rec.format.__name__),
selected=str(fmt_rec.format == default_format).lower())
select.append(option)
fmt_when = XMLNode('when',
value=galaxy_esc(fmt_rec.format.__name__))
fmt_conditional.append(fmt_when)
_add_format_ui(fmt_when, fmt_rec)
conditional.append(when)
outputs = XMLNode('outputs')
outputs.append(
XMLNode('data',
name='imported_data',
format='qza',
from_work_dir='imported_data.qza'))
tool = XMLNode('tool',
id=tool_id,
name=make_tool_name_from_id(tool_id),
version=make_builtin_version(plugins))
tool.append(inputs)
tool.append(outputs)
tool.append(XMLNode('command', "q2galaxy run tools import '$inputs'"))
tool.append(_make_config())
tool.append(XMLNode('description', 'Import data into a QIIME 2 artifact'))
tool.append(make_citations())
tool.append(make_requirements(meta, *[p.project_name for p in plugins]))
tool.append(_make_help(known_formats))
return tool
def template_all_iter(directory):
pm = _sdk.PluginManager()
for plugin in pm.plugins.values():
yield from template_plugin_iter(plugin, directory)
yield from template_builtins_iter(directory)
def _get_plugin(plugin_id):
if plugin_id == 'mystery_stew':
return get_mystery_stew()
else:
pm = sdk.PluginManager()
return pm.get_plugin(id=plugin_id)
def action_to_galaxy_xml(action):
meta = _environment.find_conda_meta()
plugin = _sdk.PluginManager().get_plugin(id=action.id)
return _templating.make_tool(meta, plugin, action)
def get_version(plugin_id):
pm = sdk.PluginManager()
plugin = pm.get_plugin(id=plugin_id)
return plugin.version
elif kwarg_type == 'metadata':
if type(arg) is list:
loaded = [_load_as_metadata(f) for f in arg]
loaded = loaded[0].merge(*loaded[1:])
else:
loaded = _load_as_metadata(arg)
elif kwarg_type == 'inputs':
if type(arg) is list:
loaded = [qiime2.Artifact.load(f) for f in arg]
else:
loaded = qiime2.Artifact.load(arg)
kwargs[param] = loaded
pm = sdk.PluginManager()
action = pm.get_plugin(id='{{ plugin }}').actions['{{ action }}']
# generate kwargs after dereferencing QZAs and MD
kwargs = {}
_dereference_kwargs(kwargs, 'inputs', action.signature.inputs)
_dereference_kwargs(kwargs, 'metadata', action.signature.parameters)
_dereference_kwargs(kwargs, 'columns', action.signature.parameters)
_dereference_kwargs(kwargs, 'params', action.signature.parameters)
for param in itertools.chain(concourse_args,
concourse_args['inputs'],
concourse_args['metadata'],
concourse_args['columns'],
concourse_args['params']):
def make_builtin_import(meta, tool_id):
pm = sdk.PluginManager()
inputs = XMLNode('inputs')
conditional = XMLNode('conditional', name='import_data')
inputs.append(conditional)
type_ = XMLNode('param',
name='type',
type='select',
label=('Type of data to import:'))
type_.append(
XMLNode('option', 'Select a QIIME 2 type to import.', value='None'))
conditional.append(type_)
when = XMLNode('when', value="None")
conditional.append(when)
plugins = set()
known_formats = set()
for record in sorted(pm.get_semantic_types().values(),
key=lambda x: str(x.semantic_type)):
plugins.add(record.plugin)
type_option = XMLNode('option',
str(record.semantic_type),
value=galaxy_esc(str(record.semantic_type)))
type_.append(type_option)
when = XMLNode('when', value=galaxy_esc(str(record.semantic_type)))
fmt_conditional = XMLNode('conditional', name='import_ui')
select = XMLNode('param',
type='select',
name='input_format',
label="QIIME 2 file format to import from:")
fmt_conditional.append(select)
when.append(fmt_conditional)
for fmt_rec in sorted(pm.get_formats(
filter=GetFormatFilters.IMPORTABLE,
semantic_type=record.semantic_type).values(),
key=lambda x: x.format.__name__):
plugins.add(fmt_rec.plugin)
known_formats.add(fmt_rec.format)
if issubclass(fmt_rec.format, model.SingleFileDirectoryFormatBase):
continue
option = XMLNode('option',
pretty_fmt_name(fmt_rec.format),
value=galaxy_esc(fmt_rec.format.__name__))
select.append(option)
fmt_when = XMLNode('when',
value=galaxy_esc(fmt_rec.format.__name__))
fmt_conditional.append(fmt_when)
_add_format_ui(fmt_when, fmt_rec)
conditional.append(when)
outputs = XMLNode('outputs')
outputs.append(XMLNode('data', name='imported_qza', type='qza'))
tool = XMLNode('tool',
id=tool_id,
name=make_tool_name_from_id(tool_id),
version=make_builtin_version(plugins))
tool.append(inputs)
tool.append(outputs)
tool.append(
XMLNode('command', "q2galaxy run builtin import_data '$inputs'"))
tool.append(_make_config())
tool.append(XMLNode('description', 'Import data into QIIME 2 artifacts'))
tool.append(XMLNode('help', ''))
return tool
def __init__(self, view_type):
self._pm = sdk.PluginManager()
self._view_type = view_type
def make_builtin_export(meta, tool_id):
pm = sdk.PluginManager()
inputs = XMLNode('inputs')
# This also works for qzvs even though the format just says qza so. . .
qza_input = 'input'
inputs.append(XMLNode('param', format="qza", name=qza_input, type='data',
label='input: The path to the artifact you '
'want to export'))
type_peek = XMLNode('param', type='select', name='type_peek',
display='radio',
label="The type of your input qza is:")
type_peek_opts = XMLNode('options')
type_peek_filter = XMLNode('filter', type='data_meta', ref=qza_input,
key='semantic_type')
type_peek_opts.append(type_peek_filter)
type_peek.append(type_peek_opts)
inputs.append(type_peek)
fmt_peek = XMLNode('param', type='select', name='fmt_peek',
display='radio', label="The current QIIME 2 format is:")
fmt_peek_opts = XMLNode('options')
fmt_peek_filter = XMLNode('filter', type='data_meta', ref=qza_input,
key='format')
fmt_peek_opts.append(fmt_peek_filter)
fmt_peek.append(fmt_peek_opts)
inputs.append(fmt_peek)
conditional = XMLNode('conditional', name='fmt_finder')
inputs.append(conditional)
type_ = XMLNode('param', name='type', type='select',
label=('To change the format, select the type indicated'
' above:'))
type_.append(XMLNode('option', 'export as is (no conversion)',
value='None', selected='true'))
conditional.append(type_)
when = XMLNode('when', value="None")
select = XMLNode('param', type='select', name='output_format',
label="QIIME 2 file format to convert to:")
select.append(XMLNode('option', 'export as is (no conversion)',
value='None', selected='true'))
when.append(select)
conditional.append(when)
plugins = set()
known_formats = set()
for record in sorted(pm.get_semantic_types().values(),
key=lambda x: str(x.semantic_type)):
plugins.add(record.plugin)
type_option = XMLNode('option', str(record.semantic_type),
value=galaxy_esc(str(record.semantic_type)))
type_.append(type_option)
when = XMLNode('when', value=galaxy_esc(str(record.semantic_type)))
select = XMLNode('param', type='select', name='output_format',
label="QIIME 2 file format to convert to:")
select.append(XMLNode('option', 'export as is (no conversion)',
value='None', selected='true'))
for fmt_rec in sorted(
pm.get_formats(filter=GetFormatFilters.EXPORTABLE,
semantic_type=record.semantic_type).values(),
key=lambda x: x.format.__name__):
plugins.add(fmt_rec.plugin)
known_formats.add(fmt_rec.format)
if not issubclass(fmt_rec.format,
model.SingleFileDirectoryFormatBase):
option = XMLNode('option', pretty_fmt_name(fmt_rec.format),
value=galaxy_esc(fmt_rec.format.__name__))
select.append(option)
when.append(select)
conditional.append(when)
outputs = XMLNode('outputs')
# default collection:
collection = XMLNode('collection', name='exported', type='list',
label='${tool.name} on ${on_string} as ${fmt_peek}')
_filter_set = repr({galaxy_esc(x.__name__) for x in known_formats})
collection.append(XMLNode('filter', "fmt_finder['output_format'] == 'None'"
f" and fmt_peek not in {_filter_set}"))
collection.append(XMLNode('discover_datasets', visible='false',
pattern='__designation_and_ext__'))
outputs.append(collection)
for fmt in sorted(known_formats, key=lambda x: x.__name__):
esc_fmt = galaxy_esc(fmt.__name__)
filter_exp = (f"fmt_finder['output_format'] == '{esc_fmt}'"
" or (fmt_finder['output_format'] == 'None' and fmt_peek"
f" == '{esc_fmt}')")
label = '${tool.name} on ${on_string} as ' + fmt.__name__
if issubclass(fmt, model.DirectoryFormat):
dyn_data = None
for field in fmt._fields: # file attrs of the dirfmt
file_attr = getattr(fmt, field)
pattern, has_ext = pathspec_to_galaxy_regex(file_attr.pathspec)
extras = {}
if not has_ext:
if issubclass(file_attr.format, model.TextFileFormat):
extras['ext'] = 'txt'
else:
extras['ext'] = 'data'
if isinstance(file_attr, model.FileCollection):
col = XMLNode('collection', type='list',
name='_'.join([esc_fmt, file_attr.name]),
label=label + f' ({file_attr.name})')
col.append(XMLNode('filter', filter_exp))
col.append(XMLNode('discover_datasets', visible='false',
pattern=pattern, **extras))
outputs.append(col)
else:
if dyn_data is None:
# init a root for all of the discoverable datasets
dyn_data = XMLNode('data', name=esc_fmt, label=label)
dyn_data.append(XMLNode('filter', filter_exp))
# only the first one, will take over the history item
extras['assign_primary_output'] = 'true'
dyn_data.append(XMLNode('discover_datasets',
visible='true', pattern=pattern,
**extras))
if dyn_data is not None:
outputs.append(dyn_data)
else:
# as it is not a directory, we don't know much about the original
# filename anymore, so we will let galaxy sort it out
# .gz may be considered the format instead of fastq.gz, but users
# can always re-sniff a history item for the format
collection = XMLNode('data', name=esc_fmt, label=label)
collection.append(XMLNode('filter', filter_exp))
collection.append(XMLNode('discover_datasets', visible='true',
assign_primary_output='true',
pattern='__designation_and_ext__'))
outputs.append(collection)
tool = XMLNode('tool', id=tool_id, name=make_tool_name_from_id(tool_id),
version=make_builtin_version(plugins))
tool.append(XMLNode('description', 'Export data from QIIME 2 artifacts'))
tool.append(XMLNode('command', "q2galaxy run tools export '$inputs'"))
tool.append(make_config())
tool.append(inputs)
tool.append(outputs)
tool.append(make_requirements(meta, *[p.project_name for p in plugins]))
tool.append(XMLNode('help', ''))
return tool
def template_all(directory):
pm = sdk.PluginManager()
for name, plugin in pm.plugins.items():
for action in plugin.actions.keys():
write_tool(directory, name.replace('-', '_'), action)
def get_version(plugin_id):
pm = sdk.PluginManager()
plugin = pm.plugins[plugin_id.replace('_', '-')]
return plugin.version
def find_all_actions():
pm = sdk.PluginManager()
for plugin in pm.plugins.values():
for action in plugin.actions.values():
yield plugin, action
