def show(self, value, mime_type=None, force_to_string=True):
if not issubclass(type(value), OutputObject):
if mime_type is not None:
value = OutputObject(value=value, type=mime_type)
elif type(value) is str:
value = TextOutput(value=value)
elif type(value) is DatasetClient:
from vizier.api.webservice import server
ds_handle = server.api.datasets.get_dataset(
project_id=self.project_id,
dataset_id=value.dataset.identifier,
offset=0,
limit=10)
value = DatasetOutput(ds_handle)
elif issubclass(type(value), BokehLayout):
value = vizier_bokeh_render(value)
value = HtmlOutput(value=value)
elif issubclass(type(value), MatplotlibFigure):
value = HtmlOutput(value=vizier_matplotlib_render(value))
elif issubclass(type(value), MatplotlibAxes):
value = HtmlOutput(
value=vizier_matplotlib_render(value.get_figure()))
else:
repr_html = getattr(value, "_repr_html_", None)
if repr_html is not None:
value = HtmlOutput(str(repr_html()))
elif force_to_string:
value = TextOutput(value=str(value))
else:
return
self.stdout.append(value)
def execute_script(self, args, context):
"""Execute a R script in the given context.
Parameters
----------
args: vizier.viztrail.command.ModuleArguments
User-provided command arguments
context: vizier.engine.task.base.TaskContext
Context in which a task is being executed
Returns
-------
vizier.engine.task.processor.ExecResult
"""
# Get R script from user arguments
source = args.get_value(cmd.PARA_R_SOURCE)
# Redirect standard output and standard error streams
out = sys.stdout
err = sys.stderr
stream = list()
sys.stdout = OutputStream(tag='out', stream=stream)
sys.stderr = OutputStream(tag='err', stream=stream)
outputs = ModuleOutputs()
mimir_table_names = dict()
for ds_name_o in context.datasets:
dataset_id = context.datasets[ds_name_o]
dataset = context.datastore.get_dataset(dataset_id)
if dataset is None:
raise ValueError('unknown dataset \'' + ds_name_o + '\'')
mimir_table_names[ds_name_o] = dataset.identifier
# Run the r code
try:
evalresp = mimir.evalR(mimir_table_names, source)
ostd = evalresp['stdout']
oerr = evalresp['stderr']
if not ostd == '':
outputs.stdout.append(HtmlOutput(ostd))
if not oerr == '':
outputs.stderr.append(TextOutput(oerr))
except Exception as ex:
outputs.error(ex)
finally:
# Make sure to reverse redirection of output streams
sys.stdout = out
sys.stderr = err
# Set module outputs
for tag, text in stream:
text = ''.join(text).strip()
if tag == 'out':
outputs.stdout.append(HtmlOutput(text))
else:
outputs.stderr.append(TextOutput(text))
provenance = ModuleProvenance()
# Return execution result
return ExecResult(
is_success=(len(outputs.stderr) == 0),
outputs=outputs,
provenance=provenance
)
def compute_unload_dataset(self, args, context):
"""Execute unload dataset command.
Parameters
----------
args: vizier.viztrail.command.ModuleArguments
User-provided command arguments
context: vizier.engine.task.base.TaskContext
Context in which a task is being executed
Returns
-------
vizier.engine.task.processor.ExecResult
"""
# Get the new dataset name. Raise exception if a dataset with the
# specified name already exsists.
ds_name = args.get_value(pckg.PARA_DATASET).lower()
if not is_valid_name(ds_name):
raise ValueError('invalid dataset name \'' + ds_name + '\'')
# Get components of the load source. Raise exception if the source
# descriptor is invalid.
unload_format = args.get_value(cmd.PARA_UNLOAD_FORMAT)
options = args.get_value(cmd.PARA_UNLOAD_OPTIONS, raise_error=False)
m_opts = []
if not options is None:
for option in options:
unload_opt_key = option.get_value(cmd.PARA_UNLOAD_OPTION_KEY)
unload_opt_val = option.get_value(cmd.PARA_UNLOAD_OPTION_VALUE)
m_opts.append({
'name': unload_opt_key,
'value': unload_opt_val
})
# Execute load command.
dataset = context.get_dataset(ds_name)
result = self.api.unload_dataset(dataset=dataset,
datastore=context.datastore,
filestore=context.filestore,
unload_format=unload_format,
options=m_opts,
resources=context.resources)
# Delete the uploaded file (of load was from file). A reference to the
# created dataset is in the resources and will be used if the module is
# re-executed.
#file_id = result.resources[apibase.RESOURCE_FILEID]
#if not file_id is None:
# context.filestore.delete_file(file_id)
# Create result object
outputhtml = HtmlOutput(''.join([
"<div><a href=\"" + config.webservice.app_path + "/projects/" +
str(context.project_id) + "/files/" + out_file.identifier +
"\" download=\"" + out_file.name + "\">Download " + out_file.name +
"</a></div>"
for out_file in result.resources[apibase.RESOURCE_FILEID]
]))
return ExecResult(outputs=ModuleOutputs(stdout=[outputhtml]),
provenance=ModuleProvenance(read={
ds_name:
context.datasets.get(ds_name.lower(), None)
},
write=dict()))
def show_html(self, value):
self.show(HtmlOutput(value))
def execute_script(self, args, context):
"""Execute a Python script in the given context.
Parameters
----------
args: vizier.viztrail.command.ModuleArguments
User-provided command arguments
context: vizier.engine.task.base.TaskContext
Context in which a task is being executed
Returns
-------
vizier.engine.task.processor.ExecResult
"""
# Get Python script from user arguments
source = args.get_value(cmd.PYTHON_SOURCE)
# Initialize the scope variables that are available to the executed
# Python script. At this point this includes only the client to access
# and manipulate datasets in the undelying datastore
client = VizierDBClient(
datastore=context.datastore,
datasets=context.datasets
)
variables = {VARS_DBCLIENT: client}
# Redirect standard output and standard error streams
out = sys.stdout
err = sys.stderr
stream = list()
sys.stdout = OutputStream(tag='out', stream=stream)
sys.stderr = OutputStream(tag='err', stream=stream)
# Keep track of exception that is thrown by the code
exception = None
# Run the Python code
try:
python_cell_preload(variables)
exec(source, variables, variables)
except Exception as ex:
exception = ex
finally:
# Make sure to reverse redirection of output streams
sys.stdout = out
sys.stderr = err
# Set module outputs
outputs = ModuleOutputs()
is_success = (exception is None)
for tag, text in stream:
text = ''.join(text).strip()
if tag == 'out':
outputs.stdout.append(HtmlOutput(text))
else:
outputs.stderr.append(TextOutput(text))
is_success = False
if is_success:
# Create provenance information. Ensure that all dictionaries
# contain elements of expected types, i.e, ensure that the user did
# not attempt anything tricky.
read = dict()
for name in client.read:
if not isinstance(name, str):
raise RuntimeError('invalid key for mapping dictionary')
if name in context.datasets:
read[name] = context.datasets[name]
if not isinstance(read[name], str):
raise RuntimeError('invalid element in mapping dictionary')
else:
read[name] = None
write = dict()
for name in client.write:
if not isinstance(name, str):
raise RuntimeError('invalid key for mapping dictionary')
ds_id = client.datasets[name]
if not ds_id is None:
if not isinstance(ds_id, str):
raise RuntimeError('invalid value in mapping dictionary')
elif ds_id in client.descriptors:
write[name] = client.descriptors[ds_id]
else:
write[name] = client.datastore.get_descriptor(ds_id)
else:
write[name] = None
provenance = ModuleProvenance(
read=read,
write=write,
delete=client.delete
)
else:
outputs.error(exception)
provenance = ModuleProvenance()
# Return execution result
return ExecResult(
is_success=is_success,
outputs=outputs,
provenance=provenance
)