def compute(self): # Required input in_dataset = self.getInputFromPort("in_dataset") new_cons = set([Constraint('extra_info', ['nino34']), Constraint('latsouth_info', ['5S']), Constraint('latnorth_info', ['5N']), Constraint('latagg_info', ['fldavg']), Constraint('lonwest_info', ['190E']), Constraint('loneast_info', ['240E']), Constraint('lonagg_info', ['fldavg']), Constraint('leveltop_info', ['surface']), Constraint('levelbottom_info', ['surface']), Constraint('anomaly_info', ['anom']), ]) # Execute the process. this_process = ProcessUnit([in_dataset], self.out_pattern, self.command, new_cons, positional_args=self.positional_args, execution_options=self._execution_options) try: this_process.execute(simulate=configuration.simulate_execution) except subprocess.CalledProcessError as e: raise vistrails_module.ModuleError(self, e.output) except Exception as e: raise vistrails_module.ModuleError(self, repr(e)) process_output = this_process.file_creator self.setResult('out_dataset', process_output)
def compute(self): # Required input future_dataset = self.getInputFromPort("future_dataset") baseline_dataset = self.getInputFromPort("baseline_dataset") # Execute the process. new_constraints = [Constraint('change_type', ['abs-change'])] this_process = ProcessUnit( [future_dataset, baseline_dataset], self.out_pattern, self.command, extra_constraints=new_constraints, execution_options=self._execution_options, map_dict={ 'fut_start': ('year_start', 0), 'fut_end': ('year_end', 0), 'hist_start': ('year_start', 1), 'hist_end': ('year_end', 1) }) try: this_process.execute(simulate=configuration.simulate_execution) except Exception as e: raise vistrails_module.ModuleError(self, e.output) process_output = this_process.file_creator self.setResult('out_dataset', process_output)
def submit(self, deps=True, noexec=False): """Submit the job to the queue. @param deps: Whether or not to submit the tasks on which this task depends. @type deps: boolean @param noexec: Whether or not to actually submit the job for execution. If True the resulting shell script is printed to stdout. @type noexec: boolean """ #Add directory creation if len(self.outdirs) > 0: self.add_pre_cmd(['mkdir', '-p'] + sorted(self.outdirs)) if noexec: log.warning("Would run script:\n\n========>\n%s\n<========\n\n" % self.to_str()) else: script_file, script_name = tempfile.mkstemp('.sh') script_file = os.fdopen(script_file, 'w+b') script_file.write(self.to_str() + '\n') script_file.close() args = ['sh', script_name] #VERBOSE = os.getenv("VERBOSE") #log.debug("SimpleJob.submit(): VERBOSE = '%s'" % VERBOSE) try: output = subprocess.check_output(args, stderr=subprocess.STDOUT) except subprocess.CalledProcessError, e: output = e.output raise vistrails_module.ModuleError(self, output) # we want the stderr finally:
def compute(self): in_dataset = self.getInputFromPort('in_dataset') method = self.getInputFromPort('method') self.positional_args = [(method, 0, 'raw'), ] self.keyword_args = {} if len(method.split(',')) > 1: agg_constraint = "".join(method.split(',')) else: agg_constraint = method new_constraints_for_output = set([Constraint('timeagg_info', [agg_constraint]), Constraint('suffix', ['nc']), ]) this_process = ProcessUnit([in_dataset], self.out_pattern, self.command, new_constraints_for_output, execution_options=self._execution_options, positional_args=self.positional_args, cons_keywords=self.keyword_args) try: this_process.execute(simulate=configuration.simulate_execution) except Exception as e: raise vistrails_module.ModuleError(self, repr(e)) process_output = this_process.file_creator self.setResult('out_dataset', process_output)
def compute(self): in_dataset1 = self.getInputFromPort('in_dataset1') in_dataset2 = self.getInputFromPort('in_dataset2') operation = self.getInputFromPort('operation') self.positional_args = [ (operation, 0, 'raw'), ] self.keyword_args = {} new_constraints_for_output = set([ Constraint('extra_info', [operation]), Constraint('suffix', ['nc']), ]) this_process = ProcessUnit([in_dataset1, in_dataset2], self.out_pattern, self.command, new_constraints_for_output, execution_options=self._execution_options, positional_args=self.positional_args, cons_keywords=self.keyword_args, merge_output=['model', 'institute']) try: this_process.execute(simulate=configuration.simulate_execution) except Exception as e: raise vistrails_module.ModuleError(self, repr(e)) process_output = this_process.file_creator self.setResult('out_dataset', process_output)
def compute(self): # Required input in_dataset = self.getInputFromPort("in_dataset") # Set up the output command for this module, adding extra options. positional_args = [] anom_label = 'anom-wrt-all' arg_number = 0 try: clim_bounds = self.getInputFromPort('clim_bounds') positional_args += [('-b', arg_number, 'raw'), (clim_bounds, arg_number + 1, 'raw')] start_date, end_date = clim_bounds.split(',') anom_label = 'anom-wrt-' + start_date + '-' + end_date arg_number += 2 except vistrails_module.ModuleError as e: pass try: timescale = self.getInputFromPort('timescale') positional_args += [('-t', arg_number, 'raw'), (timescale, arg_number + 1, 'raw')] anom_label = timescale + anom_label except vistrails_module.ModuleError as e: pass cons_for_output = set([ Constraint('suffix', ['nc']), Constraint('anomaly_info', [anom_label]) ]) # Execute the process. this_process = ProcessUnit([in_dataset], self.out_pattern, self.command, cons_for_output, positional_args=positional_args, execution_options=self._execution_options) try: this_process.execute(simulate=configuration.simulate_execution) except Exception as e: raise vistrails_module.ModuleError(self, repr(e)) process_output = this_process.file_creator self.setResult('out_dataset', process_output)
def compute(self): in_dataset = self.getInputFromPort('in_dataset') # Change the file_type constraint from nc to xml cons_for_output = set([Constraint('suffix', ['xml'])]) # Execute the cdscan this_process = ProcessUnit([in_dataset], self.out_pattern, self.command, cons_for_output, execution_options=self._execution_options) try: this_process.execute(simulate=configuration.simulate_execution) except subprocess.CalledProcessError, e: raise vistrails_module.ModuleError(self, e.output)
def compute(self): in_dataset = self.getInputFromPort('in_dataset') method = self.getInputFromPort('method') seas_list = {'mon': 'ymon', 'seas': 'yseas', 'ann': 'tim'} ### loop over seas_list to generate all 3 season files ### for seas in seas_list.keys(): self.positional_args = [ ('%s%s' % (seas_list[seas], method), 0, 'raw'), ] self.keyword_args = {} if len(method.split(',')) > 1: agg_constraint = "".join(method.split(',')) else: agg_constraint = method new_constraints_for_output = set([ Constraint( 'timeagg_info', ['%s%s' % (seas_list[seas], method)], ), Constraint('suffix', ['nc']), ]) this_process = ProcessUnit( [in_dataset], self.out_pattern, self.command, new_constraints_for_output, execution_options=self._execution_options, positional_args=self.positional_args, cons_keywords=self.keyword_args) try: this_process.execute(simulate=configuration.simulate_execution) except Exception as e: raise vistrails_module.ModuleError(self, repr(e)) process_output = this_process.file_creator self.setResult('out_dataset_%s' % seas, process_output)
def compute(self): in_dataset = self.getInputFromPort('in_dataset') method = self.getInputFromPort('method') grid = self.getInputFromPort('grid') self.positional_args = [ (method, 0, 'raw'), (grid, 1, 'raw'), ] self.keyword_args = {} grid = grid.split('/')[-1] if len(grid.split('.') ) > 1: # i.e. a weights file as opposed to pre-defined grid grid_constraint = method + '-' + grid.split('.')[0] else: grid_constraint = method + '-' + grid new_constraints_for_output = set([ Constraint('grid_info', [grid_constraint]), Constraint('suffix', ['nc']), ]) this_process = ProcessUnit([in_dataset], self.out_pattern, self.command, new_constraints_for_output, execution_options=self._execution_options, positional_args=self.positional_args, cons_keywords=self.keyword_args) try: this_process.execute(simulate=configuration.simulate_execution) except Exception as e: raise vistrails_module.ModuleError(self, repr(e)) process_output = this_process.file_creator self.setResult('out_dataset', process_output)
def compute(self): in_dataset1 = self.getInputFromPort('in_dataset1') in_dataset2 = self.getInputFromPort('in_dataset2') self.positional_args = [] self.keyword_args = {} new_constraints_for_output = set([ Constraint('extra_info', ['timcor']), Constraint('suffix', ['nc']), ]) merge_val = self.getInputFromPort('merge_constraints') if merge_val: extra_merge = [ cons_name.strip() for cons_name in merge_val.split(',') ] else: extra_merge = [] this_process = ProcessUnit([in_dataset1, in_dataset2], self.out_pattern, self.command, new_constraints_for_output, execution_options=self._execution_options, positional_args=self.positional_args, cons_keywords=self.keyword_args, merge_output=extra_merge) try: this_process.execute(simulate=configuration.simulate_execution) except Exception as e: raise vistrails_module.ModuleError(self, repr(e)) process_output = this_process.file_creator self.setResult('out_dataset', process_output)
def compute(self): # Required input in_dataset = self.getInputFromPort("in_dataset") # Set up the output command for this module, adding extra options. positional_args = [('variable', 0)] port_names = [ "timestart", "timeend", "lonwest", "loneast", "latsouth", "latnorth", "levelbottom", "leveltop" ] port_vals = {} for name in port_names: try: port_vals[name + "_info"] = self.getInputFromPort(name) except vistrails_module.ModuleError as e: port_vals[name + "_info"] = None arg_number = 3 cons_for_output = set([Constraint('suffix', ['nc'])]) if port_vals["timestart_info"] and port_vals["timeend_info"]: positional_args += [('--time_bounds', arg_number, 'raw'), ('timestart_info', arg_number + 1), ('timeend_info', arg_number + 2)] arg_number += 3 cons_for_output |= set([ Constraint('timestart_info', [port_vals["timestart_info"]]), Constraint('timeend_info', [port_vals["timeend_info"]]) ]) if port_vals["loneast_info"] and port_vals["lonwest_info"]: positional_args += [('--lon_bounds', arg_number, 'raw'), ('lonwest_info', arg_number + 1), ('loneast_info', arg_number + 2)] arg_number += 3 lonwest_text = longitude_label(port_vals["lonwest_info"]) loneast_text = longitude_label(port_vals["loneast_info"]) cons_for_output |= set([ Constraint('lonwest_info', [lonwest_text]), Constraint('loneast_info', [loneast_text]) ]) if port_vals["latsouth_info"] and port_vals["latnorth_info"]: positional_args += [('--lat_bounds', arg_number, 'raw'), ('latsouth_info', arg_number + 1), ('latnorth_info', arg_number + 2)] arg_number += 3 latsouth_text = latitude_label(port_vals["latsouth_info"]) latnorth_text = latitude_label(port_vals["latnorth_info"]) cons_for_output |= set([ Constraint('latsouth_info', [latsouth_text]), Constraint('latnorth_info', [latnorth_text]) ]) if port_vals["levelbottom_info"] and port_vals["leveltop_info"]: positional_args += [('--level_bounds', arg_number, 'raw'), ('levelbottom_info', arg_number + 1), ('leveltop_info', arg_number + 2)] arg_number += 3 cons_for_output |= set([ Constraint('levelbottom_info', [port_vals["levelbottom_info"]]), Constraint('leveltop_info', [port_vals["leveltop_info"]]) ]) # Execute the xml_to_nc process. this_process = ProcessUnit([in_dataset], self.out_pattern, self.command, cons_for_output, positional_args=positional_args, execution_options=self._execution_options) try: this_process.execute(simulate=configuration.simulate_execution) except Exception as e: raise vistrails_module.ModuleError(self, repr(e)) process_output = this_process.file_creator self.setResult('out_dataset', process_output)
def compute(self): # Required input in_dataset = self.getInputFromPort("in_dataset") variable = self.getInputFromPort("variable") #self.positional_args=[(variable_name, 0, 'raw')] plot_type = self.getInputFromPort("plot_type") #self.positional_args=[(plot_type, 1, 'raw')] title = self.getInputFromPort("title") #self.positional_args=[(title, 2, 'raw')] region = self.getInputFromPort("region") #self.positional_args=[(plot_type, 3, 'raw')] colormap = self.getInputFromPort("colormap") #self.positional_args=[(colormap, 4, 'raw')] ticks = self.getInputFromPort("ticks") #self.positional_args=[(plot_type, 5, 'raw')] conv_units = self.getInputFromPort("conv_units") #self.positional_args=[(conv_units, 6, 'raw')] cons_for_output = set([Constraint('suffix', ['png'])]) run_opts = '' if variable: run_opts = run_opts + ' --variable %s' %variable if plot_type: run_opts = run_opts + ' --plot_type %s' %plot_type if title: run_opts = run_opts + ' --title %s' %title if region: run_opts = run_opts + ' --region %s' %region if colormap: run_opts = run_opts + ' --colourmap %s' %colormap if ticks: run_opts = run_opts + " --ticks '%s'" %ticks if conv_units: run_opts = run_opts + " --conv_units '%s'" %conv_units # Execute plotting process. this_process = ProcessUnit([in_dataset], self.out_pattern, self.command, cons_for_output, execution_options=self._execution_options, #positional_args=self.positional_args, kw_string=run_opts) #kw_string="--title '${model}_${experiment}'") try: process_output = this_process.execute(simulate=configuration.simulate_execution) except Exception as e: raise vistrails_module.ModuleError(self, repr(e)) self.setResult('out_dataset', process_output)