def typeChecking(self, module, inputPorts, inputList): """ Function used to check if the types of the input list element and of the inputPort of 'module' match. """ for elementList in inputList: if len(elementList) != len(inputPorts): raise ModuleError(self, 'The number of input values and input ports ' 'are not the same.') for element, inputPort in izip(elementList, inputPorts): p_modules = module.moduleInfo['pipeline'].modules p_module = p_modules[module.moduleInfo['moduleId']] port_spec = p_module.get_port_spec(inputPort, 'input') v_module = create_module(element, port_spec.signature) if v_module is not None: if not self.compare(port_spec, v_module, inputPort): raise ModuleError(self, 'The type of a list element does ' 'not match with the type of the ' 'port %s.' % inputPort) del v_module else: break
def compute(self): self.check_input("baseName") n = self.get_input("baseName") if self.has_input("file_hdr") and \ self.has_input("file_img"): n1 = self.get_input("file_hdr").name n2 = self.get_input("file_img").name if n1.endswith('.hdr'): n1base = n1.rsplit('.hdr',1)[0] else: n1base = n1 if n2.endswith('.img'): n2base = n2.rsplit('.img',1)[0] else: n2base = n2 if n1base != n2base: o = self.interpreter.filePool.create_file() o1 = o.name + '.hdr' o2 = o.name + '.img' try: core.system.link_or_copy(n1, o1) except OSError, e: msg = "error creating tmp file '%s'" % o1 raise ModuleError(self, msg) try: core.system.link_or_copy(n2, o2) except OSError, e: msg = "error creating tmp file '%s'" % o2 raise ModuleError(self, msg)
def git_get_hash(name, version="HEAD", path_type=None): if path_type is None: path_type = PersistentPath.git_get_type(name, version) if path_type == 'blob': cmd_list = [ PersistentPath.git_command() + ["ls-files", "--stage", str(version), str(name)] ] debug_print('executing commands', cmd_list) result, output, errs = execute_piped_cmdlines(cmd_list) debug_print('stdout:', type(output), output) debug_print('stderr:', type(errs), errs) if result != 0: # check output for error messages raise ModuleError(self, "Error retrieving path '%s'\n" % name + errs) return output.split(None, 2)[1] elif path_type == 'tree': cmd_list = [ PersistentPath.git_command() + ["ls-tree", "-d", str(version), str(name)] ] debug_print('executing commands', cmd_list) result, output, errs = execute_piped_cmdlines(cmd_list) debug_print('stdout:', type(output), output) debug_print('stderr:', type(errs), errs) if result != 0: # check output for error messages raise ModuleError(self, "Error retrieving path '%s'\n" % name + errs) return output.split(None, 3)[2] return None
def get_header_annotations(self): """Returns the header information for the file using the AIR scanheader tool.""" process = subprocess.Popen(global_airpath + 'scanheader ' + self.name, shell=True, stdout=subprocess.PIPE) waitCode = 0 try: waitCode = process.wait() except OSError: pass if waitCode != 0: raise ModuleError(self, "Could not open header file " + self.name) result = {} lines = core.utils.no_interrupt(process.stdout.readlines) for l in lines: l = l[:-1] if not l: continue entries = l.split('=') if len(entries) != 2: raise ModuleError(self, "Error parsing line '%s' of header %s" % (l[:-1], self.name)) result[entries[0]] = entries[1] return result
def map_ports(module, port_map): args = {} for port, (flag, access, required) in port_map.iteritems(): if required or module.has_input(port): #breakpoint() value = module.force_get_input_list(port) if len(value) > 1: raise ModuleError( module, 'Multiple items found from Port ' + port + '. Only single entry handled. Please remove extraneous items.' ) elif len(value) == 0: raise ModuleError( module, 'Multiple items found from Port ' + port + '. Only single entry handled. Please remove extraneous items.' ) value = module.force_get_input(port) if access is not None: value = access(value) if isinstance(value, File) or \ isinstance(value, Directory) or \ isinstance(value, Path): value = path_port(module, port) args[flag] = value return args
def compute(self): self.checkInputPort('input_file') self.checkInputPort('iterations') iters = self.getInputFromPort('iterations') if iters < 1: raise ModuleError(self, 'iterations must be >=1') input_file = self.getInputFromPort('input_file') if self.hasInputFromPort('output_format'): output_suffix = self.getInputFromPort('output_format') else: output_suffix = self.guess_input_format(input_file.name) if not output_suffix: output_suffix = '.off' output_file = self.interpreter.filePool.create_file( suffix=output_suffix) values = [ _mesh_filter_path, input_file.name, '-subdiv', output_file.name ] cmdline = list2cmdline(values) print cmdline result = os.system(cmdline) if result != 0: raise ModuleError(self, 'Execution failed') for i in xrange(iters - 1): cmdline = '%s %s -subdiv %s' % (_mesh_filter_path, output_file.name, output_file.name) print cmdline result = os.system(cmdline) if result != 0: raise ModuleError(self, 'Execution failed') self.setResult('output_file', output_file)
def compute(self): """ 1. use type input as object or create new 2. add other inputs to obj 3. set obj and parts as outputs """ if self.wstype.enum: # only makes sure the enum is one of the valid values p = self.wstype.parts['value'] if self.hasInputFromPort(p.name): obj = self.getInputFromPort(p.name) else: obj = p.enum[0] if len(p.enum) else '' if self.hasInputFromPort('value'): obj = self.getInputFromPort('value') if obj not in p.enum: raise ModuleError( self, "'%s' is not one of the valid enums: %s" % (obj, str(p.enum))) self.setResult(self.wstype.qname[0], obj) self.setResult('value', obj) return if self.hasInputFromPort(self.wstype.qname[0]): obj = self.getInputFromPort(self.wstype.qname[0]) else: obj = {} s = "{%s}%s" % (self.wstype.qname[1], self.wstype.qname[0]) try: obj = self.service.service.factory.create(s) except (suds.TypeNotFound, suds.BuildError): raise ModuleError("Type not found: %s" % s) for part in self.wstype.parts.itervalues(): # if obj.__class__.__name__ == 'UberClass': # UberClass is a placeholder and its value is assumed # to be the correct attribute value if len(self.wstype.parts) == 1: setattr(obj, part.name, obj.value) else: # update each attribute if hasattr(obj.value, part.name): setattr(obj, part.name, getattr(obj.value, part.name)) if self.hasInputFromPort(part.name): p = self.getInputFromPort(part.name) if hasattr(obj, part.name): setattr(obj, part.name, p) else: # do it anyway - assume attribute missing in template setattr(obj, part.name, p) if hasattr(obj, part.name): # res = getattr(obj, part.name) self.setResult(part.name, res) self.setResult(self.wstype.qname[0], obj)
def get_locator_and_version(self): self.locator = self.moduleInfo['locator'] self.version = self.moduleInfo['version'] self.pipeline = self.moduleInfo['pipeline'] if self.locator is None: raise ModuleError(self, 'could not get the locator for this pipeline') if self.version is None: raise ModuleError( self, 'could not get the version number of this peline')
def compute(self): self.checkInputPort('csvFile') csv_file = self.getInputFromPort('csvFile') if not LoadAppLogic.IsExistsCSVFile(csv_file.file_entry): raise ModuleError(self, "IsExistsCSVFile failed") self.annotate({'used_files': str([csv_file.file_entry.HeaderPath])}) csv_file.file_entry = \ LoadAppLogic.ReadCSVFileColumnNames(csv_file.file_entry) if not LoadAppLogic.IsMatchCSVFileColumnNames(csv_file.file_entry): raise ModuleError(self, "IsMatchCSVFileColumnNames failed") self.setResult('csvFile', csv_file)
def do_input(self): r = [] if self.hasInputFromPort('in1_file'): r += [self.getInputFromPort('in1_file').name] elif self.hasInputFromPort('in1_value'): r += [self.getInputFromPort('in1_value')] else: raise ModuleError(self, "Needs either in1_file or in1_value") if self.hasInputFromPort('in2_file'): r += [self.getInputFromPort('in2_file').name] elif self.hasInputFromPort('in2_value'): r += [self.getInputFromPort('in2_value')] else: raise ModuleError(self, "Needs either in2_file or in2_value") return r
def compute(self): """ compute() -> None Dispatch the vtkRenderer to the actual rendering widget """ # Check required input ports if not self.hasInputFromPort('slab1'): raise ModuleError(self, "'slab1' is mandatory.") if not self.hasInputFromPort('template'): raise ModuleError(self, "'template' is mandatory.") if not self.hasInputFromPort('plotType'): raise ModuleError(self, "'plotType' is mandatory.") # Build up the argument list args = [] slab1 = self.getInputFromPort('slab1') args.append(self.getInputFromPort('slab1')) if self.hasInputFromPort('slab2'): args.append(self.getInputFromPort('slab2')) args.append(self.getInputFromPort('template')) args.append(self.getInputFromPort('plotType')) if self.hasInputFromPort('gmName'): args.append(self.getInputFromPort('gmName')) # Build up plot keyword args ... kwargs = {} if self.hasInputFromPort('continents'): kwargs['continents'] = self.getInputFromPort('continents') # Set the cell row / col self.location = CellLocation() if self.hasInputFromPort('row'): self.location.row = self.getInputFromPort('row') if self.hasInputFromPort('col'): self.location.col = self.getInputFromPort('col') canvas = None if self.hasInputFromPort('canvas'): canvas = self.getInputFromPort('canvas') gm = None if self.hasInputFromPort('gm'): gm = self.getInputFromPort('gm') # Plot into the cell inputPorts = (canvas, gm, args, kwargs) self.cellWidget = self.displayAndWait(QCDATWidget, inputPorts) if self.cellWidget is not None: self.setResult('canvas', self.cellWidget.canvas)
def compute(self): # *** IMPORTANT *** # Once someone figures out how to pass the tvariable object, to this # module none of the computation in this method is necessary # Check ports if not self.has_input('cdmsfile'): raise ModuleError(self, "'cdmsfile' is mandatory.") if not self.has_input('id'): raise ModuleError(self, "'id' is mandatory.") # Get input from ports cdmsfile = self.get_input('cdmsfile') id = self.get_input('id') axes = self.force_get_input('axes') # None if no input axesOperations = self.force_get_input( 'axesOperations') # None if no input # Get the variable varType = self.getVarType(id, cdmsfile) if (varType == 'variable'): var = cdmsfile.__call__(id) elif (varType == 'axis'): varID = self.getAxisID(id) axis = getattr(cdmsfile, 'axes')[varID] var = MV2.array(axis) var.setAxis(0, axis) elif (varType == 'weighted-axis'): varID, axisID = self.getVarAndAxisID(id) var = cdmsfile.__call__(varID) var = genutil.getAxisWeightByName(var, axisID) var.id = varID + '_' + axisID + '_weight' else: var = None # Eval the variable with the axes if axes is not None and var is not None: try: kwargs = eval(axes) var = var(**kwargs) except: raise ModuleError(self, "Invalid 'axes' specification", axes) # Apply axes ops to the variable if axesOperations is not None: var = self.applyAxesOperations(var, axesOperations) self.set_output('variable', var)
def open(self): retry = True while retry: config = {'host': self.host, 'port': self.port, 'user': self.user} # unfortunately keywords are not standard across libraries if self.protocol == 'mysql': config['db'] = self.db_name if self.password is not None: config['passwd'] = self.password elif self.protocol == 'postgresql': config['database'] = self.db_name if self.password is not None: config['password'] = self.password try: self.conn = self.get_db_lib().connect(**config) break except self.get_db_lib().Error, e: debug.warning(str(e)) if (e[0] == 1045 or self.get_db_lib().OperationalError and self.password is None): passwd_dlg = QPasswordEntry() if passwd_dlg.exec_(): self.password = passwd_dlg.get_password() else: retry = False else: raise ModuleError(self, str(e))
def compute(self): list1 = self.getInputFromPort("List_1") list2 = self.getInputFromPort("List_2") lenght1 = len(list1) lenght2 = len(list2) result = [] if lenght1 != lenght2: raise ModuleError(self, 'Both lists must have the same size.') if self.hasInputFromPort("CombineTuple") and (not self.getInputFromPort\ ("CombineTuple")): for i in xrange(lenght1): tuple_ = (list1[i], list2[i]) result.append(tuple_) else: for i in xrange(lenght1): if type(list1[i]) == tuple and type(list2[i]) == tuple: tuple_ = list1[i] + list2[i] result.append(tuple_) elif type(list1[i]) == tuple and type(list2[i]) != tuple: tuple_ = list1[i] + (list2[i], ) result.append(tuple_) elif type(list1[i]) != tuple and type(list2[i]) == tuple: tuple_ = (list1[i], ) + list2[i] result.append(tuple_) else: tuple_ = (list1[i], list2[i]) result.append(tuple_) self.setResult("Result", result)
def joboperation_compute(self): job = self.getInputFromPort('job') queue = job.queue # print "TERMINAL ID", id(queue.terminal), id(queue.local_terminal) self.anno_counter = 1 self.anno_dict = {} def annotate(fncself, *args, **kwargs): if len(args) == 1: self.anno_dict.update({"note%d" % self.anno_counter: args[0]}) elif len(kwargs) == 0: self.anno_dict.update( {"note%d" % self.anno_counter: " ".join(args)}) else: self.anno_dict.updateself.annotate(kwargs) self.anno_counter += 1 return None function = getattr(queue, self.function_name) pnt = function.Qprint function.Qprint = annotate ret = function().val() function.Qprint = pnt ## TODO: annotate does not seem to work self.annotate(self.anno_dict) if isinstance(ret, FunctionMessage) and ret.code != 0: raise ModuleSuspended(self, ret.message) if ret.code > 0 \ else ModuleError(self,ret.message) self.setResult("job", job) self.setResult("result", ret) self.setResult("string", str(ret))
def runRScript(script, args, module=None): global config r_path = config.r_path program = os.path.join(r_path, "i386", "Rterm.exe") #-q prevents program from running scriptFile = os.path.join(getModelsPath(), script) command = program + " --vanilla -f " + scriptFile + " --args " + args p = subprocess.Popen(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE) writetolog("\nStarting R Processing of " + script, True) writetolog(" args: " + args, False, False) writetolog(" command: " + command, False, False) ret = p.communicate() if 'Error' in ret[1]: msg = "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" msg +="\n An error was encountered in the R script for this module." msg += "\n The R error message is below: \n" msg += ret[1] writetolog(msg) if module: raise ModuleError(module, msg) else: raise RuntimeError , msg if 'Warning' in ret[1]: msg = "The R scipt returned the following warning(s). The R warning message is below - \n" msg += ret[1] writetolog(msg) del(ret) writetolog("\nFinished R Processing of " + script, True)
def _parse_url(self, url): s = url.split('/') try: self.host = s[2] self.filename = '/' + '/'.join(s[3:]) except: raise ModuleError(self, "Malformed URL: %s" % url)
def compute(self): ds = self.get_input("VTK Data") ar = self.get_input("Scalars") v_name = self.force_get_input("Array Name") if v_name == None: v_name = 'scalars' num_times = ds.vtkInstance.GetNumberOfTimeSteps() ar_ = ar.get_array() if ar_.shape[0] != num_times: raise ModuleError("Cannot process array with num timesteps = " + str(ar_.shape[0]) + " and vtkdata with " + str(num_times) + " timesteps") for i in range(num_times): self.update_progress(i, num_times) s_ar = ar_[i,::].squeeze().flatten() vtk_ar = vtk.vtkDoubleArray() vtk_ar.SetNumberOfValues(s_ar.size) vtk_ar.SetName(v_name) for j in range(s_ar.size): vtk_ar.InsertNextValue(s_ar[j]) ds.vtkInstance.GetTimeStep(i).GetPointData().AddArray(vtk_ar) ds.vtkInstance.GetTimeStep(i).GetPointData().SetScalars(vtk_ar) self.set_output("Output", ds)
def get_db_lib(self): if self.protocol == 'mysql': return MySQLdb elif self.protocol == 'postgresql': return psycopg2 else: raise ModuleError(self, "Currently no support for '%s'" % protocol)
class DirectorySink(NotCacheable, Module): """DirectorySink takes a directory and writes it to a user-specified location in the file system. The directory is stored at location specified by the outputPath. The overwrite flag allows users to specify whether an existing path should be overwritten.""" def compute(self): input_dir = self.getInputFromPort("dir") output_path = self.getInputFromPort("outputPath") full_path = output_path.name if os.path.exists(full_path): if (self.hasInputFromPort("overwrite") and self.getInputFromPort("overwrite")): try: if os.path.isfile(full_path): os.remove(full_path) else: shutil.rmtree(full_path) except OSError, e: msg = ('Could not delete existing path "%s" ' '(overwrite on)' % full_path) raise ModuleError(self, msg + '\n' + str(e)) else: msg = ('Could not write to existing path "%s" ' '(overwrite off)' % full_path) raise ModuleError(self, msg) try: shutil.copytree(input_dir.name, full_path) except OSError, e: msg = 'Could not copy path from "%s" to "%s"' % \ (input_dir.name, full_path) raise ModuleError(self, msg + '\n' + str(e))
def getR_application(module=None): global r_path #are we in 64 or 32 bit? If 64 use the 64 bit install of R otherwise 32 bit. #if we don't have the matching version and the other exists use it. version_dirs = ["i386", "x64"] possible_exes = [ os.path.join(r_path, version_dir, "Rterm.exe") for version_dir in version_dirs ] if sys.maxsize > 2**32 and os.path.exists(possible_exes[1]): program = possible_exes[1] elif os.path.exists(possible_exes[0]): program = possible_exes[0] elif os.path.exists(possible_exes[1]): program = possible_exes[1] else: #no R exe found we can't go on msg = "No R executable found.\nPlease check the install folder: " msg += r_path + "\nfor either a ..\i386\Rterm.exe or a ..\x64\Rterm.exe" if module: raise ModuleError(module, msg) else: raise RuntimeError, msg return program
def compute(self): o = self.create_output_file() r = self.getInputFromPort("r") g = self.getInputFromPort("g") b = self.getInputFromPort("b") a = self.getInputFromPort("a") path = None if configuration.check('path'): path = configuration.path if path: cmd = os.path.join(path, 'convert') else: cmd = 'convert' cmd = [ cmd, '-channel', 'RGBA', '-combine', r.name, g.name, b.name, a.name, o.name ] if not configuration.quiet: debug.log(cmd) cmdline = list2cmdline(cmd) result = os.system(cmdline) if result != 0: raise ModuleError(self, "system call failed: '%s'" % cmdline) self.setResult("output", o)
def write_raw(self, fn, data): try: fid = open(fn, 'wb') scipy.io.fwrite(fid, data.size, data) fid.close() except: raise ModuleError("Could not write .raw file!")
def git_get_dir(name, version="HEAD", out_dirname=None, out_suffix=''): global temp_persist_files, tar_bin if out_dirname is None: # create a temporary directory out_dirname = tempfile.mkdtemp(suffix=out_suffix, prefix='vt_persist') temp_persist_files.append(out_dirname) elif not os.path.exists(out_dirname): os.makedirs(out_dirname) if systemType == "Windows": cmd_list = [PersistentPath.git_command() + \ ["archive", str(version + ':' + name)], ["%s:" % out_dirname[0], "&&", "cd", "%s"%out_dirname, "&&", tar_bin, '-xf-']] else: cmd_list = [PersistentPath.git_command() + \ ["archive", str(version + ':' + name)], [tar_bin, '-C', out_dirname, '-xf-']] debug_print('executing commands', cmd_list) result, output, errs = execute_piped_cmdlines(cmd_list) debug_print('stdout:', type(output), output) debug_print('stderr:', type(errs), errs) if result != 0: # check output for error messages raise ModuleError(self, "Error retrieving file '%s'\n" % name + errs) return out_dirname
def compute(self): if self.hasInputFromPort('title'): title = self.getInputFromPort('title') else: title = 'VisTrails Dialog' if self.hasInputFromPort('label'): label = self.getInputFromPort('label') else: label = '' if self.password: label = 'Password' if self.hasInputFromPort('default'): default = QtCore.QString(self.getInputFromPort('default')) else: default = QtCore.QString('') if self.hasInputFromPort('cacheable') and self.getInputFromPort( 'cacheable'): self.cacheable_dialog = True else: self.cacheable_dialog = False mode = QtGui.QLineEdit.Normal if self.password: mode = QtGui.QLineEdit.Password (result, ok) = QtGui.QInputDialog.getText(None, title, label, mode, default) if not ok: raise ModuleError(self, "Canceled") self.setResult('result', str(result))
def updateUpstream(self, is_input=None, path_type=None): global local_db, db_access if is_input is None: if not self.hasInputFromPort('value'): is_input = True else: # FIXME: check if the signature is the signature of # the value if so we know that it's an input... is_input = False self.persistent_ref = None self.persistent_path = None if not is_input: # can check updateUpstream if not hasattr(self, 'signature'): raise ModuleError(self, 'Module has no signature') if not self.hasInputFromPort('ref'): # create new reference with no name or tags ref = PersistentRef() ref.signature = self.signature debug_print('searching for signature', self.signature) sig_ref = db_access.search_by_signature(self.signature) debug_print('sig_ref:', sig_ref) if sig_ref: debug_print('setting persistent_ref') ref.id, ref.version, ref.name = sig_ref self.persistent_ref = ref # else: # ref.id = uuid.uuid1() else: # update single port self.updateUpstreamPort('ref') ref = self.getInputFromPort('ref') if db_access.ref_exists(ref.id, ref.version): if ref.version is None: ref.version = self.git_get_latest_version(ref.id) signature = db_access.get_signature(ref.id, ref.version) if signature == self.signature: # don't need to create a new version self.persistent_ref = ref # copy as normal # don't copy if equal # FIXME also need to check that the file actually exists here! if self.persistent_ref is not None: _, suffix = os.path.splitext(self.persistent_ref.name) self.persistent_path = \ self.git_get_path(self.persistent_ref.id, self.persistent_ref.version, out_suffix=suffix) debug_print("FOUND persistent path") debug_print(self.persistent_path) debug_print(self.persistent_ref.local_path) if self.persistent_ref is None or self.persistent_path is None: debug_print("NOT FOUND persistent path") Module.updateUpstream(self)
def compute(self): n = self.get_name() if (self.hasInputFromPort("create_directory") and self.getInputFromPort("create_directory")): try: core.system.mkdir(n) except Exception, e: raise ModuleError(self, 'mkdir: ' + str(e))
def compute(self): """ compute() -> None Dispatch the vtkRenderer to the actual rendering widget """ renderView = self.force_get_input('SetRenderView') if renderView == None: raise ModuleError(self, 'A vtkRenderView input is required.') self.cellWidget = self.displayAndWait(QVTKViewWidget, (renderView, ))
def compute(self): """ compute() -> None Dispatch the vtkRenderer to the actual rendering widget """ renderers = self.forceGetInputListFromPort('AddRenderer') renderViews = self.forceGetInputListFromPort('SetRenderView') if len(renderViews)>1: raise ModuleError(self, 'There can only be one vtkRenderView ' 'per cell') if len(renderViews)==1 and len(renderers)>0: raise ModuleError(self, 'Cannot set both vtkRenderView ' 'and vtkRenderer to a cell') renderView = self.forceGetInputFromPort('SetRenderView') iHandlers = self.forceGetInputListFromPort('InteractionHandler') iStyle = self.forceGetInputFromPort('InteractorStyle') picker = self.forceGetInputFromPort('AddPicker') self.cellWidget = self.displayAndWait(QVTKWidget, (renderers, renderView, iHandlers, iStyle, picker))
def run(self, cmd): if not self.__quiet: print cmd if self.__programsQuiet: cmd = '(' + cmd + ') 2>&1 >/dev/null' r = os.system(cmd) if r != 0: raise ModuleError(self, "system call failed: '%s'" % cmd)