def execute_tdl_job(self, _tdlmod, ns, func, name, job_id): """executes a predefined TDL job given by func""" self._tdlexec_menu.hide() try: # log job TDLOptions.dump_log( "running TDL job '%s' (%s)" % (name, ("job id '%s'" % job_id if job_id else "no job id"))) busy = BusyIndicator() func(meqds.mqs(), self) # no errors, so clear error list, if any self.clear_errors() self.show_message("TDL job '" + name + "' executed.", transient=True) busy = None except: (etype, exc, tb) = sys.exc_info() _dprint(0, 'exception running TDL job', func.__name__) traceback.print_exception(etype, exc, tb) # use TDL add_error() to process the error, since this automatically # adds location information. However, we want to remove ourselves # from the stack traceback first tb = traceback.extract_tb(tb) # pop everything leading up to our filename while tb[0][0] != _MODULE_FILENAME: tb.pop(0) # pop frame with our filename, this should leave only TDL-code frames tb.pop(0) ns.AddError(exc, tb, error_limit=None) msg = "TDL job '" + name + "' failed" self._error_window.set_errors(ns.GetErrors(), message=msg) self.emit(PYSIGNAL("showEditor()"), self) busy = None
def _save_profile(self, name): if self.profiles.has_section(name): if QMessageBox.warning( self, "Overwriting profile", """<P>Do you really want to overwrite the existing profile '%s'?</P>""" % name, QMessageBox.Ok | QMessageBox.Cancel, QMessageBox.Ok) != QMessageBox.Ok: return newprof = False else: self.profiles.add_section(name) newprof = True TDLOptions.save_to_config(self.profiles, name) try: ff = file(PROFILE_FILE, "wt") for section in sorted(self.profiles.sections()): ff.write("[%s]\n" % section) for opt, value in sorted(self.profiles.items(section)): ff.write("%s = %s\n" % (opt.lower(), value)) ff.write("\n") ff.close() except: dprintf(0, "error writing %s" % PROFILE_FILE) traceback.print_exc() QMessageBox.warning( self, "Error saving profile", """<P>There was an error writing to the %s file, profile was not saved.</P>""" % PROFILE_FILE) if newprof: self.emit(PYSIGNAL("refreshProfiles()"))
def _save_profile (self,name): if self.profiles.has_section(name): if QMessageBox.warning(self,"Overwriting profile","""<P>Do you really want to overwrite the existing profile '%s'?</P>"""%name, QMessageBox.Ok|QMessageBox.Cancel,QMessageBox.Ok) != QMessageBox.Ok: return; newprof = False; else: self.profiles.add_section(name); newprof = True; TDLOptions.save_to_config(self.profiles,name); try: ff = file(PROFILE_FILE,"wt"); for section in sorted(self.profiles.sections()): ff.write("[%s]\n"%section); for opt,value in sorted(self.profiles.items(section)): ff.write("%s = %s\n"%(opt.lower(),value)); ff.write("\n"); ff.close(); except: dprintf(0,"error writing %s"%PROFILE_FILE); traceback.print_exc(); QMessageBox.warning(self,"Error saving profile","""<P>There was an error writing to the %s file, profile was not saved.</P>"""%PROFILE_FILE); if newprof: self.emit(PYSIGNAL("refreshProfiles()"));
def execute_tdl_job (self,_tdlmod,ns,func,name,job_id): """executes a predefined TDL job given by func"""; self._tdlexec_menu.hide(); try: # log job TDLOptions.dump_log("running TDL job '%s' (%s)"%(name,("job id '%s'"%job_id if job_id else "no job id"))); busy = BusyIndicator(); func(meqds.mqs(),self); # no errors, so clear error list, if any self.clear_errors(); self.show_message("TDL job '"+name+"' executed.",transient=True); busy = None; except: (etype,exc,tb) = sys.exc_info(); _dprint(0,'exception running TDL job',func.__name__); traceback.print_exception(etype,exc,tb); # use TDL add_error() to process the error, since this automatically # adds location information. However, we want to remove ourselves # from the stack traceback first tb = traceback.extract_tb(tb); # pop everything leading up to our filename while tb[0][0] != _MODULE_FILENAME: tb.pop(0); # pop frame with our filename, this should leave only TDL-code frames tb.pop(0); ns.AddError(exc,tb,error_limit=None); msg = "TDL job '"+name+"' failed"; self._error_window.set_errors(ns.GetErrors(),message=msg); self.emit(PYSIGNAL("showEditor()"),self); busy = None;
def _load_profile (self,name): if not self.profiles.has_section(name): QMessageBox.warning(self,"Error loading profile","""<P>Profile '%s' not found, check your %s file.</P>"""%(name,PROFILE_FILE)); return; for name,value in self.profiles.items(name): try: TDLOptions.set_option(name,value,save=False,from_str=True); except ValueError: dprintf(0,"can't set '%s' to %s, ignoring",name,value); TDLOptions.save_config();
def compile_script(self, need_mqs=False): """compiles the script, if not already compiled. If need_mqs=True, starts a meqserver and builds the tree as well""" if not self._module: self._compile_failed = True # will reset to False if all goes well\ self.log_progress("compile") # start meqserver if required if (need_mqs or int(self.get_option("start_meqserver", 0))) and not self._mqs: from Timba.Apps import meqserver # get multithreading option mt = int(self.get_option("multithreaded", 0)) if mt > 1: extra = ["-mt", str(mt)] else: extra = [] _dprint(1, "starting meqserver", extra) self._mqs = meqserver.default_mqs(wait_init=10, extra=extra) from Timba.TDL import Compile from Timba.TDL import TDLOptions # load config file tdlconf = self.get_option("tdlconf", ".tdl.conf") TDLOptions.config.read(tdlconf) TDLOptions.init_options(self.name, save=False) # compile TDL module _dprint(1, "compiling TDL script", self.name) try: (self._module, ns, msg) = Compile.compile_file(self._mqs, self.name) except: excinfo = sys.exc_info() self.log_exc(level=2, *excinfo) self.fail("compile failed") excinfo = None return # success self.log(msg, level=2) self.success("compile") self._compile_failed = False pass
def compile_script (self,need_mqs=False): """compiles the script, if not already compiled. If need_mqs=True, starts a meqserver and builds the tree as well"""; if not self._module: self._compile_failed = True; # will reset to False if all goes well\ self.log_progress("compile"); # start meqserver if required if ( need_mqs or int(self.get_option("start_meqserver",0)) ) and not self._mqs: from Timba.Apps import meqserver # get multithreading option mt = int(self.get_option("multithreaded",0)); if mt>1: extra = [ "-mt",str(mt) ]; else: extra = [] _dprint(1,"starting meqserver",extra); self._mqs = meqserver.default_mqs(wait_init=10,extra=extra); from Timba.TDL import Compile from Timba.TDL import TDLOptions # load config file tdlconf = self.get_option("tdlconf",".tdl.conf"); TDLOptions.config.read(tdlconf); TDLOptions.init_options(self.name,save=False); # compile TDL module _dprint(1,"compiling TDL script",self.name); try: (self._module,ns,msg) = Compile.compile_file(self._mqs,self.name); except: excinfo = sys.exc_info(); self.log_exc(level=2,*excinfo); self.fail("compile failed"); excinfo = None; return; # success self.log(msg,level=2); self.success("compile"); self._compile_failed = False; pass;
mod,ns,msg = Compile.compile_file(mqs,script); # 'mod' now refers to a Python object that is the compiled module. To execute the _test_forest job within that # module, we do as follows: print "Running TDL job"; mod._test_forest(mqs,None,wait=True); # The wait=True argument causes the thing to not return until the job has been completed. # None for the second argument tells it that we're running headless (without a GUI parent.) ### Now for some variations # This shows how to use a different section in the config file. print "Recompiling and running for test_a"; mod,ns,msg = Compile.compile_file(mqs,script,config="test_a"); # and this shows how to locate and call a "TDL Job" by its job ID TDLOptions.get_job_func('job1')(mqs,None,wait=True); print "Recompiling and running for test_b"; mod,ns,msg = Compile.compile_file(mqs,script,config="test_b"); # we can also find jobs by their long names TDLOptions.get_job_func('Run job 2')(mqs,None,wait=True); # finally, this is expected to fail print "Trying to call a non-existing TDL job 'job3'"; try: TDLOptions.get_job_func('job3')(mqs,None,wait=True); except NameError: print "Indeed, 'job3' does not exist." # This shows how to change configuration on-the-fly. # TDLOptions.set_option() changes the value of a configuration variable "in memory".
def compile_content(self): # import content first, and return if failed if not self.import_content(force=True): return None _dprint(1, self._filename, "compiling forest") # clear predefined functions self._tb_tdlexec.hide() # try the compilation busy = BusyIndicator() try: (_tdlmod,ns,msg) = \ TDL.Compile.run_forest_definition( meqds.mqs(),self._filename,self._tdlmod,self._tdltext, parent=self,wait=False) # catch compilation errors except TDL.CumulativeError as value: _dprint(0, "caught cumulative error, length", len(value.args)) self._error_window.set_errors(value.args, message="TDL import failed") busy = None return None except Exception as value: _dprint(0, "caught other error, traceback follows") traceback.print_exc() self._error_window.set_errors([value]) busy = None return None # refresh the nodelist meqds.request_nodelist(sync=True) # restore publishing nodes for name in (self._pub_nodes or []): if name in ns.AllNodes(): _dprint(2, "reenabling publishing of", name) meqds.enable_node_publish_by_name(name, sync=True, get_state=True) else: _dprint(2, "not reenabling publishing of", name, ", as it no longer exists?") # clear publisher list so it's re-saved next time in import_content() self._pub_nodes = None ### NB: presume this all was successful for now # insert node scope into TDL module setattr(_tdlmod, '_tdl_nodescope', ns) # does the script define an explicit job list? joblist = getattr(_tdlmod, '_tdl_job_list', []) if not joblist: joblist = [] # try to build it from implicit function names for (name, func) in _tdlmod.__dict__.items(): if name.startswith("_tdl_job_") and callable( func) and not TDLOptions.is_jobfunc_defined(func): joblist.append(func) # does the script define a testing function? testfunc = getattr(_tdlmod, '_test_forest', None) if not callable(testfunc): testfunc = getattr(_tdlmod, 'test_forest', None) if callable(testfunc): res = QMessageBox.warning( self, "Deprecated method", """Your script contains a test_forest() method. This is deprecated and will be disabled in the future. Please rename it to _test_forest(). """, QMessageBox.Ok) if callable(testfunc): joblist.append(testfunc) from past.builtins import cmp from functools import cmp_to_key joblist.sort(key=cmp_to_key(lambda a, b: cmp(str(a), str(b)))) # create list of job actions opts = TDLOptions.get_runtime_options() self._tdlexec_menu.clear() if joblist or opts: if opts: self._job_executor = curry(self.execute_tdl_job, _tdlmod, ns) ## new style: try: TDLOptions.populate_option_treewidget( self._tdlexec_menu.treeWidget(), opts, executor=self._job_executor) except Exception as value: _dprint(0, "error setting up TDL options GUI") traceback.print_exc() self._error_window.set_errors( [value], message="Error setting up TDL options GUI") busy = None return None if joblist: for func in joblist: name = re.sub("^_tdl_job_", "", func.__name__) name = name.replace('_', ' ') self._tdlexec_menu.addAction(name, curry(self.execute_tdl_job, _tdlmod, ns, func, name, func.__name__), icon=pixmaps.gear) self.emit(PYSIGNAL("hasRuntimeOptions()"), self, True) self._tdlexec_menu.adjustSizes() self._tb_tdlexec.show() else: self.emit(PYSIGNAL("hasRuntimeOptions()"), self, False) self._tb_tdlexec.hide() if joblist: msg += " %d predefined function(s) available, please use the Exec menu to run them." % ( len(joblist), ) self.show_message(msg, transient=True) busy = None return True
class TDLEditor(QFrame, PersistentCurrier): SupportsLineNumbers = False # a single editor always has the focus current_editor = None # flag: sync to external editor external_sync = True def set_external_sync(value): global _external_sync _external_sync = value set_external_sync = staticmethod(set_external_sync) def __init__(self, parent, close_button=False, error_window=None): QFrame.__init__(self, parent) self._enabled = True toplo = QVBoxLayout(self) toplo.setContentsMargins(0, 0, 0, 0) toplo.setSpacing(0) self.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding) splitter = QSplitter(Qt.Vertical, self) toplo.addWidget(splitter) splitter.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding) splitter.setChildrenCollapsible(False) # figure out our app_gui parent self._appgui = app_proxy_gui.appgui(parent) # create an editor box editor_box = QFrame(splitter) lo = QVBoxLayout(editor_box) lo.setContentsMargins(0, 0, 0, 0) lo.setSpacing(0) # find main window to associate our toolbar with mainwin = parent while mainwin and not isinstance(mainwin, QMainWindow): mainwin = mainwin.parent() self._toolbar = QToolBar(mainwin or self) lo.addWidget(self._toolbar) self._toolbar.setIconSize(QSize(16, 16)) #### populate toolbar # Exec button and menu self._tb_tdlexec = QToolButton(self._toolbar) self._toolbar.addWidget(self._tb_tdlexec) self._tb_tdlexec.setIcon(pixmaps.gear.icon()) self._tb_tdlexec.setText("Exec") self._tb_tdlexec.setToolButtonStyle(Qt.ToolButtonTextBesideIcon) self._tb_tdlexec.setToolTip( "Accesses run-time options & jobs defined by this TDL script") self._tb_tdlexec.hide() jobs = self._tdlexec_menu = TDLOptionsDialog(self) jobs.setWindowTitle("TDL Jobs & Runtime Options") jobs.setWindowIcon(pixmaps.gear.icon()) jobs.hide() QObject.connect(self._tb_tdlexec, SIGNAL("clicked()"), jobs.exec_) # save menu and button self._tb_save = QToolButton(self._toolbar) self._toolbar.addWidget(self._tb_save) self._tb_save.setIcon(pixmaps.file_save.icon()) self._tb_save.setToolTip( "Saves script. Click on the down-arrow for other options.") savemenu = QMenu(self) self._tb_save.setMenu(savemenu) self._tb_save.setPopupMode(QToolButton.MenuButtonPopup) self._tb_save._modified_color = QColor("yellow") qa_save = savemenu.addAction(pixmaps.file_save.icon(), "&Save script", self._save_file) qa_save.setShortcut(Qt.ALT + Qt.Key_S) QObject.connect(self._tb_save, SIGNAL("clicked()"), self._save_file) qa_save_as = savemenu.addAction( pixmaps.file_save.icon(), "Save script &as...", self.curry(self._save_file, save_as=True)) qa_revert = self._qa_revert = savemenu.addAction( "Revert to saved", self._revert_to_saved) # run menu and button self._qa_run = self._toolbar.addAction( pixmaps.blue_round_reload.icon(), "&Save & reload", self._import_main_file) self._qa_run.setShortcut(Qt.ALT + Qt.Key_R) # Compile-time options and menu self._tb_opts = QToolButton(self._toolbar) self._toolbar.addWidget(self._tb_opts) self._tb_opts.setIcon(pixmaps.wrench.icon()) self._tb_opts.setText("Options") self._tb_opts.setToolButtonStyle(Qt.ToolButtonTextBesideIcon) self._tb_opts.setToolTip( "Accesses compile-time options for this TDL script") # self._tb_opts.hide(); opts = self._options_menu = TDLOptionsDialog( self, ok_label="Compile", ok_icon=pixmaps.blue_round_reload) opts.setWindowTitle("TDL Compile-time Options") opts.setWindowIcon(pixmaps.wrench.icon()) QObject.connect(opts, PYSIGNAL("accepted()"), self._compile_main_file) QObject.connect(TDLOptions.OptionObject, SIGNAL("mandatoryOptionsSet"), self.mark_mandatory_options_set) opts.hide() QObject.connect(self._tb_opts, SIGNAL("clicked()"), opts.show) # cross-connect the rereshProfiles() signals, so that dialogs can update each other's # profile menus QObject.connect(self._options_menu, PYSIGNAL("refreshProfiles()"), self._tdlexec_menu.refresh_profiles) QObject.connect(self._tdlexec_menu, PYSIGNAL("refreshProfiles()"), self._options_menu.refresh_profiles) self._toolbar.addSeparator() # cursor position indicator self._poslabel = QLabel(self._toolbar) #wa = QWidgetAction(self._toolbar); #wa.setDefaultWidget(self._poslabel); #self._toolbar.addAction(wa); #self._toolbar.addWidget(self._poslabel); self._toolbar.addWidget(self._poslabel) width = self._poslabel.fontMetrics().width("L:999 C:999") self._poslabel.setMinimumWidth(width) self._poslabel.setText("L:0 C:0") # filename indicator self._pathlabel = QLabel(self._toolbar) #wa = QWidgetAction(self._toolbar); #wa.setDefaultWidget(self._pathlabel); self._toolbar.addWidget(self._pathlabel) #self._toolbar.addAction(wa); self._pathlabel.show() self._pathlabel.setAlignment(Qt.AlignRight | Qt.AlignVCenter) self._pathlabel.setIndent(10) self._pathlabel.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Minimum) if close_button: if not isinstance(close_button, QIcon): close_button = pixmaps.remove.icon() self._qa_close = self._toolbar.addAction(close_button, "&Close file", self._file_closed) self._qa_close.setShortcut(Qt.ALT + Qt.Key_W) self._pathlabel.setText("(no file)") #### add editor window self._editor = editor = QTextEdit(editor_box) lo.addWidget(self._editor) editor.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding) editor.setAcceptRichText(False) editor.setLineWrapMode(QTextEdit.NoWrap) self._document = QTextDocument(self) editor.setDocument(self._document) QObject.connect(self._document, SIGNAL("modificationChanged(bool)"), self._text_modified) QObject.connect(self._editor, SIGNAL("cursorPositionChanged()"), self._display_cursor_position) # QObject.connect(self._editor,SIGNAL("textChanged()"),self._clear_transients); # add character formats for error display self._format_error = QTextCharFormat(self._editor.currentCharFormat()) self._format_error.setBackground(QBrush(QColor("lightpink"))) self._format_suberror = QTextCharFormat( self._editor.currentCharFormat()) self._format_suberror.setBackground(QBrush(QColor("lightgrey"))) self._format_current_error = QTextCharFormat( self._editor.currentCharFormat()) self._format_current_error.setBackground(QBrush(QColor("orangered"))) # add message window self._message_box = QFrame(editor_box) lo.addWidget(self._message_box) self._message_box.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Preferred) self._message_box.setFrameStyle(QFrame.Panel + QFrame.Sunken) self._message_box.setLineWidth(2) mblo = QVBoxLayout(self._message_box) mblo.setContentsMargins(0, 0, 0, 0) msgb1 = QHBoxLayout() mblo.addLayout(msgb1) msgb1.setContentsMargins(0, 0, 0, 0) msgb1.setSpacing(0) self._message_icon = QToolButton(self._message_box) msgb1.addWidget(self._message_icon) self._message = QLabel(self._message_box) msgb1.addWidget(self._message) self._message.setTextFormat(Qt.RichText) self._message.setWordWrap(True) self._message.setMargin(0) self._message.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Preferred) # self._message_icon.setAlignment(Qt.AlignTop); # self._message_icon.setMargin(4); self._message_icon.setAutoRaise(True) self._message_icon.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed) QObject.connect(self._message_icon, SIGNAL("clicked()"), self.clear_message) self._message_icon.setToolTip("Click here to clear the message") self._message_widgets = [] self._message_transient = False # figure out if we already have an error box to attach to self._error_window = error_window or getattr( parent, '_tdlgui_error_window', None) if self._error_window: #self._resize_errwin = False; pass else: # otherwise create an error floater self._error_window = TDLErrorFloat(parent) setattr(parent, '_tdlgui_error_window', self._error_window) # self._resize_errwin = True; QObject.connect(self._error_window, PYSIGNAL("hasErrors()"), self._reset_errors) QObject.connect(self._error_window, PYSIGNAL("showError()"), self.show_error) # set filename self._filename = None # "official" path of file (None if new script not yet saved) self._mainfile = None # if not None, then we're "slaved" to a main file (see below) self._file_disktime = None # modtime on disk when file was loaded self._basename = None self._modified = False self._closed = False self._error_at_line = {} self._is_tree_in_sync = True self._pub_nodes = None def __del__(self): self.has_focus(False) def disable_editor(self): """Called before disabling the editor, as on some versions of PyQt the object is not destroyed properly and keeps receving signals""" self._enabled = False QObject.disconnect(TDLOptions.OptionObject, SIGNAL("mandatoryOptionsSet"), self.mark_mandatory_options_set) def show_compile_options(self): self._options_menu.show() def mark_mandatory_options_set(self, enabled): self._options_menu.enableOkButton(enabled) def show_runtime_options(self): self._tdlexec_menu.show() def tree_is_in_sync(self, sync=True): """Tells the editor wheether the current tree is in sync with the content of the script. This is indicated by a visual cue on the toolbar. """ if sync: self._tb_tdlexec.setIcon(pixmaps.gear.icon()) self._tb_tdlexec.setToolTip( "Access run-time options & jobs defined by this TDL script") else: self._tb_tdlexec.setIcon(pixmaps.exclaim_yellow_warning.icon()) self._tb_tdlexec.setToolTip( """Access run-time options & jobs defined by this TDL script. Warning! You have modified the script since it was last compiled, so the tree may be out of date.""" ) def _file_closed(self): self.emit(PYSIGNAL("fileClosed()"), self) def hideEvent(self, ev): self.emit(PYSIGNAL("hidden()")) self.emit(PYSIGNAL("visible()"), False) return QFrame.hideEvent(self, ev) def showEvent(self, ev): self.emit(PYSIGNAL("shown()")) self.emit(PYSIGNAL("visible()"), True) return QFrame.showEvent(self, ev) def hide_jobs_menu(self, dum=False): if self._closed: return self._tb_tdlexec.hide() self.clear_message() def show_line_numbers(self, show): pass def show_run_control(self, show=True): if self._closed: return self._qa_run.setVisible(show) def enable_controls(self, enable=True): if self._closed: return self._qa_run.setEnabled(enable) self._tb_tdlexec.setEnabled(enable) if not enable: self.clear_message() def disable_controls(self, disable=True): if self._closed: return self._qa_run.setDisabled(disable) self._tb_tdlexec.setDisabled(disable) if disable: self.clear_message() def get_filename(self): return self._filename def get_mainfile(self): return self._mainfile def _import_main_file(self): # self._tb_opts.setOn(False); self.clear_errors() if self._document.isModified(): self._save_file() self.emit(PYSIGNAL("importFile()"), self, self._mainfile or self._filename) def _compile_main_file(self): # self._tb_opts.setOn(False); self.clear_errors() if self._document.isModified(): self._save_file() self.emit(PYSIGNAL("compileFile()"), self, self._mainfile or self._filename) def _clear_transients(self): """if message box contains a transient message, clears it""" if self._message_transient: self.clear_message() def _display_cursor_position(self): cursor = self._editor.textCursor() pos = cursor.position() col = cursor.columnNumber() line = cursor.blockNumber() self._poslabel.setText("L:<b>%d</b> C:<b>%d</b>" % (line + 1, col + 1)) self._poslabel.repaint() def _text_modified(self, mod=True): self._modified = mod self.emit(PYSIGNAL("textModified()"), self, bool(mod)) self._tb_save.setAutoRaise(not mod) self._qa_revert.setEnabled(mod) #if mod: #self._tb_save.setBackgroundRole(QPalette.ToolTipBase); #else: #self._tb_save.setBackgroundRole(QPalette.Button); if self._filename: label = '<b>' + self._basename + '</b>' self._pathlabel.setToolTip(self._filename) else: label = '' self._pathlabel.setToolTip('') if self._readonly: label = '[r/o] ' + label if mod: self._clear_transients() label = '[mod] ' + label self._pathlabel.setText(label) def clear_message(self): # traceback.print_stack(); self._message_box.hide() self._message.setText('') self._message_icon.setText('') self._message_transient = False if self._message_widgets: dum = QWidget() for w in self._message_widgets: w.reparent(dum) self._message_widgets = [] def show_message(self, msg, error=False, icon=None, transient=False): """Shows message in box. If icon is not None, overrides standard icon. If error=True, uses error icon (icon overrides this). If transient=True, message will be cleared when text is edited. """ self._message.setText(msg) if not icon: if error: icon = pixmaps.red_round_cross.icon() else: icon = pixmaps.info_blue_round.icon() self._message_icon.setIcon(icon) self._message_box.show() self._message_transient = transient self.clear_errors() def messagebox(): return self._message_box def add_message_widget(self, widget): self._mblo.addWidget(widget) self._message_widgets.append(widget) def clear_errors(self): self._error_window.clear_errors(emit_signal=True) def _find_block_by_linenumber(self, line): block = block0 = self._document.begin() nblock = 1 while block and nblock < line: block0 = block block = block0.next() nblock += 1 return block or block0 def _reset_errors(self, nerr): """helper method, resets error markers and such. Usually tied to a hasErrors() signal from an error window""" if not self._enabled: return self._error_at_line = {} self._error_selections = [] self._current_error_selection = QTextEdit.ExtraSelection() nerr_local = 0 if nerr: error_locations = self._error_window.get_error_locations() for err_num, filename, line, column, suberror in error_locations: if filename == self._filename: nerr_local += 1 # make cursor corresponding to line containing the error cursor = QTextCursor(self._find_block_by_linenumber(line)) cursor.select(QTextCursor.LineUnderCursor) # make selection object qsel = QTextEdit.ExtraSelection() qsel.cursor = cursor if suberror: qsel.format = self._format_suberror else: qsel.format = self._format_error # insert into error_at_line list self._error_at_line[line - 1] = len( self._error_selections), cursor # append to list of error selections self._error_selections.append(qsel) self._editor.setExtraSelections(self._error_selections) else: self._editor.setExtraSelections([]) self.emit(PYSIGNAL("hasErrors()"), self, nerr_local) def show_error(self, err_num, filename, line, column): """Shows error at the given position, but only if the filename matches. Can be directly connected to a showError() signal from an error window""" if filename == self._filename: errnum, cursor = self._error_at_line.get(line - 1, (None, None)) if cursor: # change selections self._current_error_selection.cursor = cursor self._current_error_selection.format = self._format_current_error sels = self._error_selections[:errnum] + self._error_selections[ (errnum + 1):] sels.append(self._current_error_selection) self._editor.setExtraSelections(sels) # move current cursor cursor = self._editor.textCursor() # scroll to line-1 and line+1 to make sure line is fully visible if line > 1: cursor.setPosition( self._find_block_by_linenumber(line - 2).position() + column) self._editor.setTextCursor(cursor) self._editor.ensureCursorVisible() if line < self._document.blockCount(): cursor.setPosition( self._find_block_by_linenumber(line).position() + column) self._editor.setTextCursor(cursor) self._editor.ensureCursorVisible() # finally, scroll to line cursor.setPosition( self._find_block_by_linenumber(line).position() + column) self._editor.setTextCursor(cursor) self._editor.ensureCursorVisible() def sync_external_file(self, filename=None, ask=True): # # NB: problem because it resets the errors filename = filename or self._filename filetime = _file_mod_time(filename) if not filetime or filetime == self._file_disktime: return True # in sync if not ask: res = QMessageBox.Discard else: res = QMessageBox.warning( self, "TDL file changed", """<p><tt>%s</tt> has been modified by another program. Would you like to save your changes and overwrite the version on disk, discard your changes and revert to the version on disk, or cancel the operation?""" % (filename, ), QMessageBox.Save | QMessageBox.Discard | QMessageBox.Cancel) if res == QMessageBox.Cancel: return None elif res == QMessageBox.Discard: if filename != self._filename: self.load_file(filename, mainfile=self._mainfile) else: self.reload_file() return True # in sync def _save_file(self, filename=None, text=None, force=False, save_as=False): """Saves text. If force=False, checks modification times in case the file has been modified by another program. If force=True, saves unconditionally. If no filename is known, asks for one. Returns True if file was successfully saved, else None.""" filename = filename or self._filename if filename and not save_as: if not force: if not self.sync_external_file(filename=filename, ask=True): return None else: # no filename, ask for one try: dialog = self._save_as_dialog dialog.setDirectory(dialog.directory()) # hopefully this rescan the directory except AttributeError: self._save_as_dialog = dialog = QFileDialog( self, "Saved TDL Script") dialog.resize(800, dialog.height()) dialog.setMode(QFileDialog.AnyFile) dialog.setNameFilters( ["TDL scripts (*.tdl *.py)", "All files (*.*)"]) dialog.setViewMode(QFileDialog.Detail) if dialog.exec_() != QDialog.Accepted: return None filename = str(dialog.selectedFiles()[0]) # save the file if text is None: text = str(self._editor.document().toPlainText()) try: outfile = file(filename, "w").write(text) except IOError: (exctype, excvalue, tb) = sys.exc_info() _dprint(0, 'exception', sys.exc_info(), 'saving TDL file', filename) self.show_message("""<b>Error writing <tt>%s</tt>: <i>%s (%s)</i></b>""" % (filename, excvalue, exctype.__name__), error=True, transient=True) return None # saved successfully, update stuff self._filename = filename self._qa_revert.setEnabled(True) self._basename = os.path.basename(filename) self._file_disktime = _file_mod_time(filename) self._document.setModified(False) self.emit(PYSIGNAL("fileSaved()"), self, filename) return self._filename def close(self): self._closed = True def confirm_close(self): if self._document.isModified(): res = QMessageBox.warning( self, "TDL file modified", """Save modified file <p><tt>%s</tt>?</p>""" % (self._filename or "", ), QMessageBox.Save | QMessageBox.Discard | QMessageBox.Cancel) if res == QMessageBox.Cancel: return False if res == QMessageBox.Save: if not self._save_file(): return False self.close() return True def _revert_to_saved(self, force=False): if not self._filename: return if not force: if QMessageBox.question( self, "Revert to saved", """Revert to saved version of <p><tt>%s</tt>?""" % (self._filename, ), QMessageBox.Ok | QMessageBox.Cancel) == QMessageBox.Cancel: return self.load_file(self._filename, mainfile=self._mainfile) def _add_menu_label(self, menu, label): tlab = QLabel("<b>" + label + "</b>", menu) tlab.setAlignment(Qt.AlignCenter) tlab.setFrameShape(QFrame.ToolBarPanel) tlab.setFrameShadow(QFrame.Sunken) menu.insertItem(tlab) def has_compile_options(self): return self._options_menu.treeWidget().topLevelItemCount() def has_runtime_options(self): return self._tdlexec_menu.treeWidget().topLevelItemCount() def import_content(self, force=False, show_options=False): """imports TDL module but does not run _define_forest(). Depending on autosync/modified state, asks to save or revert. If module is already imported, does nothing, unless force=True, in which case it imports unconditionally. If do_compile=True, proceeds to show compile-time options on success, or to compile directly if there are no options Return value: True on successful import None if cancelled by user. Import errors are posted to the error dialog. """ # save list of publishers (since forest will be cleared on import) if self._pub_nodes is None: if TDL.Compile.last_imported_file() == self._filename: self._pub_nodes = [ node.name for node in meqds.nodelist.iternodes() if node.is_publishing() ] _dprint(2, "last imported file matches, saving", len(self._pub_nodes), "publishers") else: self._pub_nodes = [] _dprint(2, "last imported file doesn't match:", TDL.Compile.last_imported_file(), self._filename) _dprint(1, self._filename, "importing") self.clear_message() self.clear_errors() self._options_menu.hide() self._tb_tdlexec.hide() self._tdlexec_menu.hide() # change the current directory to where the file is # os.chdir(os.path.dirname(self._filename)); # The Python imp module expects text to reside in a disk file, which is # a pain in the ass for us if we're dealing with modified text or text # entered on-the-fly. So, either save or sync before proceeding global _external_sync if self._document.isModified() or not self._filename: if not self._save_file(): return None else: if not self.sync_external_file(ask=False): return None # if we already have an imported module and disk file hasn't changed, skip # the importing step busy = BusyIndicator() if force or self._tdlmod is None or self._tdlmod_filetime == self._file_disktime: # reset data members _dprint(2, self._filename, "emitting signal for 0 compile-time options") self.emit(PYSIGNAL("hasCompileOptions()"), self, 0) self._options_menu.hide() self._options_menu.clear() self._tdlmod = None # get text from editor tdltext = str(self._document.toPlainText()) try: tdlmod, tdltext = TDL.Compile.import_tdl_module( self._filename, tdltext) # catch import errors except TDL.CumulativeError, value: _dprint(0, "caught cumulative error, length", len(value.args)) self._error_window.set_errors(value.args, message="TDL import failed") busy = None return None except Exception, value: _dprint(0, "caught other error, traceback follows") traceback.print_exc() self._error_window.set_errors([value], message="TDL import failed") busy = None return None # remember module and nodescope self._tdlmod = tdlmod self._tdltext = tdltext self._tdlmod_filetime = self._file_disktime # build options menu from compile-time options opt_tw = self._options_menu.treeWidget() opts = TDLOptions.get_compile_options() if opts: # add options try: TDLOptions.populate_option_treewidget(opt_tw, opts) except Exception, value: _dprint(0, "error setting up TDL options GUI") traceback.print_exc() self._error_window.set_errors( [value], message="Error setting up TDL options GUI") busy = None return None # self._tb_opts.show(); _dprint(2, self._filename, "emitting signal for", len(opts), "compile-time options") self.emit(PYSIGNAL("hasCompileOptions()"), self, len(opts))
# or was interrupted by an exception midway through, we use a try...finally block. try: TDLOptions.config.read("batch_sim_example.tdl.conf"); script = "example-sim.py"; print "========== Compiling batch job 1"; mod,ns,msg = Compile.compile_file(mqs,script,config="batch job 1"); print "========== Running batch job 1"; mod._tdl_job_1_simulate_MS(mqs,None,wait=True); print "========== Compiling batch job 2"; mod,ns,msg = Compile.compile_file(mqs,script,config="batch job 2"); print "========== Running batch job 2"; mod._tdl_job_1_simulate_MS(mqs,None,wait=True); print "========== Compiling batch job 2 with modified config"; TDLOptions.init_options("batch job 2",save=False); TDLOptions.set_option("me.enable_G",False); mod,ns,msg = Compile.compile_file(mqs,script,config=None); print "========== Running batch job 2"; mod._tdl_job_1_simulate_MS(mqs,None,wait=True); ### Cleanup time finally: print "Stopping meqserver"; # this halts the meqserver meqserver.stop_default_mqs(); # now we can exit print "Bye!";
opt = optstr.split("=") + ['1'] context, level = opt[:2] debuglevels[context] = int(level) # tell verbosity class to not parse argv -- we do it ourselves here Timba.utils.verbosity.disable_argv() for optstr in (options.verbose or []): opt = optstr.split("=") + ['1'] context, level = opt[:2] Timba.utils.verbosity.set_verbosity_level(context, int(level)) # start meqserver from Timba.Apps import meqserver from Timba.TDL import Compile from Timba.TDL import TDLOptions TDLOptions.enable_save_config(False) print "### Starting meqserver" mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", str(options.mt)]) retcode = 0 # use a try...finally block to exit meqserver cleanly at the end try: if not os.path.exists(options.config): print "Config file %s doesn't exist" % options.config sys.exit(1) print "### Attaching to configuration file", options.config TDLOptions.config.read(options.config) # disable the writing-out of configuration TDLOptions.config.set_save_filename(None)
# This starts a meqserver. Note how we pass the "-mt 2" option to run two threads. # A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv). print "Starting meqserver"; mqs = meqserver.default_mqs(wait_init=10,extra=["-mt","2"]); try: ## make simulation with perfect MODEL_DATA script = path("testing-sim.py"); print "========== Compiling",script; TDLOptions.config.read(path("testing.tdl.conf")); mod,ns,msg = Compile.compile_file(mqs,script,config="simulate-model"); print "========== Simulating MODEL_DATA "; mod._tdl_job_1_simulate_MS(mqs,None,wait=True); print "========== Imaging MODEL_DATA "; TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False); ## compare against reference image print "========== Verifying test image "; verify_image('WSRT.MS.MODEL_DATA.channel.1ch.fits',path('test-refimage.fits'),maxdelta=1e-3); print "========== Compiling script with modified config"; TDLOptions.init_options("simulate-model",save=False); TDLOptions.set_option("me.g_enable",True); mod,ns,msg = Compile.compile_file(mqs,script,config=None); print "========== Simulating DATA "; TDLOptions.set_option("ms_sel.output_column","DATA"); mod._tdl_job_1_simulate_MS(mqs,None,wait=True); print "========== Imaging DATA "; TDLOptions.set_option("img_sel.imaging_column","DATA"); TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
print "Starting meqserver" mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", "2"]) try: ## make simulation with perfect MODEL_DATA script = path("testing-sim.py") print "========== Compiling", script TDLOptions.config.read(path("testing.tdl.conf")) mod, ns, msg = Compile.compile_file(mqs, script, config="simulate-model") print "========== Simulating MODEL_DATA " mod._tdl_job_1_simulate_MS(mqs, None, wait=True) print "========== Imaging MODEL_DATA " TDLOptions.get_job_func('make_dirty_image')(mqs, None, wait=True, run_viewer=False) ## compare against reference image print "========== Verifying test image " verify_image('WSRT.MS.MODEL_DATA.channel.1ch.fits', path('test-refimage.fits'), maxdelta=1e-3) print "========== Compiling script with modified config" TDLOptions.init_options("simulate-model", save=False) TDLOptions.set_option("me.g_enable", True) mod, ns, msg = Compile.compile_file(mqs, script, config=None) print "========== Simulating DATA " TDLOptions.set_option("ms_sel.output_column", "DATA") mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
def testMeqtreesBatchJob(): trace_sync = True; # sys.settrace(trace_lines); if len(sys.argv) > 1: newdir = PACKAGE_TEST_DIR; print("========== Changing working directory to",newdir); os.chdir(newdir); print("========== Making required symlinks"); run("rm {0:s}/WSRT_ANTENNA ; ln -s {1:s}".format(PACKAGE_TEST_DIR, path(os.path.join(PACKAGE_TEST_DIR, "WSRT_ANTENNA")))); run("rm {0:s}/test-lsm.txt; ln -s {1:s}".format(PACKAGE_TEST_DIR, path(os.path.join(path("test-lsm.txt"))))); if not os.access(".",os.R_OK|os.W_OK): print("Directory",os.getcwd(),"not writable, can't run tests in here.") print("You may choose to run the tests in a different directory by giving it as an argument to this script.") sys.exit(1); ## check if we have owlcat or owlcat.sh owlcat = ""; for dirname in os.environ['PATH'].split(':'): for binary in "owlcat","owlcat.sh": tmp = os.path.join(dirname,binary); if os.path.exists(tmp): owlcat = tmp; break; if owlcat: break; if not owlcat: raise RuntimeError("Can't locate owlcat or owlcat.sh"); ## make simulated MS print("========== Removing files"); run("rm -fr {0:s}/WSRT.MS* {0:s}/WSRT*img {0:s}/WSRT*fits".format(PACKAGE_TEST_DIR)); print("========== Running makems"); run("makems %s" % path(os.path.join(PACKAGE_TEST_DIR, "WSRT_makems.cfg"))); run("mv {0:s}/WSRT.MS_p0 {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR)); os.environ["MEQTREES_CATTERY_PATH"] = Cattery.__path__[0] run("pyxis {0:s}/WSRT.MS ms.prep".format(PACKAGE_TEST_DIR)); #TODO: this is hacky, bug in CASAcore run("ls -ld {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR)); run("{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS".format(owlcat, PACKAGE_TEST_DIR)); run("lwimager ms={0:s}/WSRT.MS data=CORRECTED_DATA mode=channel weight=natural npix=10".format(PACKAGE_TEST_DIR)); # make test LSMs run("""tigger-convert {0:s}/test-lsm.txt --rename --format "ra_d dec_d i q u v" --center 0.1deg,60.5deg -f""".format(PACKAGE_TEST_DIR)); run("""tigger-convert {0:s}/test-lsm.lsm.html {0:s}/test-lsm1.txt --output-format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""".format(PACKAGE_TEST_DIR)); run("""cut -d " " -f 1-10 {0:s}/test-lsm1.txt >{0:s}/test-lsm1.txt.tmp""".format(PACKAGE_TEST_DIR)); run("""diff {0:s}/test-lsm1.txt.tmp {1:s}""".format(PACKAGE_TEST_DIR, path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm1.txt.reference')))); run("""tigger-convert {0:s}/test-lsm1.txt --format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""".format(PACKAGE_TEST_DIR)); run("""{0:s} plot-ms {1:s}/WSRT.MS DATA:I -o data_i.png""".format(owlcat, PACKAGE_TEST_DIR)); run("""{0:s} run-imager ms={1:s}/WSRT.MS name_dirty=tmp""".format(owlcat, PACKAGE_TEST_DIR)); print("importing meqserver") from Timba.Apps import meqserver print("importing Compile") from Timba.TDL import Compile print("importing TDLOptions") from Timba.TDL import TDLOptions # This starts a meqserver. Note how we pass the "-mt 2" option to run two threads. # A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv). print("Starting meqserver"); mqs = meqserver.default_mqs(wait_init=10,extra=["-mt","2"]); try: ## make simulation with perfect MODEL_DATA script = path(os.path.join(PACKAGE_TEST_DIR, "sim.py")); print("========== Compiling",script); TDLOptions.config.read(path(os.path.join(PACKAGE_TEST_DIR, "testing.tdl.conf"))); TDLOptions.config.set("calibrate", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "ms_sel.msname"))) TDLOptions.config.set("calibrate", "tiggerlsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "tiggerlsm.filename"))) TDLOptions.config.set("calibrate", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "lsm.filename"))) TDLOptions.config.set("calibrate", "cal_g_diag.g_diag.table_name", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_diag.g_diag.table_name"))) TDLOptions.config.set("calibrate", "cal_g_offdiag.g_offdiag.table_name", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_offdiag.g_offdiag.table_name"))) TDLOptions.config.set("simulate-model", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "lsm.filename"))) TDLOptions.config.set("simulate-model", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "ms_sel.msname"))) TDLOptions.config.set("simulate-model", "tiggerlsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "tiggerlsm.filename"))) TDLOptions.config.set("calibrate", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits")) TDLOptions.config.set("simulate-model", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits")) with open(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"), "w") as f: TDLOptions.config.write(f) TDLOptions.config.read(path(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"))); # needs to re-read because of a Timba perculiarity mod,ns,msg = Compile.compile_file(mqs, script, config="simulate-model"); print("========== Simulating MODEL_DATA "); mod._tdl_job_1_simulate_MS(mqs,None,wait=True); print("========== Imaging MODEL_DATA "); TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False); ## compare against reference image print("========== Verifying test image "); if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist") if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")): raise RuntimeError("Reference FITS file does not exist") verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits"), path(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")), maxdelta=1e-3); print("========== Compiling script with modified config"); TDLOptions.init_options("simulate-model",save=False); TDLOptions.set_option("me.g_enable",True); mod,ns,msg = Compile.compile_file(mqs,script,config=None); print("========== Simulating DATA "); TDLOptions.set_option("ms_sel.output_column","DATA"); mod._tdl_job_1_simulate_MS(mqs,None,wait=True); print("========== Imaging DATA "); TDLOptions.set_option("img_sel.imaging_column","DATA"); TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False); ## calibrate script = path(os.path.join(PACKAGE_TEST_DIR, "cal.py")); print("========== Compiling",script); mod,ns,msg = Compile.compile_file(mqs,script,config="calibrate"); print("========== Calibrating "); TDLOptions.get_job_func('cal_G_diag')(mqs,None,wait=True); print("========== Imaging MODEL_DATA "); TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False); finally: print("Stopping meqserver"); # this halts the meqserver meqserver.stop_default_mqs(); print("========== Making plots of solutions "); run("""{0:s} plot-ms {1:s}/WSRT.MS CORRECTED_DATA:I -I ">0" -o {1:s}/corrected_data_i.png""".format(owlcat, PACKAGE_TEST_DIR)); run("""{0:s} plot-parms -l {1:s}/WSRT.MS/G_diag.fmep""".format(owlcat, PACKAGE_TEST_DIR)); run("""{0:s} plot-parms {1:s}/WSRT.MS/G_diag.fmep G:*/norm -o {1:s}/parmplot.png""".format(owlcat, PACKAGE_TEST_DIR)); run("""{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS""".format(owlcat, PACKAGE_TEST_DIR)); ## compare against reference image print("========== Verifying residual image "); if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist") if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits")): raise RuntimeError("Reference FITS file does not exist") verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits"), os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits"), maxdelta=1e-3); ## all tests succeeded print("========== Break out the bubbly, this hog is airborne!"); # now we can exit print("Bye!");
meqds.enable_node_publish_by_name(name,sync=True,get_state=True); else: _dprint(2,"not reenabling publishing of",name,", as it no longer exists?"); # clear publisher list so it's re-saved next time in import_content() self._pub_nodes = None; ### NB: presume this all was successful for now # insert node scope into TDL module setattr(_tdlmod,'_tdl_nodescope',ns); # does the script define an explicit job list? joblist = getattr(_tdlmod,'_tdl_job_list',[]); if not joblist: joblist = []; # try to build it from implicit function names for (name,func) in _tdlmod.__dict__.iteritems(): if name.startswith("_tdl_job_") and callable(func) and not TDLOptions.is_jobfunc_defined(func): joblist.append(func); # does the script define a testing function? testfunc = getattr(_tdlmod,'_test_forest',None); if not callable(testfunc): testfunc = getattr(_tdlmod,'test_forest',None); if callable(testfunc): res = QMessageBox.warning(self,"Deprecated method", """Your script contains a test_forest() method. This is deprecated and will be disabled in the future. Please rename it to _test_forest(). """, QMessageBox.Ok); if callable(testfunc): joblist.append(testfunc); joblist.sort(lambda a,b:cmp(str(a),str(b)));
def main(): for optstr in (options.debug or []): opt = optstr.split("=") + ['1'] context, level = opt[:2] debuglevels[context] = int(level) Timba.utils.verbosity.disable_argv() # tell verbosity class to not parse its argv for optstr in (options.verbose or []): opt = optstr.split("=") + ['1'] context, level = opt[:2] Timba.utils.verbosity.set_verbosity_level(context, level) if not args: parser.print_help() sys.exit(1) if debuglevels: octopussy.set_debug(debuglevels) script = args[0] tdljob = (len(args) > 1 and args[1]) or None from Timba.Apps import meqserver from Timba.TDL import Compile from Timba.TDL import TDLOptions # this starts a kernel. if options.compile_only: mqs = None else: mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", options.mt]) TDLOptions.config.read(options.config) TDLOptions.init_options(script) print(("************************ Compiling TDL script", script)) # this compiles a script as a TDL module. Any errors will be thrown as # and exception, so this always returns successfully (mod, ns, msg) = Compile.compile_file(mqs, script) if options.compile_only: print(msg) sys.exit(0) # if a solve job is not specified, try to find one if tdljob: jobfunc = getattr(mod, tdljob, None) if not jobfunc: print(("Cannot find TDL job named", tdljob)) sys.exit(1) else: # does the script define an explicit job list? joblist = getattr(mod, '_tdl_job_list', []) if not joblist: joblist = [] # try to build it from implicit function names for (name, func) in list(mod.__dict__.items()): if name.startswith("_tdl_job_") and callable(func): joblist.append(func) # does the script define a testing function? testfunc = getattr(mod, '_test_forest', None) if testfunc: joblist.insert(0, testfunc) if not joblist: print(("No TDL jobs found in script", script)) sys.exit(1) jobfunc = joblist[0] tdljob = jobfunc.__name__ # this runs the appropriate job. wait=True is needed to wait print(("************************ Running TDL job", tdljob)) # check if job takes a "wait" argument (fargs, fvarargs, fvarkw, fdefaults) = inspect.getargspec(jobfunc) if 'wait' in fargs or fvarkw: jobopts = dict(wait=True) else: jobopts = {} jobfunc(mqs, None, **jobopts)
mod, ns, msg = Compile.compile_file(mqs, script) # 'mod' now refers to a Python object that is the compiled module. To execute the _test_forest job within that # module, we do as follows: print "Running TDL job" mod._test_forest(mqs, None, wait=True) # The wait=True argument causes the thing to not return until the job has been completed. # None for the second argument tells it that we're running headless (without a GUI parent.) ### Now for some variations # This shows how to use a different section in the config file. print "Recompiling and running for test_a" mod, ns, msg = Compile.compile_file(mqs, script, config="test_a") # and this shows how to locate and call a "TDL Job" by its job ID TDLOptions.get_job_func('job1')(mqs, None, wait=True) print "Recompiling and running for test_b" mod, ns, msg = Compile.compile_file(mqs, script, config="test_b") # we can also find jobs by their long names TDLOptions.get_job_func('Run job 2')(mqs, None, wait=True) # finally, this is expected to fail print "Trying to call a non-existing TDL job 'job3'" try: TDLOptions.get_job_func('job3')(mqs, None, wait=True) except NameError: print "Indeed, 'job3' does not exist." # This shows how to change configuration on-the-fly. # TDLOptions.set_option() changes the value of a configuration variable "in memory".
def run_forest_definition(mqs, filename, tdlmod, text, parent=None, wait=True, predef_args={}, define_args={}, postdef_args={}): """Compiles a TDL script and sends it to meqserver given by mqs. Parameters: mqs: a meqserver object (None to run without a meqserver) filename: the filename of the script (used for error reporting) tdlmod: the imported TDL module, as returned by import_tdl_module() text: script text for putting into forest parent: parent widget passed to TDL script (if a GUI is running) wait: if True, waits for forest to build before returning predef_args: dict of extra arguments for _tdl_predefine() define_args: dict of extra arguments for _define_forest() postdef_args: dict of extra arguments for _tdl_postdefine() Return value: a tuple of (module,ns,message), where module is the newly-imported TDL module, ns is a NodeScope object, and message is an informational message. Exceptions thrown: Any compilation error results in an exception. This is always a TDL.CumulativeError exception containing an error list. Other exceptions may indicate an internal failure. """ _dprint(1, "defining forest") try: TDLOptions.enable_save_config(False) ns = TDL.NodeScope() # module here, call functions errlist = [] # find define_forest func define_func = getattr(tdlmod, '_define_forest', None) if not callable(define_func): raise TDL.TDLError("No _define_forest() function found", filename=filename, lineno=1) # now find predefine function and call it predefine_func = getattr(tdlmod, '_tdl_predefine', None) if callable(predefine_func): predef_result = predefine_func(mqs, parent, **predef_args) _update_modlist() else: predef_result = None # inspect the define function to support older scripts that only # defined a define_forest(ns), i.e., with a single argument (fargs, fvarargs, fvarkw, fdefaults) = inspect.getargspec(define_func) if not fargs: raise TDL.TDLError( "invalid _define_forest() function: must have at least a single argument ('ns')", filename=filename, lineno=1) # function must have either a single argument, or allow keyword arguments if len(fargs) > 1 and not fvarkw: raise TDL.TDLError( "invalid _define_forest() function: must have a **kwargs parameter", filename=filename, lineno=1) # if no support for keyword arguments, pass an empty dict, else use valid dict if fvarkw: args = define_args.copy() args['parent'] = parent if isinstance(predef_result, dict): args.update(predef_result) else: args = {} # call the define function define_func(ns, **args) _update_modlist() # resolve the nodescope ns.Resolve() # do we have an error list? show it errlist = ns.GetErrors() if errlist: raise TDL.CumulativeError(*errlist) allnodes = ns.AllNodes() num_nodes = len(allnodes) # no nodes? return if not num_nodes: return ( tdlmod, ns, "TDL script successfully compiled, but no nodes were defined.") # try to run stuff if mqs is not None: meqds.clear_forest() # is a forest state defined? send it on then fst = getattr(Timba.TDL.Settings, 'forest_state', record()) # add in source code fst.tdl_source = record(**{os.path.basename(filename): text}) mqs.meq('Set.Forest.State', record(state=fst, get_forest_status=0)) if num_nodes: mqs.meq( 'Create.Node.Batch', record(script_name=os.path.basename(filename), batch=map(lambda nr: nr.initrec(), allnodes.itervalues()))) # mqs.meq('Init.Node.Batch',record(name=list(ns.RootNodes().iterkeys())),wait=wait); msg = """TDL script successfully compiled. %d node definitions (of which %d are root nodes) sent to meqserver.""" \ % (num_nodes,len(ns.RootNodes())) else: msg = "TDL script successfully compiled, but no nodes were defined." else: msg = "TDL script successfully compiled, %d nodes were defined." % num_nodes # call the post-define function postdefine_func = getattr(tdlmod, '_tdl_postdefine', None) if callable(postdefine_func): res = postdefine_func(mqs, parent, **postdef_args) _update_modlist() if isinstance(res, str): msg += "\n" + res TDLOptions.enable_save_config(True) TDLOptions.save_config() return (tdlmod, ns, msg) # CumulativeError exceptions returned as is except TDL.CumulativeError: _update_modlist() TDLOptions.enable_save_config(True) TDLOptions.save_config() _dprint(0, 'cumulative error compiling TDL script:', filename) traceback.print_exc() args = sys.exc_info()[1].args _dprint(0, 'number of errors in list:', len(args)) _dprint(1, 'errors are: {', args, '}') raise # Other exceptions wrapped in a CumulativeError, and # location information is added in except: _update_modlist() TDLOptions.enable_save_config(True) TDLOptions.save_config() (etype, exc, tb) = sys.exc_info() _dprint(0, 'exception defining forest from TDL file:', filename) traceback.print_exception(etype, exc, tb) # use TDL add_error() to process the error, since this automatically # adds location information ns.AddError(exc, traceback.extract_tb(tb), error_limit=None) # re-raise as a CumulativeError raise TDL.CumulativeError(*ns.GetErrors())
def main (): for optstr in (options.debug or []): opt = optstr.split("=") + ['1']; context,level = opt[:2]; debuglevels[context] = int(level); Timba.utils.verbosity.disable_argv(); # tell verbosity class to not parse its argv for optstr in (options.verbose or []): opt = optstr.split("=") + ['1']; context,level = opt[:2]; Timba.utils.verbosity.set_verbosity_level(context,level); if not args: parser.print_help(); sys.exit(1); if debuglevels: octopussy.set_debug(debuglevels); script = args[0]; tdljob = (len(args)>1 and args[1]) or None; from Timba.Apps import meqserver from Timba.TDL import Compile from Timba.TDL import TDLOptions # this starts a kernel. if options.compile_only: mqs = None; else: mqs = meqserver.default_mqs(wait_init=10,extra=["-mt",options.mt]); TDLOptions.config.read(options.config); TDLOptions.init_options(script); print "************************ Compiling TDL script",script; # this compiles a script as a TDL module. Any errors will be thrown as # and exception, so this always returns successfully (mod,ns,msg) = Compile.compile_file(mqs,script); if options.compile_only: print msg; sys.exit(0); # if a solve job is not specified, try to find one if tdljob: jobfunc = getattr(mod,tdljob,None); if not jobfunc: print "Cannot find TDL job named",tdljob; sys.exit(1); else: # does the script define an explicit job list? joblist = getattr(mod,'_tdl_job_list',[]); if not joblist: joblist = []; # try to build it from implicit function names for (name,func) in mod.__dict__.iteritems(): if name.startswith("_tdl_job_") and callable(func): joblist.append(func); # does the script define a testing function? testfunc = getattr(mod,'_test_forest',None); if testfunc: joblist.insert(0,testfunc); if not joblist: print "No TDL jobs found in script",script; sys.exit(1); jobfunc = joblist[0]; tdljob = jobfunc.__name__; # this runs the appropriate job. wait=True is needed to wait print "************************ Running TDL job",tdljob; # check if job takes a "wait" argument (fargs,fvarargs,fvarkw,fdefaults) = inspect.getargspec(jobfunc); if 'wait' in fargs or fvarkw: jobopts = dict(wait=True); else: jobopts = {}; jobfunc(mqs,None,**jobopts);
def import_tdl_module(filename, text=None, config=0): """Imports a TDL module. Parameters: filename: script location text: text of module. If none, file will be re-read. Return value: a tuple of (module,text), where module is the newly-imported TDL module, and text is the module text. Exceptions thrown: Any import error results in an exception. This is always a TDL.CumulativeError exception containing an error list. Other exceptions may indicate an internal failure. """ _dprint(1, "importing", filename) global _current_filename _current_filename = filename # initialize global nodescope (and repository) meqds.clear_forest() try: reload(Timba.TDL.Settings) # reset TDL script options, unless config=None ('0' is used as default, causing the filename to be substituted) TDLOptions.init_script(filename) if config is not None: TDLOptions.init_options(config or filename) # remove .pyo file as that can have cached paths and directories may have changed # (see bug 677) try: os.unlink(os.path.splitext(filename)[0] + '.pyo') except: pass # open file infile = file(filename, 'r') if text is None: text = infile.read() infile.seek(0) # infile is now an open input file object, and text is the script # flush all modules imported via previous TDL run global _tdlmodlist _dprint(1, 'clearing out TDL-imported modules', _tdlmodlist) for m in _tdlmodlist: try: del sys.modules[m] except KeyError: pass # remember which modules are imported global _prior_compile_modules _prior_compile_modules = set(sys.modules.iterkeys()) modname = '__tdlruntime' try: TDLOptions.enable_save_config(False) imp.acquire_lock() _tdlmod = imp.load_source(modname, filename, infile) finally: TDLOptions.enable_save_config(True) TDLOptions.save_config() imp.release_lock() infile.close() _update_modlist() return (_tdlmod, text) # CumulativeError exceptions returned as is except TDL.CumulativeError: _dprint(0, 'cumulative error importing TDL file:', filename) traceback.print_exc() args = sys.exc_info()[1].args _dprint(0, 'number of errors in list:', len(args)) _dprint(1, 'errors are: {', args, '}') raise # Other exceptions wrapped in a CumulativeError, and # location information is added in except: (etype, exc, tb) = sys.exc_info() _dprint(0, 'exception importing TDL file:', filename) traceback.print_exception(etype, exc, tb) # use TDL add_error() to process the error, since this automatically # adds location information ns = TDL.NodeScope() ns.AddError(exc, traceback.extract_tb(tb), error_limit=None) # re-raise as a CumulativeError raise TDL.CumulativeError(*ns.GetErrors())
for optstr in (options.debug or []): opt = optstr.split("=") + ['1'] context, level = opt[:2] # tell verbosity class to not parse argv -- we do it ourselves here Timba.utils.verbosity.disable_argv() for optstr in (options.verbose or []): opt = optstr.split("=") + ['1'] context, level = opt[:2] Timba.utils.verbosity.set_verbosity_level(context, int(level)) # start meqserver from Timba.Apps import meqserver from Timba.TDL import Compile from Timba.TDL import TDLOptions TDLOptions.enable_save_config(False) print("### Starting meqserver") mqs = meqserver.default_mqs( wait_init=10, extra=["-mt", str(options.mt)] + (["-python_memprof"] if options.memprof else [])) retcode = 0 # use a try...finally block to exit meqserver cleanly at the end try: if not os.path.exists(options.config): print(("Config file %s doesn't exist" % options.config)) sys.exit(1) print(("### Attaching to configuration file", options.config)) TDLOptions.config.read(options.config)
def run_forest_definition (mqs,filename,tdlmod,text, parent=None,wait=True, predef_args={},define_args={},postdef_args={}): """Compiles a TDL script and sends it to meqserver given by mqs. Parameters: mqs: a meqserver object (None to run without a meqserver) filename: the filename of the script (used for error reporting) tdlmod: the imported TDL module, as returned by import_tdl_module() text: script text for putting into forest parent: parent widget passed to TDL script (if a GUI is running) wait: if True, waits for forest to build before returning predef_args: dict of extra arguments for _tdl_predefine() define_args: dict of extra arguments for _define_forest() postdef_args: dict of extra arguments for _tdl_postdefine() Return value: a tuple of (module,ns,message), where module is the newly-imported TDL module, ns is a NodeScope object, and message is an informational message. Exceptions thrown: Any compilation error results in an exception. This is always a TDL.CumulativeError exception containing an error list. Other exceptions may indicate an internal failure. """; _dprint(1,"defining forest"); try: TDLOptions.enable_save_config(False); ns = TDL.NodeScope(); # module here, call functions errlist = []; # find define_forest func define_func = getattr(tdlmod,'_define_forest',None); if not callable(define_func): raise TDL.TDLError("No _define_forest() function found",filename=filename,lineno=1); # now find predefine function and call it predefine_func = getattr(tdlmod,'_tdl_predefine',None); if callable(predefine_func): predef_result = predefine_func(mqs,parent,**predef_args); _update_modlist(); else: predef_result = None; # inspect the define function to support older scripts that only # defined a define_forest(ns), i.e., with a single argument (fargs,fvarargs,fvarkw,fdefaults) = inspect.getargspec(define_func); if not fargs: raise TDL.TDLError("invalid _define_forest() function: must have at least a single argument ('ns')",filename=filename,lineno=1); # function must have either a single argument, or allow keyword arguments if len(fargs) > 1 and not fvarkw: raise TDL.TDLError("invalid _define_forest() function: must have a **kwargs parameter",filename=filename,lineno=1); # if no support for keyword arguments, pass an empty dict, else use valid dict if fvarkw: args = define_args.copy(); args['parent'] = parent; if isinstance(predef_result,dict): args.update(predef_result); else: args = {}; # call the define function define_func(ns,**args); _update_modlist(); # resolve the nodescope ns.Resolve(); # do we have an error list? show it errlist = ns.GetErrors(); if errlist: raise TDL.CumulativeError(*errlist); allnodes = ns.AllNodes(); num_nodes = len(allnodes); # no nodes? return if not num_nodes: return (tdlmod,ns,"TDL script successfully compiled, but no nodes were defined."); # try to run stuff if mqs is not None: meqds.clear_forest(); # is a forest state defined? send it on then fst = getattr(Timba.TDL.Settings,'forest_state',record()); # add in source code fst.tdl_source = record(**{os.path.basename(filename):text}); mqs.meq('Set.Forest.State',record(state=fst,get_forest_status=0)); if num_nodes: mqs.meq('Create.Node.Batch', record(script_name=os.path.basename(filename), batch=map(lambda nr:nr.initrec(),allnodes.itervalues()))); # mqs.meq('Init.Node.Batch',record(name=list(ns.RootNodes().iterkeys())),wait=wait); msg = """TDL script successfully compiled. %d node definitions (of which %d are root nodes) sent to meqserver.""" \ % (num_nodes,len(ns.RootNodes())); else: msg = "TDL script successfully compiled, but no nodes were defined."; else: msg = "TDL script successfully compiled, %d nodes were defined."%num_nodes; # call the post-define function postdefine_func = getattr(tdlmod,'_tdl_postdefine',None); if callable(postdefine_func): res = postdefine_func(mqs,parent,**postdef_args); _update_modlist(); if isinstance(res,str): msg += "\n" + res; TDLOptions.enable_save_config(True); TDLOptions.save_config(); return (tdlmod,ns,msg); # CumulativeError exceptions returned as is except TDL.CumulativeError: _update_modlist(); TDLOptions.enable_save_config(True); TDLOptions.save_config(); _dprint(0,'cumulative error compiling TDL script:',filename); traceback.print_exc(); args = sys.exc_info()[1].args; _dprint(0,'number of errors in list:',len(args)); _dprint(1,'errors are: {',args,'}'); raise; # Other exceptions wrapped in a CumulativeError, and # location information is added in except: _update_modlist(); TDLOptions.enable_save_config(True); TDLOptions.save_config(); (etype,exc,tb) = sys.exc_info(); _dprint(0,'exception defining forest from TDL file:',filename); traceback.print_exception(etype,exc,tb); # use TDL add_error() to process the error, since this automatically # adds location information ns.AddError(exc,traceback.extract_tb(tb),error_limit=None); # re-raise as a CumulativeError raise TDL.CumulativeError(*ns.GetErrors());
# or was interrupted by an exception midway through, we use a try...finally block. try: TDLOptions.config.read("batch_sim_example.tdl.conf") script = "example-sim.py" print("========== Compiling batch job 1") mod, ns, msg = Compile.compile_file(mqs, script, config="batch job 1") print("========== Running batch job 1") mod._tdl_job_1_simulate_MS(mqs, None, wait=True) print("========== Compiling batch job 2") mod, ns, msg = Compile.compile_file(mqs, script, config="batch job 2") print("========== Running batch job 2") mod._tdl_job_1_simulate_MS(mqs, None, wait=True) print("========== Compiling batch job 2 with modified config") TDLOptions.init_options("batch job 2", save=False) TDLOptions.set_option("me.enable_G", False) mod, ns, msg = Compile.compile_file(mqs, script, config=None) print("========== Running batch job 2") mod._tdl_job_1_simulate_MS(mqs, None, wait=True) ### Cleanup time finally: print("Stopping meqserver") # this halts the meqserver meqserver.stop_default_mqs() # now we can exit print("Bye!")
def import_tdl_module (filename,text=None,config=0): """Imports a TDL module. Parameters: filename: script location text: text of module. If none, file will be re-read. Return value: a tuple of (module,text), where module is the newly-imported TDL module, and text is the module text. Exceptions thrown: Any import error results in an exception. This is always a TDL.CumulativeError exception containing an error list. Other exceptions may indicate an internal failure. """; _dprint(1,"importing",filename); global _current_filename; _current_filename = filename; # initialize global nodescope (and repository) meqds.clear_forest(); try: reload(Timba.TDL.Settings); # reset TDL script options, unless config=None ('0' is used as default, causing the filename to be substituted) TDLOptions.init_script(filename); if config is not None: TDLOptions.init_options(config or filename); # remove .pyo file as that can have cached paths and directories may have changed # (see bug 677) try: os.unlink(os.path.splitext(filename)[0]+'.pyo'); except: pass; # open file infile = file(filename,'r'); if text is None: text = infile.read(); infile.seek(0); # infile is now an open input file object, and text is the script # flush all modules imported via previous TDL run global _tdlmodlist; _dprint(1,'clearing out TDL-imported modules',_tdlmodlist); for m in _tdlmodlist: try: del sys.modules[m]; except KeyError: pass; # remember which modules are imported global _prior_compile_modules; _prior_compile_modules = set(sys.modules.iterkeys()); modname = '__tdlruntime'; try: TDLOptions.enable_save_config(False); imp.acquire_lock(); _tdlmod = imp.load_source(modname,filename,infile); finally: TDLOptions.enable_save_config(True); TDLOptions.save_config(); imp.release_lock(); infile.close(); _update_modlist(); return (_tdlmod,text); # CumulativeError exceptions returned as is except TDL.CumulativeError: _dprint(0,'cumulative error importing TDL file:',filename); traceback.print_exc(); args = sys.exc_info()[1].args; _dprint(0,'number of errors in list:',len(args)); _dprint(1,'errors are: {',args,'}'); raise; # Other exceptions wrapped in a CumulativeError, and # location information is added in except: (etype,exc,tb) = sys.exc_info(); _dprint(0,'exception importing TDL file:',filename); traceback.print_exception(etype,exc,tb); # use TDL add_error() to process the error, since this automatically # adds location information ns = TDL.NodeScope(); ns.AddError(exc,traceback.extract_tb(tb),error_limit=None); # re-raise as a CumulativeError raise TDL.CumulativeError(*ns.GetErrors());
def testMeqtreesBatchJob(): trace_sync = True # sys.settrace(trace_lines); if len(sys.argv) > 1: newdir = PACKAGE_TEST_DIR print("========== Changing working directory to", newdir) os.chdir(newdir) print("========== Making required symlinks") run("rm {0:s}/WSRT_ANTENNA ; ln -s {1:s}".format( PACKAGE_TEST_DIR, path(os.path.join(PACKAGE_TEST_DIR, "WSRT_ANTENNA")))) run("rm {0:s}/test-lsm.txt; ln -s {1:s}".format( PACKAGE_TEST_DIR, path(os.path.join(path("test-lsm.txt"))))) if not os.access(".", os.R_OK | os.W_OK): print("Directory", os.getcwd(), "not writable, can't run tests in here.") print( "You may choose to run the tests in a different directory by giving it as an argument to this script." ) sys.exit(1) ## check if we have owlcat or owlcat.sh owlcat = "" for dirname in os.environ['PATH'].split(':'): for binary in "owlcat", "owlcat.sh": tmp = os.path.join(dirname, binary) if os.path.exists(tmp): owlcat = tmp break if owlcat: break if not owlcat: raise RuntimeError("Can't locate owlcat or owlcat.sh") ## make simulated MS print("========== Removing files") run("rm -fr {0:s}/WSRT.MS* {0:s}/WSRT*img {0:s}/WSRT*fits".format( PACKAGE_TEST_DIR)) print("========== Running makems") run("makems %s" % path(os.path.join(PACKAGE_TEST_DIR, "WSRT_makems.cfg"))) run("mv {0:s}/WSRT.MS_p0 {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR)) os.environ["MEQTREES_CATTERY_PATH"] = Cattery.__path__[0] run("pyxis {0:s}/WSRT.MS ms.prep".format(PACKAGE_TEST_DIR)) #TODO: this is hacky, bug in CASAcore run("ls -ld {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR)) run("{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS".format( owlcat, PACKAGE_TEST_DIR)) run("lwimager ms={0:s}/WSRT.MS data=CORRECTED_DATA mode=channel weight=natural npix=10" .format(PACKAGE_TEST_DIR)) # make test LSMs run("""tigger-convert {0:s}/test-lsm.txt --rename --format "ra_d dec_d i q u v" --center 0.1deg,60.5deg -f""" .format(PACKAGE_TEST_DIR)) run("""tigger-convert {0:s}/test-lsm.lsm.html {0:s}/test-lsm1.txt --output-format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""" .format(PACKAGE_TEST_DIR)) run("""cut -d " " -f 1-10 {0:s}/test-lsm1.txt >{0:s}/test-lsm1.txt.tmp""". format(PACKAGE_TEST_DIR)) run("""diff {0:s}/test-lsm1.txt.tmp {1:s} || diff {0:s}/test-lsm1.txt.tmp {2:s}""" .format( PACKAGE_TEST_DIR, path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm1.txt.reference')), path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm2.txt.reference')))) run("""tigger-convert {0:s}/test-lsm1.txt --format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""" .format(PACKAGE_TEST_DIR)) run("""{0:s} plot-ms {1:s}/WSRT.MS DATA:I -o data_i.png""".format( owlcat, PACKAGE_TEST_DIR)) run("""{0:s} run-imager ms={1:s}/WSRT.MS name_dirty=tmp""".format( owlcat, PACKAGE_TEST_DIR)) print("importing meqserver") from Timba.Apps import meqserver print("importing Compile") from Timba.TDL import Compile print("importing TDLOptions") from Timba.TDL import TDLOptions # This starts a meqserver. Note how we pass the "-mt 2" option to run two threads. # A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv). print("Starting meqserver") mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", "2"]) try: ## make simulation with perfect MODEL_DATA script = path(os.path.join(PACKAGE_TEST_DIR, "sim.py")) print("========== Compiling", script) TDLOptions.config.read( path(os.path.join(PACKAGE_TEST_DIR, "testing.tdl.conf"))) TDLOptions.config.set( "calibrate", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "ms_sel.msname"))) TDLOptions.config.set( "calibrate", "tiggerlsm.filename", os.path.join( PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "tiggerlsm.filename"))) TDLOptions.config.set( "calibrate", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "lsm.filename"))) TDLOptions.config.set( "calibrate", "cal_g_diag.g_diag.table_name", os.path.join( PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_diag.g_diag.table_name"))) TDLOptions.config.set( "calibrate", "cal_g_offdiag.g_offdiag.table_name", os.path.join( PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_offdiag.g_offdiag.table_name"))) TDLOptions.config.set( "simulate-model", "lsm.filename", os.path.join( PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "lsm.filename"))) TDLOptions.config.set( "simulate-model", "ms_sel.msname", os.path.join( PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "ms_sel.msname"))) TDLOptions.config.set( "simulate-model", "tiggerlsm.filename", os.path.join( PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "tiggerlsm.filename"))) TDLOptions.config.set( "calibrate", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits")) TDLOptions.config.set( "simulate-model", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits")) with open(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"), "w") as f: TDLOptions.config.write(f) TDLOptions.config.read( path(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"))) # needs to re-read because of a Timba perculiarity mod, ns, msg = Compile.compile_file(mqs, script, config="simulate-model") print("========== Simulating MODEL_DATA ") mod._tdl_job_1_simulate_MS(mqs, None, wait=True) print("========== Imaging MODEL_DATA ") TDLOptions.get_job_func('make_dirty_image')(mqs, None, wait=True, run_viewer=False) ## compare against reference image print("========== Verifying test image ") if not os.path.exists( os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist") if not os.path.exists( os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")): raise RuntimeError("Reference FITS file does not exist") verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits"), path(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")), maxdelta=1e-3) print("========== Compiling script with modified config") TDLOptions.init_options("simulate-model", save=False) TDLOptions.set_option("me.g_enable", True) mod, ns, msg = Compile.compile_file(mqs, script, config=None) print("========== Simulating DATA ") TDLOptions.set_option("ms_sel.output_column", "DATA") mod._tdl_job_1_simulate_MS(mqs, None, wait=True) print("========== Imaging DATA ") TDLOptions.set_option("img_sel.imaging_column", "DATA") TDLOptions.get_job_func('make_dirty_image')(mqs, None, wait=True, run_viewer=False) ## calibrate script = path(os.path.join(PACKAGE_TEST_DIR, "cal.py")) print("========== Compiling", script) mod, ns, msg = Compile.compile_file(mqs, script, config="calibrate") print("========== Calibrating ") TDLOptions.get_job_func('cal_G_diag')(mqs, None, wait=True) print("========== Imaging MODEL_DATA ") TDLOptions.get_job_func('make_dirty_image')(mqs, None, wait=True, run_viewer=False) finally: print("Stopping meqserver") # this halts the meqserver meqserver.stop_default_mqs() print("========== Making plots of solutions ") run("""{0:s} plot-ms {1:s}/WSRT.MS CORRECTED_DATA:I -I ">0" -o {1:s}/corrected_data_i.png""" .format(owlcat, PACKAGE_TEST_DIR)) run("""{0:s} plot-parms -l {1:s}/WSRT.MS/G_diag.fmep""".format( owlcat, PACKAGE_TEST_DIR)) run("""{0:s} plot-parms {1:s}/WSRT.MS/G_diag.fmep G:*/norm -o {1:s}/parmplot.png""" .format(owlcat, PACKAGE_TEST_DIR)) run("""{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS""".format( owlcat, PACKAGE_TEST_DIR)) ## compare against reference image print("========== Verifying residual image ") if not os.path.exists( os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist") if not os.path.exists( os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits")): raise RuntimeError("Reference FITS file does not exist") verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits"), os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits"), maxdelta=1e-3) ## all tests succeeded print("========== Break out the bubbly, this hog is airborne!") # now we can exit print("Bye!")
def import_content(self, force=False, show_options=False): """imports TDL module but does not run _define_forest(). Depending on autosync/modified state, asks to save or revert. If module is already imported, does nothing, unless force=True, in which case it imports unconditionally. If do_compile=True, proceeds to show compile-time options on success, or to compile directly if there are no options Return value: True on successful import None if cancelled by user. Import errors are posted to the error dialog. """ # save list of publishers (since forest will be cleared on import) if self._pub_nodes is None: if TDL.Compile.last_imported_file() == self._filename: self._pub_nodes = [ node.name for node in meqds.nodelist.iternodes() if node.is_publishing() ] _dprint(2, "last imported file matches, saving", len(self._pub_nodes), "publishers") else: self._pub_nodes = [] _dprint(2, "last imported file doesn't match:", TDL.Compile.last_imported_file(), self._filename) _dprint(1, self._filename, "importing") self.clear_message() self.clear_errors() self._options_menu.hide() self._tb_tdlexec.hide() self._tdlexec_menu.hide() # change the current directory to where the file is # os.chdir(os.path.dirname(self._filename)); # The Python imp module expects text to reside in a disk file, which is # a pain in the ass for us if we're dealing with modified text or text # entered on-the-fly. So, either save or sync before proceeding global _external_sync if self._document.isModified() or not self._filename: if not self._save_file(): return None else: if not self.sync_external_file(ask=False): return None # if we already have an imported module and disk file hasn't changed, skip # the importing step busy = BusyIndicator() if force or self._tdlmod is None or self._tdlmod_filetime == self._file_disktime: # reset data members _dprint(2, self._filename, "emitting signal for 0 compile-time options") self.emit(PYSIGNAL("hasCompileOptions()"), self, 0) self._options_menu.hide() self._options_menu.clear() self._tdlmod = None # get text from editor tdltext = str(self._document.toPlainText()) try: tdlmod, tdltext = TDL.Compile.import_tdl_module( self._filename, tdltext) # catch import errors except TDL.CumulativeError as value: _dprint(0, "caught cumulative error, length", len(value.args)) self._error_window.set_errors(value.args, message="TDL import failed") busy = None return None except Exception as value: _dprint(0, "caught other error, traceback follows") traceback.print_exc() self._error_window.set_errors([value], message="TDL import failed") busy = None return None # remember module and nodescope self._tdlmod = tdlmod self._tdltext = tdltext self._tdlmod_filetime = self._file_disktime # build options menu from compile-time options opt_tw = self._options_menu.treeWidget() opts = TDLOptions.get_compile_options() if opts: # add options try: TDLOptions.populate_option_treewidget(opt_tw, opts) except Exception as value: _dprint(0, "error setting up TDL options GUI") traceback.print_exc() self._error_window.set_errors( [value], message="Error setting up TDL options GUI") busy = None return None # self._tb_opts.show(); _dprint(2, self._filename, "emitting signal for", len(opts), "compile-time options") self.emit(PYSIGNAL("hasCompileOptions()"), self, len(opts)) # success, show options or compile if show_options and self.has_compile_options(): self._options_menu.adjustSizes() self._options_menu.show() busy = None return True
context,level = opt[:2]; debuglevels[context] = int(level); # tell verbosity class to not parse argv -- we do it ourselves here Timba.utils.verbosity.disable_argv(); for optstr in (options.verbose or []): opt = optstr.split("=") + ['1']; context,level = opt[:2]; Timba.utils.verbosity.set_verbosity_level(context,int(level)); # start meqserver from Timba.Apps import meqserver from Timba.TDL import Compile from Timba.TDL import TDLOptions TDLOptions.enable_save_config(False); print "### Starting meqserver"; mqs = meqserver.default_mqs(wait_init=10,extra=["-mt",str(options.mt)]); retcode = 0; # use a try...finally block to exit meqserver cleanly at the end try: if not os.path.exists(options.config): print "Config file %s doesn't exist"%options.config; sys.exit(1); print "### Attaching to configuration file",options.config; TDLOptions.config.read(options.config); # disable the writing-out of configuration TDLOptions.config.set_save_filename(None);
_dprint(2, "not reenabling publishing of", name, ", as it no longer exists?") # clear publisher list so it's re-saved next time in import_content() self._pub_nodes = None ### NB: presume this all was successful for now # insert node scope into TDL module setattr(_tdlmod, '_tdl_nodescope', ns) # does the script define an explicit job list? joblist = getattr(_tdlmod, '_tdl_job_list', []) if not joblist: joblist = [] # try to build it from implicit function names for (name, func) in _tdlmod.__dict__.iteritems(): if name.startswith("_tdl_job_") and callable( func) and not TDLOptions.is_jobfunc_defined(func): joblist.append(func) # does the script define a testing function? testfunc = getattr(_tdlmod, '_test_forest', None) if not callable(testfunc): testfunc = getattr(_tdlmod, 'test_forest', None) if callable(testfunc): res = QMessageBox.warning( self, "Deprecated method", """Your script contains a test_forest() method. This is deprecated and will be disabled in the future. Please rename it to _test_forest(). """, QMessageBox.Ok) if callable(testfunc): joblist.append(testfunc) joblist.sort(lambda a, b: cmp(str(a), str(b)))