def execute_tdl_job(self, _tdlmod, ns, func, name, job_id):
"""executes a predefined TDL job given by func"""
self._tdlexec_menu.hide()
try:
# log job
TDLOptions.dump_log(
"running TDL job '%s' (%s)" %
(name, ("job id '%s'" % job_id if job_id else "no job id")))
busy = BusyIndicator()
func(meqds.mqs(), self)
# no errors, so clear error list, if any
self.clear_errors()
self.show_message("TDL job '" + name + "' executed.",
transient=True)
busy = None
except:
(etype, exc, tb) = sys.exc_info()
_dprint(0, 'exception running TDL job', func.__name__)
traceback.print_exception(etype, exc, tb)
# use TDL add_error() to process the error, since this automatically
# adds location information. However, we want to remove ourselves
# from the stack traceback first
tb = traceback.extract_tb(tb)
# pop everything leading up to our filename
while tb[0][0] != _MODULE_FILENAME:
tb.pop(0)
# pop frame with our filename, this should leave only TDL-code frames
tb.pop(0)
ns.AddError(exc, tb, error_limit=None)
msg = "TDL job '" + name + "' failed"
self._error_window.set_errors(ns.GetErrors(), message=msg)
self.emit(PYSIGNAL("showEditor()"), self)
busy = None
def _save_profile(self, name):
if self.profiles.has_section(name):
if QMessageBox.warning(
self, "Overwriting profile", """<P>Do you really want to
overwrite the existing profile '%s'?</P>""" % name,
QMessageBox.Ok | QMessageBox.Cancel,
QMessageBox.Ok) != QMessageBox.Ok:
return
newprof = False
else:
self.profiles.add_section(name)
newprof = True
TDLOptions.save_to_config(self.profiles, name)
try:
ff = file(PROFILE_FILE, "wt")
for section in sorted(self.profiles.sections()):
ff.write("[%s]\n" % section)
for opt, value in sorted(self.profiles.items(section)):
ff.write("%s = %s\n" % (opt.lower(), value))
ff.write("\n")
ff.close()
except:
dprintf(0, "error writing %s" % PROFILE_FILE)
traceback.print_exc()
QMessageBox.warning(
self, "Error saving profile",
"""<P>There was an error writing to the %s file,
profile was not saved.</P>""" % PROFILE_FILE)
if newprof:
self.emit(PYSIGNAL("refreshProfiles()"))
def _save_profile (self,name):
if self.profiles.has_section(name):
if QMessageBox.warning(self,"Overwriting profile","""<P>Do you really want to
overwrite the existing profile '%s'?</P>"""%name,
QMessageBox.Ok|QMessageBox.Cancel,QMessageBox.Ok) != QMessageBox.Ok:
return;
newprof = False;
else:
self.profiles.add_section(name);
newprof = True;
TDLOptions.save_to_config(self.profiles,name);
try:
ff = file(PROFILE_FILE,"wt");
for section in sorted(self.profiles.sections()):
ff.write("[%s]\n"%section);
for opt,value in sorted(self.profiles.items(section)):
ff.write("%s = %s\n"%(opt.lower(),value));
ff.write("\n");
ff.close();
except:
dprintf(0,"error writing %s"%PROFILE_FILE);
traceback.print_exc();
QMessageBox.warning(self,"Error saving profile","""<P>There was an error writing to the %s file,
profile was not saved.</P>"""%PROFILE_FILE);
if newprof:
self.emit(PYSIGNAL("refreshProfiles()"));
def execute_tdl_job (self,_tdlmod,ns,func,name,job_id):
"""executes a predefined TDL job given by func""";
self._tdlexec_menu.hide();
try:
# log job
TDLOptions.dump_log("running TDL job '%s' (%s)"%(name,("job id '%s'"%job_id if job_id else "no job id")));
busy = BusyIndicator();
func(meqds.mqs(),self);
# no errors, so clear error list, if any
self.clear_errors();
self.show_message("TDL job '"+name+"' executed.",transient=True);
busy = None;
except:
(etype,exc,tb) = sys.exc_info();
_dprint(0,'exception running TDL job',func.__name__);
traceback.print_exception(etype,exc,tb);
# use TDL add_error() to process the error, since this automatically
# adds location information. However, we want to remove ourselves
# from the stack traceback first
tb = traceback.extract_tb(tb);
# pop everything leading up to our filename
while tb[0][0] != _MODULE_FILENAME:
tb.pop(0);
# pop frame with our filename, this should leave only TDL-code frames
tb.pop(0);
ns.AddError(exc,tb,error_limit=None);
msg = "TDL job '"+name+"' failed";
self._error_window.set_errors(ns.GetErrors(),message=msg);
self.emit(PYSIGNAL("showEditor()"),self);
busy = None;
def _load_profile (self,name):
if not self.profiles.has_section(name):
QMessageBox.warning(self,"Error loading profile","""<P>Profile '%s' not found,
check your %s file.</P>"""%(name,PROFILE_FILE));
return;
for name,value in self.profiles.items(name):
try:
TDLOptions.set_option(name,value,save=False,from_str=True);
except ValueError:
dprintf(0,"can't set '%s' to %s, ignoring",name,value);
TDLOptions.save_config();
def _load_profile (self,name):
if not self.profiles.has_section(name):
QMessageBox.warning(self,"Error loading profile","""<P>Profile '%s' not found,
check your %s file.</P>"""%(name,PROFILE_FILE));
return;
for name,value in self.profiles.items(name):
try:
TDLOptions.set_option(name,value,save=False,from_str=True);
except ValueError:
dprintf(0,"can't set '%s' to %s, ignoring",name,value);
TDLOptions.save_config();
def compile_script(self, need_mqs=False):
"""compiles the script, if not already compiled.
If need_mqs=True, starts a meqserver and builds the tree as well"""
if not self._module:
self._compile_failed = True
# will reset to False if all goes well\
self.log_progress("compile")
# start meqserver if required
if (need_mqs or int(self.get_option("start_meqserver",
0))) and not self._mqs:
from Timba.Apps import meqserver
# get multithreading option
mt = int(self.get_option("multithreaded", 0))
if mt > 1:
extra = ["-mt", str(mt)]
else:
extra = []
_dprint(1, "starting meqserver", extra)
self._mqs = meqserver.default_mqs(wait_init=10, extra=extra)
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
# load config file
tdlconf = self.get_option("tdlconf", ".tdl.conf")
TDLOptions.config.read(tdlconf)
TDLOptions.init_options(self.name, save=False)
# compile TDL module
_dprint(1, "compiling TDL script", self.name)
try:
(self._module, ns,
msg) = Compile.compile_file(self._mqs, self.name)
except:
excinfo = sys.exc_info()
self.log_exc(level=2, *excinfo)
self.fail("compile failed")
excinfo = None
return
# success
self.log(msg, level=2)
self.success("compile")
self._compile_failed = False
pass
def compile_script (self,need_mqs=False):
"""compiles the script, if not already compiled.
If need_mqs=True, starts a meqserver and builds the tree as well""";
if not self._module:
self._compile_failed = True; # will reset to False if all goes well\
self.log_progress("compile");
# start meqserver if required
if ( need_mqs or int(self.get_option("start_meqserver",0)) ) and not self._mqs:
from Timba.Apps import meqserver
# get multithreading option
mt = int(self.get_option("multithreaded",0));
if mt>1:
extra = [ "-mt",str(mt) ];
else:
extra = []
_dprint(1,"starting meqserver",extra);
self._mqs = meqserver.default_mqs(wait_init=10,extra=extra);
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
# load config file
tdlconf = self.get_option("tdlconf",".tdl.conf");
TDLOptions.config.read(tdlconf);
TDLOptions.init_options(self.name,save=False);
# compile TDL module
_dprint(1,"compiling TDL script",self.name);
try:
(self._module,ns,msg) = Compile.compile_file(self._mqs,self.name);
except:
excinfo = sys.exc_info();
self.log_exc(level=2,*excinfo);
self.fail("compile failed");
excinfo = None;
return;
# success
self.log(msg,level=2);
self.success("compile");
self._compile_failed = False;
pass;
mod,ns,msg = Compile.compile_file(mqs,script);
# 'mod' now refers to a Python object that is the compiled module. To execute the _test_forest job within that
# module, we do as follows:
print "Running TDL job";
mod._test_forest(mqs,None,wait=True);
# The wait=True argument causes the thing to not return until the job has been completed.
# None for the second argument tells it that we're running headless (without a GUI parent.)
### Now for some variations
# This shows how to use a different section in the config file.
print "Recompiling and running for test_a";
mod,ns,msg = Compile.compile_file(mqs,script,config="test_a");
# and this shows how to locate and call a "TDL Job" by its job ID
TDLOptions.get_job_func('job1')(mqs,None,wait=True);
print "Recompiling and running for test_b";
mod,ns,msg = Compile.compile_file(mqs,script,config="test_b");
# we can also find jobs by their long names
TDLOptions.get_job_func('Run job 2')(mqs,None,wait=True);
# finally, this is expected to fail
print "Trying to call a non-existing TDL job 'job3'";
try:
TDLOptions.get_job_func('job3')(mqs,None,wait=True);
except NameError:
print "Indeed, 'job3' does not exist."
# This shows how to change configuration on-the-fly.
# TDLOptions.set_option() changes the value of a configuration variable "in memory".
def compile_content(self):
# import content first, and return if failed
if not self.import_content(force=True):
return None
_dprint(1, self._filename, "compiling forest")
# clear predefined functions
self._tb_tdlexec.hide()
# try the compilation
busy = BusyIndicator()
try:
(_tdlmod,ns,msg) = \
TDL.Compile.run_forest_definition(
meqds.mqs(),self._filename,self._tdlmod,self._tdltext,
parent=self,wait=False)
# catch compilation errors
except TDL.CumulativeError as value:
_dprint(0, "caught cumulative error, length", len(value.args))
self._error_window.set_errors(value.args,
message="TDL import failed")
busy = None
return None
except Exception as value:
_dprint(0, "caught other error, traceback follows")
traceback.print_exc()
self._error_window.set_errors([value])
busy = None
return None
# refresh the nodelist
meqds.request_nodelist(sync=True)
# restore publishing nodes
for name in (self._pub_nodes or []):
if name in ns.AllNodes():
_dprint(2, "reenabling publishing of", name)
meqds.enable_node_publish_by_name(name,
sync=True,
get_state=True)
else:
_dprint(2, "not reenabling publishing of", name,
", as it no longer exists?")
# clear publisher list so it's re-saved next time in import_content()
self._pub_nodes = None
### NB: presume this all was successful for now
# insert node scope into TDL module
setattr(_tdlmod, '_tdl_nodescope', ns)
# does the script define an explicit job list?
joblist = getattr(_tdlmod, '_tdl_job_list', [])
if not joblist:
joblist = []
# try to build it from implicit function names
for (name, func) in _tdlmod.__dict__.items():
if name.startswith("_tdl_job_") and callable(
func) and not TDLOptions.is_jobfunc_defined(func):
joblist.append(func)
# does the script define a testing function?
testfunc = getattr(_tdlmod, '_test_forest', None)
if not callable(testfunc):
testfunc = getattr(_tdlmod, 'test_forest', None)
if callable(testfunc):
res = QMessageBox.warning(
self, "Deprecated method",
"""Your script contains a test_forest() method. This is deprecated
and will be disabled in the future. Please rename it to
_test_forest().
""", QMessageBox.Ok)
if callable(testfunc):
joblist.append(testfunc)
from past.builtins import cmp
from functools import cmp_to_key
joblist.sort(key=cmp_to_key(lambda a, b: cmp(str(a), str(b))))
# create list of job actions
opts = TDLOptions.get_runtime_options()
self._tdlexec_menu.clear()
if joblist or opts:
if opts:
self._job_executor = curry(self.execute_tdl_job, _tdlmod, ns)
## new style:
try:
TDLOptions.populate_option_treewidget(
self._tdlexec_menu.treeWidget(),
opts,
executor=self._job_executor)
except Exception as value:
_dprint(0, "error setting up TDL options GUI")
traceback.print_exc()
self._error_window.set_errors(
[value], message="Error setting up TDL options GUI")
busy = None
return None
if joblist:
for func in joblist:
name = re.sub("^_tdl_job_", "", func.__name__)
name = name.replace('_', ' ')
self._tdlexec_menu.addAction(name,
curry(self.execute_tdl_job,
_tdlmod, ns, func, name,
func.__name__),
icon=pixmaps.gear)
self.emit(PYSIGNAL("hasRuntimeOptions()"), self, True)
self._tdlexec_menu.adjustSizes()
self._tb_tdlexec.show()
else:
self.emit(PYSIGNAL("hasRuntimeOptions()"), self, False)
self._tb_tdlexec.hide()
if joblist:
msg += " %d predefined function(s) available, please use the Exec menu to run them." % (
len(joblist), )
self.show_message(msg, transient=True)
busy = None
return True
class TDLEditor(QFrame, PersistentCurrier):
SupportsLineNumbers = False
# a single editor always has the focus
current_editor = None
# flag: sync to external editor
external_sync = True
def set_external_sync(value):
global _external_sync
_external_sync = value
set_external_sync = staticmethod(set_external_sync)
def __init__(self, parent, close_button=False, error_window=None):
QFrame.__init__(self, parent)
self._enabled = True
toplo = QVBoxLayout(self)
toplo.setContentsMargins(0, 0, 0, 0)
toplo.setSpacing(0)
self.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding)
splitter = QSplitter(Qt.Vertical, self)
toplo.addWidget(splitter)
splitter.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding)
splitter.setChildrenCollapsible(False)
# figure out our app_gui parent
self._appgui = app_proxy_gui.appgui(parent)
# create an editor box
editor_box = QFrame(splitter)
lo = QVBoxLayout(editor_box)
lo.setContentsMargins(0, 0, 0, 0)
lo.setSpacing(0)
# find main window to associate our toolbar with
mainwin = parent
while mainwin and not isinstance(mainwin, QMainWindow):
mainwin = mainwin.parent()
self._toolbar = QToolBar(mainwin or self)
lo.addWidget(self._toolbar)
self._toolbar.setIconSize(QSize(16, 16))
#### populate toolbar
# Exec button and menu
self._tb_tdlexec = QToolButton(self._toolbar)
self._toolbar.addWidget(self._tb_tdlexec)
self._tb_tdlexec.setIcon(pixmaps.gear.icon())
self._tb_tdlexec.setText("Exec")
self._tb_tdlexec.setToolButtonStyle(Qt.ToolButtonTextBesideIcon)
self._tb_tdlexec.setToolTip(
"Accesses run-time options & jobs defined by this TDL script")
self._tb_tdlexec.hide()
jobs = self._tdlexec_menu = TDLOptionsDialog(self)
jobs.setWindowTitle("TDL Jobs & Runtime Options")
jobs.setWindowIcon(pixmaps.gear.icon())
jobs.hide()
QObject.connect(self._tb_tdlexec, SIGNAL("clicked()"), jobs.exec_)
# save menu and button
self._tb_save = QToolButton(self._toolbar)
self._toolbar.addWidget(self._tb_save)
self._tb_save.setIcon(pixmaps.file_save.icon())
self._tb_save.setToolTip(
"Saves script. Click on the down-arrow for other options.")
savemenu = QMenu(self)
self._tb_save.setMenu(savemenu)
self._tb_save.setPopupMode(QToolButton.MenuButtonPopup)
self._tb_save._modified_color = QColor("yellow")
qa_save = savemenu.addAction(pixmaps.file_save.icon(), "&Save script",
self._save_file)
qa_save.setShortcut(Qt.ALT + Qt.Key_S)
QObject.connect(self._tb_save, SIGNAL("clicked()"), self._save_file)
qa_save_as = savemenu.addAction(
pixmaps.file_save.icon(), "Save script &as...",
self.curry(self._save_file, save_as=True))
qa_revert = self._qa_revert = savemenu.addAction(
"Revert to saved", self._revert_to_saved)
# run menu and button
self._qa_run = self._toolbar.addAction(
pixmaps.blue_round_reload.icon(), "&Save & reload",
self._import_main_file)
self._qa_run.setShortcut(Qt.ALT + Qt.Key_R)
# Compile-time options and menu
self._tb_opts = QToolButton(self._toolbar)
self._toolbar.addWidget(self._tb_opts)
self._tb_opts.setIcon(pixmaps.wrench.icon())
self._tb_opts.setText("Options")
self._tb_opts.setToolButtonStyle(Qt.ToolButtonTextBesideIcon)
self._tb_opts.setToolTip(
"Accesses compile-time options for this TDL script")
# self._tb_opts.hide();
opts = self._options_menu = TDLOptionsDialog(
self, ok_label="Compile", ok_icon=pixmaps.blue_round_reload)
opts.setWindowTitle("TDL Compile-time Options")
opts.setWindowIcon(pixmaps.wrench.icon())
QObject.connect(opts, PYSIGNAL("accepted()"), self._compile_main_file)
QObject.connect(TDLOptions.OptionObject, SIGNAL("mandatoryOptionsSet"),
self.mark_mandatory_options_set)
opts.hide()
QObject.connect(self._tb_opts, SIGNAL("clicked()"), opts.show)
# cross-connect the rereshProfiles() signals, so that dialogs can update each other's
# profile menus
QObject.connect(self._options_menu, PYSIGNAL("refreshProfiles()"),
self._tdlexec_menu.refresh_profiles)
QObject.connect(self._tdlexec_menu, PYSIGNAL("refreshProfiles()"),
self._options_menu.refresh_profiles)
self._toolbar.addSeparator()
# cursor position indicator
self._poslabel = QLabel(self._toolbar)
#wa = QWidgetAction(self._toolbar);
#wa.setDefaultWidget(self._poslabel);
#self._toolbar.addAction(wa);
#self._toolbar.addWidget(self._poslabel);
self._toolbar.addWidget(self._poslabel)
width = self._poslabel.fontMetrics().width("L:999 C:999")
self._poslabel.setMinimumWidth(width)
self._poslabel.setText("L:0 C:0")
# filename indicator
self._pathlabel = QLabel(self._toolbar)
#wa = QWidgetAction(self._toolbar);
#wa.setDefaultWidget(self._pathlabel);
self._toolbar.addWidget(self._pathlabel)
#self._toolbar.addAction(wa);
self._pathlabel.show()
self._pathlabel.setAlignment(Qt.AlignRight | Qt.AlignVCenter)
self._pathlabel.setIndent(10)
self._pathlabel.setSizePolicy(QSizePolicy.Expanding,
QSizePolicy.Minimum)
if close_button:
if not isinstance(close_button, QIcon):
close_button = pixmaps.remove.icon()
self._qa_close = self._toolbar.addAction(close_button,
"&Close file",
self._file_closed)
self._qa_close.setShortcut(Qt.ALT + Qt.Key_W)
self._pathlabel.setText("(no file)")
#### add editor window
self._editor = editor = QTextEdit(editor_box)
lo.addWidget(self._editor)
editor.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding)
editor.setAcceptRichText(False)
editor.setLineWrapMode(QTextEdit.NoWrap)
self._document = QTextDocument(self)
editor.setDocument(self._document)
QObject.connect(self._document, SIGNAL("modificationChanged(bool)"),
self._text_modified)
QObject.connect(self._editor, SIGNAL("cursorPositionChanged()"),
self._display_cursor_position)
# QObject.connect(self._editor,SIGNAL("textChanged()"),self._clear_transients);
# add character formats for error display
self._format_error = QTextCharFormat(self._editor.currentCharFormat())
self._format_error.setBackground(QBrush(QColor("lightpink")))
self._format_suberror = QTextCharFormat(
self._editor.currentCharFormat())
self._format_suberror.setBackground(QBrush(QColor("lightgrey")))
self._format_current_error = QTextCharFormat(
self._editor.currentCharFormat())
self._format_current_error.setBackground(QBrush(QColor("orangered")))
# add message window
self._message_box = QFrame(editor_box)
lo.addWidget(self._message_box)
self._message_box.setSizePolicy(QSizePolicy.Expanding,
QSizePolicy.Preferred)
self._message_box.setFrameStyle(QFrame.Panel + QFrame.Sunken)
self._message_box.setLineWidth(2)
mblo = QVBoxLayout(self._message_box)
mblo.setContentsMargins(0, 0, 0, 0)
msgb1 = QHBoxLayout()
mblo.addLayout(msgb1)
msgb1.setContentsMargins(0, 0, 0, 0)
msgb1.setSpacing(0)
self._message_icon = QToolButton(self._message_box)
msgb1.addWidget(self._message_icon)
self._message = QLabel(self._message_box)
msgb1.addWidget(self._message)
self._message.setTextFormat(Qt.RichText)
self._message.setWordWrap(True)
self._message.setMargin(0)
self._message.setSizePolicy(QSizePolicy.Expanding,
QSizePolicy.Preferred)
# self._message_icon.setAlignment(Qt.AlignTop);
# self._message_icon.setMargin(4);
self._message_icon.setAutoRaise(True)
self._message_icon.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
QObject.connect(self._message_icon, SIGNAL("clicked()"),
self.clear_message)
self._message_icon.setToolTip("Click here to clear the message")
self._message_widgets = []
self._message_transient = False
# figure out if we already have an error box to attach to
self._error_window = error_window or getattr(
parent, '_tdlgui_error_window', None)
if self._error_window:
#self._resize_errwin = False;
pass
else:
# otherwise create an error floater
self._error_window = TDLErrorFloat(parent)
setattr(parent, '_tdlgui_error_window', self._error_window)
# self._resize_errwin = True;
QObject.connect(self._error_window, PYSIGNAL("hasErrors()"),
self._reset_errors)
QObject.connect(self._error_window, PYSIGNAL("showError()"),
self.show_error)
# set filename
self._filename = None
# "official" path of file (None if new script not yet saved)
self._mainfile = None
# if not None, then we're "slaved" to a main file (see below)
self._file_disktime = None
# modtime on disk when file was loaded
self._basename = None
self._modified = False
self._closed = False
self._error_at_line = {}
self._is_tree_in_sync = True
self._pub_nodes = None
def __del__(self):
self.has_focus(False)
def disable_editor(self):
"""Called before disabling the editor, as on some versions of PyQt
the object is not destroyed properly and keeps receving signals"""
self._enabled = False
QObject.disconnect(TDLOptions.OptionObject,
SIGNAL("mandatoryOptionsSet"),
self.mark_mandatory_options_set)
def show_compile_options(self):
self._options_menu.show()
def mark_mandatory_options_set(self, enabled):
self._options_menu.enableOkButton(enabled)
def show_runtime_options(self):
self._tdlexec_menu.show()
def tree_is_in_sync(self, sync=True):
"""Tells the editor wheether the current tree is in sync with the content of the script.
This is indicated by a visual cue on the toolbar.
"""
if sync:
self._tb_tdlexec.setIcon(pixmaps.gear.icon())
self._tb_tdlexec.setToolTip(
"Access run-time options & jobs defined by this TDL script")
else:
self._tb_tdlexec.setIcon(pixmaps.exclaim_yellow_warning.icon())
self._tb_tdlexec.setToolTip(
"""Access run-time options & jobs defined by this TDL script.
Warning! You have modified the script since it was last compiled, so the tree may be out of date."""
)
def _file_closed(self):
self.emit(PYSIGNAL("fileClosed()"), self)
def hideEvent(self, ev):
self.emit(PYSIGNAL("hidden()"))
self.emit(PYSIGNAL("visible()"), False)
return QFrame.hideEvent(self, ev)
def showEvent(self, ev):
self.emit(PYSIGNAL("shown()"))
self.emit(PYSIGNAL("visible()"), True)
return QFrame.showEvent(self, ev)
def hide_jobs_menu(self, dum=False):
if self._closed:
return
self._tb_tdlexec.hide()
self.clear_message()
def show_line_numbers(self, show):
pass
def show_run_control(self, show=True):
if self._closed:
return
self._qa_run.setVisible(show)
def enable_controls(self, enable=True):
if self._closed:
return
self._qa_run.setEnabled(enable)
self._tb_tdlexec.setEnabled(enable)
if not enable:
self.clear_message()
def disable_controls(self, disable=True):
if self._closed:
return
self._qa_run.setDisabled(disable)
self._tb_tdlexec.setDisabled(disable)
if disable:
self.clear_message()
def get_filename(self):
return self._filename
def get_mainfile(self):
return self._mainfile
def _import_main_file(self):
# self._tb_opts.setOn(False);
self.clear_errors()
if self._document.isModified():
self._save_file()
self.emit(PYSIGNAL("importFile()"), self, self._mainfile
or self._filename)
def _compile_main_file(self):
# self._tb_opts.setOn(False);
self.clear_errors()
if self._document.isModified():
self._save_file()
self.emit(PYSIGNAL("compileFile()"), self, self._mainfile
or self._filename)
def _clear_transients(self):
"""if message box contains a transient message, clears it"""
if self._message_transient:
self.clear_message()
def _display_cursor_position(self):
cursor = self._editor.textCursor()
pos = cursor.position()
col = cursor.columnNumber()
line = cursor.blockNumber()
self._poslabel.setText("L:<b>%d</b> C:<b>%d</b>" % (line + 1, col + 1))
self._poslabel.repaint()
def _text_modified(self, mod=True):
self._modified = mod
self.emit(PYSIGNAL("textModified()"), self, bool(mod))
self._tb_save.setAutoRaise(not mod)
self._qa_revert.setEnabled(mod)
#if mod:
#self._tb_save.setBackgroundRole(QPalette.ToolTipBase);
#else:
#self._tb_save.setBackgroundRole(QPalette.Button);
if self._filename:
label = '<b>' + self._basename + '</b>'
self._pathlabel.setToolTip(self._filename)
else:
label = ''
self._pathlabel.setToolTip('')
if self._readonly:
label = '[r/o] ' + label
if mod:
self._clear_transients()
label = '[mod] ' + label
self._pathlabel.setText(label)
def clear_message(self):
# traceback.print_stack();
self._message_box.hide()
self._message.setText('')
self._message_icon.setText('')
self._message_transient = False
if self._message_widgets:
dum = QWidget()
for w in self._message_widgets:
w.reparent(dum)
self._message_widgets = []
def show_message(self, msg, error=False, icon=None, transient=False):
"""Shows message in box.
If icon is not None, overrides standard icon.
If error=True, uses error icon (icon overrides this).
If transient=True, message will be cleared when text is edited.
"""
self._message.setText(msg)
if not icon:
if error:
icon = pixmaps.red_round_cross.icon()
else:
icon = pixmaps.info_blue_round.icon()
self._message_icon.setIcon(icon)
self._message_box.show()
self._message_transient = transient
self.clear_errors()
def messagebox():
return self._message_box
def add_message_widget(self, widget):
self._mblo.addWidget(widget)
self._message_widgets.append(widget)
def clear_errors(self):
self._error_window.clear_errors(emit_signal=True)
def _find_block_by_linenumber(self, line):
block = block0 = self._document.begin()
nblock = 1
while block and nblock < line:
block0 = block
block = block0.next()
nblock += 1
return block or block0
def _reset_errors(self, nerr):
"""helper method, resets error markers and such. Usually tied to a hasErrors() signal
from an error window"""
if not self._enabled:
return
self._error_at_line = {}
self._error_selections = []
self._current_error_selection = QTextEdit.ExtraSelection()
nerr_local = 0
if nerr:
error_locations = self._error_window.get_error_locations()
for err_num, filename, line, column, suberror in error_locations:
if filename == self._filename:
nerr_local += 1
# make cursor corresponding to line containing the error
cursor = QTextCursor(self._find_block_by_linenumber(line))
cursor.select(QTextCursor.LineUnderCursor)
# make selection object
qsel = QTextEdit.ExtraSelection()
qsel.cursor = cursor
if suberror:
qsel.format = self._format_suberror
else:
qsel.format = self._format_error
# insert into error_at_line list
self._error_at_line[line - 1] = len(
self._error_selections), cursor
# append to list of error selections
self._error_selections.append(qsel)
self._editor.setExtraSelections(self._error_selections)
else:
self._editor.setExtraSelections([])
self.emit(PYSIGNAL("hasErrors()"), self, nerr_local)
def show_error(self, err_num, filename, line, column):
"""Shows error at the given position, but only if the filename matches.
Can be directly connected to a showError() signal from an error window"""
if filename == self._filename:
errnum, cursor = self._error_at_line.get(line - 1, (None, None))
if cursor:
# change selections
self._current_error_selection.cursor = cursor
self._current_error_selection.format = self._format_current_error
sels = self._error_selections[:errnum] + self._error_selections[
(errnum + 1):]
sels.append(self._current_error_selection)
self._editor.setExtraSelections(sels)
# move current cursor
cursor = self._editor.textCursor()
# scroll to line-1 and line+1 to make sure line is fully visible
if line > 1:
cursor.setPosition(
self._find_block_by_linenumber(line - 2).position() +
column)
self._editor.setTextCursor(cursor)
self._editor.ensureCursorVisible()
if line < self._document.blockCount():
cursor.setPosition(
self._find_block_by_linenumber(line).position() +
column)
self._editor.setTextCursor(cursor)
self._editor.ensureCursorVisible()
# finally, scroll to line
cursor.setPosition(
self._find_block_by_linenumber(line).position() + column)
self._editor.setTextCursor(cursor)
self._editor.ensureCursorVisible()
def sync_external_file(self, filename=None, ask=True):
#
# NB: problem because it resets the errors
filename = filename or self._filename
filetime = _file_mod_time(filename)
if not filetime or filetime == self._file_disktime:
return True
# in sync
if not ask:
res = QMessageBox.Discard
else:
res = QMessageBox.warning(
self, "TDL file changed",
"""<p><tt>%s</tt> has been modified by another program.
Would you like to save your changes and overwrite the version on disk,
discard your changes and revert to the version on disk, or cancel the operation?"""
% (filename, ),
QMessageBox.Save | QMessageBox.Discard | QMessageBox.Cancel)
if res == QMessageBox.Cancel:
return None
elif res == QMessageBox.Discard:
if filename != self._filename:
self.load_file(filename, mainfile=self._mainfile)
else:
self.reload_file()
return True
# in sync
def _save_file(self, filename=None, text=None, force=False, save_as=False):
"""Saves text. If force=False, checks modification times in case
the file has been modified by another program.
If force=True, saves unconditionally.
If no filename is known, asks for one.
Returns True if file was successfully saved, else None."""
filename = filename or self._filename
if filename and not save_as:
if not force:
if not self.sync_external_file(filename=filename, ask=True):
return None
else: # no filename, ask for one
try:
dialog = self._save_as_dialog
dialog.setDirectory(dialog.directory())
# hopefully this rescan the directory
except AttributeError:
self._save_as_dialog = dialog = QFileDialog(
self, "Saved TDL Script")
dialog.resize(800, dialog.height())
dialog.setMode(QFileDialog.AnyFile)
dialog.setNameFilters(
["TDL scripts (*.tdl *.py)", "All files (*.*)"])
dialog.setViewMode(QFileDialog.Detail)
if dialog.exec_() != QDialog.Accepted:
return None
filename = str(dialog.selectedFiles()[0])
# save the file
if text is None:
text = str(self._editor.document().toPlainText())
try:
outfile = file(filename, "w").write(text)
except IOError:
(exctype, excvalue, tb) = sys.exc_info()
_dprint(0, 'exception', sys.exc_info(), 'saving TDL file',
filename)
self.show_message("""<b>Error writing <tt>%s</tt>:
<i>%s (%s)</i></b>""" % (filename, excvalue, exctype.__name__),
error=True,
transient=True)
return None
# saved successfully, update stuff
self._filename = filename
self._qa_revert.setEnabled(True)
self._basename = os.path.basename(filename)
self._file_disktime = _file_mod_time(filename)
self._document.setModified(False)
self.emit(PYSIGNAL("fileSaved()"), self, filename)
return self._filename
def close(self):
self._closed = True
def confirm_close(self):
if self._document.isModified():
res = QMessageBox.warning(
self, "TDL file modified",
"""Save modified file <p><tt>%s</tt>?</p>""" %
(self._filename or "", ),
QMessageBox.Save | QMessageBox.Discard | QMessageBox.Cancel)
if res == QMessageBox.Cancel:
return False
if res == QMessageBox.Save:
if not self._save_file():
return False
self.close()
return True
def _revert_to_saved(self, force=False):
if not self._filename:
return
if not force:
if QMessageBox.question(
self, "Revert to saved",
"""Revert to saved version of <p><tt>%s</tt>?""" %
(self._filename, ),
QMessageBox.Ok | QMessageBox.Cancel) == QMessageBox.Cancel:
return
self.load_file(self._filename, mainfile=self._mainfile)
def _add_menu_label(self, menu, label):
tlab = QLabel("<b>" + label + "</b>", menu)
tlab.setAlignment(Qt.AlignCenter)
tlab.setFrameShape(QFrame.ToolBarPanel)
tlab.setFrameShadow(QFrame.Sunken)
menu.insertItem(tlab)
def has_compile_options(self):
return self._options_menu.treeWidget().topLevelItemCount()
def has_runtime_options(self):
return self._tdlexec_menu.treeWidget().topLevelItemCount()
def import_content(self, force=False, show_options=False):
"""imports TDL module but does not run _define_forest().
Depending on autosync/modified state, asks to save or revert.
If module is already imported, does nothing, unless force=True,
in which case it imports unconditionally.
If do_compile=True, proceeds to show compile-time options on success,
or to compile directly if there are no options
Return value:
True on successful import
None if cancelled by user.
Import errors are posted to the error dialog.
"""
# save list of publishers (since forest will be cleared on import)
if self._pub_nodes is None:
if TDL.Compile.last_imported_file() == self._filename:
self._pub_nodes = [
node.name for node in meqds.nodelist.iternodes()
if node.is_publishing()
]
_dprint(2, "last imported file matches, saving",
len(self._pub_nodes), "publishers")
else:
self._pub_nodes = []
_dprint(2, "last imported file doesn't match:",
TDL.Compile.last_imported_file(), self._filename)
_dprint(1, self._filename, "importing")
self.clear_message()
self.clear_errors()
self._options_menu.hide()
self._tb_tdlexec.hide()
self._tdlexec_menu.hide()
# change the current directory to where the file is
# os.chdir(os.path.dirname(self._filename));
# The Python imp module expects text to reside in a disk file, which is
# a pain in the ass for us if we're dealing with modified text or text
# entered on-the-fly. So, either save or sync before proceeding
global _external_sync
if self._document.isModified() or not self._filename:
if not self._save_file():
return None
else:
if not self.sync_external_file(ask=False):
return None
# if we already have an imported module and disk file hasn't changed, skip
# the importing step
busy = BusyIndicator()
if force or self._tdlmod is None or self._tdlmod_filetime == self._file_disktime:
# reset data members
_dprint(2, self._filename,
"emitting signal for 0 compile-time options")
self.emit(PYSIGNAL("hasCompileOptions()"), self, 0)
self._options_menu.hide()
self._options_menu.clear()
self._tdlmod = None
# get text from editor
tdltext = str(self._document.toPlainText())
try:
tdlmod, tdltext = TDL.Compile.import_tdl_module(
self._filename, tdltext)
# catch import errors
except TDL.CumulativeError, value:
_dprint(0, "caught cumulative error, length", len(value.args))
self._error_window.set_errors(value.args,
message="TDL import failed")
busy = None
return None
except Exception, value:
_dprint(0, "caught other error, traceback follows")
traceback.print_exc()
self._error_window.set_errors([value],
message="TDL import failed")
busy = None
return None
# remember module and nodescope
self._tdlmod = tdlmod
self._tdltext = tdltext
self._tdlmod_filetime = self._file_disktime
# build options menu from compile-time options
opt_tw = self._options_menu.treeWidget()
opts = TDLOptions.get_compile_options()
if opts:
# add options
try:
TDLOptions.populate_option_treewidget(opt_tw, opts)
except Exception, value:
_dprint(0, "error setting up TDL options GUI")
traceback.print_exc()
self._error_window.set_errors(
[value], message="Error setting up TDL options GUI")
busy = None
return None
# self._tb_opts.show();
_dprint(2, self._filename, "emitting signal for", len(opts),
"compile-time options")
self.emit(PYSIGNAL("hasCompileOptions()"), self, len(opts))
# or was interrupted by an exception midway through, we use a try...finally block.
try:
TDLOptions.config.read("batch_sim_example.tdl.conf");
script = "example-sim.py";
print "========== Compiling batch job 1";
mod,ns,msg = Compile.compile_file(mqs,script,config="batch job 1");
print "========== Running batch job 1";
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print "========== Compiling batch job 2";
mod,ns,msg = Compile.compile_file(mqs,script,config="batch job 2");
print "========== Running batch job 2";
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print "========== Compiling batch job 2 with modified config";
TDLOptions.init_options("batch job 2",save=False);
TDLOptions.set_option("me.enable_G",False);
mod,ns,msg = Compile.compile_file(mqs,script,config=None);
print "========== Running batch job 2";
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
### Cleanup time
finally:
print "Stopping meqserver";
# this halts the meqserver
meqserver.stop_default_mqs();
# now we can exit
print "Bye!";
opt = optstr.split("=") + ['1']
context, level = opt[:2]
debuglevels[context] = int(level)
# tell verbosity class to not parse argv -- we do it ourselves here
Timba.utils.verbosity.disable_argv()
for optstr in (options.verbose or []):
opt = optstr.split("=") + ['1']
context, level = opt[:2]
Timba.utils.verbosity.set_verbosity_level(context, int(level))
# start meqserver
from Timba.Apps import meqserver
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
TDLOptions.enable_save_config(False)
print "### Starting meqserver"
mqs = meqserver.default_mqs(wait_init=10, extra=["-mt",
str(options.mt)])
retcode = 0
# use a try...finally block to exit meqserver cleanly at the end
try:
if not os.path.exists(options.config):
print "Config file %s doesn't exist" % options.config
sys.exit(1)
print "### Attaching to configuration file", options.config
TDLOptions.config.read(options.config)
# disable the writing-out of configuration
TDLOptions.config.set_save_filename(None)
# This starts a meqserver. Note how we pass the "-mt 2" option to run two threads.
# A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv).
print "Starting meqserver";
mqs = meqserver.default_mqs(wait_init=10,extra=["-mt","2"]);
try:
## make simulation with perfect MODEL_DATA
script = path("testing-sim.py");
print "========== Compiling",script;
TDLOptions.config.read(path("testing.tdl.conf"));
mod,ns,msg = Compile.compile_file(mqs,script,config="simulate-model");
print "========== Simulating MODEL_DATA ";
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print "========== Imaging MODEL_DATA ";
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
## compare against reference image
print "========== Verifying test image ";
verify_image('WSRT.MS.MODEL_DATA.channel.1ch.fits',path('test-refimage.fits'),maxdelta=1e-3);
print "========== Compiling script with modified config";
TDLOptions.init_options("simulate-model",save=False);
TDLOptions.set_option("me.g_enable",True);
mod,ns,msg = Compile.compile_file(mqs,script,config=None);
print "========== Simulating DATA ";
TDLOptions.set_option("ms_sel.output_column","DATA");
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print "========== Imaging DATA ";
TDLOptions.set_option("img_sel.imaging_column","DATA");
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
print "Starting meqserver"
mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", "2"])
try:
## make simulation with perfect MODEL_DATA
script = path("testing-sim.py")
print "========== Compiling", script
TDLOptions.config.read(path("testing.tdl.conf"))
mod, ns, msg = Compile.compile_file(mqs,
script,
config="simulate-model")
print "========== Simulating MODEL_DATA "
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print "========== Imaging MODEL_DATA "
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
## compare against reference image
print "========== Verifying test image "
verify_image('WSRT.MS.MODEL_DATA.channel.1ch.fits',
path('test-refimage.fits'),
maxdelta=1e-3)
print "========== Compiling script with modified config"
TDLOptions.init_options("simulate-model", save=False)
TDLOptions.set_option("me.g_enable", True)
mod, ns, msg = Compile.compile_file(mqs, script, config=None)
print "========== Simulating DATA "
TDLOptions.set_option("ms_sel.output_column", "DATA")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
def testMeqtreesBatchJob():
trace_sync = True;
# sys.settrace(trace_lines);
if len(sys.argv) > 1:
newdir = PACKAGE_TEST_DIR;
print("========== Changing working directory to",newdir);
os.chdir(newdir);
print("========== Making required symlinks");
run("rm {0:s}/WSRT_ANTENNA ; ln -s {1:s}".format(PACKAGE_TEST_DIR,
path(os.path.join(PACKAGE_TEST_DIR, "WSRT_ANTENNA"))));
run("rm {0:s}/test-lsm.txt; ln -s {1:s}".format(PACKAGE_TEST_DIR,
path(os.path.join(path("test-lsm.txt")))));
if not os.access(".",os.R_OK|os.W_OK):
print("Directory",os.getcwd(),"not writable, can't run tests in here.")
print("You may choose to run the tests in a different directory by giving it as an argument to this script.")
sys.exit(1);
## check if we have owlcat or owlcat.sh
owlcat = "";
for dirname in os.environ['PATH'].split(':'):
for binary in "owlcat","owlcat.sh":
tmp = os.path.join(dirname,binary);
if os.path.exists(tmp):
owlcat = tmp;
break;
if owlcat:
break;
if not owlcat:
raise RuntimeError("Can't locate owlcat or owlcat.sh");
## make simulated MS
print("========== Removing files");
run("rm -fr {0:s}/WSRT.MS* {0:s}/WSRT*img {0:s}/WSRT*fits".format(PACKAGE_TEST_DIR));
print("========== Running makems");
run("makems %s" % path(os.path.join(PACKAGE_TEST_DIR, "WSRT_makems.cfg")));
run("mv {0:s}/WSRT.MS_p0 {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR));
os.environ["MEQTREES_CATTERY_PATH"] = Cattery.__path__[0]
run("pyxis {0:s}/WSRT.MS ms.prep".format(PACKAGE_TEST_DIR)); #TODO: this is hacky, bug in CASAcore
run("ls -ld {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR));
run("{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS".format(owlcat, PACKAGE_TEST_DIR));
run("lwimager ms={0:s}/WSRT.MS data=CORRECTED_DATA mode=channel weight=natural npix=10".format(PACKAGE_TEST_DIR));
# make test LSMs
run("""tigger-convert {0:s}/test-lsm.txt --rename --format "ra_d dec_d i q u v" --center 0.1deg,60.5deg -f""".format(PACKAGE_TEST_DIR));
run("""tigger-convert {0:s}/test-lsm.lsm.html {0:s}/test-lsm1.txt --output-format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""".format(PACKAGE_TEST_DIR));
run("""cut -d " " -f 1-10 {0:s}/test-lsm1.txt >{0:s}/test-lsm1.txt.tmp""".format(PACKAGE_TEST_DIR));
run("""diff {0:s}/test-lsm1.txt.tmp {1:s}""".format(PACKAGE_TEST_DIR, path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm1.txt.reference'))));
run("""tigger-convert {0:s}/test-lsm1.txt --format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""".format(PACKAGE_TEST_DIR));
run("""{0:s} plot-ms {1:s}/WSRT.MS DATA:I -o data_i.png""".format(owlcat, PACKAGE_TEST_DIR));
run("""{0:s} run-imager ms={1:s}/WSRT.MS name_dirty=tmp""".format(owlcat, PACKAGE_TEST_DIR));
print("importing meqserver")
from Timba.Apps import meqserver
print("importing Compile")
from Timba.TDL import Compile
print("importing TDLOptions")
from Timba.TDL import TDLOptions
# This starts a meqserver. Note how we pass the "-mt 2" option to run two threads.
# A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv).
print("Starting meqserver");
mqs = meqserver.default_mqs(wait_init=10,extra=["-mt","2"]);
try:
## make simulation with perfect MODEL_DATA
script = path(os.path.join(PACKAGE_TEST_DIR, "sim.py"));
print("========== Compiling",script);
TDLOptions.config.read(path(os.path.join(PACKAGE_TEST_DIR, "testing.tdl.conf")));
TDLOptions.config.set("calibrate", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "ms_sel.msname")))
TDLOptions.config.set("calibrate", "tiggerlsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "tiggerlsm.filename")))
TDLOptions.config.set("calibrate", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "lsm.filename")))
TDLOptions.config.set("calibrate", "cal_g_diag.g_diag.table_name", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_diag.g_diag.table_name")))
TDLOptions.config.set("calibrate", "cal_g_offdiag.g_offdiag.table_name", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_offdiag.g_offdiag.table_name")))
TDLOptions.config.set("simulate-model", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "lsm.filename")))
TDLOptions.config.set("simulate-model", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "ms_sel.msname")))
TDLOptions.config.set("simulate-model", "tiggerlsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "tiggerlsm.filename")))
TDLOptions.config.set("calibrate", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits"))
TDLOptions.config.set("simulate-model", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits"))
with open(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"), "w") as f:
TDLOptions.config.write(f)
TDLOptions.config.read(path(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"))); # needs to re-read because of a Timba perculiarity
mod,ns,msg = Compile.compile_file(mqs, script, config="simulate-model");
print("========== Simulating MODEL_DATA ");
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print("========== Imaging MODEL_DATA ");
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
## compare against reference image
print("========== Verifying test image ");
if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist")
if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")): raise RuntimeError("Reference FITS file does not exist")
verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits"),
path(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")),
maxdelta=1e-3);
print("========== Compiling script with modified config");
TDLOptions.init_options("simulate-model",save=False);
TDLOptions.set_option("me.g_enable",True);
mod,ns,msg = Compile.compile_file(mqs,script,config=None);
print("========== Simulating DATA ");
TDLOptions.set_option("ms_sel.output_column","DATA");
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print("========== Imaging DATA ");
TDLOptions.set_option("img_sel.imaging_column","DATA");
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
## calibrate
script = path(os.path.join(PACKAGE_TEST_DIR, "cal.py"));
print("========== Compiling",script);
mod,ns,msg = Compile.compile_file(mqs,script,config="calibrate");
print("========== Calibrating ");
TDLOptions.get_job_func('cal_G_diag')(mqs,None,wait=True);
print("========== Imaging MODEL_DATA ");
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
finally:
print("Stopping meqserver");
# this halts the meqserver
meqserver.stop_default_mqs();
print("========== Making plots of solutions ");
run("""{0:s} plot-ms {1:s}/WSRT.MS CORRECTED_DATA:I -I ">0" -o {1:s}/corrected_data_i.png""".format(owlcat, PACKAGE_TEST_DIR));
run("""{0:s} plot-parms -l {1:s}/WSRT.MS/G_diag.fmep""".format(owlcat, PACKAGE_TEST_DIR));
run("""{0:s} plot-parms {1:s}/WSRT.MS/G_diag.fmep G:*/norm -o {1:s}/parmplot.png""".format(owlcat, PACKAGE_TEST_DIR));
run("""{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS""".format(owlcat, PACKAGE_TEST_DIR));
## compare against reference image
print("========== Verifying residual image ");
if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist")
if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits")): raise RuntimeError("Reference FITS file does not exist")
verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits"),
os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits"),
maxdelta=1e-3);
## all tests succeeded
print("========== Break out the bubbly, this hog is airborne!");
# now we can exit
print("Bye!");
meqds.enable_node_publish_by_name(name,sync=True,get_state=True);
else:
_dprint(2,"not reenabling publishing of",name,", as it no longer exists?");
# clear publisher list so it's re-saved next time in import_content()
self._pub_nodes = None;
### NB: presume this all was successful for now
# insert node scope into TDL module
setattr(_tdlmod,'_tdl_nodescope',ns);
# does the script define an explicit job list?
joblist = getattr(_tdlmod,'_tdl_job_list',[]);
if not joblist:
joblist = [];
# try to build it from implicit function names
for (name,func) in _tdlmod.__dict__.iteritems():
if name.startswith("_tdl_job_") and callable(func) and not TDLOptions.is_jobfunc_defined(func):
joblist.append(func);
# does the script define a testing function?
testfunc = getattr(_tdlmod,'_test_forest',None);
if not callable(testfunc):
testfunc = getattr(_tdlmod,'test_forest',None);
if callable(testfunc):
res = QMessageBox.warning(self,"Deprecated method",
"""Your script contains a test_forest() method. This is deprecated
and will be disabled in the future. Please rename it to
_test_forest().
""",
QMessageBox.Ok);
if callable(testfunc):
joblist.append(testfunc);
joblist.sort(lambda a,b:cmp(str(a),str(b)));
def main():
for optstr in (options.debug or []):
opt = optstr.split("=") + ['1']
context, level = opt[:2]
debuglevels[context] = int(level)
Timba.utils.verbosity.disable_argv()
# tell verbosity class to not parse its argv
for optstr in (options.verbose or []):
opt = optstr.split("=") + ['1']
context, level = opt[:2]
Timba.utils.verbosity.set_verbosity_level(context, level)
if not args:
parser.print_help()
sys.exit(1)
if debuglevels:
octopussy.set_debug(debuglevels)
script = args[0]
tdljob = (len(args) > 1 and args[1]) or None
from Timba.Apps import meqserver
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
# this starts a kernel.
if options.compile_only:
mqs = None
else:
mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", options.mt])
TDLOptions.config.read(options.config)
TDLOptions.init_options(script)
print(("************************ Compiling TDL script", script))
# this compiles a script as a TDL module. Any errors will be thrown as
# and exception, so this always returns successfully
(mod, ns, msg) = Compile.compile_file(mqs, script)
if options.compile_only:
print(msg)
sys.exit(0)
# if a solve job is not specified, try to find one
if tdljob:
jobfunc = getattr(mod, tdljob, None)
if not jobfunc:
print(("Cannot find TDL job named", tdljob))
sys.exit(1)
else:
# does the script define an explicit job list?
joblist = getattr(mod, '_tdl_job_list', [])
if not joblist:
joblist = []
# try to build it from implicit function names
for (name, func) in list(mod.__dict__.items()):
if name.startswith("_tdl_job_") and callable(func):
joblist.append(func)
# does the script define a testing function?
testfunc = getattr(mod, '_test_forest', None)
if testfunc:
joblist.insert(0, testfunc)
if not joblist:
print(("No TDL jobs found in script", script))
sys.exit(1)
jobfunc = joblist[0]
tdljob = jobfunc.__name__
# this runs the appropriate job. wait=True is needed to wait
print(("************************ Running TDL job", tdljob))
# check if job takes a "wait" argument
(fargs, fvarargs, fvarkw, fdefaults) = inspect.getargspec(jobfunc)
if 'wait' in fargs or fvarkw:
jobopts = dict(wait=True)
else:
jobopts = {}
jobfunc(mqs, None, **jobopts)
mod, ns, msg = Compile.compile_file(mqs, script)
# 'mod' now refers to a Python object that is the compiled module. To execute the _test_forest job within that
# module, we do as follows:
print "Running TDL job"
mod._test_forest(mqs, None, wait=True)
# The wait=True argument causes the thing to not return until the job has been completed.
# None for the second argument tells it that we're running headless (without a GUI parent.)
### Now for some variations
# This shows how to use a different section in the config file.
print "Recompiling and running for test_a"
mod, ns, msg = Compile.compile_file(mqs, script, config="test_a")
# and this shows how to locate and call a "TDL Job" by its job ID
TDLOptions.get_job_func('job1')(mqs, None, wait=True)
print "Recompiling and running for test_b"
mod, ns, msg = Compile.compile_file(mqs, script, config="test_b")
# we can also find jobs by their long names
TDLOptions.get_job_func('Run job 2')(mqs, None, wait=True)
# finally, this is expected to fail
print "Trying to call a non-existing TDL job 'job3'"
try:
TDLOptions.get_job_func('job3')(mqs, None, wait=True)
except NameError:
print "Indeed, 'job3' does not exist."
# This shows how to change configuration on-the-fly.
# TDLOptions.set_option() changes the value of a configuration variable "in memory".
def run_forest_definition(mqs,
filename,
tdlmod,
text,
parent=None,
wait=True,
predef_args={},
define_args={},
postdef_args={}):
"""Compiles a TDL script and sends it to meqserver given by mqs.
Parameters:
mqs: a meqserver object (None to run without a meqserver)
filename: the filename of the script (used for error reporting)
tdlmod: the imported TDL module, as returned by import_tdl_module()
text: script text for putting into forest
parent: parent widget passed to TDL script (if a GUI is running)
wait: if True, waits for forest to build before returning
predef_args: dict of extra arguments for _tdl_predefine()
define_args: dict of extra arguments for _define_forest()
postdef_args: dict of extra arguments for _tdl_postdefine()
Return value:
a tuple of (module,ns,message), where module is the newly-imported
TDL module, ns is a NodeScope object, and message is an
informational message.
Exceptions thrown:
Any compilation error results in an exception. This is always
a TDL.CumulativeError exception containing an error list.
Other exceptions may indicate an internal failure.
"""
_dprint(1, "defining forest")
try:
TDLOptions.enable_save_config(False)
ns = TDL.NodeScope()
# module here, call functions
errlist = []
# find define_forest func
define_func = getattr(tdlmod, '_define_forest', None)
if not callable(define_func):
raise TDL.TDLError("No _define_forest() function found",
filename=filename,
lineno=1)
# now find predefine function and call it
predefine_func = getattr(tdlmod, '_tdl_predefine', None)
if callable(predefine_func):
predef_result = predefine_func(mqs, parent, **predef_args)
_update_modlist()
else:
predef_result = None
# inspect the define function to support older scripts that only
# defined a define_forest(ns), i.e., with a single argument
(fargs, fvarargs, fvarkw, fdefaults) = inspect.getargspec(define_func)
if not fargs:
raise TDL.TDLError(
"invalid _define_forest() function: must have at least a single argument ('ns')",
filename=filename,
lineno=1)
# function must have either a single argument, or allow keyword arguments
if len(fargs) > 1 and not fvarkw:
raise TDL.TDLError(
"invalid _define_forest() function: must have a **kwargs parameter",
filename=filename,
lineno=1)
# if no support for keyword arguments, pass an empty dict, else use valid dict
if fvarkw:
args = define_args.copy()
args['parent'] = parent
if isinstance(predef_result, dict):
args.update(predef_result)
else:
args = {}
# call the define function
define_func(ns, **args)
_update_modlist()
# resolve the nodescope
ns.Resolve()
# do we have an error list? show it
errlist = ns.GetErrors()
if errlist:
raise TDL.CumulativeError(*errlist)
allnodes = ns.AllNodes()
num_nodes = len(allnodes)
# no nodes? return
if not num_nodes:
return (
tdlmod, ns,
"TDL script successfully compiled, but no nodes were defined.")
# try to run stuff
if mqs is not None:
meqds.clear_forest()
# is a forest state defined? send it on then
fst = getattr(Timba.TDL.Settings, 'forest_state', record())
# add in source code
fst.tdl_source = record(**{os.path.basename(filename): text})
mqs.meq('Set.Forest.State', record(state=fst, get_forest_status=0))
if num_nodes:
mqs.meq(
'Create.Node.Batch',
record(script_name=os.path.basename(filename),
batch=map(lambda nr: nr.initrec(),
allnodes.itervalues())))
# mqs.meq('Init.Node.Batch',record(name=list(ns.RootNodes().iterkeys())),wait=wait);
msg = """TDL script successfully compiled. %d node definitions
(of which %d are root nodes) sent to meqserver.""" \
% (num_nodes,len(ns.RootNodes()))
else:
msg = "TDL script successfully compiled, but no nodes were defined."
else:
msg = "TDL script successfully compiled, %d nodes were defined." % num_nodes
# call the post-define function
postdefine_func = getattr(tdlmod, '_tdl_postdefine', None)
if callable(postdefine_func):
res = postdefine_func(mqs, parent, **postdef_args)
_update_modlist()
if isinstance(res, str):
msg += "\n" + res
TDLOptions.enable_save_config(True)
TDLOptions.save_config()
return (tdlmod, ns, msg)
# CumulativeError exceptions returned as is
except TDL.CumulativeError:
_update_modlist()
TDLOptions.enable_save_config(True)
TDLOptions.save_config()
_dprint(0, 'cumulative error compiling TDL script:', filename)
traceback.print_exc()
args = sys.exc_info()[1].args
_dprint(0, 'number of errors in list:', len(args))
_dprint(1, 'errors are: {', args, '}')
raise
# Other exceptions wrapped in a CumulativeError, and
# location information is added in
except:
_update_modlist()
TDLOptions.enable_save_config(True)
TDLOptions.save_config()
(etype, exc, tb) = sys.exc_info()
_dprint(0, 'exception defining forest from TDL file:', filename)
traceback.print_exception(etype, exc, tb)
# use TDL add_error() to process the error, since this automatically
# adds location information
ns.AddError(exc, traceback.extract_tb(tb), error_limit=None)
# re-raise as a CumulativeError
raise TDL.CumulativeError(*ns.GetErrors())
def main ():
for optstr in (options.debug or []):
opt = optstr.split("=") + ['1'];
context,level = opt[:2];
debuglevels[context] = int(level);
Timba.utils.verbosity.disable_argv(); # tell verbosity class to not parse its argv
for optstr in (options.verbose or []):
opt = optstr.split("=") + ['1'];
context,level = opt[:2];
Timba.utils.verbosity.set_verbosity_level(context,level);
if not args:
parser.print_help();
sys.exit(1);
if debuglevels:
octopussy.set_debug(debuglevels);
script = args[0];
tdljob = (len(args)>1 and args[1]) or None;
from Timba.Apps import meqserver
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
# this starts a kernel.
if options.compile_only:
mqs = None;
else:
mqs = meqserver.default_mqs(wait_init=10,extra=["-mt",options.mt]);
TDLOptions.config.read(options.config);
TDLOptions.init_options(script);
print "************************ Compiling TDL script",script;
# this compiles a script as a TDL module. Any errors will be thrown as
# and exception, so this always returns successfully
(mod,ns,msg) = Compile.compile_file(mqs,script);
if options.compile_only:
print msg;
sys.exit(0);
# if a solve job is not specified, try to find one
if tdljob:
jobfunc = getattr(mod,tdljob,None);
if not jobfunc:
print "Cannot find TDL job named",tdljob;
sys.exit(1);
else:
# does the script define an explicit job list?
joblist = getattr(mod,'_tdl_job_list',[]);
if not joblist:
joblist = [];
# try to build it from implicit function names
for (name,func) in mod.__dict__.iteritems():
if name.startswith("_tdl_job_") and callable(func):
joblist.append(func);
# does the script define a testing function?
testfunc = getattr(mod,'_test_forest',None);
if testfunc:
joblist.insert(0,testfunc);
if not joblist:
print "No TDL jobs found in script",script;
sys.exit(1);
jobfunc = joblist[0];
tdljob = jobfunc.__name__;
# this runs the appropriate job. wait=True is needed to wait
print "************************ Running TDL job",tdljob;
# check if job takes a "wait" argument
(fargs,fvarargs,fvarkw,fdefaults) = inspect.getargspec(jobfunc);
if 'wait' in fargs or fvarkw:
jobopts = dict(wait=True);
else:
jobopts = {};
jobfunc(mqs,None,**jobopts);
def import_tdl_module(filename, text=None, config=0):
"""Imports a TDL module.
Parameters:
filename: script location
text: text of module. If none, file will be re-read.
Return value:
a tuple of (module,text), where module is the newly-imported
TDL module, and text is the module text.
Exceptions thrown:
Any import error results in an exception. This is always
a TDL.CumulativeError exception containing an error list.
Other exceptions may indicate an internal failure.
"""
_dprint(1, "importing", filename)
global _current_filename
_current_filename = filename
# initialize global nodescope (and repository)
meqds.clear_forest()
try:
reload(Timba.TDL.Settings)
# reset TDL script options, unless config=None ('0' is used as default, causing the filename to be substituted)
TDLOptions.init_script(filename)
if config is not None:
TDLOptions.init_options(config or filename)
# remove .pyo file as that can have cached paths and directories may have changed
# (see bug 677)
try:
os.unlink(os.path.splitext(filename)[0] + '.pyo')
except:
pass
# open file
infile = file(filename, 'r')
if text is None:
text = infile.read()
infile.seek(0)
# infile is now an open input file object, and text is the script
# flush all modules imported via previous TDL run
global _tdlmodlist
_dprint(1, 'clearing out TDL-imported modules', _tdlmodlist)
for m in _tdlmodlist:
try:
del sys.modules[m]
except KeyError:
pass
# remember which modules are imported
global _prior_compile_modules
_prior_compile_modules = set(sys.modules.iterkeys())
modname = '__tdlruntime'
try:
TDLOptions.enable_save_config(False)
imp.acquire_lock()
_tdlmod = imp.load_source(modname, filename, infile)
finally:
TDLOptions.enable_save_config(True)
TDLOptions.save_config()
imp.release_lock()
infile.close()
_update_modlist()
return (_tdlmod, text)
# CumulativeError exceptions returned as is
except TDL.CumulativeError:
_dprint(0, 'cumulative error importing TDL file:', filename)
traceback.print_exc()
args = sys.exc_info()[1].args
_dprint(0, 'number of errors in list:', len(args))
_dprint(1, 'errors are: {', args, '}')
raise
# Other exceptions wrapped in a CumulativeError, and
# location information is added in
except:
(etype, exc, tb) = sys.exc_info()
_dprint(0, 'exception importing TDL file:', filename)
traceback.print_exception(etype, exc, tb)
# use TDL add_error() to process the error, since this automatically
# adds location information
ns = TDL.NodeScope()
ns.AddError(exc, traceback.extract_tb(tb), error_limit=None)
# re-raise as a CumulativeError
raise TDL.CumulativeError(*ns.GetErrors())
for optstr in (options.debug or []):
opt = optstr.split("=") + ['1']
context, level = opt[:2]
# tell verbosity class to not parse argv -- we do it ourselves here
Timba.utils.verbosity.disable_argv()
for optstr in (options.verbose or []):
opt = optstr.split("=") + ['1']
context, level = opt[:2]
Timba.utils.verbosity.set_verbosity_level(context, int(level))
# start meqserver
from Timba.Apps import meqserver
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
TDLOptions.enable_save_config(False)
print("### Starting meqserver")
mqs = meqserver.default_mqs(
wait_init=10,
extra=["-mt", str(options.mt)] +
(["-python_memprof"] if options.memprof else []))
retcode = 0
# use a try...finally block to exit meqserver cleanly at the end
try:
if not os.path.exists(options.config):
print(("Config file %s doesn't exist" % options.config))
sys.exit(1)
print(("### Attaching to configuration file", options.config))
TDLOptions.config.read(options.config)
def run_forest_definition (mqs,filename,tdlmod,text,
parent=None,wait=True,
predef_args={},define_args={},postdef_args={}):
"""Compiles a TDL script and sends it to meqserver given by mqs.
Parameters:
mqs: a meqserver object (None to run without a meqserver)
filename: the filename of the script (used for error reporting)
tdlmod: the imported TDL module, as returned by import_tdl_module()
text: script text for putting into forest
parent: parent widget passed to TDL script (if a GUI is running)
wait: if True, waits for forest to build before returning
predef_args: dict of extra arguments for _tdl_predefine()
define_args: dict of extra arguments for _define_forest()
postdef_args: dict of extra arguments for _tdl_postdefine()
Return value:
a tuple of (module,ns,message), where module is the newly-imported
TDL module, ns is a NodeScope object, and message is an
informational message.
Exceptions thrown:
Any compilation error results in an exception. This is always
a TDL.CumulativeError exception containing an error list.
Other exceptions may indicate an internal failure.
""";
_dprint(1,"defining forest");
try:
TDLOptions.enable_save_config(False);
ns = TDL.NodeScope();
# module here, call functions
errlist = [];
# find define_forest func
define_func = getattr(tdlmod,'_define_forest',None);
if not callable(define_func):
raise TDL.TDLError("No _define_forest() function found",filename=filename,lineno=1);
# now find predefine function and call it
predefine_func = getattr(tdlmod,'_tdl_predefine',None);
if callable(predefine_func):
predef_result = predefine_func(mqs,parent,**predef_args);
_update_modlist();
else:
predef_result = None;
# inspect the define function to support older scripts that only
# defined a define_forest(ns), i.e., with a single argument
(fargs,fvarargs,fvarkw,fdefaults) = inspect.getargspec(define_func);
if not fargs:
raise TDL.TDLError("invalid _define_forest() function: must have at least a single argument ('ns')",filename=filename,lineno=1);
# function must have either a single argument, or allow keyword arguments
if len(fargs) > 1 and not fvarkw:
raise TDL.TDLError("invalid _define_forest() function: must have a **kwargs parameter",filename=filename,lineno=1);
# if no support for keyword arguments, pass an empty dict, else use valid dict
if fvarkw:
args = define_args.copy();
args['parent'] = parent;
if isinstance(predef_result,dict):
args.update(predef_result);
else:
args = {};
# call the define function
define_func(ns,**args);
_update_modlist();
# resolve the nodescope
ns.Resolve();
# do we have an error list? show it
errlist = ns.GetErrors();
if errlist:
raise TDL.CumulativeError(*errlist);
allnodes = ns.AllNodes();
num_nodes = len(allnodes);
# no nodes? return
if not num_nodes:
return (tdlmod,ns,"TDL script successfully compiled, but no nodes were defined.");
# try to run stuff
if mqs is not None:
meqds.clear_forest();
# is a forest state defined? send it on then
fst = getattr(Timba.TDL.Settings,'forest_state',record());
# add in source code
fst.tdl_source = record(**{os.path.basename(filename):text});
mqs.meq('Set.Forest.State',record(state=fst,get_forest_status=0));
if num_nodes:
mqs.meq('Create.Node.Batch',
record(script_name=os.path.basename(filename),
batch=map(lambda nr:nr.initrec(),allnodes.itervalues())));
# mqs.meq('Init.Node.Batch',record(name=list(ns.RootNodes().iterkeys())),wait=wait);
msg = """TDL script successfully compiled. %d node definitions
(of which %d are root nodes) sent to meqserver.""" \
% (num_nodes,len(ns.RootNodes()));
else:
msg = "TDL script successfully compiled, but no nodes were defined.";
else:
msg = "TDL script successfully compiled, %d nodes were defined."%num_nodes;
# call the post-define function
postdefine_func = getattr(tdlmod,'_tdl_postdefine',None);
if callable(postdefine_func):
res = postdefine_func(mqs,parent,**postdef_args);
_update_modlist();
if isinstance(res,str):
msg += "\n" + res;
TDLOptions.enable_save_config(True);
TDLOptions.save_config();
return (tdlmod,ns,msg);
# CumulativeError exceptions returned as is
except TDL.CumulativeError:
_update_modlist();
TDLOptions.enable_save_config(True);
TDLOptions.save_config();
_dprint(0,'cumulative error compiling TDL script:',filename);
traceback.print_exc();
args = sys.exc_info()[1].args;
_dprint(0,'number of errors in list:',len(args));
_dprint(1,'errors are: {',args,'}');
raise;
# Other exceptions wrapped in a CumulativeError, and
# location information is added in
except:
_update_modlist();
TDLOptions.enable_save_config(True);
TDLOptions.save_config();
(etype,exc,tb) = sys.exc_info();
_dprint(0,'exception defining forest from TDL file:',filename);
traceback.print_exception(etype,exc,tb);
# use TDL add_error() to process the error, since this automatically
# adds location information
ns.AddError(exc,traceback.extract_tb(tb),error_limit=None);
# re-raise as a CumulativeError
raise TDL.CumulativeError(*ns.GetErrors());
# or was interrupted by an exception midway through, we use a try...finally block.
try:
TDLOptions.config.read("batch_sim_example.tdl.conf")
script = "example-sim.py"
print("========== Compiling batch job 1")
mod, ns, msg = Compile.compile_file(mqs, script, config="batch job 1")
print("========== Running batch job 1")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print("========== Compiling batch job 2")
mod, ns, msg = Compile.compile_file(mqs, script, config="batch job 2")
print("========== Running batch job 2")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print("========== Compiling batch job 2 with modified config")
TDLOptions.init_options("batch job 2", save=False)
TDLOptions.set_option("me.enable_G", False)
mod, ns, msg = Compile.compile_file(mqs, script, config=None)
print("========== Running batch job 2")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
### Cleanup time
finally:
print("Stopping meqserver")
# this halts the meqserver
meqserver.stop_default_mqs()
# now we can exit
print("Bye!")
def import_tdl_module (filename,text=None,config=0):
"""Imports a TDL module.
Parameters:
filename: script location
text: text of module. If none, file will be re-read.
Return value:
a tuple of (module,text), where module is the newly-imported
TDL module, and text is the module text.
Exceptions thrown:
Any import error results in an exception. This is always
a TDL.CumulativeError exception containing an error list.
Other exceptions may indicate an internal failure.
""";
_dprint(1,"importing",filename);
global _current_filename;
_current_filename = filename;
# initialize global nodescope (and repository)
meqds.clear_forest();
try:
reload(Timba.TDL.Settings);
# reset TDL script options, unless config=None ('0' is used as default, causing the filename to be substituted)
TDLOptions.init_script(filename);
if config is not None:
TDLOptions.init_options(config or filename);
# remove .pyo file as that can have cached paths and directories may have changed
# (see bug 677)
try:
os.unlink(os.path.splitext(filename)[0]+'.pyo');
except:
pass;
# open file
infile = file(filename,'r');
if text is None:
text = infile.read();
infile.seek(0);
# infile is now an open input file object, and text is the script
# flush all modules imported via previous TDL run
global _tdlmodlist;
_dprint(1,'clearing out TDL-imported modules',_tdlmodlist);
for m in _tdlmodlist:
try: del sys.modules[m];
except KeyError: pass;
# remember which modules are imported
global _prior_compile_modules;
_prior_compile_modules = set(sys.modules.iterkeys());
modname = '__tdlruntime';
try:
TDLOptions.enable_save_config(False);
imp.acquire_lock();
_tdlmod = imp.load_source(modname,filename,infile);
finally:
TDLOptions.enable_save_config(True);
TDLOptions.save_config();
imp.release_lock();
infile.close();
_update_modlist();
return (_tdlmod,text);
# CumulativeError exceptions returned as is
except TDL.CumulativeError:
_dprint(0,'cumulative error importing TDL file:',filename);
traceback.print_exc();
args = sys.exc_info()[1].args;
_dprint(0,'number of errors in list:',len(args));
_dprint(1,'errors are: {',args,'}');
raise;
# Other exceptions wrapped in a CumulativeError, and
# location information is added in
except:
(etype,exc,tb) = sys.exc_info();
_dprint(0,'exception importing TDL file:',filename);
traceback.print_exception(etype,exc,tb);
# use TDL add_error() to process the error, since this automatically
# adds location information
ns = TDL.NodeScope();
ns.AddError(exc,traceback.extract_tb(tb),error_limit=None);
# re-raise as a CumulativeError
raise TDL.CumulativeError(*ns.GetErrors());
def testMeqtreesBatchJob():
trace_sync = True
# sys.settrace(trace_lines);
if len(sys.argv) > 1:
newdir = PACKAGE_TEST_DIR
print("========== Changing working directory to", newdir)
os.chdir(newdir)
print("========== Making required symlinks")
run("rm {0:s}/WSRT_ANTENNA ; ln -s {1:s}".format(
PACKAGE_TEST_DIR,
path(os.path.join(PACKAGE_TEST_DIR, "WSRT_ANTENNA"))))
run("rm {0:s}/test-lsm.txt; ln -s {1:s}".format(
PACKAGE_TEST_DIR, path(os.path.join(path("test-lsm.txt")))))
if not os.access(".", os.R_OK | os.W_OK):
print("Directory", os.getcwd(),
"not writable, can't run tests in here.")
print(
"You may choose to run the tests in a different directory by giving it as an argument to this script."
)
sys.exit(1)
## check if we have owlcat or owlcat.sh
owlcat = ""
for dirname in os.environ['PATH'].split(':'):
for binary in "owlcat", "owlcat.sh":
tmp = os.path.join(dirname, binary)
if os.path.exists(tmp):
owlcat = tmp
break
if owlcat:
break
if not owlcat:
raise RuntimeError("Can't locate owlcat or owlcat.sh")
## make simulated MS
print("========== Removing files")
run("rm -fr {0:s}/WSRT.MS* {0:s}/WSRT*img {0:s}/WSRT*fits".format(
PACKAGE_TEST_DIR))
print("========== Running makems")
run("makems %s" % path(os.path.join(PACKAGE_TEST_DIR, "WSRT_makems.cfg")))
run("mv {0:s}/WSRT.MS_p0 {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR))
os.environ["MEQTREES_CATTERY_PATH"] = Cattery.__path__[0]
run("pyxis {0:s}/WSRT.MS ms.prep".format(PACKAGE_TEST_DIR))
#TODO: this is hacky, bug in CASAcore
run("ls -ld {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR))
run("{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS".format(
owlcat, PACKAGE_TEST_DIR))
run("lwimager ms={0:s}/WSRT.MS data=CORRECTED_DATA mode=channel weight=natural npix=10"
.format(PACKAGE_TEST_DIR))
# make test LSMs
run("""tigger-convert {0:s}/test-lsm.txt --rename --format "ra_d dec_d i q u v" --center 0.1deg,60.5deg -f"""
.format(PACKAGE_TEST_DIR))
run("""tigger-convert {0:s}/test-lsm.lsm.html {0:s}/test-lsm1.txt --output-format "name ra_h dec_d i q u v freq0 spi rm tags..." -f"""
.format(PACKAGE_TEST_DIR))
run("""cut -d " " -f 1-10 {0:s}/test-lsm1.txt >{0:s}/test-lsm1.txt.tmp""".
format(PACKAGE_TEST_DIR))
run("""diff {0:s}/test-lsm1.txt.tmp {1:s} || diff {0:s}/test-lsm1.txt.tmp {2:s}"""
.format(
PACKAGE_TEST_DIR,
path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm1.txt.reference')),
path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm2.txt.reference'))))
run("""tigger-convert {0:s}/test-lsm1.txt --format "name ra_h dec_d i q u v freq0 spi rm tags..." -f"""
.format(PACKAGE_TEST_DIR))
run("""{0:s} plot-ms {1:s}/WSRT.MS DATA:I -o data_i.png""".format(
owlcat, PACKAGE_TEST_DIR))
run("""{0:s} run-imager ms={1:s}/WSRT.MS name_dirty=tmp""".format(
owlcat, PACKAGE_TEST_DIR))
print("importing meqserver")
from Timba.Apps import meqserver
print("importing Compile")
from Timba.TDL import Compile
print("importing TDLOptions")
from Timba.TDL import TDLOptions
# This starts a meqserver. Note how we pass the "-mt 2" option to run two threads.
# A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv).
print("Starting meqserver")
mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", "2"])
try:
## make simulation with perfect MODEL_DATA
script = path(os.path.join(PACKAGE_TEST_DIR, "sim.py"))
print("========== Compiling", script)
TDLOptions.config.read(
path(os.path.join(PACKAGE_TEST_DIR, "testing.tdl.conf")))
TDLOptions.config.set(
"calibrate", "ms_sel.msname",
os.path.join(PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate", "ms_sel.msname")))
TDLOptions.config.set(
"calibrate", "tiggerlsm.filename",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate", "tiggerlsm.filename")))
TDLOptions.config.set(
"calibrate", "lsm.filename",
os.path.join(PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate", "lsm.filename")))
TDLOptions.config.set(
"calibrate", "cal_g_diag.g_diag.table_name",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate",
"cal_g_diag.g_diag.table_name")))
TDLOptions.config.set(
"calibrate", "cal_g_offdiag.g_offdiag.table_name",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate",
"cal_g_offdiag.g_offdiag.table_name")))
TDLOptions.config.set(
"simulate-model", "lsm.filename",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("simulate-model", "lsm.filename")))
TDLOptions.config.set(
"simulate-model", "ms_sel.msname",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("simulate-model", "ms_sel.msname")))
TDLOptions.config.set(
"simulate-model", "tiggerlsm.filename",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("simulate-model", "tiggerlsm.filename")))
TDLOptions.config.set(
"calibrate", "img_sel.output_fitsname",
os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.CORRECTED_DATA.channel.1ch.fits"))
TDLOptions.config.set(
"simulate-model", "img_sel.output_fitsname",
os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.MODEL_DATA.channel.1ch.fits"))
with open(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"),
"w") as f:
TDLOptions.config.write(f)
TDLOptions.config.read(
path(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf")))
# needs to re-read because of a Timba perculiarity
mod, ns, msg = Compile.compile_file(mqs,
script,
config="simulate-model")
print("========== Simulating MODEL_DATA ")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print("========== Imaging MODEL_DATA ")
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
## compare against reference image
print("========== Verifying test image ")
if not os.path.exists(
os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.MODEL_DATA.channel.1ch.fits")):
raise RuntimeError("Output FITS file does not exist")
if not os.path.exists(
os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")):
raise RuntimeError("Reference FITS file does not exist")
verify_image(os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.MODEL_DATA.channel.1ch.fits"),
path(os.path.join(PACKAGE_TEST_DIR,
"test-refimage.fits")),
maxdelta=1e-3)
print("========== Compiling script with modified config")
TDLOptions.init_options("simulate-model", save=False)
TDLOptions.set_option("me.g_enable", True)
mod, ns, msg = Compile.compile_file(mqs, script, config=None)
print("========== Simulating DATA ")
TDLOptions.set_option("ms_sel.output_column", "DATA")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print("========== Imaging DATA ")
TDLOptions.set_option("img_sel.imaging_column", "DATA")
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
## calibrate
script = path(os.path.join(PACKAGE_TEST_DIR, "cal.py"))
print("========== Compiling", script)
mod, ns, msg = Compile.compile_file(mqs, script, config="calibrate")
print("========== Calibrating ")
TDLOptions.get_job_func('cal_G_diag')(mqs, None, wait=True)
print("========== Imaging MODEL_DATA ")
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
finally:
print("Stopping meqserver")
# this halts the meqserver
meqserver.stop_default_mqs()
print("========== Making plots of solutions ")
run("""{0:s} plot-ms {1:s}/WSRT.MS CORRECTED_DATA:I -I ">0" -o {1:s}/corrected_data_i.png"""
.format(owlcat, PACKAGE_TEST_DIR))
run("""{0:s} plot-parms -l {1:s}/WSRT.MS/G_diag.fmep""".format(
owlcat, PACKAGE_TEST_DIR))
run("""{0:s} plot-parms {1:s}/WSRT.MS/G_diag.fmep G:*/norm -o {1:s}/parmplot.png"""
.format(owlcat, PACKAGE_TEST_DIR))
run("""{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS""".format(
owlcat, PACKAGE_TEST_DIR))
## compare against reference image
print("========== Verifying residual image ")
if not os.path.exists(
os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.CORRECTED_DATA.channel.1ch.fits")):
raise RuntimeError("Output FITS file does not exist")
if not os.path.exists(
os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits")):
raise RuntimeError("Reference FITS file does not exist")
verify_image(os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.CORRECTED_DATA.channel.1ch.fits"),
os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits"),
maxdelta=1e-3)
## all tests succeeded
print("========== Break out the bubbly, this hog is airborne!")
# now we can exit
print("Bye!")
def import_content(self, force=False, show_options=False):
"""imports TDL module but does not run _define_forest().
Depending on autosync/modified state, asks to save or revert.
If module is already imported, does nothing, unless force=True,
in which case it imports unconditionally.
If do_compile=True, proceeds to show compile-time options on success,
or to compile directly if there are no options
Return value:
True on successful import
None if cancelled by user.
Import errors are posted to the error dialog.
"""
# save list of publishers (since forest will be cleared on import)
if self._pub_nodes is None:
if TDL.Compile.last_imported_file() == self._filename:
self._pub_nodes = [
node.name for node in meqds.nodelist.iternodes()
if node.is_publishing()
]
_dprint(2, "last imported file matches, saving",
len(self._pub_nodes), "publishers")
else:
self._pub_nodes = []
_dprint(2, "last imported file doesn't match:",
TDL.Compile.last_imported_file(), self._filename)
_dprint(1, self._filename, "importing")
self.clear_message()
self.clear_errors()
self._options_menu.hide()
self._tb_tdlexec.hide()
self._tdlexec_menu.hide()
# change the current directory to where the file is
# os.chdir(os.path.dirname(self._filename));
# The Python imp module expects text to reside in a disk file, which is
# a pain in the ass for us if we're dealing with modified text or text
# entered on-the-fly. So, either save or sync before proceeding
global _external_sync
if self._document.isModified() or not self._filename:
if not self._save_file():
return None
else:
if not self.sync_external_file(ask=False):
return None
# if we already have an imported module and disk file hasn't changed, skip
# the importing step
busy = BusyIndicator()
if force or self._tdlmod is None or self._tdlmod_filetime == self._file_disktime:
# reset data members
_dprint(2, self._filename,
"emitting signal for 0 compile-time options")
self.emit(PYSIGNAL("hasCompileOptions()"), self, 0)
self._options_menu.hide()
self._options_menu.clear()
self._tdlmod = None
# get text from editor
tdltext = str(self._document.toPlainText())
try:
tdlmod, tdltext = TDL.Compile.import_tdl_module(
self._filename, tdltext)
# catch import errors
except TDL.CumulativeError as value:
_dprint(0, "caught cumulative error, length", len(value.args))
self._error_window.set_errors(value.args,
message="TDL import failed")
busy = None
return None
except Exception as value:
_dprint(0, "caught other error, traceback follows")
traceback.print_exc()
self._error_window.set_errors([value],
message="TDL import failed")
busy = None
return None
# remember module and nodescope
self._tdlmod = tdlmod
self._tdltext = tdltext
self._tdlmod_filetime = self._file_disktime
# build options menu from compile-time options
opt_tw = self._options_menu.treeWidget()
opts = TDLOptions.get_compile_options()
if opts:
# add options
try:
TDLOptions.populate_option_treewidget(opt_tw, opts)
except Exception as value:
_dprint(0, "error setting up TDL options GUI")
traceback.print_exc()
self._error_window.set_errors(
[value], message="Error setting up TDL options GUI")
busy = None
return None
# self._tb_opts.show();
_dprint(2, self._filename, "emitting signal for", len(opts),
"compile-time options")
self.emit(PYSIGNAL("hasCompileOptions()"), self, len(opts))
# success, show options or compile
if show_options and self.has_compile_options():
self._options_menu.adjustSizes()
self._options_menu.show()
busy = None
return True
context,level = opt[:2];
debuglevels[context] = int(level);
# tell verbosity class to not parse argv -- we do it ourselves here
Timba.utils.verbosity.disable_argv();
for optstr in (options.verbose or []):
opt = optstr.split("=") + ['1'];
context,level = opt[:2];
Timba.utils.verbosity.set_verbosity_level(context,int(level));
# start meqserver
from Timba.Apps import meqserver
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
TDLOptions.enable_save_config(False);
print "### Starting meqserver";
mqs = meqserver.default_mqs(wait_init=10,extra=["-mt",str(options.mt)]);
retcode = 0;
# use a try...finally block to exit meqserver cleanly at the end
try:
if not os.path.exists(options.config):
print "Config file %s doesn't exist"%options.config;
sys.exit(1);
print "### Attaching to configuration file",options.config;
TDLOptions.config.read(options.config);
# disable the writing-out of configuration
TDLOptions.config.set_save_filename(None);
_dprint(2, "not reenabling publishing of", name,
", as it no longer exists?")
# clear publisher list so it's re-saved next time in import_content()
self._pub_nodes = None
### NB: presume this all was successful for now
# insert node scope into TDL module
setattr(_tdlmod, '_tdl_nodescope', ns)
# does the script define an explicit job list?
joblist = getattr(_tdlmod, '_tdl_job_list', [])
if not joblist:
joblist = []
# try to build it from implicit function names
for (name, func) in _tdlmod.__dict__.iteritems():
if name.startswith("_tdl_job_") and callable(
func) and not TDLOptions.is_jobfunc_defined(func):
joblist.append(func)
# does the script define a testing function?
testfunc = getattr(_tdlmod, '_test_forest', None)
if not callable(testfunc):
testfunc = getattr(_tdlmod, 'test_forest', None)
if callable(testfunc):
res = QMessageBox.warning(
self, "Deprecated method",
"""Your script contains a test_forest() method. This is deprecated
and will be disabled in the future. Please rename it to
_test_forest().
""", QMessageBox.Ok)
if callable(testfunc):
joblist.append(testfunc)
joblist.sort(lambda a, b: cmp(str(a), str(b)))
