def OK(self):
if self.datadir is None:
utils.show_message("Please choose data directory !")
return False
if self.dirname is None:
utils.show_message("Please choose work directory !")
return False
# check & set jss
if str(self.ui.comboBox.currentText()).strip()[:2] == "NO":
self.jss = None
elif str(self.ui.comboBox.currentText()).strip()[:3] == "PBS":
# check if PBS is working
if not utils.check_PBS():
utils.show_message("NO PBS detected !")
return False
self.jss = "PBS"
else:
utils.show_message("I can not recognize the job submitting system")
return False
# check data format
self.format_index = self.ui.comboBox_2.currentIndex()
# check subDir
if self.ui.checkBox.checkState() == QtCore.Qt.Checked:
self.subDir = True
# check data dir
if not os.path.exists(os.path.join(self.datadir)):
utils.show_message("Data directory is invalid !")
return False
else:
stat = utils.check_datadir(self.datadir, self.format_index,
self.namespace['data_format'],
self.subDir)
# checking results, interact with user
if stat[0] == 0 and stat[1] == 0:
re = utils.show_warning("I can not find any %s files in the data folder, still go on ?" \
% self.namespace['data_format'][self.format_index])
if re == 1:
pass
else:
return False
elif stat[0] == -1 and stat[1] == -1:
utils.show_message("Some error occurred ! Please report it")
return False
elif stat[0] == 1:
if stat[1] == 1:
pass
elif stat[1] == -1:
subdre = ''
if self.subDir:
subdre = ' no'
re = utils.show_warning(
"It seems there are%s sub directories in data folder, still go on ?"
% subdre)
if re == 1:
pass
else:
return False
else:
utils.show_message(
"Some error occurred ! Please report it")
return False
elif type(stat[0]) == str:
if stat[1] == 1:
re = utils.show_warning("It seems the data format is '%s', still use '%s' ?" \
% (stat[0], self.namespace['data_format'][self.format_index]) )
if re == 1:
pass
else:
return False
elif stat[1] == -1:
subdre = ''
if self.subDir:
subdre = ' no'
re = utils.show_warning("It seems there are%s sub directories in data folder, \
and the data format is '%s', still go on and use '%s' ?" \
% (subdre, stat[0], self.namespace['data_format'][self.format_index]) )
if re == 1:
pass
else:
return False
else:
utils.show_message(
"Some error occurred ! Please report it")
return False
# make dir
# if not os.path.exists(os.path.join(self.dirname, self.namespace['ini'])):
self.makedirs()
# copy darkcal to Process/config
project_darkcal_path = os.path.join(
self.dirname, self.namespace['project_structure'][0],
"config/darkcal_default.ini")
if not os.path.exists(project_darkcal_path):
shutil.copyfile(
os.path.join(
os.path.split(os.path.realpath(__file__))[0],
"darkcal.ini"), project_darkcal_path)
subprocess.check_call( "ln -fs %s %s" % ( project_darkcal_path, \
os.path.join(self.dirname, self.namespace['project_structure'][0], "config/darkcal.ini") ), shell=True )
# write project.ini
config_name = self.namespace['project_ini'][0]
config_item = self.namespace['project_ini'][1].split(',')
utils.write_config(os.path.join(self.dirname, self.namespace['ini']),\
{config_name:{config_item[0]:self.datadir, config_item[1]:self.jss, \
config_item[2]:self.format_index, config_item[3]:self.subDir, \
config_item[4]:self.job_control[0], config_item[5]:self.job_control[1]}}, 'w')
utils.print2projectLog(self.dirname,
"Select data dir %s" % self.datadir)
utils.print2projectLog(self.dirname,
"Choose work dir %s" % self.dirname)
utils.print2projectLog(self.dirname, "Spipy GUI opened successfully")
# close and open main gui
self.mainapp.setup(self.dirname, self.datadir, self.jss, self.subDir,
self.format_index)
self.mainapp.show()
self.close()
def view_job(self): jobview.show_jobView(self) utils.print2projectLog(self.dirname, "Open job viewer.")
def table_menu(self, position):
"""
entrance to submit jobs
"""
# selected cells
selected_runs = []
selected_tag_remarks = {}
selected_datafile = {}
for pos in self.ui.tableWidget.selectionModel().selection().indexes():
row = pos.row()
run_name = str(self.ui.tableWidget.item(row, 0).text())
if len(self.process_data[run_name][0]) == 0:
utils.show_message("Run %s does not contain any data. Skip." % run_name)
continue
tag_remarks_name = str(self.ui.tableWidget.item(row, 1).text())
selected_runs.append(run_name)
selected_tag_remarks[run_name] = self.split_tag_remarks(tag_remarks_name)
selected_datafile[run_name] = self.process_data[run_name][0]
# show menu
if len(selected_runs) > 0:
# get assignments
assignments = str(self.ui.comboBox.currentText())
# show menu
menu = QtGui.QMenu()
a1 = menu.addAction("Run %s" % assignments)
menu.addSeparator()
# a2 = menu.addAction("Terminate all")
menu_sub = menu.addMenu("Terminate")
b = []
if len(selected_runs) > 1:
for assign in self.namespace['process_assignments']:
b.append(menu_sub.addAction(assign))
if selected_tag_remarks[selected_runs[0]][0] != "--":
menu.addSeparator()
a4 = menu.addAction("Open %s results in data viewer" % assignments)
elif len(selected_runs) == 1:
run_tag_remarks = self.get_existing_runtags(assignments, selected_runs[0])
for tr in run_tag_remarks:
tr_status = self.JobCenter.get_run_status(selected_runs[0], self.namespace['project_structure'][0], assignments, tr[0], tr[1])
if tr_status == self.JobCenter.RUN:
b.append( menu_sub.addAction("%s.%s.%s.%s" % (assignments, selected_runs[0], tr[0], tr[1])) )
menu.addSeparator()
if selected_tag_remarks[selected_runs[0]][0] == "darkcal":
a4 = menu.addAction("Set as current darkcal")
elif selected_tag_remarks[selected_runs[0]][0] != "--":
a4 = menu.addAction("Open %s results in data viewer" % assignments)
else:
a4 = 0
else:
pass
# exec
action = menu.exec_(self.ui.tableWidget.mapToGlobal(position))
# parse selection
job_type = self.namespace['project_structure'][0] + "/" + assignments
if action == a1:
utils.print2projectLog(self.dirname, "Choose %s on %s" % (assignments, str(selected_runs)))
self.JobCenter.TableRun_showoff(job_type, selected_runs, selected_datafile, selected_tag_remarks)
#elif action == a2:
# print("Terminate all jobs of %s" % str(selected_runs))
elif len(selected_runs) == 1 and action == a4:
if selected_tag_remarks[selected_runs[0]][0] == "darkcal":
# re-link current-darkcal.h5
tmp_darkfile = utils.fmt_job_dir(selected_runs[0], \
selected_tag_remarks[selected_runs[0]][0], selected_tag_remarks[selected_runs[0]][1])
tmp_darkfile = os.path.join(self.dirname, self.namespace['project_structure'][0], self.namespace['process_HF'], tmp_darkfile)
try:
tmp_darkfile = glob.glob(os.path.join(tmp_darkfile, "*darkcal.h5"))[0]
except:
utils.show_message("I cannot find ?.darkcal.h5 in this run. Fail to set current darkcal.")
return
tmp_curr_darklink = os.path.join(self.dirname, self.namespace['project_structure'][0], self.namespace['process_HF'], self.namespace['darkcal'])
prev_dark_run = None
if os.path.exists(tmp_curr_darklink):
prev_dark_run = subprocess.check_output("ls -l %s | awk -F'->' '{print $2}' | tr -d ' '" % tmp_curr_darklink, shell=True)
prev_dark_run = os.path.split(prev_dark_run.strip("\n"))[-1].split('.')[0]
subprocess.check_call("ln -fs %s %s" % (tmp_darkfile, tmp_curr_darklink), shell=True)
# refresh table info
if prev_dark_run is not None:
self.process_data[prev_dark_run][6] = "--"
self.process_data[selected_runs[0]][6] = "Current-Darkcal"
utils.print2projectLog(self.dirname, "Set %s as current darkcal" % selected_runs[0])
# draw table
self.draw_table()
else:
# open data viewer and add files
tmp = utils.fmt_job_dir(selected_runs[0], \
selected_tag_remarks[selected_runs[0]][0], selected_tag_remarks[selected_runs[0]][1])
tmp = os.path.join(self.dirname, self.namespace['project_structure'][0], assignments, tmp, '*.h5')
tmp = glob.glob(tmp)
if not data_viewer.is_shown():
data_viewer.show_data_viewer(self)
data_viewer.add_files(tmp)
utils.print2projectLog(self.dirname, "Add %s results of %s to data viewer." % (assignments, selected_runs[0]))
elif len(selected_runs) == 1 and action in b:
tmp = str(action.text()).split('.')
jid = self.JobCenter.get_jid(tmp[0], tmp[1], tmp[2], tmp[3])
re = utils.show_warning("Terminate job %d of %s ?" % (jid, str(selected_runs)))
if re == 1:
tmp = self.JobCenter.kill_job(jid)
if tmp[0] == 1:
utils.print2projectLog(self.dirname, "Terminate job %d of %s" % (jid, str(selected_runs)))
utils.show_message("Job %s is successfully terminated" % str(action.text()))
else:
utils.print2projectLog(self.dirname, "Fail to terminate job %d of %s" % (jid, str(selected_runs)))
utils.show_message("**FAIL** to terminate job !" % str(action.text()))
# refresh
self.update_table_runs()
self.draw_table()
elif len(selected_runs) > 1 and action in b:
assign = str(action.text())
all_jid = []
for run_name in selected_runs:
tag_remarks = self.get_existing_runtags(assign, run_name)
for tr in tag_remarks:
tr_status = self.JobCenter.get_run_status(run_name, self.namespace['project_structure'][0], assign, tr[0], tr[1])
if tr_status == self.JobCenter.RUN:
tmp_jid = self.JobCenter.run_view[utils.fmt_runview_key(assign, run_name, tr[0], tr[1])]
all_jid.append(tmp_jid)
re = utils.show_warning("Terminate all these %d jobs ?" % len(all_jid))
if re == 1:
killed = 0
for tmp_jid in all_jid:
killed += self.JobCenter.kill_job(jid)
utils.print2projectLog("Terminate %d %s jobs of %s" % (killed, assign, str(selected_runs)))
utils.show_message("Successfully terminate %d jobs" % killed)
# refresh
self.update_table_runs()
self.draw_table()
elif len(selected_runs) > 1 and action == a4:
if not data_viewer.is_shown():
data_viewer.show_data_viewer(self)
for selected_run in selected_runs:
tmp = utils.fmt_job_dir(selected_run, \
selected_tag_remarks[selected_run][0], selected_tag_remarks[selected_run][1])
tmp = os.path.join(self.dirname, self.namespace['project_structure'][0], assignments, tmp, '*.h5')
tmp = glob.glob(tmp)
data_viewer.add_files(tmp)
utils.print2projectLog(self.dirname, "Add %s results of %s to data viewer." % (assignments, str(selected_runs)))
else:
pass
else:
menu = QtGui.QMenu()
if self.ui.tableWidget.horizontalHeader().resizeMode(0) == QtGui.QHeaderView.Stretch:
a1 = menu.addAction("Unfill table window")
else:
a1 = menu.addAction("Fill table window")
menu.addSeparator()
a2 = menu.addAction("Set 'force overwrite' to %s" % str(not self.JobCenter.force_overwrite))
action = menu.exec_(self.ui.tableWidget.mapToGlobal(position))
if action == a1:
if self.ui.tableWidget.horizontalHeader().resizeMode(0) == QtGui.QHeaderView.Stretch:
self.ui.tableWidget.horizontalHeader().setResizeMode(QtGui.QHeaderView.Interactive)
else:
self.ui.tableWidget.horizontalHeader().setResizeMode(QtGui.QHeaderView.Stretch)
elif action == a2:
self.JobCenter.reverseForceOverwrite()
else:
pass
def closeEvent(self, event): #self.JobCenter.write_job_hub() utils.print2projectLog(self.dirname, "Close spipy GUI.") QtGui.qApp.quit()
def run(self):
# update self.run_view
# update run_tag, config, run_remarks, pid, savepath, stime, status in self.jobs
# read tag, remarks and darkcal
title = str(self.windowTitle())
module, assignments = title.split('.')
config_file_name = str(self.ui.comboBox.currentText()) # 'xxx.ini'
config_file = os.path.join(self.rootdir, module, "config",
config_file_name)
tag_name = self.extract_tag(config_file_name)
self.submit_queue = str(self.ui.lineEdit_3.text())
if not os.path.isfile(config_file):
utils.show_message("Choose a configuration file !")
return
remarks = utils.fmt_remarks(str(self.ui.lineEdit.text()))
if len(remarks) == 0:
remarks = "default"
# hit-finding has darkcal
if assignments == self.namespace['process_HF']:
if config_file_name == "darkcal.ini":
if len(str(self.ui.lineEdit_2.text())) == 0:
utils.show_message(
"Dark calibration needs data location in raw h5/cxi file."
)
return
if not str(self.ui.lineEdit_2.text()) == self.darkcal_inh5:
self.darkcal_inh5 = str(self.ui.lineEdit_2.text())
utils.write_config(
os.path.join(self.rootdir,
self.namespace['project_structure'][0],
'config/darkcal.ini'),
{'darkcal': {
'inh5': self.darkcal_inh5
}}, "a")
else:
if not os.path.isfile(str(self.ui.lineEdit_2.text())):
re = utils.show_warning(
"Dark calibration file is invalid, \
which may affect the hit-finding accuracy. Ignore and go on?")
if re == 0:
return
else:
self.ui.lineEdit_2.setText("None")
while not self.job_queue['waiting'].empty():
wjid = self.job_queue['waiting'].get()
jobdir = utils.fmt_job_dir(self.jobs[wjid].run_name, tag_name,
remarks)
self.jobs[wjid].run_tag = tag_name
self.jobs[wjid].config = config_file
self.jobs[wjid].run_remarks = remarks
self.jobs[wjid].savepath = os.path.join(self.jobs[wjid].savepath,
jobdir)
# submit
pobj = self.packSubmit(self.jobs[wjid])
if pobj is not None:
self.jobs[wjid].jss = self.jss
self.jobs[wjid].submit_time = time.time()
self.jobs[wjid].process_obj = pobj
# self.run_view
run_view_key = utils.fmt_runview_key(self.jobs[wjid].assignments, \
self.jobs[wjid].run_name, tag_name, remarks)
self.run_view[run_view_key] = wjid
# write to project log
utils.print2projectLog(
self.rootdir, "Submit %s job on %s : pid is %s" %
(self.jobs[wjid].assignments, self.jobs[wjid].run_name,
str(pobj.pid)))
# write job infomation into jobHub
self.write_job_hub(wjid)
# refresh
self.main_gui.update_table_runs()
self.main_gui.draw_table()
else:
utils.show_message("Fail to submit job %s" % jobdir)
utils.print2projectLog(
self.rootdir, "Fail to submit job : %s (%s.%s)" %
(self.jobs[wjid].assignments, self.jobs[wjid].run_name,
self.jobs[wjid].run_tag))
self.jobs.pop(wjid)
self.close()
pass
def reverseForceOverwrite(self):
self.force_overwrite = not self.force_overwrite
utils.print2projectLog(
self.rootdir,
"Set project force overwrite to %s" % str(force_overwrite))