def test_combining_scripts(self):
s1 = Script()
s2 = Script()
s1.add("a")
s1.add("b")
s2.add("c")
s2.add("d")
assert "a\nb\nc\nd" == str(s1 + s2)
def test_combining_scripts(self):
s1 = Script()
s2 = Script()
s1.add("a")
s1.add("b")
s2.add("c")
s2.add("d")
assert "a\nb\nc\nd" == str(s1 + s2)
def test_cp(self):
s = Script()
s.cp("a", "b")
self.assertEqual("cp a b", str(s))
def test_add_two_argument_command(self):
s = Script()
s._add_two_argument_command("command", "a", "b")
self.assertEqual("command a b", str(s))
def test_add_two_argument_command_multiple_times(self):
s = Script()
s._add_two_argument_command("cmd1", "a", "b")
s._add_two_argument_command("cmd2", "c", "d")
s._add_two_argument_command("cmd3", "e", "f")
self.assertEqual("cmd1 a b\ncmd2 c d\ncmd3 e f", str(s))
def test_str(self):
s = Script()
s.add("a")
s.add("b")
self.assertEqual("a\nb", str(s))
def test_clear(self):
s = Script()
s.add("a")
s.clear()
self.assertEqual("", str(s))
def test_separator(self):
s = Script()
s.separator()
self.assertEqual('echo "' + "=" * 42 + '"', str(s))
def test_add_two_argument_command(self):
s = Script()
s._add_two_argument_command("command", "a", "b")
self.assertEqual('command a b', str(s))
def test_mkdir(self):
s = Script()
s.mkdir("/path/to/file")
self.assertEqual('mkdir -p "/path/to/file"', str(s))
def test_iget(self):
s = Script()
s.iget("/path/to/file")
self.assertEqual('iget -v "/path/to/file"', str(s))
def test_echo(self):
s = Script()
s.echo("test")
self.assertEqual('echo "test"', str(s))
def test_separator(self):
s = Script()
s.separator()
self.assertEqual('echo "' + "=" * 42 + '"', str(s))
def test_mv(self):
s = Script()
s.mv("a", "b")
self.assertEqual('mv a b', str(s))
def test_cp(self):
s = Script()
s.cp("a", "b")
self.assertEqual('cp a b', str(s))
def test_add_two_argument_command_multiple_times(self):
s = Script()
s._add_two_argument_command("cmd1", "a", "b")
s._add_two_argument_command("cmd2", "c", "d")
s._add_two_argument_command("cmd3", "e", "f")
self.assertEqual('cmd1 a b\ncmd2 c d\ncmd3 e f', str(s))
def test_mv(self):
s = Script()
s.mv("a", "b")
self.assertEqual("mv a b", str(s))
def test_iget(self):
s = Script()
s.iget("/path/to/file")
self.assertEqual('iget -v "/path/to/file"', str(s))
def test_echo(self):
s = Script()
s.echo("test")
self.assertEqual('echo "test"', str(s))
def main():
args = docopt(__doc__)
DET_ID = int(args["-d"])
TMAX = int(args["-t"])
ET_PER_RUN = int(args["-e"]) * 60 # [s]
RUNS_PER_JOB = int(args["-n"])
VMEM = args["-m"]
CWD = os.getcwd()
LOG_PATH = args["-l"]
JOB_NAME = args["-j"]
CALIB_PATH = os.path.join(CWD, args["OUTPUT_PATH"])
RUN_SUBSTR = args["-s"]
DRYRUN = args["-q"]
if not os.path.exists(CALIB_PATH):
os.makedirs(CALIB_PATH)
run_table = km3db.StreamDS(container="pd").get("run_table", det_id=DET_ID)
phys_run_table = run_table[run_table.RUNSETUPNAME.str.contains(RUN_SUBSTR)]
phys_runs = set(phys_run_table.RUN)
processed_runs = set(
int(re.search("_\\d{8}_(\\d{8})", s).group(1))
for s in glob(os.path.join(CALIB_PATH, "*.k40_cal.p"))
)
remaining_runs = list(phys_runs - processed_runs)
print("Remaining runs: {}".format(remaining_runs))
s = Script()
for job_id, runs_chunk in enumerate(kp.tools.chunks(remaining_runs, RUNS_PER_JOB)):
n_runs = len(runs_chunk)
print("Preparing batch script for a chunk of {} runs.".format(len(runs_chunk)))
s.add("cd $TMPDIR; mkdir -p $USER; cd $USER")
for run in runs_chunk:
s.add("echo Processing {}:".format(run))
irods_path = kp.tools.irods_path(DET_ID, run)
root_filename = os.path.basename(irods_path)
calib_filename = root_filename + ".k40_cal.p"
s.add("iget -v {}".format(irods_path))
s.add(
"CTMIN=$(JPrint -f {}|grep '^ctMin'|awk '{{print $2}}')".format(
root_filename
)
)
s.add(
"k40calib {} {} -t {} -c $CTMIN -o {}".format(
root_filename, DET_ID, TMAX, calib_filename
)
)
s.add("cp {} {}".format(calib_filename, CALIB_PATH))
s.add("rm -f {}".format(root_filename))
s.add("rm -f {}".format(calib_filename))
s.add("echo Run {} processed.".format(run))
s.add("echo " + 42 * "=")
walltime = time.strftime("%H:%M:%S", time.gmtime(ET_PER_RUN * n_runs))
qsub(
s,
"{}_{}".format(JOB_NAME, job_id),
walltime=walltime,
vmem=VMEM,
log_path=LOG_PATH,
irods=True,
dryrun=DRYRUN,
)
if DRYRUN:
break
s.clear()
def test_mkdir(self):
s = Script()
s.mkdir("/path/to/file")
self.assertEqual('mkdir -p "/path/to/file"', str(s))
def test_clear(self):
s = Script()
s.add("a")
s.clear()
self.assertEqual('', str(s))
def test_add(self):
s = Script()
s.add("a")
s.add("b")
def test_str(self):
s = Script()
s.add("a")
s.add("b")
self.assertEqual("a\nb", str(s))
def main():
args = docopt(__doc__)
DET_ID = int(args['-d'])
TMAX = int(args['-t'])
ET_PER_RUN = int(args['-e']) * 60 # [s]
RUNS_PER_JOB = int(args['-n'])
VMEM = args['-m']
CWD = os.getcwd()
LOG_PATH = args['-l']
JOB_NAME = args['-j']
CALIB_PATH = os.path.join(CWD, args['OUTPUT_PATH'])
RUN_SUBSTR = args['-s']
DRYRUN = args['-q']
if not os.path.exists(CALIB_PATH):
os.makedirs(CALIB_PATH)
db = kp.db.DBManager()
run_table = db.run_table(det_id=DET_ID)
phys_run_table = run_table[run_table.RUNSETUPNAME.str.contains(RUN_SUBSTR)]
phys_runs = set(phys_run_table.RUN)
processed_runs = set(
int(re.search("_\\d{8}_(\\d{8})", s).group(1))
for s in glob(os.path.join(CALIB_PATH, '*.k40_cal.p'))
)
remaining_runs = list(phys_runs - processed_runs)
print("Remaining runs: {}".format(remaining_runs))
s = Script()
for job_id, runs_chunk in enumerate(kp.tools.chunks(remaining_runs,
RUNS_PER_JOB)):
n_runs = len(runs_chunk)
print(
"Preparing batch script for a chunk of {} runs.".format(
len(runs_chunk)
)
)
s.add("cd $TMPDIR; mkdir -p $USER; cd $USER")
for run in runs_chunk:
s.add("echo Processing {}:".format(run))
irods_path = kp.tools.irods_filepath(DET_ID, run)
root_filename = os.path.basename(irods_path)
calib_filename = root_filename + '.k40_cal.p'
s.add("iget -v {}".format(irods_path))
s.add(
"CTMIN=$(JPrint -f {}|grep '^ctMin'|awk '{{print $2}}')".
format(root_filename)
)
s.add(
"k40calib {} {} -t {} -c $CTMIN -o {}".format(
root_filename, DET_ID, TMAX, calib_filename
)
)
s.add("cp {} {}".format(calib_filename, CALIB_PATH))
s.add("rm -f {}".format(root_filename))
s.add("rm -f {}".format(calib_filename))
s.add("echo Run {} processed.".format(run))
s.add("echo " + 42 * "=")
walltime = time.strftime('%H:%M:%S', time.gmtime(ET_PER_RUN * n_runs))
qsub(
s,
'{}_{}'.format(JOB_NAME, job_id),
walltime=walltime,
vmem=VMEM,
log_path=LOG_PATH,
irods=True,
dryrun=DRYRUN
)
if DRYRUN:
break
s.clear()
def test_add(self):
s = Script()
s.add("a")
s.add("b")