def list_smf(self):
print "Getting the list of SMF files"
try:
utils.shell("find {catpath} -name '*.smf' > {cat_filelist} 2> {logdir}/cat-filelist.log".format(catpath=self.catpath, cat_filelist=self.cat_filelist, logdir=self.logdir))
catsmf = [ s.strip() for s in open(self.cat_filelist).xreadlines() ]
print "\tSMF files in catalog: %d" % len(catsmf)
except subprocess.CalledProcessError:
print >>sys.stderr, "Enumeration of catlog files failed. See cat-filelist.log for details."
print >>sys.stderr, "Snippet:"
for k, l in enumerate(file(os.path.join(self.logdir, 'cat-filelist.log')).xreadlines()):
print >>sys.stderr, " ", l.strip()
if k == 10: break
exit()
def postprocess(self):
print "Post-processing"
newsmf = [ s.strip() for s in open(self.new_filelist).xreadlines() ]
if len(newsmf) != 0:
pp = r"""
#!/bin/bash
{environ_source}
SMFLIST={smffiles}
SURVEY={survey}
export LSD_DB={db}
{lsd_prefix}lsd-import-smf -m postprocess $SURVEY ps1_det ps1_exp $SMFLIST > {logdir}/postprocess.log 2>&1
""".format(db=self.dbpath, survey=self.survey, logdir=self.logdir, smffiles=self.new_filelist, environ_source=self.environ_source, lsd_prefix=self.lsd_prefix)
pp = textwrap.dedent(pp).lstrip()
with open(self.postprocess_sh, 'w') as fp: fp.write(pp)
try:
out = utils.shell("bash %s" % self.postprocess_sh)
except subprocess.CalledProcessError:
print >>sys.stderr, "Error while postprocessing. See postprocess.log for details."
else:
print "\tNo new files imported, no need to postprocess."
def bsub(self):
print "Submitting LSF job"
newsmf = [ s.strip() for s in open(self.new_filelist).xreadlines() ]
if len(newsmf) != 0:
# Start a transaction. While lsd-import-smf would also start
# a transaction, it would also join one (perhaps even a
# failed one), if it exists. So we start it here to ensure
# no other transaction is running and we're starting with
# a clean slate.
db = lsd.DB(self.dbpath)
db.begin_transaction()
out = utils.shell("bsub < %s" % self.import_bsub).strip()
#out = "Job <27448054> is submitted to queue <itc>"
print "\t%s" % out
# Try to parse out the job number
m = re.match(r"Job <(\d+)> is submitted to queue <(\w+)>", out)
(jobid, queue) = map(m.group, [1, 2])
with open(self.jobid_fn, 'w') as fp:
fp.write("%d %s\n" % (int(jobid), queue))
else:
print "\tNo new files, not starting import."
try:
os.unlink(self.jobid_fn)
except:
pass
def build_objects(self):
print "Building object catalog"
newsmf = [s.strip() for s in open(self.new_filelist).xreadlines()]
if len(newsmf) != 0:
pp = r"""
#!/bin/bash
{environ_source}
export LSD_DB={db}
{lsd_prefix}lsd-make-object-catalog --auto --fov-radius=2 ps1_obj ps1_det ps1_exp > {logdir}/build_objects.log 2>&1
""".format(db=self.dbpath,
logdir=self.logdir,
environ_source=self.environ_source,
lsd_prefix=self.lsd_prefix)
pp = textwrap.dedent(pp).lstrip()
with open(self.build_objects_sh, 'w') as fp:
fp.write(pp)
try:
out = utils.shell("bash %s" % self.build_objects_sh)
except subprocess.CalledProcessError:
print >> sys.stderr, "Error while postprocessing. See build-objects.log for details."
else:
print "\tNo new files imported, no need to postprocess."
def ingest(self):
print "Loading new data"
newsmf = [ s.strip() for s in open(self.new_filelist).xreadlines() ]
if len(newsmf) != 0:
pp = r"""
#!/bin/bash
{environ_source}
SMFLIST={smffiles}
SURVEY={survey}
export LSD_DB={db}
export PIXLEVEL=6
{lsd_prefix}lsd-import-smf -c -f $SURVEY ps1_det ps1_exp $SMFLIST > {logdir}/ingest.log 2>&1
""".format(db=self.dbpath, survey=self.survey, logdir=self.logdir, smffiles=self.new_filelist, environ_source=self.environ_source, lsd_prefix=self.lsd_prefix)
pp = textwrap.dedent(pp).lstrip()
with open(self.ingest_sh, 'w') as fp: fp.write(pp)
try:
out = utils.shell("bash %s" % self.ingest_sh)
except subprocess.CalledProcessError:
print >>sys.stderr, "Error while loading new data. See ingest.log for details."
else:
print "\tNo new files to import."
def postprocess(self):
print "Post-processing"
newsmf = [s.strip() for s in open(self.new_filelist).xreadlines()]
if len(newsmf) != 0:
pp = r"""
#!/bin/bash
{environ_source}
SMFLIST={smffiles}
SURVEY={survey}
export LSD_DB={db}
{lsd_prefix}lsd-import-smf -m postprocess $SURVEY ps1_det ps1_exp $SMFLIST > {logdir}/postprocess.log 2>&1
""".format(db=self.dbpath,
survey=self.survey,
logdir=self.logdir,
smffiles=self.new_filelist,
environ_source=self.environ_source,
lsd_prefix=self.lsd_prefix)
pp = textwrap.dedent(pp).lstrip()
with open(self.postprocess_sh, 'w') as fp:
fp.write(pp)
try:
out = utils.shell("bash %s" % self.postprocess_sh)
except subprocess.CalledProcessError:
print >> sys.stderr, "Error while postprocessing. See postprocess.log for details."
else:
print "\tNo new files imported, no need to postprocess."
def bsub(self):
print "Submitting LSF job"
newsmf = [s.strip() for s in open(self.new_filelist).xreadlines()]
if len(newsmf) != 0:
# Start a transaction. While lsd-import-smf would also start
# a transaction, it would also join one (perhaps even a
# failed one), if it exists. So we start it here to ensure
# no other transaction is running and we're starting with
# a clean slate.
db = lsd.DB(self.dbpath)
db.begin_transaction()
out = utils.shell("bsub < %s" % self.import_bsub).strip()
#out = "Job <27448054> is submitted to queue <itc>"
print "\t%s" % out
# Try to parse out the job number
m = re.match(r"Job <(\d+)> is submitted to queue <(\w+)>", out)
(jobid, queue) = map(m.group, [1, 2])
with open(self.jobid_fn, 'w') as fp:
fp.write("%d %s\n" % (int(jobid), queue))
else:
print "\tNo new files, not starting import."
try:
os.unlink(self.jobid_fn)
except:
pass
def ingest(self):
print "Loading new data"
newsmf = [s.strip() for s in open(self.new_filelist).xreadlines()]
if len(newsmf) != 0:
pp = r"""
#!/bin/bash
{environ_source}
SMFLIST={smffiles}
SURVEY={survey}
export LSD_DB={db}
export PIXLEVEL=6
{lsd_prefix}lsd-import-smf -c -f $SURVEY ps1_det ps1_exp $SMFLIST > {logdir}/ingest.log 2>&1
""".format(db=self.dbpath,
survey=self.survey,
logdir=self.logdir,
smffiles=self.new_filelist,
environ_source=self.environ_source,
lsd_prefix=self.lsd_prefix)
pp = textwrap.dedent(pp).lstrip()
with open(self.ingest_sh, 'w') as fp:
fp.write(pp)
try:
out = utils.shell("bash %s" % self.ingest_sh)
except subprocess.CalledProcessError:
print >> sys.stderr, "Error while loading new data. See ingest.log for details."
else:
print "\tNo new files to import."
def list_smf(self):
print "Getting the list of SMF files"
try:
utils.shell(
"find {catpath} -name '*.smf' > {cat_filelist} 2> {logdir}/cat-filelist.log"
.format(catpath=self.catpath,
cat_filelist=self.cat_filelist,
logdir=self.logdir))
catsmf = [s.strip() for s in open(self.cat_filelist).xreadlines()]
print "\tSMF files in catalog: %d" % len(catsmf)
except subprocess.CalledProcessError:
print >> sys.stderr, "Enumeration of catlog files failed. See cat-filelist.log for details."
print >> sys.stderr, "Snippet:"
for k, l in enumerate(
file(os.path.join(self.logdir,
'cat-filelist.log')).xreadlines()):
print >> sys.stderr, " ", l.strip()
if k == 10: break
exit()
def wait(self):
print "Waiting for completion"
try:
jobid, queue = file(self.jobid_fn).read().strip().split()
except IOError as e:
if e.errno != errno.ENOENT: raise
print "\tNot running."
return
nretry = 30
report = ""
while True:
## "27043806 jsteine RUN itc iliadaccess hero4207 *_a_aas4_2 Feb 23 18:12"
try:
out = utils.shell(
"bjobs -q {queue} {jobid} 2> /dev/null | grep {jobid}".
format(queue=queue, jobid=jobid)).strip().split('\n')
except subprocess.CalledProcessError:
# It takes some time for a job to become visible
nretry -= 1
if nretry == 0: raise
time.sleep(2)
continue
stats = defaultdict(int)
for line in out:
state = line.split()[2]
stats[state] += 1
newreport = ', '.join("%s: %d/%d" % (state, njobs, self.njobs)
for state, njobs in stats.iteritems())
if newreport == report:
continue
report = newreport
print "\t%s" % report
if stats['DONE'] == self.njobs:
os.unlink(self.jobid_fn)
break
else:
time.sleep(5)
def wait(self):
print "Waiting for completion"
try:
jobid, queue = file(self.jobid_fn).read().strip().split()
except IOError as e:
if e.errno != errno.ENOENT: raise
print "\tNot running."
return
nretry = 30
report = ""
while True:
## "27043806 jsteine RUN itc iliadaccess hero4207 *_a_aas4_2 Feb 23 18:12"
try:
out = utils.shell("bjobs -q {queue} {jobid} 2> /dev/null | grep {jobid}".format(queue=queue, jobid=jobid)).strip().split('\n')
except subprocess.CalledProcessError:
# It takes some time for a job to become visible
nretry -= 1
if nretry == 0: raise
time.sleep(2)
continue
stats = defaultdict(int)
for line in out:
state = line.split()[2]
stats[state] += 1
newreport = ', '.join("%s: %d/%d" % (state, njobs, self.njobs) for state, njobs in stats.iteritems())
if newreport == report:
continue
report = newreport
print "\t%s" % report
if stats['DONE'] == self.njobs:
os.unlink(self.jobid_fn)
break
else:
time.sleep(5)
def build_objects(self):
print "Building object catalog"
newsmf = [ s.strip() for s in open(self.new_filelist).xreadlines() ]
if len(newsmf) != 0:
pp = r"""
#!/bin/bash
{environ_source}
export LSD_DB={db}
{lsd_prefix}lsd-make-object-catalog --auto --fov-radius=2 ps1_obj ps1_det ps1_exp > {logdir}/build_objects.log 2>&1
""".format(db=self.dbpath, logdir=self.logdir, environ_source=self.environ_source, lsd_prefix=self.lsd_prefix)
pp = textwrap.dedent(pp).lstrip()
with open(self.build_objects_sh, 'w') as fp: fp.write(pp)
try:
out = utils.shell("bash %s" % self.build_objects_sh)
except subprocess.CalledProcessError:
print >>sys.stderr, "Error while postprocessing. See build-objects.log for details."
else:
print "\tNo new files imported, no need to postprocess."