def can_add_file(fn, verbose=False):
"""Checks a file to see if it should be added to the 'files'
table in the jobtracker DB.
Input:
fn: The file to check.
verbose: Print messages to stdout. (Default: be silent).
Outputs:
can_add: Boolean value. True if the file should be added.
False otherwise.
"""
import datafile
try:
datafile_type = datafile.get_datafile_type([fn])
except datafile.DataFileError, e:
if verbose:
print "Unrecognized data file type: %s" % fn
return False
def copy_zaplist(fns, workdir):
# Copy zaplist to working directory
filetype = datafile.get_datafile_type(fns)
parsed = filetype.fnmatch(fns[0]).groupdict()
if 'date' not in parsed.keys():
parsed['date'] = "%04d%02d%02d" % \
astro_utils.calendar.MJD_to_date(int(parsed['mjd']))
customzapfns = []
# First, try to find a custom zaplist for this specific data file
customzapfns.append(fns[0].replace(".fits", ".zaplist"))
# Next, try to find custom zaplist for this beam
customzapfns.append("%s.%s.b%s.zaplist" % \
(parsed['projid'], parsed['date'], parsed['beam']))
# Try to find custom zaplist for this MJD
customzapfns.append("%s.%s.all.zaplist" % (parsed['projid'], parsed['date']))
zaptar = tarfile.open(os.path.join(config.processing.zaplistdir, \
"zaplists.tar.gz"), mode='r')
members = zaptar.getmembers()
for customzapfn in customzapfns:
matches = [mem for mem in members \
if mem.name.endswith(customzapfn)]
if matches:
ti = matches[0] # The first TarInfo object found
# that matches the file name
# Write custom zaplist to workdir
localfn = os.path.join(workdir, customzapfn)
f = open(localfn, 'w')
f.write(zaptar.extractfile(ti).read())
f.close()
print "Copied custom zaplist: %s" % customzapfn
break
else:
# The member we searched for doesn't exist, try next one
pass
else:
# Copy default zaplist
shutil.copy(config.processing.default_zaplist, workdir)
print "No custom zaplist found. Copied default zaplist: %s" % \
config.processing.default_zaplist
zaptar.close()
def can_add_file(fn, verbose=False):
"""Checks a file to see if it should be added to the 'files'
table in the jobtracker DB.
Input:
fn: The file to check.
verbose: Print messages to stdout. (Default: be silent).
Outputs:
can_add: Boolean value. True if the file should be added.
False otherwise.
"""
import jobtracker
import datafile
try:
datafile_type = datafile.get_datafile_type([fn])
except datafile.DataFileError, e:
if verbose:
print "Unrecognized data file type: %s" % fn
return False
def copy_zaplist(fns, workdir):
# Copy zaplist to working directory
filetype = datafile.get_datafile_type(fns)
parsed = filetype.fnmatch(fns[0]).groupdict()
#if 'date' not in parsed.keys():
# parsed['date'] = "%04d%02d%02d" % \
# astro_utils.calendar.MJD_to_date(int(parsed['mjd']))
customzapfns = []
# First, try to find a custom zaplist for this specific data file
#customzapfns.append(fns[0].replace(".fits", ".zaplist"))
# Next, try to find custom zaplist for this beam
#customzapfns.append("%s.%s.b%s.zaplist" % \
# (parsed['projid'], parsed['date'], parsed['beam']))
# Try to find custom zaplist for this MJD
#customzapfns.append("%s.%s.all.zaplist" % (parsed['projid'], parsed['date']))
#zaptar_fn = pipeline_utils.get_zaplist_tarball(no_check=True)
#zaptar = tarfile.open(zaptar_fn, mode='r')
if 1:
# Copy default zaplist
if filetype == datafile.WappPsrfitsData:
zapfn = config.processing.default_wapp_zaplist
elif (filetype == datafile.MockPsrfitsData
or filetype == datafile.MergedMockPsrfitsData):
zapfn = config.processing.default_mock_zaplist
elif filetype == datafile.NuppiPsrfitsData:
zapfn = config.processing.default_span512_zaplist
elif filetype == datafile.PFFTSPsrfitsData:
zapfn = config.processing.default_pffts_zaplist
else:
raise ValueError("No default zaplist for data files of type %s" % \
filetype.__name__)
shutil.copy(zapfn, workdir)
print "No custom zaplist found. Copied default zaplist: %s" % \
zapfn
def create_file_entries(request):
"""Given a row from the requests table in the job-tracker DB
check the FTP server for its files and create entries in
the files table.
Input:
request: A row from the requests table.
Outputs:
None
"""
cftp = CornellFTP.CornellFTP()
try:
files = cftp.get_files(request['guid'])
except CornellFTP.M2Crypto.ftpslib.error_perm:
exctype, excvalue, exctb = sys.exc_info()
dlm_cout.outs("FTP error getting file information.\n" \
"\tGUID: %s\n\tError: %s" % \
(request['guid'], \
"".join(traceback.format_exception_only(exctype, excvalue)).strip()))
files = []
total_size = 0
num_files = 0
queries = []
for fn, size in files:
# Check if file is from the phantom beam (beam 7)
datafile_type = datafile.get_datafile_type([fn])
parsedfn = datafile_type.fnmatch(fn)
if parsedfn.groupdict().setdefault('beam', '-1') == '7':
print "Ignoring beam 7 data: %s" % fn
continue
# Insert entry into DB's files table
queries.append("INSERT INTO files ( " \
"request_id, " \
"remote_filename, " \
"filename, " \
"status, " \
"created_at, " \
"updated_at, " \
"size) " \
"VALUES ('%s', '%s', '%s', '%s', '%s', '%s', %d)" % \
(request['id'], fn, os.path.join(config.download.datadir, fn), \
'new', jobtracker.nowstr(), jobtracker.nowstr(), size))
total_size += size
num_files += 1
if num_files:
dlm_cout.outs("Request (GUID: %s) has succeeded.\n" \
"\tNumber of files to be downloaded: %d" % \
(request['guid'], num_files))
queries.append("UPDATE requests " \
"SET size=%d, " \
"updated_at='%s', " \
"status='finished', " \
"details='Request has been filled' " \
"WHERE id=%d" % \
(total_size, jobtracker.nowstr(), request['id']))
else:
dlm_cout.outs("Request (GUID: %s) has failed.\n" \
"\tThere are no files to be downloaded." % \
request['guid'])
queries.append("UPDATE requests " \
"SET updated_at='%s', " \
"status='failed', " \
"details='No files to download' " \
"WHERE id=%d" % \
(jobtracker.nowstr(), request['id']))
jobtracker.query(queries)
def copy_zaplist(fns, workdir):
# Copy zaplist to working directory
filetype = datafile.get_datafile_type(fns)
parsed = filetype.fnmatch(fns[0]).groupdict()
if 'date' not in parsed.keys():
parsed['date'] = "%04d%02d%02d" % \
astro_utils.calendar.MJD_to_date(int(parsed['mjd']))
customzapfns = []
# First, try to find a custom zaplist for this specific data file
customzapfns.append(fns[0].replace(".fits", ".zaplist"))
# Next, try to find custom zaplist for this beam
customzapfns.append("%s.%s.b%s.zaplist" % \
(parsed['projid'], parsed['date'], parsed['beam']))
# Try to find custom zaplist for this MJD
customzapfns.append("%s.%s.all.zaplist" %
(parsed['projid'], parsed['date']))
zaptar_fn = pipeline_utils.get_zaplist_tarball(no_check=True)
zaptar = tarfile.open(zaptar_fn, mode='r')
members = zaptar.getmembers()
for customzapfn in customzapfns:
radar_samples_fn = os.path.splitext(
customzapfn)[0] + '_merged_radar_samples.txt'
matches = [mem for mem in members \
if mem.name.endswith(customzapfn)]
radar_matches = [mem for mem in members \
if mem.name.endswith(radar_samples_fn)]
if matches:
ti = matches[0] # The first TarInfo object found
# that matches the file name
# Write custom zaplist to workdir
localfn = os.path.join(workdir, customzapfn)
f = open(localfn, 'w')
f.write(zaptar.extractfile(ti).read())
f.close()
print "Copied custom zaplist: %s" % customzapfn
if radar_matches:
radar_ti = radar_matches[0]
# Write radar samples list to workdir
radar_localfn = os.path.join(workdir, radar_samples_fn)
f = open(radar_localfn, 'w')
f.write(zaptar.extractfile(radar_ti).read())
f.close()
print "Copied radar samples list: %s" % radar_samples_fn
break
else:
# The member we searched for doesn't exist, try next one
pass
else:
# Copy default zaplist
if filetype == datafile.WappPsrfitsData:
zapfn = config.processing.default_wapp_zaplist
elif (filetype == datafile.MockPsrfitsData
or filetype == datafile.MergedMockPsrfitsData):
zapfn = config.processing.default_mock_zaplist
else:
raise ValueError("No default zaplist for data files of type %s" % \
filetype.__name__)
shutil.copy(zapfn, workdir)
print "No custom zaplist found. Copied default zaplist: %s" % \
zapfn
zaptar.close()
def create_file_entries(request):
"""Given a row from the requests table in the job-tracker DB
check the FTP server for its files and create entries in
the files table.
Input:
request: A row from the requests table.
Outputs:
None
"""
cftp = CornellFTP.CornellFTP()
try:
files = cftp.get_files(request['guid'])
except CornellFTP.M2Crypto.ftpslib.error_perm:
exctype, excvalue, exctb = sys.exc_info()
dlm_cout.outs("FTP error getting file information.\n" \
"\tGUID: %s\n\tError: %s" % \
(request['guid'], \
"".join(traceback.format_exception_only(exctype, excvalue)).strip()))
files = []
total_size = 0
num_files = 0
queries = []
for fn, size in files:
# Check if file is from the phantom beam (beam 7)
datafile_type = datafile.get_datafile_type([fn])
parsedfn = datafile_type.fnmatch(fn)
if parsedfn.groupdict().setdefault('beam', '-1') == '7':
print "Ignoring beam 7 data: %s" % fn
continue
# Insert entry into DB's files table
queries.append("INSERT INTO files ( " \
"request_id, " \
"remote_filename, " \
"filename, " \
"status, " \
"created_at, " \
"updated_at, " \
"size) " \
"VALUES ('%s', '%s', '%s', '%s', '%s', '%s', %d)" % \
(request['id'], fn, os.path.join(config.download.temp, fn), \
'new', jobtracker.nowstr(), jobtracker.nowstr(), size))
total_size += size
num_files += 1
if num_files:
dlm_cout.outs("Request (GUID: %s) has succeeded.\n" \
"\tNumber of files to be downloaded: %d" % \
(request['guid'], num_files))
queries.append("UPDATE requests " \
"SET size=%d, " \
"updated_at='%s', " \
"status='finished', " \
"details='Request has been filled' " \
"WHERE id=%d" % \
(total_size, jobtracker.nowstr(), request['id']))
else:
dlm_cout.outs("Request (GUID: %s) has failed.\n" \
"\tThere are no files to be downloaded." % \
request['guid'])
queries.append("UPDATE requests " \
"SET updated_at='%s', " \
"status='failed', " \
"details='No files to download' " \
"WHERE id=%d" % \
(jobtracker.nowstr(), request['id']))
jobtracker.query(queries)
