def make_request():
"""Make a request for data to be restored by connecting to the
web services at Cornell.
"""
dlm_cout.outs("Requesting data")
num_beams = 1
web_service = CornellWebservice.Client()
guid = web_service.Restore(username=config.download.api_username, \
pw=config.download.api_password, \
pipeline=config.basic.pipeline.lower(), \
number=num_beams, \
bits=config.download.request_numbits, \
fileType=config.download.request_datatype)
if guid == "fail":
raise pipeline_utils.PipelineError(
"Request for restore returned 'fail'.")
requests = jobtracker.query("SELECT * FROM requests " \
"WHERE guid='%s'" % guid)
if requests:
# Entries in the requests table exist with this GUID!?
raise pipeline_utils.PipelineError("There are %d requests in the " \
"job-tracker DB with this GUID %s" % \
(len(requests), guid))
jobtracker.query("INSERT INTO requests ( " \
"guid, " \
"created_at, " \
"updated_at, " \
"status, " \
"details) " \
"VALUES ('%s', '%s', '%s', '%s', '%s')" % \
(guid, jobtracker.nowstr(), jobtracker.nowstr(), 'waiting', \
'Newly created request'))
def get_output_dir(fns):
"""Given a list of data files, 'fns', generate path to output results.
path is:
{base_results_directory}/{mjd}/{obs_name}/{beam_num}/{proc_date}/
Note: 'base_results_directory' is defined in the config file.
'mjd', 'obs_name', and 'beam_num' are from parsing
the job's datafiles. 'proc_date' is the current date
in YYMMDD format.
"""
# Check that files exist
missingfiles = [fn for fn in fns if not os.path.exists(fn)]
if missingfiles:
errormsg = "The following files cannot be found:\n"
for missing in missingfiles:
errormsg += "\t%s\n" % missing
raise pipeline_utils.PipelineError(errormsg)
# Get info from datafile headers
data = datafile.autogen_dataobj([fns[0]])
if not isinstance(data, datafile.PsrfitsData):
errormsg = "Data must be of PSRFITS format.\n"
errormsg += "\tData type: %s\n" % type(data)
raise pipeline_utils.PipelineError(errormsg)
# Generate output directory
mjd = int(data.timestamp_mjd)
beam_num = data.beam_id
obs_name = data.obs_name
proc_date = datetime.datetime.now().strftime('%y%m%d')
outdir = os.path.join(config.jobpooler.base_results_directory, \
str(mjd), str(obs_name), \
str(beam_num), proc_date)
return outdir
def submit(self, datafiles, outdir):
"""Submits a job to the queue to be processed.
Returns a unique identifier for the job.
Inputs:
datafiles: A list of the datafiles being processed.
outdir: The directory where results will be copied to.
Output:
jobid: A unique job identifier.
*** NOTE: A pipeline_utils.PipelineError is raised if
the queue submission fails.
"""
searchscript = os.path.join(config.basic.pipelinedir, 'bin',
'search.py')
cmd = 'qsub -M %s -V -v DATAFILES="%s",OUTDIR="%s" -l %s -N %s -e %s -o /dev/null %s' % \
(config.email.recipient, ';'.join(datafiles), outdir, self.resource_list, \
self.job_basename, self.qsublogdir, searchscript)
pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, \
stdin=subprocess.PIPE)
queue_id = pipe.communicate()[0].strip()
pipe.stdin.close()
if not queue_id:
errormsg = "No job identifier returned by qsub!\n"
errormsg += "\tCommand executed: %s\n" % cmd
raise pipeline_utils.PipelineError(errormsg)
else:
# There is occasionally a short delay between submission and
# the job appearing on the queue, so sleep for 1 second.
time.sleep(1)
return queue_id
def delete(self, queue_id):
"""Remove the job identified by 'queue_id' from the queue.
Input:
queue_id: Unique identifier for a job.
Output:
None
*** NOTE: A pipeline_utils.PipelineError is raised if
the job removal fails.
"""
cmd = "canceljob %s" % queue_id
pipe = subprocess.Popen(cmd, shell=True)
# Wait a few seconds a see if the job is still being tracked by
# the queue manager, or if it marked as exiting.
time.sleep(5)
#force queue update
queue, comm_err = self._showq(update_time=0)
state = self._check_job_state(queue_id)
if ('Completed' not in state) and ('Canceling'
not in state) and ('DNE'
not in state):
errormsg = "The job (%s) is still in the queue " % queue_id
errormsg += "and is marked as state '%s'!\n" % state
raise pipeline_utils.PipelineError(errormsg)
def get_output_dir(fns):
"""Given a list of data files, 'fns', generate path to output results.
path is:
{base_results_directory}/{mjd}/{obs_name}/{beam_num}/{proc_date}/
Note: 'base_results_directory' is defined in the processing config file.
'mjd', 'obs_name', and 'beam_num' are from parsing
the job's datafiles. 'proc_date' is the current date
in yymmddThhmmss format.
"""
# Get info from datafile headers
data = datafile.autogen_dataobj([fns[0]])
if not isinstance(data, datafile.PsrfitsData):
errormsg = "Data must be of PSRFITS format.\n"
errormsg += "\tData type: %s\n" % type(data)
raise pipeline_utils.PipelineError(errormsg)
# Generate output directory
mjd = int(data.timestamp_mjd)
beam_num = data.beam_id
obs_name = data.obs_name
proc_date = datetime.datetime.now().strftime('%y%m%dT%H%M%S')
baseoutdir = os.path.join(config.processing.base_results_directory, \
str(mjd), str(obs_name), \
str(beam_num), proc_date)
outdir = baseoutdir
# Make sure our output directory doesn't already exist
counter = 0
while os.path.exists(outdir):
counter += 1
outdir = "%s_%d" % (baseoutdir, counter)
# Make the directory immediately so the pipeline knows it's taken
try:
os.makedirs(outdir)
os.chmod(outdir, 0777)
created = True
return outdir, created
except:
current_usr = getpass.getuser()
errormsg = "Permission denied to %s when attempting to create output directory %s" % (
current_usr, outdir)
f = open(
'/scratch/eparent/eparent/PALFA4/results/missing_permission.out',
'a')
f.write(errormsg + '\n')
f.close()
print errormsg
created = False
return outdir, created
def submit(self, datafiles, outdir, \
script=os.path.join(config.basic.pipelinedir, 'bin', 'search.py')):
"""Submits a job to the queue to be processed.
Returns a unique identifier for the job.
Inputs:
datafiles: A list of the datafiles being processed.
outdir: The directory where results will be copied to.
script: The script to submit to the queue. (Default:
'{config.basic.pipelinedir}/bin/search.py')
Output:
jobid: A unique job identifier.
*** NOTE: A pipeline_utils.PipelineError is raised if
the queue submission fails.
"""
node = self._get_submit_node()
if node is None:
errormsg = "No nodes to accept job submission!\n"
raise pipeline_utils.PipelineError(errormsg)
errorlog = os.path.join(config.basic.qsublog_dir, "'$PBS_JOBID'.ER")
stdoutlog = os.devnull
cmd = "qsub -V -v DATAFILES='%s',OUTDIR='%s' -l nodes=%s:ppn=1 -N %s -e %s -o %s %s" % \
(';'.join(datafiles), outdir, node, \
self.job_basename, errorlog, stdoutlog, script)
pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, \
stdin=subprocess.PIPE)
queue_id = pipe.communicate()[0].strip()
pipe.stdin.close()
if not queue_id:
errormsg = "No job identifier returned by qsub!\n"
errormsg += "\tCommand executed: %s\n" % cmd
raise pipeline_utils.PipelineError(errormsg)
else:
# There is occasionally a short delay between submission and
# the job appearing on the queue, so sleep for 1 second.
time.sleep(1)
return queue_id
def get_output_dir(fns):
"""Given a list of data files, 'fns', generate path to output results.
path is:
{base_results_directory}/{mjd}/{obs_name}/{beam_num}/{proc_date}/
Note: 'base_results_directory' is defined in the processing config file.
'mjd', 'obs_name', and 'beam_num' are from parsing
the job's datafiles. 'proc_date' is the current date
in yymmddThhmmss format.
"""
# Get info from datafile headers
data = datafile.autogen_dataobj([fns[0]])
if not isinstance(data, datafile.PsrfitsData):
errormsg = "Data must be of PSRFITS format.\n"
errormsg += "\tData type: %s\n" % type(data)
raise pipeline_utils.PipelineError(errormsg)
# Generate output directory
mjd = int(data.timestamp_mjd)
beam_num = data.beam_id
obs_name = data.obs_name
proc_date = datetime.datetime.now().strftime('%y%m%dT%H%M%S')
baseoutdir = os.path.join(config.processing.base_results_directory, \
str(mjd), str(obs_name), \
str(beam_num), proc_date)
outdir = baseoutdir
# Make sure our output directory doesn't already exist
counter = 0
while os.path.exists(outdir):
counter += 1
outdir = "%s_%d" % (baseoutdir, counter)
# Make the directory immediately so the pipeline knows it's taken
os.makedirs(outdir)
# Send an email if our first choice for outdir wasn't available
if counter:
errormsg = "The first-choice output directory '%s' " \
"already existed. Had to settle for '%s' " \
"after %d tries. \n\n " \
"Data files:\n " \
"\t%s" % (baseoutdir, outdir, counter, "\n\t".join(fns))
notification = mailer.ErrorMailer(errormsg, \
subject="Job outdir existance warning")
notification.send()
return outdir
def make_request():
"""Make a request for data to be restored by connecting to the
web services at Cornell.
"""
dlm_cout.outs("Requesting data")
num_beams = 1
web_service = suds.client.Client(config.download.api_service_url).service
try:
guid = web_service.Restore(username=config.download.api_username, \
pw=config.download.api_password, \
number=num_beams, \
bits=config.download.request_numbits, \
fileType=config.download.request_datatype)
except urllib2.URLError, e:
raise pipeline_utils.PipelineError("urllib2.URLError caught when " \
"making a request for restore: %s" % \
str(e))
def presubmission_check(fns):
"""Check to see if datafiles meet the critera for submission.
"""
# Check that files exist
missingfiles = [fn for fn in fns if not os.path.exists(fn)]
if missingfiles:
errormsg = "The following files cannot be found:\n"
for missing in missingfiles:
errormsg += "\t%s\n" % missing
raise pipeline_utils.PipelineError(
errormsg) # if files missing want to crash
try:
#for WAPP, check if coords are in table
data = datafile.autogen_dataobj([fns[0]])
if not isinstance(data, datafile.PsrfitsData):
errormsg = "Data must be of PSRFITS format.\n"
errormsg += "\tData type: %s\n" % type(data)
raise FailedPreCheckError(errormsg)
except datafile.DataFileError, e:
raise FailedPreCheckError(e)
def delete(self, queue_id):
"""Remove the job identified by 'queue_id' from the queue.
Input:
queue_id: Unique identifier for a job.
Output:
None
*** NOTE: A pipeline_utils.PipelineError is raised if
the job removal fails.
"""
cmd = "qsig -s SIGINT %s" % queue_id
pipe = subprocess.Popen(cmd, shell=True)
# Wait a few seconds a see if the job is still being tracked by
# the queue manager, or if it marked as exiting.
time.sleep(5)
batch = PBSQuery.PBSQuery().getjobs()
if (queue_id in batch) and ('E' not in batch[queue_id]['job_state']):
errormsg = "The job (%s) is still in the queue " % queue_id
errormsg += "and is not marked as exiting (status = 'E')!\n"
raise pipeline_utils.PipelineError(errormsg)
"""
dlm_cout.outs("Requesting data")
num_beams = 1
web_service = suds.client.Client(config.download.api_service_url).service
try:
guid = web_service.Restore(username=config.download.api_username, \
pw=config.download.api_password, \
number=num_beams, \
bits=config.download.request_numbits, \
fileType=config.download.request_datatype)
except urllib2.URLError, e:
raise pipeline_utils.PipelineError("urllib2.URLError caught when " \
"making a request for restore: %s" % \
str(e))
if guid == "fail":
raise pipeline_utils.PipelineError(
"Request for restore returned 'fail'.")
requests = jobtracker.query("SELECT * FROM requests " \
"WHERE guid='%s'" % guid)
if requests:
# Entries in the requests table exist with this GUID!?
raise pipeline_utils.PipelineError("There are %d requests in the " \
"job-tracker DB with this GUID %s" % \
(len(requests), guid))
jobtracker.query("INSERT INTO requests ( " \
"guid, " \
"created_at, " \
"updated_at, " \
"status, " \
"details) " \
break
# remove temporary dir for PFDs
shutil.rmtree(tempdir)
# remove temporary dir for SPDs
shutil.rmtree(tempdir_sp)
if attempts >= 5:
errmsg = "FTP upload failed after %d connection failures!\n" % attempts
sys.stderr.write(errmsg)
sys.stderr.write(
"\tRolling back DB transaction and raising error.\n")
# Rolling back changes (just last uncommited FTP).
db.rollback()
raise pipeline_utils.PipelineError(errmsg)
else:
# Update database statuses
queries = []
queries.append("UPDATE job_submits " \
"SET status='uploaded', " \
"details='Upload successful (header_id=%d)', " \
"updated_at='%s' " \
"WHERE id=%d" %
(header_id, jobtracker.nowstr(), job_submit['id']))
queries.append("UPDATE jobs " \
"SET status='uploaded', " \
"details='Upload successful (header_id=%d)', " \
"updated_at='%s' " \
"WHERE id=%d" % \