def get_trackfile(fileset, target_directory):
"""
Copy track files corresponding to the given
fileset into the target directory.
"""
LOG.info("Building list of previous track files...")
if not (os.path.isdir(target_directory)):
LOG.error("%s does not exist!", target_directory)
return []
trackset = []
for current_file in fileset:
current_track = current_file.split("_feat.csv")[0] + "_track.csv"
if ((current_track.startswith("ftp"))
or (current_track.startwidth("http"))):
target = download_file(current_track,
target_directory=target_directory,
tries=1,
wait=0,
quiet=False)
if (os.path.isfile(target)):
trackset.append(target)
else:
if (os.path.isfile(current_track)):
if (target_directory != os.path.dirname(current_track)):
shutil.copy(current_track, target_directory)
trackset.append(target)
return trackset
def show_info(self, key=None, output_dir=None):
url = self.build_show_info(key=key)
get_stream = False
if (output_dir is None):
get_stream = True
if (self.verbose): print "Fetching %s" % (url)
target = download_file(url,
target_directory=output_dir,
get_stream=get_stream)
return target
def fetch(self,
operation,
protocol=None,
method=None,
requestid=None,
format=None):
url = self.build_fetch(operation,
protocol=protocol,
method=method,
requestid=requestid,
format=format)
if (self.verbose): print "Fetching %s" % (url)
resp = download_file(url, get_stream=True)
if not (resp.startswith("{")):
print "Empty jsoc_fetch response!"
print "Query was: %s" % (url)
return None
else:
fetch_resp = self.parse_json(resp)
self.fetch_resp = fetch_resp
return fetch_resp
def get_fits(self, output_dir=CURRENT_DIR, timeout=TIMEOUT, wait=WAIT):
res = ''
fetch_resp = self.fetch("exp_request",
protocol=PROTOCOL,
method=METHOD)
if (fetch_resp is None): sys.exit(0)
if (fetch_resp['status'] == 0):
#downloading file from info given in the response
data = fetch_resp['data'][0]
filename = data['record']
filename = filename.replace('][2]{', '.')
filename = filename.replace('[', '.')
filename = filename.replace('}', '')
# filename = filename.replace(':', '_')
filename = filename + '.fits'
download_url = 'http://jsoc.stanford.edu' + data['filename']
if (self.verbose):
print "Downloading %s to %s ..." % (download_url,
output_dir + filename)
res = download_file(download_url,
target_directory=output_dir,
filename=filename)
else:
try:
requestid = fetch_resp['requestid']
except KeyError:
print "Error: no JSOC request ID: %s" % (self.fetch_resp)
return ''
time.sleep(1)
t0 = time.time()
remaining_sec = timeout - int(time.time() - t0)
while (remaining_sec >= 0):
fetch_resp = self.fetch("exp_status",
requestid=requestid,
format=FORMAT)
if (self.verbose):
print "%s (remaining time: %i sec.)" % (self.fetch_resp,
remaining_sec)
if (fetch_resp is None):
print "Fetching error!"
res = ''
break
try:
status = int(fetch_resp['status'])
except KeyError:
print "Error: no status in response: %s" % (
self.fetch_resp)
res = ''
break
if (status == 0):
try:
size = int(fetch_resp['size'])
except KeyError:
print "Error: no size in response: %s" % (
self.fetch_resp)
res = ''
break
if (size > 0):
data = fetch_resp['data'][0]
filename = data['record']
filename = filename.replace('][2]', '')
filename = filename.replace('[', '.')
tmp = data['filename'].split('.')
filename = filename + '.' + tmp[3] + '.fits'
download_url = 'http://jsoc.stanford.edu' + fetch_resp[
'dir'] + '/' + data['filename']
if (self.verbose):
print "Downloading %s to %s ..." % (
download_url, output_dir + filename)
res = download_file(download_url,
target_directory=output_dir,
filename=filename)
else:
res = ''
break
else:
print "Status: %i for %s" % (status, requestid)
time.sleep(wait)
remaining_sec = remaining_sec = timeout - int(time.time() - t0)
return res
def run(self):
if not (os.path.exists(self.idl_exe_path)):
LOG.error("Given path of the IDL executable (%s) does not exist!",
self.idl_exe_path)
self.end()
return False
if not (os.path.exists(self.sdoss_idl_bin)):
LOG.error("%s does not exist!", self.sdoss_idl_bin)
self.end()
return False
if (len(self.fileset) != 2):
log.error("Input fileset must have two elements!")
self.end()
return False
#If input files are urls then download data files.
if (self.fileset[0].startswith("http:")) or \
(self.fileset[0].startswith("ftp:")):
ic_url = self.fileset[0]
self.fileid[0] = ic_url
#try to download from jsoc
LOG.info("Job[#%i] downloading ic_file from JSOC %s",
self.thread_id, self.date_obs[0])
j_soc = jsoc("hmi.ic_45s",
starttime=self.date_obs[0],
endtime=self.date_obs[0],
verbose=True,
notify='*****@*****.**')
ic_file = j_soc.get_fits(output_dir=self.data_directory)
if (ic_file):
LOG.info("Job[#%i]: %s downloaded from JSOC.", self.thread_id,
ic_file)
self.fileset[0] = ic_file
else:
LOG.info("Job[#%i]: Downloading from VSO for %s %s...",
self.thread_id, self.date_obs[0], ic_url)
ic_file = download_file(ic_url,
target_directory=self.data_directory,
timeout=60,
wait=30,
quiet=False)
if (ic_file):
LOG.info("Job[#%i]: %s downloaded from VSO.",
self.thread_id, ic_file)
self.fileset[0] = ic_file
else:
ic_url = None
ic_file = self.fileset[0]
self.fileid[0] = ic_file
if not (os.path.isfile(ic_file)):
LOG.error("Job[#%i]: ic_file %s does not exist!", self.thread_id,
ic_file)
self.end()
return False
if (self.fileset[1].startswith("http:")) or \
(self.fileset[1].startswith("ftp:")):
m_url = self.fileset[1]
self.fileid[1] = m_url
#try to download from jsoc
LOG.info("Job[#%i] downloading m_file from JSOC %s",
self.thread_id, self.date_obs[0])
j_soc = jsoc("hmi.m_45s",
starttime=self.date_obs[0],
endtime=self.date_obs[0],
verbose=True,
notify='*****@*****.**')
m_file = j_soc.get_fits(output_dir=self.data_directory)
if (m_file):
LOG.info("Job[#%i]: %s downloaded from JSOC.", self.thread_id,
m_file)
self.fileset[1] = m_file
else:
LOG.info("Job[#%i]: Downloading from VSO for %s %s...",
self.thread_id, self.date_obs[0], m_url)
m_file = download_file(m_url,
target_directory=self.data_directory,
timeout=60,
wait=30,
quiet=False)
if (m_file):
LOG.info("Job[#%i]: %s downloaded from VSO.",
self.thread_id, m_file)
self.fileset[1] = m_file
else:
m_url = None
m_file = self.fileList[1]
self.fileid[1] = m_file
if not (os.path.isfile(m_file)):
LOG.error("Job[#%i]: m_file %s does not exist!", self.thread_id,
m_file)
self.end()
return False
idl_args = [
self.config_file, ic_file, m_file,
"data_dir=" + self.data_directory,
"output_dir=" + self.output_directory
]
if (ic_url is not None): idl_args.append("fnc_url=" + ic_url)
if (m_url is not None): idl_args.append("fnm_url=" + m_url)
if (self.quicklook): idl_args.append("/QUICKLOOK")
if (self.verbose): idl_args.append("/VERBOSE")
#build idl command line
idl_cmd = [self.idl_exe_path
] + ["-quiet", "-rt=" + self.sdoss_idl_bin, "-args"]
#idl_cmd = [self.idl_exe_path]+["-rt="+self.sdoss_idl_bin,"-args"]
idl_cmd.extend(idl_args)
LOG.info("Executing --> " + " ".join(idl_cmd))
idl_process = subprocess.Popen(idl_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, errors = idl_process.communicate()
if idl_process.wait() == 0:
#LOG.info("Sucessfully ran idl command %s, output: %s, errors: %s",
# ' '.join(idl_cmd), str(output), str(errors))
if (len(errors) == 0): self.success = True
else:
LOG.error("Error running idl command %s, output: %s, errors: %s",
' '.join(idl_cmd), str(output), str(errors))
self.end()
time.sleep(3)