def get_trackfile(fileset, target_directory): """ Copy track files corresponding to the given fileset into the target directory. """ LOG.info("Building list of previous track files...") if not (os.path.isdir(target_directory)): LOG.error("%s does not exist!", target_directory) return [] trackset = [] for current_file in fileset: current_track = current_file.split("_feat.csv")[0] + "_track.csv" if ((current_track.startswith("ftp")) or (current_track.startwidth("http"))): target = download_file(current_track, target_directory=target_directory, tries=1, wait=0, quiet=False) if (os.path.isfile(target)): trackset.append(target) else: if (os.path.isfile(current_track)): if (target_directory != os.path.dirname(current_track)): shutil.copy(current_track, target_directory) trackset.append(target) return trackset
def show_info(self, key=None, output_dir=None): url = self.build_show_info(key=key) get_stream = False if (output_dir is None): get_stream = True if (self.verbose): print "Fetching %s" % (url) target = download_file(url, target_directory=output_dir, get_stream=get_stream) return target
def fetch(self, operation, protocol=None, method=None, requestid=None, format=None): url = self.build_fetch(operation, protocol=protocol, method=method, requestid=requestid, format=format) if (self.verbose): print "Fetching %s" % (url) resp = download_file(url, get_stream=True) if not (resp.startswith("{")): print "Empty jsoc_fetch response!" print "Query was: %s" % (url) return None else: fetch_resp = self.parse_json(resp) self.fetch_resp = fetch_resp return fetch_resp
def get_fits(self, output_dir=CURRENT_DIR, timeout=TIMEOUT, wait=WAIT): res = '' fetch_resp = self.fetch("exp_request", protocol=PROTOCOL, method=METHOD) if (fetch_resp is None): sys.exit(0) if (fetch_resp['status'] == 0): #downloading file from info given in the response data = fetch_resp['data'][0] filename = data['record'] filename = filename.replace('][2]{', '.') filename = filename.replace('[', '.') filename = filename.replace('}', '') # filename = filename.replace(':', '_') filename = filename + '.fits' download_url = 'http://jsoc.stanford.edu' + data['filename'] if (self.verbose): print "Downloading %s to %s ..." % (download_url, output_dir + filename) res = download_file(download_url, target_directory=output_dir, filename=filename) else: try: requestid = fetch_resp['requestid'] except KeyError: print "Error: no JSOC request ID: %s" % (self.fetch_resp) return '' time.sleep(1) t0 = time.time() remaining_sec = timeout - int(time.time() - t0) while (remaining_sec >= 0): fetch_resp = self.fetch("exp_status", requestid=requestid, format=FORMAT) if (self.verbose): print "%s (remaining time: %i sec.)" % (self.fetch_resp, remaining_sec) if (fetch_resp is None): print "Fetching error!" res = '' break try: status = int(fetch_resp['status']) except KeyError: print "Error: no status in response: %s" % ( self.fetch_resp) res = '' break if (status == 0): try: size = int(fetch_resp['size']) except KeyError: print "Error: no size in response: %s" % ( self.fetch_resp) res = '' break if (size > 0): data = fetch_resp['data'][0] filename = data['record'] filename = filename.replace('][2]', '') filename = filename.replace('[', '.') tmp = data['filename'].split('.') filename = filename + '.' + tmp[3] + '.fits' download_url = 'http://jsoc.stanford.edu' + fetch_resp[ 'dir'] + '/' + data['filename'] if (self.verbose): print "Downloading %s to %s ..." % ( download_url, output_dir + filename) res = download_file(download_url, target_directory=output_dir, filename=filename) else: res = '' break else: print "Status: %i for %s" % (status, requestid) time.sleep(wait) remaining_sec = remaining_sec = timeout - int(time.time() - t0) return res
def run(self): if not (os.path.exists(self.idl_exe_path)): LOG.error("Given path of the IDL executable (%s) does not exist!", self.idl_exe_path) self.end() return False if not (os.path.exists(self.sdoss_idl_bin)): LOG.error("%s does not exist!", self.sdoss_idl_bin) self.end() return False if (len(self.fileset) != 2): log.error("Input fileset must have two elements!") self.end() return False #If input files are urls then download data files. if (self.fileset[0].startswith("http:")) or \ (self.fileset[0].startswith("ftp:")): ic_url = self.fileset[0] self.fileid[0] = ic_url #try to download from jsoc LOG.info("Job[#%i] downloading ic_file from JSOC %s", self.thread_id, self.date_obs[0]) j_soc = jsoc("hmi.ic_45s", starttime=self.date_obs[0], endtime=self.date_obs[0], verbose=True, notify='*****@*****.**') ic_file = j_soc.get_fits(output_dir=self.data_directory) if (ic_file): LOG.info("Job[#%i]: %s downloaded from JSOC.", self.thread_id, ic_file) self.fileset[0] = ic_file else: LOG.info("Job[#%i]: Downloading from VSO for %s %s...", self.thread_id, self.date_obs[0], ic_url) ic_file = download_file(ic_url, target_directory=self.data_directory, timeout=60, wait=30, quiet=False) if (ic_file): LOG.info("Job[#%i]: %s downloaded from VSO.", self.thread_id, ic_file) self.fileset[0] = ic_file else: ic_url = None ic_file = self.fileset[0] self.fileid[0] = ic_file if not (os.path.isfile(ic_file)): LOG.error("Job[#%i]: ic_file %s does not exist!", self.thread_id, ic_file) self.end() return False if (self.fileset[1].startswith("http:")) or \ (self.fileset[1].startswith("ftp:")): m_url = self.fileset[1] self.fileid[1] = m_url #try to download from jsoc LOG.info("Job[#%i] downloading m_file from JSOC %s", self.thread_id, self.date_obs[0]) j_soc = jsoc("hmi.m_45s", starttime=self.date_obs[0], endtime=self.date_obs[0], verbose=True, notify='*****@*****.**') m_file = j_soc.get_fits(output_dir=self.data_directory) if (m_file): LOG.info("Job[#%i]: %s downloaded from JSOC.", self.thread_id, m_file) self.fileset[1] = m_file else: LOG.info("Job[#%i]: Downloading from VSO for %s %s...", self.thread_id, self.date_obs[0], m_url) m_file = download_file(m_url, target_directory=self.data_directory, timeout=60, wait=30, quiet=False) if (m_file): LOG.info("Job[#%i]: %s downloaded from VSO.", self.thread_id, m_file) self.fileset[1] = m_file else: m_url = None m_file = self.fileList[1] self.fileid[1] = m_file if not (os.path.isfile(m_file)): LOG.error("Job[#%i]: m_file %s does not exist!", self.thread_id, m_file) self.end() return False idl_args = [ self.config_file, ic_file, m_file, "data_dir=" + self.data_directory, "output_dir=" + self.output_directory ] if (ic_url is not None): idl_args.append("fnc_url=" + ic_url) if (m_url is not None): idl_args.append("fnm_url=" + m_url) if (self.quicklook): idl_args.append("/QUICKLOOK") if (self.verbose): idl_args.append("/VERBOSE") #build idl command line idl_cmd = [self.idl_exe_path ] + ["-quiet", "-rt=" + self.sdoss_idl_bin, "-args"] #idl_cmd = [self.idl_exe_path]+["-rt="+self.sdoss_idl_bin,"-args"] idl_cmd.extend(idl_args) LOG.info("Executing --> " + " ".join(idl_cmd)) idl_process = subprocess.Popen(idl_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, errors = idl_process.communicate() if idl_process.wait() == 0: #LOG.info("Sucessfully ran idl command %s, output: %s, errors: %s", # ' '.join(idl_cmd), str(output), str(errors)) if (len(errors) == 0): self.success = True else: LOG.error("Error running idl command %s, output: %s, errors: %s", ' '.join(idl_cmd), str(output), str(errors)) self.end() time.sleep(3)