def test_strftime_array_2(): tstrings = [['1998-01-01 00:00:01', '1998-01-01 00:00:02'], ['1998-01-01 00:00:03', '1995-12-31 23:59:60']] tstrings = np.array(tstrings) t = Time(tstrings) for format in t.FORMATS: t.format = format assert np.all(t.strftime('%Y-%m-%d %H:%M:%S') == tstrings) assert t.strftime('%Y-%m-%d %H:%M:%S').shape == tstrings.shape
def build_hor_map(ra_v,dec_v,site,timestr): # Build tools for Horizontal coordinates if site == "lenghu": location = EarthLocation(lat=38.4*u.deg, lon=+93.3*u.deg, height=2650*u.m) obs_time = Time(timestr) # UTC time str_time = Time.strftime(obs_time,"%b-%d-%Y %H:%M") stepaz = round(360./np.shape(ra_v)[0]) stepalt = round(180./np.shape(dec_v)[0]) az_v = np.arange(0,360,stepaz) nl = np.shape(az_v)[0] alt_v = np.arange(-90,90,stepalt) nb = np.shape(alt_v)[0] az = np.repeat(az_v,nb) az = np.reshape(az,(nl,nb)) alt = np.tile(alt_v,(nl,1)) temp_hor = np.zeros(shape=np.shape(alt)) # Now loop on RaDec map and extract temp values for i, ra in enumerate(ra_v): print("***",i,"/",len(ra_v)) for j, dec in enumerate(dec_v): c = SkyCoord(ra, dec, frame='icrs', unit=(u.radian, u.radian)) altaz = c.transform_to(AltAz(obstime=obs_time,location=location)) th = altaz.alt.degree phi = altaz.az.degree ii = np.argmin(abs(phi-az_v)) jj = np.argmin(abs(th-alt_v)) temp_hor[ii,jj] = temp[i,j] return az,alt,temp_hor
def get_anneal_name(anneal_start): t = Time(anneal_start, format='mjd') anneal_name = t.strftime('%Y%b%d') print('Anneal start date: ', anneal_name) return anneal_name
def test_strftime_leapsecond(): time_string = '1995-12-31 23:59:60' t = Time(time_string) for format in t.FORMATS: t.format = format assert t.strftime('%Y-%m-%d %H:%M:%S') == time_string
def test_strftime_array(): tstrings = ['2010-09-03 00:00:00', '2005-09-03 06:00:00', '1995-12-31 23:59:60'] t = Time(tstrings) for format in t.FORMATS: t.format = format assert t.strftime('%Y-%m-%d %H:%M:%S').tolist() == tstrings
def write_fits(skysubrect, specrect, errorrect, header): hdulist = [] for image, ftp, nn in zip([skysubrect, specrect, errorrect], [fits.PrimaryHDU, fits.ImageHDU, fits.ImageHDU], ['skysub', 'spectra', 'error']): hdu = ftp(np.array(image, dtype='float32')) hdu.header['CRVAL2'] = 1 hdu.header['CRVAL1'] = np.log(def_wave)[0] hdu.header['CRPIX2'] = 1 hdu.header['CRPIX1'] = 1 hdu.header['CTYPE2'] = 'fiber' hdu.header['CTYPE1'] = 'ln(wave)' hdu.header['CDELT2'] = 1 hdu.header['CDELT1'] = np.log(def_wave)[1] - np.log(def_wave)[0] for key in header.keys(): if key in hdu.header: continue if ('CCDSEC' in key) or ('DATASEC' in key): continue if ('BSCALE' in key) or ('BZERO' in key): continue try: hdu.header[key] = header[key] except: continue t = Time(header['DATE-OBS']) objname = '_'.join(str(header['OBJECT']).split()) iname = '_'.join([objname, t.strftime('%Y%m%dT%H%M%S'), 'multi']) hdu.header['EXTNAME'] = nn hdulist.append(hdu) fits.HDUList(hdulist).writeto(op.join(outfolder, iname + '.fits'), overwrite=True)
def test_strftime_scalar(): """Test of Time.strftime """ time_string = '2010-09-03 06:00:00' t = Time(time_string) for format in t.FORMATS: t.format = format assert t.strftime('%Y-%m-%d %H:%M:%S') == time_string
def wind_avg_max(wind_tab0, h0): ''' Compute average, maximum and avg direction of wind from wind data for each observation period Parameters ---------- wind_tab0 : astropy.table.Table Astropy table containing young and young2 wind data h0 : astropy.io.fits.header.Header FITS header Returns ------- h0 : astropy.io.fits.header.Header Updated FITS header Notes ----- Created by Chun Ly, 1 March 2017 ''' t_start = Time(h0['DATE-OBS']).to_datetime(timezone=utc_mst) t_stop = t_start + timedelta(seconds=h0['EXPTIME']) # Note: mjd_start/mjd_stop is MJD associated with MST time not UTC mjd_start = Time(t_start.strftime('%Y-%m-%d %H:%M%:%S')).mjd mjd_stop = Time(t_stop.strftime('%Y-%m-%d %H:%M%:%S')).mjd time0 = Time(wind_tab0['MST_time']) mjd0 = time0.mjd t_idx = np.where((mjd0 >= mjd_start) & (mjd0 <= mjd_stop))[0] avg1 = np.average(wind_tab0['speed1'][t_idx]*mph_conv) max1 = np.max(wind_tab0['speed1'][t_idx]*mph_conv) dir1 = np.average(wind_tab0['direct1'][t_idx]) avg2 = np.average(wind_tab0['speed2'][t_idx]*mph_conv) max2 = np.max(wind_tab0['speed2'][t_idx]*mph_conv) dir2 = np.average(wind_tab0['direct2'][t_idx]) #young1 = {'avg':avg1, 'max':max1, 'dir':dir1} #young2 = {'avg':avg2, 'max':max2, 'dir':dir2} #return young1, young2 # Update header with wind data h0.set('Y1_AVG', avg1, 'YOUNG1 avg wind speed [mph]') h0.set('Y1_MAX', max1, 'YOUNG1 max wind speed [mph]') h0.set('Y1_DIR', dir1, 'YOUNG1 avg wind direction [deg]') h0.set('Y2_AVG', avg2, 'YOUNG2 avg wind speed [mph]') h0.set('Y2_MAX', max2, 'YOUNG2 max wind speed [mph]') h0.set('Y2_DIR', dir2, 'YOUNG1 avg wind direction [deg]') return h0
def GPSTime2Local(time): timeLocal0 = [] for i in range(len(time)): tgps0 = time[i] tgps = tgps0 - 19 # Added correction t = Time(tgps, format='gps', precision=3) t = Time(t, format='iso', precision=3) t = t.strftime("%H:%M:%S") timeLocal0.append(t) # print(str(tgps) + ' --> ' + str(t)) return timeLocal0
def jd2cal(jd, precision=0, DT_FMT='iso'): t = Time(jd, format='jd', precision=precision) if DT_FMT == 'iso': t.format = 'iso' return t.value else: if type(jd) == type(0.0): dt = t.strftime(DT_FMT) else: dt = [x.tt.datetime.strftime(DT_FMT) for x in t] return np.asarray(dt)
def get_twilights(night, alt=-18., verbose=False): ''' Calculate and return twilight values for the start and end of the night. Parameters ---------- night : int night in form 20210320 for 20 March 2021 alt : float sun altitude for twilight (default 18 deg twilight) verbose : bool print verbose output? Returns ------- twibeg : float time of twilight in MJD at beginning of the night twiend : float time of twilight in MJD at the end of the night ''' night = int(night) nightstr = str(night) timestr = "{0}-{1}-{2}T00:00:00.0".format(nightstr[:4], nightstr[4:6], nightstr[6:8]) t = Time(timestr, format='isot', scale='utc') + 1 # Set observatory lat,long to calculate twilight desi = ephem.Observer() desi.lon = '-111.59989' desi.lat = '31.96403' desi.elev = 2097. desi.date = t.strftime('%Y/%m/%d 7:00') # Calculate twilight times desi.horizon = str(alt) beg_twilight=desi.previous_setting(ephem.Sun(), use_center=True) # End astro twilight end_twilight=desi.next_rising(ephem.Sun(), use_center=True) # Begin astro twilight twibeg = Time( beg_twilight.datetime(), format='datetime').mjd twiend = Time( end_twilight.datetime(), format='datetime').mjd if verbose: print("Evening twilight: ", beg_twilight, "UT") print("Morning twilight: ", end_twilight, "UT") return twibeg, twiend
def build_hor_map2(ra,dec,temp,site,timestr): # First transfer RaDec map to Horizontal coordinates if site == "lenghu": location = EarthLocation(lat=38.4*u.deg, lon=+93.3*u.deg, height=2650*u.m) obs_time = Time(timestr, scale='utc',location=location) # UTC time str_time = Time.strftime(obs_time,"%b-%d-%Y %H:%M") LST = obs_time.sidereal_time('mean') LST_hour = str(LST).split('h')[0] print("Local Sideral Time:",LST) print("Building LFMap in Hor coordinates for LST hour =",LST_hour,"h.") c = SkyCoord(ra, dec, frame='icrs', unit=(u.radian, u.radian)) altaz = c.transform_to(AltAz(obstime=obs_time,location=location)) # Now build final (ordered) map in Horizonatl coordinates nl = np.shape(ra)[0] nb = np.shape(ra)[1] stepaz = round(360./nl) stepalt = round(180./nb) az_v = np.arange(0,360,stepaz) * u.degree alt_v = np.arange(-90,90,stepalt) * u.degree az = np.repeat(az_v,nb) az = np.reshape(az,(nl,nb)) alt = np.tile(alt_v,(nl,1)) altaz_fin = SkyCoord(az, alt, frame = 'altaz', obstime=obs_time,location=location) # Now find corresponding cells and fill temperature matrix accordingly sep = altaz.separation(altaz_fin) temp_hor = np.zeros(shape=np.shape(alt)) for i in range(nl): print('***',i,'/',nl) for j in range(nb): sep = altaz.separation(altaz_fin[i,j]).degree sepi = np.where(sep == np.amin(sep)) a = sepi[0][0] b = sepi[1][0] #print('Minimal distance =',np.amin(sep),'deg in cell',a,b) #print('(',altaz_fin[i,j].az.degree,altaz_fin[i,j].alt.degree,') vs (',altaz[a,b].az.degree,altaz[a,b].alt.degree,')') temp_hor[i,j] = temp[a,b] return np.array(az),np.array(alt),temp_hor
def get_mst(h0): ''' Get string-formatted MST time from UTC datetime Parameters ---------- h0 : astropy.io.fits.header.Header FITS header Returns ------- t.strftime : string Time in HH:MM:SS format Notes ----- Created by Chun Ly, 26 February 2017 ''' t = Time(h0['DATE-OBS']).to_datetime(timezone=utc_mst) return t.strftime('%H:%M:%S')
def main(year=None, month=None, day=None, ndays=1, overwritejp2=False, overwritefits=False): # tst = datetime.strptime("2017-04-01", "%Y-%m-%d") # ted = datetime.strptime("2019-12-31", "%Y-%m-%d") if year: ted = datetime(year, month, day) else: ted = datetime.now() - timedelta(days=2) tst = Time(np.fix(Time(ted).mjd) - ndays + 1, format='mjd').datetime print("Running pipeline_fitsutils for date from {} to {}".format( tst.strftime("%Y-%m-%d"), ted.strftime("%Y-%m-%d"))) dateobs = tst while dateobs <= ted: datestr = dateobs.strftime("%Y-%m-%d") rewriteImageFits(datestr, verbose=True, writejp2=True, overwritejp2=overwritejp2, overwritefits=overwritefits) dateobs = dateobs + timedelta(days=1)
def fetch(self, qres, path=None, error_callback=None, **kwargs): """ Download a set of results. Parameters ---------- qres : `~sunpy.net.dataretriever.QueryResponse` Results to download. Returns ------- Results Object """ urls = [qrblock['url'] for qrblock in qres] filenames = [] local_filenames = [] for i, [url, qre] in enumerate(zip(urls, qres)): name = url.split('/')[-1] day = Time(qre['Time'].start.strftime('%Y-%m-%d')) + TimeDelta( i * u.day) if name not in filenames: filenames.append(name) if name.endswith('.gz'): local_filenames.append('{}SRS.txt'.format( day.strftime('%Y%m%d'))) else: local_filenames.append(name) if path is not None: path = pathlib.Path(path) # Files to be actually downloaded paths = self._get_full_filenames(qres, filenames, path) # Those files that will be present after get returns local_paths = self._get_full_filenames(qres, local_filenames, path) # remove duplicate urls. This will make paths and urls to have same number of elements. # OrderedDict is required to maintain ordering because it will be zipped with paths later urls = list(OrderedDict.fromkeys(urls)) dobj = Downloader(max_conn=5) for aurl, fname in zip(urls, paths): dobj.enqueue_file(aurl, filename=fname) paths = dobj.download() outfiles = [] for fname, srs_filename in zip(local_paths, local_filenames): name = fname.name past_year = False for i, fname2 in enumerate(paths): fname2 = pathlib.Path(fname2) if fname2.name.endswith('.txt'): continue year = fname2.name.split('_SRS')[0] if year in name: TarFile = tarfile.open(fname2) filepath = fname.parent member = TarFile.getmember('SRS/' + srs_filename) member.name = name TarFile.extract(member, path=filepath) TarFile.close() outfiles.append(fname) past_year = True break if past_year is False: outfiles.append(fname) paths.data = list(map(str, outfiles)) return paths
max_bl = int(np.max(baseline_lengths)) lb_threshold = { 'B3': 750, 'B6': 780, } TM = ('TM1' if max_bl < lb_threshold[band] else 'TM2') if antennadiameter == 12: array_config = stacked[(obstime > start_times) & (obstime < end_times)]['Approx\xa0Config.'] if len(array_config) == 0: array_config = ['unknown'] else: TM = array_config = '7M' key = obstime.strftime('%Y-%m-%d') if fields[0] in results: # account for if key in results[fields[0]]: key = key + "_" results[fields[0]][key] = { 'array': array_config[0], 'TM': TM, 'mous': mous, 'band': band, 'onsource': onsource_time / 3600, 'exptime': integration_time.to(u.hour).value } else: results[fields[0]] = { key: {
def fetch(self, qres, path=None, error_callback=None, **kwargs): """ Download a set of results. Parameters ---------- qres : `~sunpy.net.dataretriever.QueryResponse` Results to download. Returns ------- Results Object """ urls = [qrblock.url for qrblock in qres] filenames = [] local_filenames = [] for i, [url, qre] in enumerate(zip(urls, qres)): name = url.split('/')[-1] # temporary fix !!! coz All QRBs have same start_time values day = Time(qre.time.start.strftime('%Y-%m-%d')) + TimeDelta( i * u.day) if name not in filenames: filenames.append(name) if name.endswith('.gz'): local_filenames.append('{}SRS.txt'.format( day.strftime('%Y%m%d'))) else: local_filenames.append(name) # Files to be actually downloaded paths = self._get_full_filenames(qres, filenames, path) # Those files that will be present after get returns local_paths = self._get_full_filenames(qres, local_filenames, path) res = Results(lambda x: None, 0, lambda map_: self._link(map_)) # remove duplicate urls. This will make paths and urls to have same number of elements. # OrderedDict is required to maintain ordering because it will be zipped with paths later urls = list(OrderedDict.fromkeys(urls)) dobj = Downloader(max_conn=len(urls), max_total=len(urls)) # We cast to list here in list(zip... to force execution of # res.require([x]) at the start of the loop. for aurl, ncall, fname in list( zip(urls, map(lambda x: res.require([x]), urls), paths)): dobj.download(aurl, fname, ncall, error_callback) res.wait() res2 = Results(lambda x: None, 0) for fname, srs_filename in zip(local_paths, local_filenames): fname = fname.args[0] name = fname.split('/')[-1] past_year = False for i, fname2 in enumerate(paths): fname2 = fname2.args[0] if fname2.endswith('.txt'): continue year = fname2.split('/')[-1] year = year.split('_SRS')[0] if year in name: TarFile = tarfile.open(fname2) filepath = fname.rpartition('/')[0] member = TarFile.getmember('SRS/' + srs_filename) member.name = name TarFile.extract(member, path=filepath) TarFile.close() callback = res2.require([fname]) callback({'path': fname}) past_year = True break if past_year is False: callback = res2.require([fname]) callback({'path': fname}) return res2
def query_mmtlog_wind(u_start, u_stop, user='******', passwd='', path0='', silent=False, verbose=True): ''' Query ops.mmto.arizona.edu's log for wind data. Note: This code requires specifying the password Parameters ---------- u_start : string UTC start time. Formatted as 'YYYY-MM-DD HH:MM:SS' u_stop : string UTC stop time. Formatted as 'YYYY-MM-DD HH:MM:SS' user : string Username to login. Default: 'webuser' passwd : string Password for user. Default: '' path0 : string Directory path to files. silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True Returns ------- tab0 : astropy.table.Table Astropy table containing young and young2 wind data Notes ----- Created by Chun Ly, 28 February 2017 Modified by Chun Ly, 01 March 2017 - Return tab0 ''' if passwd == '': log.error('Must specify password!') log.error('Exiting!!!') return if silent == False: log.info('### Begin query_mmtlog_wind: '+systime()) m_start = Time(u_start).to_datetime(timezone=utc_mst) m_start = m_start.strftime('%Y-%m-%d %H:%M%:%S') m_stop = Time(u_stop).to_datetime(timezone=utc_mst) + \ timedelta(seconds=10*60.0) # Add 10 min. to have enough buffer m_stop = m_stop.strftime('%Y-%m-%d %H:%M%:%S') conn = pymysql.connect(host='ops.mmto.arizona.edu', user=user, passwd=passwd, db='mmtlogs') cur = conn.cursor() sql1 = "SELECT timestamp,young_wind_speed,young_wind_direction FROM "+\ "young_background_log where timestamp >= '"+m_start+"' AND "+\ "timestamp < '"+m_stop+"'" # print sql1 n_entries = cur.execute(sql1) results1 = cur.fetchall() sql2 = sql1.replace('young', 'young2') cur.execute(sql2) results2 = cur.fetchall() time0 = np.repeat('XXXX-XX-XX XX:XX:XX', n_entries) speed1 = np.zeros(n_entries) direct1 = np.zeros(n_entries) speed2 = np.zeros(n_entries) direct2 = np.zeros(n_entries) for nn in xrange(n_entries): time0[nn] = results1[nn][0].isoformat() speed1[nn] = results1[nn][1] direct1[nn] = results1[nn][2] speed2[nn] = results2[nn][1] direct2[nn] = results2[nn][2] outfile = path0+'wind_data.tbl' vec0 = [time0, speed1, direct1, speed2, direct2] names0 = ('MST_time','speed1', 'direct1', 'speed2', 'direct2') tab0 = Table(vec0, names=names0) if silent == False: log.info('## Writing : '+outfile) asc.write(tab0, outfile, format='fixed_width_two_line', overwrite=True) if silent == False: log.info('### End query_mmtlog_wind: '+systime()) return tab0
class lcogt(object): def __init__(self, username, password, progid, telescop, start_date, end_date): uri_base = 'https://observe.lco.global/' archive_base = 'https://archive-api.lco.global/' self.params = { 'uri': { 'archive': { 'authorization': archive_base + 'api-token-auth/', 'frames': archive_base + 'frames/', 'token': None }, 'request': { 'authorization': uri_base + 'api/api-token-auth/', 'request': uri_base + 'api/requestgroups/', 'token': None } }, 'date_format': '%Y-%m-%d %H:%M:%S', 'constraints': { 'max_airmass': 2.5, 'min_lunar_distance': 15 }, 'strategy': { 'default': { 'type': 'default', 'proposal': [ { 'name': progid, 'obstype': 'NORMAL' }, ], 'filters': ['up', 'gp', 'rp', 'ip'], 'min_exposure': { 'default': 45, 'up': 150 }, 'max_exposure': 540, # SNR strategy are pairwise mag, snr values. # If current source magnitude is < mag, then use given snr. 'snr': [[16, 40], [18, 20], [99, 10]], 'cadence': 4, 'telescope_class': '1m0', 'instrument_type': '1M0-SCICAM-SINISTRO', 'acquisition_config': 'OFF', 'guiding_config': 'ON', 'window': 1.0, 'ipp': 1.0 }, 'spectroscopy': { 'type': 'spectroscopy', 'proposal': [ { 'name': progid, 'obstype': 'NORMAL' }, ], 'min_exposure': 300, 'max_exposure': 2400, 'telescope_class': '2m0', 'instrument_type': '2M0-FLOYDS-SCICAM', 'slit': 'slit_1.6as', 'acquisition_config': 'OFF', 'guiding_config': 'ON', 'window': 1.0, 'ipp': 1.0 }, 'photometry': { 'type': 'photometry', 'proposal': [ { 'name': progid, 'obstype': 'NORMAL' }, ], 'filters': ['up', 'gp', 'rp', 'ip'], 'min_exposure': { 'default': 45, 'up': 150 }, 'max_exposure': 540, # SNR strategy are pairwise mag, snr values. first is mag # and second is snr. If mag_source < mag, then use snr. 'snr': [[16, 40], [18, 20], [99, 10]], 'cadence': 4, 'telescope_class': '1m0', 'instrument_type': '1M0-SCICAM-SINISTRO', 'acquisition_config': 'OFF', 'guiding_config': 'ON', 'window': 1.0, 'ipp': 1.01 } } } self.start_date = Time(start_date).datetime self.end_date = Time(end_date).datetime self.telescope = telescop if 'soar' in telescop: self.params['strategy']['spectroscopy'] = { 'type': 'spectroscopy', 'proposal': [ { 'name': progid, 'obstype': 'NORMAL' }, ], 'min_exposure': 300, 'max_exposure': 2400, 'telescope_class': '4m0', 'instrument_type': 'SOAR_GHTS_REDCAM', 'slit': 'slit_1.0as', 'acquisition_config': 'OFF', 'guiding_config': 'ON', 'window': 6.0, 'ipp': 1.0 } self.params['constraints'] = { 'max_airmass': 1.7, 'min_lunar_distance': 30 } # List of all proposals to which I have access. Some of these are old # proposals where the data are now public self.proposals = [ 'NOAO2019A-020', 'NOAO2017AB-012', 'NOAO2018A-005', 'NOAO2019A-020-TC', 'NOAO2017AB-005', 'NOAO2018A-007', 'NOAO2018B-022', 'NOAO2019A-001', 'NOAO2018B-022b', 'NOAO2019B-008', 'NOAO2019B-004', 'KEY2017AB-001', 'NOAO2019B-009', 'NAOC2017AB-001', 'LCO2018A-004', 'FTPEPO2014A-004', 'ARI2017AB-002', progid ] # These are relevant to SINISTRO on the 1m. Will update with Spectral # and MuSCAT3 zeropoints self.constants = { 'zpt': { 'up': 20.5665, 'gp': 23.2249, 'rp': 23.1314, 'ip': 22.8465 } } self.username = username self.password = password self.format = {'token': 'Token {token}'} def get_username_password(self): if (self.username is not None and self.password is not None): return (self.username, self.password) else: # Get username password out of shibboleth return (None, None) def get_token_header(self, username, password, auth_type='archive'): # Check that we're getting the right token if auth_type not in self.params['uri'].keys(): return (None) params = self.params['uri'][auth_type] # Check if header token has already been defined if params['token']: # Return the heade that we need fmt = self.format['token'] header = {'Authorization': fmt.format(token=params['token'])} return (header) else: # Need to generate a new token data = {'username': username, 'password': password} uri = params['authorization'] # Now run a request response = requests.post(uri, data=data).json() # Check that the request worked if 'token' in response.keys(): self.params['uri'][auth_type]['token'] = response['token'] fmt = self.format['token'] header = {'Authorization': fmt.format(token=response['token'])} return (header) else: # There was some problem with authentication/connection return (None) # Download spectral calibration files for a specific telid and date def get_spectral_calibrations(self, date, telid, site, outrootdir='', funpack=True): # Get username and password username, password = self.get_username_password() # Get authorization token headers = self.get_token_header(username, password) params = {'limit': 100} results = [] params['TELID'] = telid params['SITEID'] = site delta = TimeDelta(1, format='jd') params['start'] = (date - delta).datetime.strftime( self.params['date_format']) params['end'] = (date + delta).datetime.strftime( self.params['date_format']) # First check for LAMPFLATs params['OBSTYPE'] = 'LAMPFLAT' response = requests.get(self.params['uri']['archive']['frames'], params=params, headers=headers) if response.status_code != 200: print(response.text) else: data = response.json() results += data['results'] # Now check for ARCs params['OBSTYPE'] = 'ARC' response = requests.get(self.params['uri']['archive']['frames'], params=params, headers=headers) if response.status_code != 200: print(response.text) else: data = response.json() results += data['results'] # Now try to download the obslist self.download_obslist(results, outrootdir=outrootdir, use_basename=True, skip_header=True, funpack=funpack) # Get a json object with complete list of observations. Optional to define # a program or date range to narrow search def get_obslist(self, propid=None, sdate=None, edate=None, telid=None, obstype=None, rlevel=None, obj=None, reqnum=None): # Get username and password username, password = self.get_username_password() # Get authorization token headers = self.get_token_header(username, password) # Get parameters for request params = {'limit': 5000} fmt = self.params['date_format'] results = [] propids = [] if rlevel is not None: params['RLEVEL'] = rlevel else: params['RLEVEL'] = 91 if propid is not None: propids = list(propid) if sdate is not None: params['start'] = sdate.datetime.strftime(fmt) if edate is not None: params['end'] = edate.datetime.strftime(fmt) if telid is not None: params['TELID'] = telid if obstype is not None: params['OBSTYPE'] = obstype if obj is not None: params['OBJECT'] = obj if reqnum is not None: params['REQNUM'] = reqnum for pid in propids: params['PROPID'] = pid # Now do request response = requests.get(self.params['uri']['archive']['frames'], params=params, headers=headers) if response.status_code != 200: print(response.text) else: data = response.json() results += data['results'] return (results) # Get a set of standard star observation for spectrophotometric calibration def get_standardobs(self, sdate=None, telid=None, rlevel=None): # Get username and password username, password = self.get_username_password() # Get authorization token headers = self.get_token_header(username, password) # Get parameters for request params = {'OBSTYPE': 'SPECTRUM', 'public': True} delta = TimeDelta(14, format='jd') date = Time(datetime.now()) start = (date - delta).datetime fmt = self.params['date_format'] if not sdate: params['start'] = start.strftime(fmt) else: params['start'] = sdate.datetime.strftime(fmt) if not rlevel: params['RLEVEL'] = 0 else: params['RLEVEL'] = rlevel objs = [ 'EGGR274', 'L745-46A', 'FEIGE110', 'HZ 44', 'BD+284211', 'GD71' ] results = [] for obj in objs: params['OBJECT'] = obj response = requests.get(self.params['uri']['archive']['frames'], params=params, headers=headers) if response.status_code != 200: print(response.text) else: data = response.json() results += data['results'] return (results) # Get recent observation requests for proposal ID, start date, end date, # observation type, and instrument type def get_requestgroups(self, propid=None, sdate=None, edate=None, obstype=None, itype=None): # Get username and password username, password = self.get_username_password() # Get authorization token headers = self.get_token_header(username, password, auth_type='request') params = {} results = [] propids = [] if propid is not None: propids = list(propid) if sdate is not None: start = sdate.datetime.strftime(self.params['date_format']) params['created_after'] = start if edate is not None: end = edate.datetime.strftime(self.params['date_format']) params['created_before'] = end for pid in propids: params['proposal'] = pid params['limit'] = 500 # Now do request response = requests.get(self.params['uri']['request']['request'], params=params, headers=headers).json() # If instrument_type, remove values that do not conform if itype is not None: for r in copy.copy(response['results']): test = r['requests'][0]['configurations'][0][ 'instrument_type'] if test != itype: response['results'].remove(r) results += response['results'] return (results) # Given an input obslist from the LCOGT archive, download the associated # files for each element def download_obslist(self, obslist, outrootdir='', use_basename=False, skip_header=False, funpack=True): for frame in obslist: filename = '' if use_basename: filename = frame['basename'] + '.fits.fz' else: target = frame['OBJECT'] # Need to sanitize target, e.g., for spaces target = target.replace(' ', '_') filt = frame['FILTER'] idnum = str(frame['id']) date = Time(frame['DATE_OBS']).datetime.strftime('ut%y%m%d') filename = target + '.' + date + '.' +\ filt + '.' + idnum + '.fits.fz' fullfilename = outrootdir + '/' + filename if not os.path.exists(outrootdir): shutil.os.makedirs(outrootdir) if ((not os.path.exists(fullfilename) and not os.path.exists(fullfilename.strip('.fz')) and funpack) or (not os.path.exists(fullfilename))): message = 'Downloading LCOGT file: {file}' print(message.format(file=fullfilename)) with open(fullfilename, 'wb') as f: f.write(requests.get(frame['url']).content) else: message = 'LCOGT file: {file} already exists!' print(message.format(file=fullfilename)) # funpack - requires cfitsio if not os.path.exists(fullfilename.strip('.fz')) and funpack: cmd = 'funpack {file}' os.system(cmd.format(file=fullfilename)) if os.path.exists(fullfilename) and funpack: os.remove(fullfilename) # Remove extraneous extension if not skip_header: hdulist = fits.open(fullfilename.strip('.fz')) newhdu = fits.HDUList() hdu = hdulist['SCI'] hdu.header['OBSTYPE'] = 'OBJECT' newhdu.append(hdu) newhdu.writeto(fullfilename.strip('.fz'), overwrite=True) # Make a location element with telescope class def make_location(self, telescope): return ({'telescope_class': telescope}) # Make an airmass/lunar distance constraint element def make_constraints(self): max_airmass = self.params['constraints']['max_airmass'] min_lunar_distance = self.params['constraints']['min_lunar_distance'] constraints = { 'max_airmass': max_airmass, 'min_lunar_distance': min_lunar_distance } return (constraints) # This sets up a target based on name, ra, dec def make_target(self, name, ra, dec): target = { 'type': 'ICRS', 'name': name, 'ra': ra, 'dec': dec, 'proper_motion_ra': 0.0, 'proper_motion_dec': 0.0, 'parallax': 0.0, 'epoch': 2000.0 } return (target) # This sets up the instrument parameters for an obs # Imaging specific def make_instrument_configs(self, filt, exptime, strat): # This sets up an imaging observation for SINISTRO. Need to update for # 2m imaging with Spectral and MuSCAT3 if strat['type'] == 'default': configuration = { 'instrument_name': '1M0-SCICAM-SINISTRO', 'optical_elements': { 'filter': filt }, 'mode': 'full_frame', 'exposure_time': exptime, 'exposure_count': 1, 'bin_x': 1, 'bin_y': 1, 'extra_params': { 'defocus': 0.0 } } # This creates a spectroscopy observing element for FLOYDS elif strat['type'] == 'spectroscopy' and 'faulkes' in self.telescope: configuration = { "bin_x": 1, "bin_y": 1, "exposure_count": 1, "exposure_time": exptime, "mode": "default", "rotator_mode": "VFLOAT", "extra_params": {}, "optical_elements": { "slit": strat['slit'] } } elif strat['type'] == 'spectroscopy' and 'soar' in self.telescope: configuration = { "bin_x": 2, "bin_y": 2, "exposure_count": 1, "exposure_time": exptime, "mode": "GHTS_R_400m1_2x2", "rotator_mode": "SKY", "extra_params": { "rotator_angle": 90 }, "optical_elements": { "slit": strat['slit'], "grating": "SYZY_400" } } return ([configuration]) # Make an acquisition element for FLOYDS spectroscopy (on coordinate vs. # acquire brightest object) def make_acquisition_config(self, strat, mode=None): if not mode: mode = strat['acquisition_config'] return ({'mode': mode}) # Should guiding be on or off def make_guiding_config(self, strat, extra_params={}): mode = strat['guiding_config'] config = {'mode': mode} for key in extra_params.keys(): config[key] = extra_params[key] return (config) """ Window object for LCOGT API v3. start time = earliest the obs can be executed duration = amount of time in which request can be executed """ def make_window(self, start, duration): fmt = self.params['date_format'] end = start + timedelta(days=duration) # Note that this is an override for YSE App window = [{ 'start': self.start_date.strftime(fmt), 'end': self.end_date.strftime(fmt) }] return (window) # Post your observing request with requests.post def post_user_request(self, user_request): # Get username and password username, password = self.get_username_password() # Get authorization token header = self.get_token_header(username, password, auth_type='request') uri = self.params['uri']['request']['request'] response = requests.post(uri, json=user_request, headers=header) return (response) # Guess the exposure time based on input magnitude of target def get_exposure_time(self, filt, mag, strat): # Get preferred snr given input magnitude # Handle 'spec' case if filt == 'spec': if mag < 14: return (300) elif mag < 15: return (600) elif mag < 16: return (900) elif mag < 17: return (1500) elif mag < 17.5: return (1800) elif mag < 18.0: return (2100) elif mag < 18.5: return (2400) else: return (None) snr = 10. for pair in strat['snr']: if mag < pair[0]: snr = pair[1] term1 = 20. * snr**2 term2 = 0.4 * (mag - self.constants['zpt'][filt]) exptime = term1 * 10**term2 print(filt, exptime) min_exposure = 45.0 if filt in strat['min_exposure'].keys(): min_exposure = strat['min_exposure'][filt] else: min_exposure = strat['min_exposure']['default'] if exptime < min_exposure: exptime = min_exposure if exptime > strat['max_exposure']: return (None) return (exptime) """ LCO API v3 configurations. configurations consist of constraints, instrument configuration, acquisition configuration, guiding configuration, target, instrument type, exposure type, and observation priority. This method will construct a series of configurations for a single target. """ def make_configurations(self, obj, ra, dec, exptime, strat): # Iterate over each filter and append to configurations list configurations = [] # If default, construct configurations for photometry strategy if strat['type'] == 'default': for i, filt in enumerate(strat['filters']): constraints = self.make_constraints() # Get exposure time and make instrument_config instrument_configs = self.make_instrument_configs( filt, exptime, strat) # Make acquisition and guiding config with strat acquisition_config = self.make_acquisition_config(strat) guiding_config = self.make_guiding_config(strat) # Make a tar object target = self.make_target(obj, ra, dec) # Compile everything into configuration configuration = { 'constraints': constraints, 'instrument_configs': instrument_configs, 'acquisition_config': acquisition_config, 'guiding_config': guiding_config, 'target': target, 'instrument_type': strat['instrument_type'], 'type': 'EXPOSE', 'priority': i + 1 } configurations.append(configuration) elif strat['type'] == 'spectroscopy' and 'faulkes' in self.telescope: # Need to construct LAMP FLAT, ARC, SPECTRUM, ARC, LAMP FLAT for obstype in [ 'LAMP_FLAT', 'ARC', 'SPECTRUM', 'ARC', 'LAMP_FLAT' ]: # Use default constraint values constraints = self.make_constraints() if obstype == 'LAMP_FLAT': exptime = 50 if obstype == 'ARC': exptime = 60 # Make acquisition and guiding config with strat instrument_configs = self.make_instrument_configs( 'spec', exptime, strat) if obstype == 'SPECTRUM': acquisition_config = self.make_acquisition_config( strat, mode='WCS') else: acquisition_config = self.make_acquisition_config(strat) guiding_config = self.make_guiding_config(strat) # Make a target object target = self.make_target(obj, ra, dec) # Compile everything into configuration configuration = { 'type': obstype, 'constraints': constraints, 'instrument_configs': instrument_configs, 'acquisition_config': acquisition_config, 'guiding_config': guiding_config, 'target': target, 'instrument_type': strat['instrument_type'] } configurations.append(configuration) elif strat['type'] == 'spectroscopy' and 'soar' in self.telescope: for i, obstype in enumerate(['ARC', 'SPECTRUM']): # Use default constraint values constraints = self.make_constraints() if obstype == 'ARC': exptime = 0.5 # Make acquisition and guiding config with strat instrument_configs = self.make_instrument_configs( 'spec', exptime, strat) if obstype == 'SPECTRUM': acquisition_config = self.make_acquisition_config( strat, mode='MANUAL') guiding_config = self.make_guiding_config( strat, extra_params={ 'optional': False, "optical_elements": {}, "exposure_time": None, "extra_params": {} }) elif obstype == 'ARC': acquisition_config = self.make_acquisition_config( strat, mode='OFF') guiding_config = self.make_guiding_config( strat, extra_params={ 'optional': True, "optical_elements": {}, "exposure_time": None, "extra_params": {} }) # Make a target object target = self.make_target(obj, ra, dec) # Compile everything into configuration configuration = { 'type': obstype, 'constraints': constraints, 'instrument_configs': instrument_configs, 'acquisition_config': acquisition_config, 'guiding_config': guiding_config, 'target': target, 'instrument_type': strat['instrument_type'], 'priority': i + 1 } configurations.append(configuration) return (configurations) """ LCO API v3 request. requests is a list of requests each defined by a location, configuration, and a window. """ def make_requests(self, obj, ra, dec, exptime, strat): location = self.make_location(strat['telescope_class']) window = self.make_window(datetime.now(), strat['window']) configurations = self.make_configurations(obj, ra, dec, exptime, strat) requests = [{ 'location': location, 'windows': window, 'configurations': configurations }] return (requests) """ LCO API v3 observation request. Each method constructs one layer of the request. Here we need to make the outermost obs request with name, proposal, ipp value, operator, observation type, and the individual requests. Send the """ def make_obs_request(self, obj, ra, dec, exptime, strategy='default', propidx=''): # Get params - strategy data strat = self.params['strategy'][strategy] proposal = propidx # Make the obs_request dictionary obs_request = { 'name': obj, 'proposal': propidx, 'ipp_value': strat['ipp'], 'operator': 'SINGLE', 'observation_type': 'NORMAL' } # Iterate through the next level of request requests = self.make_requests(obj, ra, dec, exptime, strat) if not requests[0]['configurations']: return (None) else: obs_request['requests'] = requests print(obs_request) response = self.post_user_request(obs_request) print(response.content) response = response.json() return (response)
def fetch(self, qres, path=None, error_callback=None, **kwargs): """ Download a set of results. Parameters ---------- qres : `~sunpy.net.dataretriever.QueryResponse` Results to download. Returns ------- Results Object """ urls = [qrblock.url for qrblock in qres] filenames = [] local_filenames = [] for i, [url, qre] in enumerate(zip(urls, qres)): name = url.split('/')[-1] # temporary fix !!! coz All QRBs have same start_time values day = Time(qre.time.start.strftime('%Y-%m-%d')) + TimeDelta(i*u.day) if name not in filenames: filenames.append(name) if name.endswith('.gz'): local_filenames.append('{}SRS.txt'.format(day.strftime('%Y%m%d'))) else: local_filenames.append(name) # Files to be actually downloaded paths = self._get_full_filenames(qres, filenames, path) # Those files that will be present after get returns local_paths = self._get_full_filenames(qres, local_filenames, path) # remove duplicate urls. This will make paths and urls to have same number of elements. # OrderedDict is required to maintain ordering because it will be zipped with paths later urls = list(OrderedDict.fromkeys(urls)) dobj = Downloader(max_conn=5) for aurl, fname in zip(urls, paths): dobj.enqueue_file(aurl, filename=fname) paths = dobj.download() outfiles = [] for fname, srs_filename in zip(local_paths, local_filenames): name = fname.name past_year = False for i, fname2 in enumerate(paths): fname2 = pathlib.Path(fname2) if fname2.name.endswith('.txt'): continue year = fname2.name.split('_SRS')[0] if year in name: TarFile = tarfile.open(fname2) filepath = fname.parent member = TarFile.getmember('SRS/' + srs_filename) member.name = name TarFile.extract(member, path=filepath) TarFile.close() outfiles.append(fname) past_year = True break if past_year is False: outfiles.append(fname) paths.data = list(map(str, outfiles)) return paths
class AllskyImage(): """Class for handling individual FITS image files.""" def __init__(self, filename, data, header): self.filename = filename # raw image filename self.datetime = None # date and time of observation self.thumbfilename = None # filename for thumbnail image self.data = data # image data array self.header = header # image header self.subregions = None # subregion arrays self.features = None # extracted features @classmethod def read_fits(cls, filename): """Create `~AllskyImage` instance from FITS image file. :return: self """ hdu = fits.open(filename)[0] self = cls(filename.split(os.path.sep)[-1], hdu.data.astype(np.float), hdu.header) try: self.datetime = Time(self.header['DATE-OBS'], format='isot') except (ValueError, KeyError): conf.logger.warning(('No time information for image file ' '{}.').format(filename)) pass return self def write_fits(self, filename): """Write `~AllskyImage` instance to FITS image file""" hdu = fits.PrimaryHDU(self.data) hdu.writeto(filename, overwrite=True) def create_overlay(self, overlaytype='srcdens', regions=None): """Create overlay for thumbnail image. Requires self.subregions to be initialized. An overlay is an array with the same dimensions as self.data` in which certain subregions get assigned certain values as defined by `overlaytype`. :param overlaytype: define data source from `self.features` from which overlay should be generated, default: 'srcdens' :param regions: list of length=len(self.subregions), highlights subregions with list element value > 0; requires `overlaytype='subregions'`, default: None :return: overlay array """ map = np.zeros(self.data.shape) for i, sub in enumerate(self.subregions): if overlaytype == 'srcdens': map += sub*self.features['srcdens'][i] elif overlaytype == 'bkgmedian': map += sub*self.features['bkgmedian'][i] elif overlaytype == 'bkgmean': map += sub*self.features['bkgmean'][i] elif overlaytype == 'bkgstd': map += sub*self.features['bkgstd'][i] elif overlaytype == 'subregions': if regions[i]: map += sub else: raise AllskyImageError('overlaytype "{}" unknown.'.format( overlaytype)) map[map == 0] = np.nan return map def write_image(self, filename, overlay=None, mask=None, overlay_alpha=0.3, overlay_color='Reds'): """Write `~AllskyImage` instance as scaled png thumbnail image file. :param filename: filename of image to be written, relative to cwd :param overlay: provide overlay or list of overlays, optional :param mask: apply image mask before writing image file :param overlay_alpha: alpha value to be applied to overlay :param overlay_color: colormap to be used with overlay :return: None """ conf.logger.info('writing thumbnail "{}"'.format(filename)) data = self.data # derive image scaling and stretching if mask is not None: norm = ImageNormalize(data[mask.data == 1], conf.THUMBNAIL_SCALE(), stretch=LinearStretch()) data[mask.data == 0] = 0 else: norm = ImageNormalize(data, conf.THUMBNAIL_SCALE(), stretch=LinearStretch()) # create figure f, ax = plt.subplots(figsize=(conf.THUMBNAIL_WIDTH, conf.THUMBNAIL_HEIGHT)) # plot image img = ax.imshow(data, origin='lower', norm=norm, cmap='gray', extent=[0, self.data.shape[1], 0, self.data.shape[0]]) # plot overlay(s) if overlay is not None: if not isinstance(overlay, list): overlay = [overlay] overlay_color = [overlay_color] overlay_img = [] for i in range(len(overlay)): overlay_img.append(ax.imshow(overlay[i], cmap=overlay_color[i], origin='lower', vmin=0, alpha=overlay_alpha, extent=[0, overlay[i].shape[1], 0, overlay[i].shape[0]])) overlay_img[i].axes.get_xaxis().set_visible(False) overlay_img[i].axes.get_yaxis().set_visible(False) # remove axis labels and ticks plt.axis('off') img.axes.get_xaxis().set_visible(False) img.axes.get_yaxis().set_visible(False) # save thumbnail image plt.savefig(filename, bbox_inches='tight', dpi=conf.THUMBNAIL_DPI, pad_inches=0) plt.close() # let thumbfilename consist of <night>/<filename> self.thumbfilename = os.path.join(*filename.split(os.path.sep)[-2:]) def apply_mask(self, mask): """Apply `~AllskyImage` mask to this instance""" self.data = self.data * mask.data def crop_image(self): """Crop this `~AllskyImage` instance to the ranges defined by ``conf.X_CROPRANGE`` and ``conf.Y_CROPRANGE``. """ self.data = self.data[conf.Y_CROPRANGE[0]:conf.Y_CROPRANGE[1], conf.X_CROPRANGE[0]:conf.X_CROPRANGE[1]] def extract_features(self, subregions, mask=None): """Extract image features for each subregion. Image should be cropped and masked. :param subregions: subregions to be used :param mask: mask to be applied in source extraction, optional :return: None """ # set internal pixel buffer sep.set_extract_pixstack(10000000) # extract time from header and derive frame properties try: time = Time(self.header['DATE-OBS'], format='isot') features = OrderedDict([ ('time', time.isot), ('filename', self.filename.split(os.path.sep)[-1]), ('moon_alt', observatory.moon_altaz(time).alt.deg), ('sun_alt', observatory.sun_altaz(time).alt.deg), ('moon_phase', 1-observatory.moon_phase(time).value/np.pi), ]) except KeyError as e: conf.logger.error('missing time data in file {}: {}.'.format( self.filename, e)) return False # derive and subtract sky background bkg = sep.Background(self.data.astype(np.float64), bw=conf.SEP_BKGBOXSIZE, bh=conf.SEP_BKGBOXSIZE, fw=conf.SEP_BKGXRANGE, fh=conf.SEP_BKGYRANGE) data_sub = self.data - bkg.back() # if mask is provided, it is applied in the proper derivation of # source brightness thresholds if mask is not None: threshold = (np.ma.median(np.ma.array(data_sub, mask=(1-mask))) + np.median(bkg.rms())*conf.SEP_SIGMA) src = sep.extract(data_sub, threshold, minarea=conf.SEP_MINAREA, mask=(1-mask), deblend_nthresh=conf.SEP_DEBLENDN, deblend_cont=conf.SEP_DEBLENDV) else: threshold = (np.median(data_sub) + np.median(bkg.rms())*conf.SEP_SIGMA) src = sep.extract(data_sub, threshold, minarea=conf.SEP_MINAREA, mask=mask, deblend_nthresh=conf.SEP_DEBLENDN, deblend_cont=conf.SEP_DEBLENDV) # apply max_flag cutoff (reject flawed sources) src = src[src['flag'] <= conf.SEP_MAXFLAG] # feature extraction per subregion features['srcdens'] = [] features['bkgmedian'] = [] features['bkgmean'] = [] features['bkgstd'] = [] for i, sub in enumerate(subregions): features['srcdens'].append(len( src[sub[src['y'].astype(np.int), src['x'].astype(np.int)]])/np.sum(sub[mask== 1])) features['bkgmedian'].append(np.median(bkg.back()[sub])) features['bkgmean'].append(np.mean(bkg.back()[sub])) features['bkgstd'].append(np.std(bkg.back()[sub])) self.subregions = subregions self.features = features def write_to_database(self): """Write extracted features to database.""" session = requests.Session() post_headers = {'Content-Type': 'application/json'} try: data = {'date': self.features['time'], 'night': int(self.datetime.strftime('%Y%m%d')), 'filearchivepath': self.thumbfilename, 'moonalt': self.features['moon_alt'], 'sunalt': self.features['sun_alt'], 'moonphase': self.features['moon_phase'], 'srcdens': self.features['srcdens'], 'bkgmean' : self.features['bkgmean'], 'bkgmedian' : self.features['bkgmedian'], 'bkgstd': self.features['bkgstd'], } except KeyError: conf.logger.error('data incomplete for file {}; reject.'.format( self.filename)) return None post_request = session.post( conf.DB_URL+'data/Unlabeled/', headers=post_headers, auth=(conf.DB_USER, conf.DB_PWD), json=data) if not ((post_request.status_code == requests.codes.ok) or (post_request.status_code == requests.codes.created)): conf.logger.error('upload to database failed with code {}; {}'.format( post_request.status_code, post_request.text)) raise ServerError('upload to database failed with code {}'.format( post_request.status_code))
def get_observations( fds_only=True, start_obsid=None, finish_obsid=None, obs_date=None, dec_pointing=None, cen_chan=None, hour_angle=None, mask_gp=False, ): df = pd.read_sql_table("observation", mdb.dbconn) if fds_only: df = df[df["obsname"].str.contains("FDS")] if mask_gp: sky = SkyCoord(df["ra_pointing"], df["dec_pointing"], unit=(u.deg, u.deg)) l = sky.galactic.l.deg b = sky.galactic.b.deg mask = ((b >= GALACTIC_PLANE_LIMITS[0]) & (b <= GALACTIC_PLANE_LIMITS[1]) & ((l <= GALACTIC_PLANE_LIMITS[2]) | (l >= GALACTIC_PLANE_LIMITS[3]))) df = df[~mask] if start_obsid is not None: df = df[df["obs_id"] >= int(start_obsid)] if finish_obsid is not None: df = df[df["obs_id"] <= int(finish_obsid)] if obs_date is not None: gps = Time(df["obs_id"], format="gps", location=MWA) gps_date = gps.strftime("%Y-%m-%d") df = df[gps_date == obs_date] if dec_pointing is not None: dec = np.array([-71.0, -55.0, -41.0, -39.0, -26.0, -12.0, 3.0, 20.0]) min_dec = lambda x: dec[np.argmin(np.abs(dec - x["dec_pointing"]))] df["Dec Strip"] = df.apply(min_dec, axis=1) df = df[df["Dec Strip"] == dec_pointing] if hour_angle is not None: gps = Time(df["obs_id"], format="gps", location=MWA) lst = gps.sidereal_time("mean") ra = df["ra_pointing"].values ha = np.round((lst.deg - ra) / 15) mask = ha > 12 ha[mask] = ha[mask] - 24.0 mask = ha < -12 ha[mask] = ha[mask] + 24.0 df = df[ha.astype(int) == hour_angle] if cen_chan is not None: df = df[df["cenchan"] == cen_chan] return df
def gps2datestr(gps): utc = Time(gps, format='gps').to_datetime() date_str = utc.strftime('%Y%m%d%H%M') return date_str
def recalc(outfile='save_all.pickle'): t0 = time.time() temptable_raw = \ load_temptable(TEMPFILE) log.info("Querying history of NuSTAR observations...") all_nustar_obs = get_obsid_list_from_heasarc() all_nustar_obs.sort('MET') all_nustar_obs['text'] = [ (f"Source: {aobs['NAME']}<br>" + f"ObsID: {aobs['OBSID']}<br>" + f"Start: {aobs['DATE']} (MJD {aobs['TIME']})<br>" + f"End: {aobs['DATE-END']} (MJD {aobs['END_TIME']})<br>" + f"Type: {aobs['OBS_TYPE']}<br>" + f"Mode: {aobs['OBSERVATION_MODE']}<br>") for aobs in all_nustar_obs ] clock_offset_table = \ load_and_flag_clock_table(clockfile=CLOCKFILE, shift_non_malindi=False) clock_offset_table_corr = \ load_and_flag_clock_table(clockfile=CLOCKFILE, shift_non_malindi=True) table_times = temptable_raw['met'] met_start = clock_offset_table['met'][0] met_stop = clock_offset_table['met'][-1] + 30 clock_jump_times = np.array( [78708320, 79657575, 81043985, 82055671, 293346772]) clock_jump_times += 30 # Sum 30 seconds to avoid to exclude these points # from previous interval gtis = find_good_time_intervals(temptable_raw, clock_jump_times) table_new = temperature_correction_table(met_start, met_stop, temptable=temptable_raw, freqchange_file=FREQFILE, time_resolution=10, craig_fit=False, hdf_dump_file='dump.hdf5') table_new = eliminate_trends_in_residuals(table_new, clock_offset_table_corr, gtis, fixed_control_points=np.arange( 291e6, 295e6, 86400)) mets = np.array(table_new['met']) start = mets[0] stop = mets[-1] good_mets = clock_offset_table['met'] < stop clock_offset_table = clock_offset_table[good_mets] # print(clock_offset_table['met'][-10:], table_new['met'][-1]) clock_mets = clock_offset_table['met'] clock_mjds = clock_offset_table['mjd'] dates = Time(clock_mjds[:-1], format='mjd') tempcorr_idx = \ np.searchsorted(table_new['met'], clock_offset_table['met']) clock_residuals = \ np.array( clock_offset_table['offset'] - table_new['temp_corr'][tempcorr_idx] ) clock_residuals_detrend = np.array( clock_offset_table['offset'] - table_new['temp_corr_detrend'][tempcorr_idx]) all_data = Table({ 'met': clock_mets[:-1], 'mjd': np.array(clock_mjds[:-1], dtype=int), 'doy': dates.strftime("%Y:%j"), 'utc': dates.strftime("%Y:%m:%d"), 'offset': clock_offset_table['offset'][:-1], 'residual': clock_residuals[:-1], 'residual_detrend': clock_residuals_detrend[:-1], 'station': clock_offset_table['station'][:-1] }) all_data.meta['clock_offset_file'] = CLOCKFILE all_data.meta['temperature_file'] = TEMPFILE all_data.meta['frequency_file'] = FREQFILE pickle.dump((all_data, table_new, gtis, all_nustar_obs), open(outfile, 'wb')) calculate_stats(all_data) log.info(f"Reprocessing done. It took {time.time() - t0} s") return all_data, table_new, gtis, all_nustar_obs
def save_observations(site: Site, obs_list: list[Observation], filename: str, new_db=True): """ Saves a list of Observation objects to a sqlite3 database and .csv file. The database contains rows formatted for: `satellite name, pass number, jd, local time, range, azimuth, elevation` Entries are ordered by the observation start time. :param Site site: Site object for the observation location :param List[Observation] obs_list: list of all observations to be saved :param filename: Name for the output .csv file :param new_db: (Optional: defaults to True) Setting to True will create a new DB for every new propagation run Setting to False will allow the DB to store data for multiple propagations """ db_name = 'outputs/observations.sqlite' db = sqlite3.connect(db_name) cursor = db.cursor() if new_db: db.execute("DROP TABLE IF EXISTS " "observations") db.execute( "CREATE TABLE IF NOT EXISTS " "observations (" "site TEXT, name TEXT, pass_number REAL, jd REAL, time TEXT, range REAL, azimuth REAL, elevation REAL" ")") obs_list.sort(key=operator.attrgetter( 't0')) # Sort the observation objects by their start times for obs in obs_list: datarows = obs.create_observation_entry() for row in datarows: sat_name, pass_number, jd, sat_range, azimuth, elevation = row # Changing the julianday into a readable datetime string t = Time(jd, format='jd') t.format = 'datetime' t += site.utc_offset print_time = t.strftime("%m/%d/%Y, %H:%M:%S") # Changing units for readability azimuth *= 180 / math.pi elevation *= 180 / math.pi cursor.execute( "INSERT INTO observations(" "site, name, pass_number, jd, time, range, azimuth, elevation" ") VALUES(?, ?, ?, ?, ?, ?, ?, ?)", (str(site.obs_location), sat_name, pass_number, jd, print_time, sat_range, azimuth, elevation)) data = cursor.execute("SELECT * FROM observations") with open(filename + '.csv', 'w') as obsfile: obs_writer = csv.writer(obsfile, delimiter=',') obs_writer.writerow([str(site.obs_location)]) obs_writer.writerow([ 'Satellite Name', 'Time', 'Range (km)', 'Azimuth (deg)', 'Elevation (deg)' ]) for site, name, pass_number, jd, time, sat_range, azimuth, elevation in data: obs_writer.writerow([ name, time, round(sat_range, 2), round(azimuth, 2), round(elevation, 2) ]) print("\nObservation details saved to:\n" + filename + '.csv') cursor.close() db.commit() db.close()
ffreff = fits.open(ffref) ffrefim = trimarrays.trimzeros(trimarrays.trimnan(ffreff[0].data)) ffreff.close() bf1 = fits.open(file1) bim1 = bf1[0].data.astype(np.float32) bdate1 = Time(bf1[0].header['DATE']).datetime bf1.close() bf2 = fits.open(file2) bim2 = bf2[0].data.astype(np.float32) bdate2 = Time(bf2[0].header['DATE']).datetime bf2.close() bim1, bim2 = trimarrays.trimto(ffrefim, bim1, bim2) bdiffs = (bim1 - bim2).flatten() absdiffs = np.abs(bdiffs) mv = np.round(absdiffs.mean()) mstd = absdiffs.std() if absval: bdiffs = absdiffs bdiffs[bdiffs > clip * mstd] = mv else: bdiffs[bdiffs < -clip * mstd] = - mv bdiffs[bdiffs > clip * mstd] = mv plt.hist(bdiffs.flatten(), bins=bins) plt.title("Compare bias" + bdate1.strftime(" %Y-%m-%d %H:%M:%S -v- ") + bdate2.strftime("%Y-%m-%d %H:%M:%S")) plt.show()
def fetch(self, qres, path=None, error_callback=None, **kwargs): """ Download a set of results. Parameters ---------- qres : `~sunpy.net.dataretriever.QueryResponse` Results to download. Returns ------- Results Object """ urls = [qrblock.url for qrblock in qres] filenames = [] local_filenames = [] for i, [url, qre] in enumerate(zip(urls, qres)): name = url.split('/')[-1] # temporary fix !!! coz All QRBs have same start_time values day = Time(qre.time.start.strftime('%Y-%m-%d')) + TimeDelta(i*u.day) if name not in filenames: filenames.append(name) if name.endswith('.gz'): local_filenames.append('{}SRS.txt'.format(day.strftime('%Y%m%d'))) else: local_filenames.append(name) # Files to be actually downloaded paths = self._get_full_filenames(qres, filenames, path) # Those files that will be present after get returns local_paths = self._get_full_filenames(qres, local_filenames, path) res = Results(lambda x: None, 0, lambda map_: self._link(map_)) # remove duplicate urls. This will make paths and urls to have same number of elements. # OrderedDict is required to maintain ordering because it will be zipped with paths later urls = list(OrderedDict.fromkeys(urls)) dobj = Downloader(max_conn=len(urls), max_total=len(urls)) # We cast to list here in list(zip... to force execution of # res.require([x]) at the start of the loop. for aurl, ncall, fname in list(zip(urls, map(lambda x: res.require([x]), urls), paths)): dobj.download(aurl, fname, ncall, error_callback) res.wait() res2 = Results(lambda x: None, 0) for fname, srs_filename in zip(local_paths, local_filenames): fname = fname.args[0] name = fname.split('/')[-1] past_year = False for i, fname2 in enumerate(paths): fname2 = fname2.args[0] if fname2.endswith('.txt'): continue year = fname2.split('/')[-1] year = year.split('_SRS')[0] if year in name: TarFile = tarfile.open(fname2) filepath = fname.rpartition('/')[0] member = TarFile.getmember('SRS/' + srs_filename) member.name = name TarFile.extract(member, path=filepath) TarFile.close() callback = res2.require([fname]) callback({'path': fname}) past_year = True break if past_year is False: callback = res2.require([fname]) callback({'path': fname}) return res2
def read_hdf5_limb_state_common_data(self, hf, lstate_id, state_in_orbit, cl_id): """SCIAMACHY level 1c common data Parameters ---------- hf : opened file Pointer to the opened level 1c HDF5 file lstate_id : int The limb state id. state_in_orbit : int The number in this batch of states for the header. cl_id : int The spectral cluster number. Returns ------- success : int 0 on success, 1 if an error occurred, for example if the measurement data set for the requested limb and cluster ids is empty. """ # MDS = measurement data set cl_mds_group_name = "/MDS/limb_{0:02d}/cluster_{1:02d}".format( lstate_id, cl_id + 1) cl_mds_group = hf.get(cl_mds_group_name) if cl_mds_group is None: return 1 # Load meta data self.metadata["calibration"] = hf.attrs["Calibration"].decode() self.metadata["l1b_product"] = hf.get("/MPH")["product_name"][0].decode() self.metadata["orbit"] = hf.get("/MPH")["abs_orbit"][0] self.metadata["state_id"] = hf.get("/ADS/STATES")["state_id"][lstate_id] self.metadata["software_version"] = hf.get( "/MPH")["software_version"][0].decode() self.metadata["keyfile_version"] = hf.get( "/SPH")["key_data_version"][0].decode() self.metadata["mfactor_version"] = hf.get( "/SPH")["m_factor_version"][0].decode() init_ver = hf.get("/SPH")["init_version"][0].decode().strip() self.metadata["init_version"], decont = init_ver.split() self.metadata["decont_flags"] = decont.lstrip("DECONT=") orb_phase = hf.get("/ADS/STATES")["orb_phase"][lstate_id] j_day_0 = 2451544.5 # 2000-01-01 dsr_d, dsr_s, dsr_us = hf.get("/ADS/STATES")["dsr_time"][lstate_id] state_dt = Time(dsr_d + j_day_0 + dsr_s / 86400. + dsr_us / (86400. * 1e6), format="jd").datetime self.metadata["date"] = state_dt.strftime("%d-%b-%Y %H:%M:%S.%f") logging.debug("applied calibrations: %s", self.metadata["calibration"]) logging.debug("product: %s, orbit_nr: %s, state_id: %s, orb_phase: %s", self.metadata["l1b_product"], self.metadata["orbit"], self.metadata["state_id"], orb_phase) logging.debug( "soft_ver: %s, key_ver: %s, mf_ver: %s, init_ver: %s, " "decont_ver: %s", self.metadata["software_version"], self.metadata["keyfile_version"], self.metadata["mfactor_version"], self.metadata["init_version"], self.metadata["decont_flags"]) ads_state = hf.get("/ADS/STATES")[lstate_id] cl_n_readouts = ads_state["clus_config"]["num_readouts"][cl_id] cl_intg_time = ads_state["clus_config"]["intg_time"][cl_id] self.metadata["nr_profile"] = 24 // (cl_intg_time * cl_n_readouts) self.metadata["act_profile"] = 0 # always zero for now # Prepare the header try: self.metadata["datatype_txt"] = _state_txt[self.metadata["state_id"]] except KeyError: logging.warn("State id %s not supported.", self.metadata["state_id"]) return 1 self.assemble_textheader() logging.debug("header:\n%s", self.textheader) # parse geolocation data gr_scia_geo = cl_mds_group.get("geoL_scia") tan_h = gr_scia_geo["tan_h"] # lat and lon are integers in degrees * 10^6 lats_all = gr_scia_geo["tang_ground_point"]["lat"] lons_all = gr_scia_geo["tang_ground_point"]["lon"] sza_all = gr_scia_geo["sun_zen_ang"] saa_all = (gr_scia_geo["sun_azi_ang"] - gr_scia_geo["los_azi_ang"]) sat_h_all = gr_scia_geo["sat_h"] earth_rad_all = gr_scia_geo["earth_rad"] # lat and lon are integers in degrees * 10^6 subsatlat_all = gr_scia_geo["sub_sat_point"]["lat"] subsatlon_all = gr_scia_geo["sub_sat_point"]["lon"] # fix longitudes to [0°, 360°) lons_all[np.where(lons_all < 0)] += 360000000 if cl_n_readouts > 2: tangent_heights = 0.5 * (tan_h[1::cl_n_readouts, 2] + tan_h[2::cl_n_readouts, 0]) tp_lats = 0.5 * (lats_all[1::cl_n_readouts, 2] + lats_all[2::cl_n_readouts, 0]) * 1e-6 tp_lons = 0.5 * (lons_all[1::cl_n_readouts, 2] + lons_all[2::cl_n_readouts, 0]) * 1e-6 sza_toa = 0.5 * (sza_all[1::cl_n_readouts, 2] + sza_all[2::cl_n_readouts, 0]) saa_toa = 0.5 * (saa_all[1::cl_n_readouts, 2] + saa_all[2::cl_n_readouts, 0]) sat_hs = sat_h_all.reshape((-1, cl_n_readouts)).mean(axis=1) earth_rads = earth_rad_all.reshape((-1, cl_n_readouts)).mean(axis=1) subsatlat = subsatlat_all.reshape( (-1, cl_n_readouts)).mean(axis=1) * 1e-6 subsatlon = subsatlon_all.reshape( (-1, cl_n_readouts)).mean(axis=1) * 1e-6 else: tangent_heights = tan_h[::cl_n_readouts, 1] tp_lats = lats_all[::cl_n_readouts, 1] * 1e-6 tp_lons = lons_all[::cl_n_readouts, 1] * 1e-6 sza_toa = sza_all[::cl_n_readouts, 1] saa_toa = saa_all[::cl_n_readouts, 1] sat_hs = sat_h_all[::cl_n_readouts] earth_rads = earth_rad_all[::cl_n_readouts] subsatlat = subsatlat_all[::cl_n_readouts] * 1e-6 subsatlon = subsatlon_all[::cl_n_readouts] * 1e-6 logging.debug("tangent altitudes: %s", tangent_heights) nalt = len(tangent_heights) centre = _middle_coord(lats_all[0, 1] * 1e-6, lons_all[0, 1] * 1e-6, lats_all[nalt - 2, 1] * 1e-6, lons_all[nalt - 2, 1] * 1e-6) cent_lat_lon = ( centre[0], # fix longitudes to [0, 360.) centre[1] if centre[1] >= 0. else 360. + centre[1], lats_all[0, 0] * 1e-6, lons_all[0, 0] * 1e-6, lats_all[0, 2] * 1e-6, lons_all[0, 2] * 1e-6, lats_all[nalt - 2, 0] * 1e-6, lons_all[nalt - 2, 0] * 1e-6, lats_all[nalt - 2, 2] * 1e-6, lons_all[nalt - 2, 2] * 1e-6) toa = 100. # to satellite first los_calc = np.degrees(np.arccos(0.0)) sza_tp_h = sza_toa.copy() saa_tp_h = saa_toa.copy() los_tp_h = np.full_like(tangent_heights, los_calc) los_toa_h = np.full_like(tangent_heights, los_calc) sza_sat_h, los_sat_h, saa_sat_h = _calc_angles(sza_toa, los_calc, saa_toa, tangent_heights, sat_hs, earth_rads) # angles toa los_calc = np.degrees( np.arcsin((tangent_heights + earth_rads) / (toa + earth_rads))) # to tangent point los_toa_l = np.full_like(tangent_heights, los_calc) sza_tp_l, los_tp_l, saa_tp_l = _calc_angles( sza_toa, los_calc, saa_toa, np.full_like(tangent_heights, toa), tangent_heights, earth_rads) # to satellite sza_sat_l, los_sat_l, saa_sat_l = _calc_angles( sza_toa, los_calc, saa_toa, np.full_like(tangent_heights, toa), sat_hs, earth_rads) sza_sat_h[np.where(tangent_heights <= toa)] = 0. sza_sat_l[np.where(tangent_heights > toa)] = 0. saa_sat_h[np.where(tangent_heights <= toa)] = 0. saa_sat_l[np.where(tangent_heights > toa)] = 0. los_sat_h[np.where(tangent_heights <= toa)] = 0. los_sat_l[np.where(tangent_heights > toa)] = 0. sza_tp_h[np.where(tangent_heights <= toa)] = 0. sza_tp_l[np.where(tangent_heights > toa)] = 0. saa_tp_h[np.where(tangent_heights <= toa)] = 0. saa_tp_l[np.where(tangent_heights > toa)] = 0. los_tp_h[np.where(tangent_heights <= toa)] = 0. los_tp_l[np.where(tangent_heights > toa)] = 0. los_toa_h[np.where(tangent_heights <= toa)] = 0. los_toa_l[np.where(tangent_heights > toa)] = 0. sza_sat = sza_sat_h + sza_sat_l saa_sat = saa_sat_h + saa_sat_l los_sat = los_sat_h + los_sat_l sza_tp = sza_tp_h + sza_tp_l saa_tp = saa_tp_h + saa_tp_l los_tp = los_tp_h + los_tp_l los_toa = los_toa_h + los_toa_l logging.debug("TP sza, saa, los: %s, %s, %s", sza_tp, saa_tp, los_tp) logging.debug("TOA sza, saa, los: %s, %s, %s", sza_toa, saa_toa, los_toa) logging.debug("SAT sza, saa, los: %s, %s, %s", sza_sat, saa_sat, los_sat) # save the data to the limb scan class self.nalt = nalt self.orbit_state = (self.metadata["orbit"], state_in_orbit, self.metadata["state_id"], self.metadata["nr_profile"], self.metadata["act_profile"]) self.date = (state_dt.year, state_dt.month, state_dt.day, state_dt.hour, state_dt.minute, state_dt.second) self.orbit_phase = orb_phase self.cent_lat_lon = cent_lat_lon # pre-set the limb_data if self._limb_data_dtype is None: self._limb_data_dtype = _limb_data_dtype[:] self.limb_data = np.zeros((self.nalt), dtype=self._limb_data_dtype) self.limb_data["sub_sat_lat"] = subsatlat self.limb_data["sub_sat_lon"] = subsatlon self.limb_data["tp_lat"] = tp_lats self.limb_data["tp_lon"] = tp_lons self.limb_data["tp_alt"] = tangent_heights self.limb_data["tp_sza"] = sza_tp self.limb_data["tp_saa"] = saa_tp self.limb_data["tp_los"] = los_tp self.limb_data["toa_sza"] = sza_toa self.limb_data["toa_saa"] = saa_toa self.limb_data["toa_los"] = los_toa self.limb_data["sat_sza"] = sza_sat self.limb_data["sat_saa"] = saa_sat self.limb_data["sat_los"] = los_sat self.limb_data["sat_alt"] = sat_hs self.limb_data["earth_rad"] = earth_rads return 0
class metacheck(): def __init__(self,ra,dec,mjd): # Transient information/search criteria self.coord = SkyCoord(ra,dec,unit=(u.hour,u.deg), frame='icrs') self.time = Time(mjd,format='mjd').datetime # Parameters for searches self.tns_radius = 5. # arcseconds self.mpc_radius = 1. # arcminutes self.ast_radius = 5. # arcseconds self.sim_radius = 10 # arcseconds self.ned_radius = 3. # arcminutes self.mdw_radius = 10 # arcseconds # Check the minor planet center's database of asteroids def checkMPC(self): table_output = [] # Hack AF url = "https://minorplanetcenter.net/cgi-bin/mpcheck.cgi" form_data = {"year":"%s" % self.time.strftime("%Y"), "month":"%s" % self.time.strftime("%m"), "day":"%s" % self.time.strftime("%d"), "which":"pos", "ra":"%s" % self.coord.ra.to_string(unit="hour",pad=True,decimal=False,sep=" "), "decl":"%s" % self.coord.dec.to_string(unit="deg",pad=True,decimal=False,sep=" "), "TextArea":"", "radius":"%s" % 10, "limit":"%s" % 24, "oc":"%s" % 500, "sort":"d", "mot":"h", "tmot":"s", "pdes":"u", "needed":"f", "ps":"n", "type":"p"} r = requests.post(url, data=form_data) soup = BeautifulSoup(r.text,'html5lib') pre = soup.find("pre") if pre is None: return(None) else: data = [] for row in [row for row in pre.contents[-1].split("\n")[3:-1]]: data.append([row[9:25],row[25:36],row[36:47]]) table = Table(list(map(list, zip(*data))),names=('name','ra','dec')) return(table) # Check TNS def checkTNS(self): url = 'https://wis-tns.weizmann.ac.il/search?' url += '&ra={ra}'.format(ra=self.coord.ra.degree) url += '&decl={dec}'.format(dec=self.coord.dec.degree) url += '&radius={rad}'.format(rad=self.tns_radius) url += '&coords_unit=arcsec' # Get page info from url r = requests.get(url) soup = BeautifulSoup(r.text,'html5lib') html_table = soup.find("tbody") if (html_table) is None: return(None) tables = html_table.find_all("tr") data = [] for table in tables: object_list = table.find_all("td") new_row = [] for element in object_list: if ('cell-name' in element['class'] or 'cell-ra' in element['class'] or 'cell-decl' in element['class'] or 'cell-discoverydate' in element['class']): new_row.append(element.get_text()) data.append(new_row) table = Table(list(map(list, zip(*data))),names=('name','ra','dec','discoverydate')) return(table) # Check OSC def checkASTROCATS(self): # Get photometry in a 10 arcsecond region around coordinates url = 'https://api.astrocats.space/catalog/photometry/time+band+magnitude?' url += 'ra={ra}'.format(ra=self.coord.ra.degree) url += '&dec={dec}'.format(dec=self.coord.dec.degree) url += '&radius={radius}'.format(radius=self.ast_radius) url += '&format=csv' r = requests.get(url) if ('No objects found' in r.text): return None else: data = [l.split(',') for l in r.text.strip("event,time,band,magnitude\n").split("\n")] table = Table(list(map(list, zip(*data))), names=('event','time','band','magnitude')) return(table) # Check Simbad def checkSimbad(self): table = Simbad.query_region(self.coord, radius = self.sim_radius * u.arcsec) return(table) # Check NED def checkNED(self): table = Ned.query_region(self.coord, radius=self.ned_radius * u.arcmin, equinox='J2000.0') # Pick only galaxies mask = table['Type'] == b'G' return(table[mask]) # Check M dwarf flare catalog def checkMDWARF(self): try: table = heasarc.query_region(self.coord, mission='mdwarfasc', radius=self.mdw_radius*u.arcsec) return(table) except: return(None)
def _get_records(self, mnemonic, starttime, endtime, time_format=None, **other_kwargs): """ Retrieve all results for a mnemonic in the requested time range. Parameters ---------- mnemonic : str The engineering mnemonic to retrieve starttime : str or astropy.time.Time The, inclusive, start time to retrieve from. endtime : str or astropy.time.Time The, inclusive, end time to retrieve from. result_format : str The format to request from the service. If None, the `default_format` is used. time_format : str The format of the input time used if the input times are strings. If None, a guess is made. other_kwargs : dict Keyword arguments not relevant to this implementation. Returns ------- records : `astropy.Table` Returns the resulting table. Notes ----- The engineering service always returns the bracketing entries before and after the requested time range. """ if not isinstance(starttime, Time): starttime = Time(starttime, format=time_format) if not isinstance(endtime, Time): endtime = Time(endtime, format=time_format) self.starttime = starttime self.endtime = endtime # Make the request mnemonic = mnemonic.strip() mnemonic = mnemonic.upper() starttime_fmt = starttime.strftime('%Y%m%dT%H%M%S') endtime_fmt = endtime.strftime('%Y%m%dT%H%M%S') uri = f'{mnemonic}-{starttime_fmt}-{endtime_fmt}.csv' self._req.params = {'uri': SERVICE_URI + uri} prepped = self._session.prepare_request(self._req) settings = self._session.merge_environment_settings( prepped.url, {}, None, None, None) logger.debug('Query: %s', prepped.url) self.response = self._session.send(prepped, **settings) self.response.raise_for_status() logger.debug('Response: %s', self.response) logger.debug('Response test: %s', self.response.text) # Convert to table. r_list = self.response.text.split('\r\n') table = Table.read(r_list, format='ascii.csv') return table