def plot_repeat_star_mags(repeats): plt.close(1) plt.figure(1, figsize=(6, 4)) aca_db = DBI(server='sybase', dbi='sybase', user='******') for agasc_id in repeats['id']: print agasc_id if agasc_id in cache: obsdata = cache[agasc_id] else: obsdata = aca_db.fetchall("select * from trak_stats_data where id = {} " "order by kalman_tstart".format(agasc_id)) cache[agasc_id] = obsdata years = DateTime(obsdata['kalman_tstart']).frac_year scatter = np.random.uniform(-0.5, 0.5, size=len(years)) dmags = obsdata['aoacmag_mean'] - np.median(obsdata['aoacmag_mean']) plt.plot(years + scatter, dmags, '.', label='ID {}'.format(agasc_id)) aca_db.conn.close() plt.xlabel('Year') plt.ylabel('Delta Mag') plt.grid() plt.ylim(-0.1, 0.1) plt.legend(loc='upper left', fontsize=10) plt.title('ACA Responsivity') plt.tight_layout() plt.savefig('responsivity.png')
def get_repeats(n_repeats=300): # default gives us 4 stars aca_db = DBI(server='sybase', dbi='sybase', user='******') repeats = aca_db.fetchall("select id, count(id) as num_obs from trak_stats_data " "group by id having (count(id) > {})".format(n_repeats)) repeats = repeats[repeats['id'] >= 20] aca_db.conn.close() return repeats
def cmd_states_fetch(self, tbegin, tend): """ Search the TA database and retrieve all the command state data between the given start/stop times. Returned - numpy array. Data types are: Data item and type ------------------ ('datestart', '|S21'), ('datestop', '|S21'), ('tstart', '<f8'), ('tstop', '<f8'), ('obsid', '<i8'), ('power_cmd', '|S10'), ('si_mode', '|S8'), ('pcad_mode', '|S4'), ('vid_board', '<i8'), ('clocking', '<i8'), ('fep_count', '<i8'), ('ccd_count', '<i8'), ('simpos', '<i8'), ('simfa_pos', '<i8'), ('pitch', '<f8'), ('ra', '<f8'), ('dec', '<f8'), ('roll', '<f8'), ('q1', '<f8'), ('q2', '<f8'), ('q3', '<f8'), ('q4', '<f8'), ('trans_keys', '|S48') ('hetg', '|S4'), ('letg', '|S4'), ('dither', '|S4') """ # convert begin and end into sybase query tstart and tstop tstart = DateTime(tbegin) tstop = DateTime(tend) # # form the query for everything, starting from tstart date to now # query = """select * from cmd_states where datestart >= '%s' and datestop <= '%s' order by datestart asc """ % ( tstart.date, tstop.date) # # set up a read to the data base # aca_read_db = DBI(dbi='sybase', server='sybase', user='******', database='aca') # Fetch all the data self.cmd_states = aca_read_db.fetchall(query) return self.cmd_states
def cmd_states_fetch(self, tbegin, tend): """ Search the TA database and retrieve all the command state data between the given start/stop times. Returned - numpy array. Data types are: Data item and type ------------------ ('datestart', '|S21'), ('datestop', '|S21'), ('tstart', '<f8'), ('tstop', '<f8'), ('obsid', '<i8'), ('power_cmd', '|S10'), ('si_mode', '|S8'), ('pcad_mode', '|S4'), ('vid_board', '<i8'), ('clocking', '<i8'), ('fep_count', '<i8'), ('ccd_count', '<i8'), ('simpos', '<i8'), ('simfa_pos', '<i8'), ('pitch', '<f8'), ('ra', '<f8'), ('dec', '<f8'), ('roll', '<f8'), ('q1', '<f8'), ('q2', '<f8'), ('q3', '<f8'), ('q4', '<f8'), ('trans_keys', '|S48')] ('hetg', '|S4'), ('letg', '|S4'), ('dither', '|S4')]) """ # convert begin and end into sybase query tstart and tstop tstart = Chandra.Time.DateTime(tbegin) tstop = Chandra.Time.DateTime(tend) # # form the query for everything, starting from tstart date to now # query = """select * from cmd_states where datestart >= '%s' and datestop <= '%s' order by datestart asc """ % ( tstart.date, tstop.date, ) # # set up a read to the data base # aca_read_db = DBI(dbi="sybase", server="sybase", user="******", database="aca") # Fetch all the data self.cmd_states = aca_read_db.fetchall(query) return self.cmd_states
def main(loadseg_rdb_dir, dryrun=False, test=False, dbi='sqlite', server='db_base.db3' ,database=None, user=None, verbose=False): """ Command Load Segment Table Updater Read RDB table from SKA arc iFOT events area and update load_segments table Meant to be run as a cront task with no arguments. Details: Reads most recent RDB file from arc data iFOT events load_segments directory. Checks loads in that file for overlap and prolonged separations Removes outdated table entries Inserts new and newly modified entries Note that dryrun mode does not show timelines which *would* be updated, as an update to the load_segments table must happen prior to get_timelines() """ dbh = DBI(dbi=dbi, server=server, database=database, user=user, verbose=verbose) ch = logging.StreamHandler() ch.setLevel(logging.WARN) if verbose: ch.setLevel(logging.DEBUG) log.addHandler(ch) if dryrun: log.info("LOAD_SEG INFO: Running in dryrun mode") loadseg_dir = loadseg_rdb_dir # get the loads from the arc ifot area all_rdb_files = glob.glob(os.path.join(loadseg_dir, "*")) rdb_file = max(all_rdb_files) log.debug("LOAD_SEG DEBUG: Updating from %s" % rdb_file) orig_rdb_loads = Ska.Table.read_ascii_table(rdb_file, datastart=3) ifot_loads = rdb_to_db_schema( orig_rdb_loads ) if len(ifot_loads): # make any scripted edits to the tables of parsed files to override directory # mapping import fix_tl_processing fix_tl_processing.repair(dbh) # make any scripted edits to the load segments table import fix_load_segments ifot_loads = fix_load_segments.repair(ifot_loads) max_timelines_id = dbh.fetchone( 'SELECT max(id) AS max_id FROM timelines')['max_id'] or 0 if max_timelines_id == 0 and test == False: raise ValueError("TIMELINES: no timelines in database.") update_loads_db( ifot_loads, dbh=dbh, test=test, dryrun=dryrun ) db_loads = dbh.fetchall("""select * from load_segments where datestart >= '%s' order by datestart """ % ( ifot_loads[0]['datestart'] ) ) update_timelines_db(loads=db_loads, dbh=dbh, max_id=max_timelines_id, dryrun=dryrun, test=test) log.removeHandler(ch)
def hrc_gain_test_obs(new_obs, test=''): """ find new AL Lac observations from a hrc obsid list Input: new_obs --- a list of hrc obsids test --- a test indicator. if it is other than "", test will run Output: "./candidate_list" which lists obsids of new AR Lac observations candidate_list it also returns the same list """ if test == "": f1 = open('./candidate_list', 'w') file = house_keeping + 'hrc_obsid_list' file2 = house_keeping + 'hrc_obsid_list~' cmd = 'cp -f ' + file + ' ' + file2 os.system(cmd) f2 = open(file, 'a') candidate_list = [] for obsid in new_obs: # #--- open sql database and extract data we need # db = DBI(dbi='sybase', server=db_server, user=db_user, passwd=db_passwd, database='axafocat') cmd = 'select obsid,targid,seq_nbr,targname,grating,instrument from target where obsid=' + obsid query_results = db.fetchall(cmd) if len(query_results): query_results = Table(query_results) line = query_results['targname'].data targname = line[0] # #--- if the observation is AR Lac, write it down in candidate_list # m1 = re.search('arlac', targname.lower()) if m1 is not None: line = obsid + '\n' candidate_list.append(obsid) if test == '': f1.write(line) f2.write(line) if test == '': f1.close() f2.close() return candidate_list
#!/usr/bin/env python import sys from Ska.DBI import DBI from Chandra.Time import DateTime import numpy as np dbh = DBI(dbi='sybase', server='sybase', user='******') mp = '/data/mpcrit1/mplogs' t = DateTime() if len(sys.argv) > 1: t = DateTime(sys.argv[1]) timelines = dbh.fetchall( """select * from timelines where datestart <= '%(date)s' and datestop > '%(date)s'""" % {'date': t.date}) if not len(timelines): timelines = dbh.fetchall( """select * from timelines where ( datestart = ( select max(datestart) from timelines where datestart < '%(date)s')) or ( datestart = ( select min(datestart) from timelines where datestart > '%(date)s'))""" % {'date': t.date}) for tdir in np.unique(timelines['dir']): print "file://%s%s%s" % (mp, tdir, 'starcheck.html')
bash('echo "False" > {}'.format(stat_file)) print "updating %s NOT POINT" % src_dir if c == 'm': bash('echo "None" > {}'.format(stat_file)) print "updating %s ..Eh " % src_dir if c == 'y': bash('echo "True" > {}'.format(stat_file)) print "updating %s POINT" % src_dir os.kill(int(proc.group(1)), 1) src = Table.read(src_file, format='ascii.tab') # append via dictionaries to avoid dealing with fact that ascdsver is # read as a float on occasion srcdict = dict(zip(src[0].colnames, src[0].data)) obs = acadb.fetchall("select * from observations where obsid = {}".format(srcdict['obsid'])) for key in ['kalman_datestart', 'kalman_datestop', 'kalman_tstart', 'kalman_tstop', 'grating', 'readmode', 'datamode', 'detector', 'sim_z', 'sim_z_offset']: srcdict[key] = obs[0][key] tilt_db = acadb.fetchall( "select max_oobagrd3 - min_oobagrd3 as rd3diff, max_oobagrd6 - min_oobagrd6 as rd6diff from obs_periscope_tilt where obsid = {}".format(srcdict['obsid'])) if not len(tilt_db): continue srcdict['oobagrd3_diff'] = tilt_db[0]['rd3diff'] srcdict['oobagrd6_diff'] = tilt_db[0]['rd6diff'] from Ska.Sun import pitch obs_pitch = pitch(obs['ra_pnt'], obs['dec_pnt'], obs['kalman_tstart']) srcdict['pitch'] = obs_pitch srcdict['point_source'] = stat srcs.append(srcdict)
LABEL = "Outstanding Targets" PLANNING_LIMIT = opt.planning_limit TASK_DATA = os.path.join(os.environ["SKA"], "data", "aca_lts_eval") db = DBI(dbi="sybase", server="sqlsao", database="axafocat", user="******") query = """SELECT t.obsid, t.ra, t.dec, t.y_det_offset as y_offset, t.z_det_offset as z_offset, t.approved_exposure_time, t.instrument, t.grating, t.obs_ao_str FROM target t WHERE ((t.status='unobserved' OR t.status='partially observed' OR t.status='untriggered' OR t.status='scheduled') AND NOT(t.ra = 0 AND t.dec = 0) AND NOT(t.ra IS NULL OR t.dec IS NULL)) ORDER BY t.obsid""" targets = Table(db.fetchall(query)) targets.write(os.path.join(OUTDIR, "requested_targets.txt"), format="ascii.fixed_width_two_line") stop = DateTime("{}-03-15".format(2000 + CYCLE)) start = stop - (365 + 210) if opt.start is not None: start = DateTime(opt.start) if opt.stop is not None: stop = DateTime(opt.stop) targets["report_start"] = start.secs targets["report_stop"] = stop.secs last_data_file = os.path.join(OUTDIR, "target_table.dat") last_data = None
def retrieve_perigee_telem(start='2009:100:00:00:00.000', stop=None, pass_data_dir='.', redo=False): """ Retrieve perigee pass and other 8x8 image telemetry. Telemetry is stored in directories named by datestart in the PASS_DATA directory. The file pass_times.txt in each directory contains the time range that has been queried for 8x8 image data :param start: Chandra.Time compatible time for beginning of range :param stop: Chandra.time compatible time for end of range :rtype: list of updated directories """ tstart = DateTime(start) # default tstop should be now if stop is None: tstop = DateTime(time.time(), format='unix') log.info("retrieve_perigee_telem(): Checking for current telemetry from %s" % tstart.date) pass_time_file = 'pass_times.txt' aca_db = DBI(dbi='sybase', server='sybase', user='******', database='aca') obsids = aca_db.fetchall("""SELECT obsid,obsid_datestart,obsid_datestop from observations where obsid_datestart > '%s' and obsid_datestart < '%s' order by obsid_datestart""" % (tstart.date, tstop.date)) # Get contiguous ER chunks, which are largely perigee passes chunks = [] chunk = {'start': None, 'stop': None} for obsid in obsids: # If a OR, end a "chunk" of ER unless undefined # (this should only append on the first OR after one or more ERs) if obsid['obsid'] < 40000: if chunk['start'] is not None and chunk['stop'] is not None: chunks.append(chunk.copy()) chunk = {'start': None, 'stop': None} else: if chunk['start'] is None: chunk['start'] = obsid['obsid_datestart'] chunk['stop'] = obsid['obsid_datestop'] pass_dirs = [] # for each ER chunk get telemetry for chunk in chunks: er_start = chunk['start'] er_stop = chunk['stop'] log.debug("checking for %s pass" % er_start) er_year = DateTime(er_start).year year_dir = os.path.join(pass_data_dir, "%s" % er_year) if not os.access(year_dir, os.R_OK): os.mkdir(year_dir) pass_dir = os.path.join(pass_data_dir, "%s" % er_year, er_start) if not os.access(pass_dir, os.R_OK): os.mkdir(pass_dir) if (DateTime(er_stop).secs - DateTime(er_start).secs > 86400 * 2): if not os.path.exists(os.path.join(pass_dir, 'warned.txt')): log.warn("Skipping %s pass, more than 48 hours long" % er_start) continue pass_dirs.append(pass_dir) made_timefile = os.path.exists(os.path.join(pass_dir, pass_time_file)) if made_timefile: pass_done = Ska.Table.read_ascii_table( os.path.join(pass_dir, pass_time_file)) if ((pass_done['obsid_datestart'] == er_start) & (pass_done['obsid_datestop'] == er_stop)): log.debug("%s times match" % pass_dir) continue else: log.info("pass %s exists but needs updating" % er_start) redo = True if not made_timefile or redo: f = open(os.path.join(pass_dir, pass_time_file), 'w') f.write("obsid_datestart,obsid_datestop\n") f.write("%s,%s\n" % (er_start, er_stop)) f.close() return pass_dirs
format="%(asctime)s %(message)s") # Get options opt = get_opt() stop = DateTime(opt.stop) start = stop - 10 if (opt.start is None) else DateTime(opt.start) logger.info('Processsing from {} to {}'.format(start.date, stop.date)) # Define file names h5_file = os.path.join(opt.data_root, 'aimpoint_asol_values.h5') obsid_file = os.path.join(opt.data_root, 'aimpoint_obsid_index.shelve') # Get obsids in date range db = DBI(dbi='sqlite', server='/data/aca/archive/obspar/archfiles.db3') obs = db.fetchall('select obsid, tstart from archfiles where tstart > {}' ' and tstart < {}' .format(start.secs, stop.secs)) db.conn.close() # Get unique obsids and then sort by tstart idx = np.unique(obs['obsid'], return_index=True)[1] obs = Table(obs[idx]) obs.sort('tstart') obs['datestart'] = Time(obs['tstart'], format='cxcsec').yday obs.pprint(max_lines=-1) obsid_index = shelve.open(obsid_file) # Go through obsids and either process or skip for obsid in obs['obsid']: if str(obsid) in obsid_index:
if c == 'n': bash('echo "False" > {}'.format(stat_file)) print "updating %s NOT POINT" % src_dir if c == 'm': bash('echo "None" > {}'.format(stat_file)) print "updating %s ..Eh " % src_dir if c == 'y': bash('echo "True" > {}'.format(stat_file)) print "updating %s POINT" % src_dir os.kill(int(proc.group(1)), 1) src = Table.read(src_file, format='ascii.tab') # append via dictionaries to avoid dealing with fact that ascdsver is # read as a float on occasion srcdict = dict(zip(src[0].colnames, src[0].data)) obs = acadb.fetchall("select * from observations where obsid = {}".format( srcdict['obsid'])) for key in [ 'kalman_datestart', 'kalman_datestop', 'kalman_tstart', 'kalman_tstop', 'grating', 'readmode', 'datamode', 'detector', 'sim_z', 'sim_z_offset' ]: srcdict[key] = obs[0][key] tilt_db = acadb.fetchall( "select max_oobagrd3 - min_oobagrd3 as rd3diff, max_oobagrd6 - min_oobagrd6 as rd6diff from obs_periscope_tilt where obsid = {}" .format(srcdict['obsid'])) if not len(tilt_db): continue srcdict['oobagrd3_diff'] = tilt_db[0]['rd3diff'] srcdict['oobagrd6_diff'] = tilt_db[0]['rd6diff'] from Ska.Sun import pitch