def generate_survey_quicklooks(in_root, out_root, start_date=None, stop_date=None, plot_length=3, last_run_time=None, line_plots = ['Lshell','altitude','lat','lon','solution_status','daylight']): logger = logging.getLogger('generate_survey_quicklooks') if not last_run_time: last_run_time=datetime.datetime.utcfromtimestamp(0) if not start_date: start_date = datetime.datetime(1980,1,1,0,0,0) if not stop_date: stop_date = datetime.datetime.now() # edges for plot times hour_segs = np.arange(21,23,1) for root, dirs, files in os.walk(in_root): for fname in files: if fname.startswith('VPM_survey_data_') and fname.endswith('.xml'): filetime = datetime.datetime.utcfromtimestamp(os.path.getmtime(os.path.join(root, fname))) if filetime < last_run_time: logger.info(f'skipping {fname} (no changes since {last_run_time})') else: day = datetime.datetime.strptime(fname, 'VPM_survey_data_%Y-%m-%d.xml') if (day >= start_date) and (day <= stop_date): filename = os.path.join(root, fname) print(f'Loading {filename}') S_data = read_survey_XML(filename) formatS = S_data[0] #for h1,h2 in zip(hour_segs[:-1], hour_segs[1:]): d1 = day + datetime.timedelta(hours=21,minutes=15) d2 = day + datetime.timedelta(hours=21,minutes=30) t1 = d1.replace(tzinfo = datetime.timezone.utc).timestamp() t2 = d2.replace(tzinfo = datetime.timezone.utc).timestamp() S_filt = list(filter(lambda x: (get_timestamp(x) >= t1) and (get_timestamp(x) < t2), S_data)) outdir = os.path.join(out_root,f'{day.year}','{:02d}'.format(day.month)) if S_filt: fig = plot_survey_data_and_metadata(S_filt,t1=d1, t2=d2, line_plots = [1], show_plots=True, lshell_file='resources/Lshell_dict.pkl') fig.suptitle(f"VPM Survey Data: {day.strftime('%D')}\n" +\ f"{d1.strftime('%H:%M:%S')} -- {d2.strftime('%H:%M:%S')} UT \n gain = " + formatS['gain'] + ", filter = " + formatS['filter']) if not os.path.exists(outdir): os.makedirs(outdir) outfile = os.path.join(outdir, f"VPM_survey_data_{d1.strftime('%Y-%m-%d_%H%M--')}{d2.strftime('%H%M')}.png") fig.savefig(outfile, dpi=120) plt.close(fig)
def save_survey_to_file_tree(S_data, out_root, file_types=['xml', 'mat']): logger = logging.getLogger('save_survey_to_file_tree') # -------------------- Filter out invalid entries ----------------------------- S_valid = list(filter(lambda x: valid_GPS_mask(x), S_data)) S_invalid = list(filter(lambda x: not valid_GPS_mask(x), S_data)) # ---------------- Quantize timestamps into range of days ----------------------------- dates = [datetime.datetime.fromtimestamp(x['GPS'][0]['timestamp'], tz=datetime.timezone.utc) for x in S_valid] days_to_do = np.unique([x.replace(hour=0, minute=0, second=0) for x in dates]) logger.info(f"Days to do: {[x.strftime('%Y-%m-%d') for x in days_to_do]}") logger.info(f'{len(S_data)} total survey products: {len(S_valid)} valid, {len(S_invalid)} rejected') for d in days_to_do: logger.info(f'doing {d}') t1 = d.timestamp() t2 = (d + datetime.timedelta(days=1)).timestamp() S_filt = list(filter(lambda x: (get_timestamp(x) >= t1) and (get_timestamp(x) < t2), S_data)) if S_filt: outpath = os.path.join(out_root,'xml',f'{d.year}', '{:02d}'.format(d.month)) fname = f"VPM_survey_data_{d.strftime('%Y-%m-%d')}.xml" # Check for previous file outfile = os.path.join(outpath, fname) # Here we're using the XML files as the master record. Load previous # entries and merge with the new ones; remove duplicates. if not os.path.exists(outpath): os.makedirs(outpath) if os.path.exists(outfile): logger.info('file exists! Loading previous entries') S_previous = read_survey_XML(outfile) logger.info(f'Joining {len(S_previous)} entries with current {len(S_filt)}') S_filt.extend(S_previous) len_pre_filt = len(S_filt) S_filt = unique_entries(S_filt) logger.info(f'rejecting {len_pre_filt - len(S_filt)} duplicate entries') logger.info(f'saving {len(S_filt)} survey entries') S_filt = sorted(S_filt, key=lambda x: get_timestamp(x)) # need to do short v long survey checking here so that is is reflected in all ftypes dt_prev = 10 # set something greater than 8 to start for si, Scheck in enumerate(S_filt): if si == len(S_filt) - 1: # last packet, assume same as last survey_type = survey_type_last else: next_data = S_filt[si+1] tdate = datetime.datetime.utcfromtimestamp(Scheck['header_timestamp']) tdate_next = datetime.datetime.utcfromtimestamp(next_data['header_timestamp']) # find tdelta dt_total = tdate_next - tdate dt = dt_total.total_seconds() if dt < 8 or dt_prev < 8: # doing this catches the last short survey packet survey_type = 'short' else: survey_type = 'long' # replace survey type with correct output Scheck['survey_type'] = survey_type survey_type_last = survey_type dt_prev = dt # Can save xml, matlab, or pickle file types: for ftype in file_types: outpath = os.path.join(out_root,ftype,f'{d.year}', '{:02d}'.format(d.month)) fname = f"VPM_survey_data_{d.strftime('%Y-%m-%d')}.{ftype}" outfile = os.path.join(outpath, fname) if not os.path.exists(outpath): os.makedirs(outpath) if ftype =='xml': # Write XML file write_survey_XML(S_filt, outfile) if ftype=='mat': # Write MAT file savemat(outfile, {'survey_data' : S_filt}) if ftype=='pkl': # Write pickle file with open(outfile,'wb') as file: pickle.dump(S_filt, file) # ---------------- Cache invalid packets ----------------------------- if S_invalid: invalid_xml_file = os.path.join(out_root,'xml','invalid_entries.xml') if os.path.exists(invalid_xml_file): logger.info('file exists! Loading previous entries') S_previous = read_survey_XML(invalid_xml_file) logger.info(f'Joining {len(S_previous)} entries with current {len(S_invalid)}') S_invalid.extend(S_previous) len_pre_filt = len(S_invalid) S_invalid = unique_entries(S_invalid) logger.info(f'rejecting {len_pre_filt - len(S_invalid)} duplicate entries') logger.info(f'saving {len(S_invalid)} rejected survey entries') S_invalid = sorted(S_invalid, key=lambda x: get_timestamp(x)) for ftype in file_types: invalid_file = os.path.join(out_root,ftype,f'invalid_entries.{ftype}') if ftype=='xml': write_survey_XML(S_invalid, invalid_file) if ftype=='mat': savemat(invalid_file, {'survey_data': S_invalid}) if ftype=='pkl': with open(outfile,'wb') as file: pickle.dump(S_invalid, file)
# ----------------------------------- return fig def get_timestamp(x): try: ts = x['GPS'][0]['timestamp'] except: print("entry is missing GPS data") ts = x['header_timestamp'] return ts data_root = '/Users/rileyannereid/macworkspace/VPM_data/issues/' S_data = read_survey_XML(data_root + 'VPM_survey_data_2020-06-28.xml') day = datetime.datetime.strptime('VPM_survey_data_2020-06-28.xml', 'VPM_survey_data_%Y-%m-%d.xml') d1 = day + datetime.timedelta(hours=15, minutes=0) d2 = day + datetime.timedelta(hours=18, minutes=0) t1 = d1.replace(tzinfo=datetime.timezone.utc).timestamp() t2 = d2.replace(tzinfo=datetime.timezone.utc).timestamp() S_filt = list( filter(lambda x: (get_timestamp(x) >= t1) and (get_timestamp(x) < t2), S_data)) # Set up the frame: # --------------- Latex Plot Beautification -------------------------- fig_width = 12
def save_survey_to_file_tree(S_data, out_root, file_types=['xml']): logger = logging.getLogger('save_survey_to_file_tree') # -------------------- Filter out invalid entries ----------------------------- S_valid = list(filter(lambda x: valid_GPS_mask(x), S_data)) S_invalid = list(filter(lambda x: not valid_GPS_mask(x), S_data)) # ---------------- Quantize timestamps into range of days ----------------------------- dates = [ datetime.datetime.fromtimestamp(x['GPS'][0]['timestamp'], tz=datetime.timezone.utc) for x in S_valid ] days_to_do = np.unique( [x.replace(hour=0, minute=0, second=0) for x in dates]) logger.info(f"Days to do: {[x.strftime('%Y-%m-%d') for x in days_to_do]}") logger.info( f'{len(S_data)} total survey products: {len(S_valid)} valid, {len(S_invalid)} rejected' ) for d in days_to_do: logger.info(f'doing {d}') t1 = d.timestamp() t2 = (d + datetime.timedelta(days=1)).timestamp() S_filt = list( filter( lambda x: (get_timestamp(x) >= t1) and (get_timestamp(x) < t2), S_data)) if S_filt: outpath = os.path.join(out_root, 'xml', f'{d.year}', '{:02d}'.format(d.month)) fname = f"VPM_survey_data_{d.strftime('%Y-%m-%d')}.xml" # Check for previous file outfile = os.path.join(outpath, fname) # Here we're using the XML files as the master record. Load previous # entries and merge with the new ones; remove duplicates. if not os.path.exists(outpath): os.makedirs(outpath) if os.path.exists(outfile): logger.info('file exists! Loading previous entries') S_previous = read_survey_XML(outfile) logger.info( f'Joining {len(S_previous)} entries with current {len(S_filt)}' ) S_filt.extend(S_previous) len_pre_filt = len(S_filt) S_filt = unique_entries(S_filt) logger.info( f'rejecting {len_pre_filt - len(S_filt)} duplicate entries' ) logger.info(f'saving {len(S_filt)} survey entries') S_filt = sorted(S_filt, key=lambda x: get_timestamp(x)) # Can save xml, matlab, or pickle file types: for ftype in file_types: outpath = os.path.join(out_root, ftype, f'{d.year}', '{:02d}'.format(d.month)) fname = f"VPM_survey_data_{d.strftime('%Y-%m-%d')}.{ftype}" outfile = os.path.join(outpath, fname) if not os.path.exists(outpath): os.makedirs(outpath) if ftype == 'xml': # Write XML file write_survey_XML(S_filt, outfile) if ftype == 'mat': # Write MAT file savemat(outfile, {'survey_data': S_filt}) if ftype == 'pkl': # Write pickle file with open(outfile, 'wb') as file: pickle.dump(S_filt, file) # ---------------- Cache invalid packets ----------------------------- if S_invalid: invalid_xml_file = os.path.join(out_root, 'xml', 'invalid_entries.xml') if os.path.exists(invalid_xml_file): logger.info('file exists! Loading previous entries') S_previous = read_survey_XML(invalid_xml_file) logger.info( f'Joining {len(S_previous)} entries with current {len(S_invalid)}' ) S_invalid.extend(S_previous) len_pre_filt = len(S_invalid) S_invalid = unique_entries(S_invalid) logger.info( f'rejecting {len_pre_filt - len(S_invalid)} duplicate entries') logger.info(f'saving {len(S_invalid)} rejected survey entries') S_invalid = sorted(S_invalid, key=lambda x: get_timestamp(x)) for ftype in file_types: invalid_file = os.path.join(out_root, ftype, f'invalid_entries.{ftype}') if ftype == 'xml': write_survey_XML(S_invalid, invalid_file) if ftype == 'mat': savemat(invalid_file, {'survey_data': S_invalid}) if ftype == 'pkl': with open(outfile, 'wb') as file: pickle.dump(S_invalid, file)
import numpy as np import pickle from scipy.io import loadmat import os import json import datetime import matplotlib.pyplot as plt from file_handlers import read_survey_XML S_data = read_survey_XML( '/Users/rileyannereid/macworkspace/special/VPM_survey_data_2020-05-11.xml') S_data = sorted(S_data, key=lambda f: f['header_timestamp']) S_with_GPS = list( filter( lambda x: (('GPS' in x) and ('timestamp' in x['GPS'][0]) and ('lat' in x['GPS'][0]) and ('horiz_speed' in x['GPS'][0])), S_data)) S_with_GPS = sorted(S_with_GPS, key=lambda f: f['GPS'][0]['timestamp']) T_gps = np.array([x['GPS'][0]['timestamp'] for x in S_with_GPS]) dts_gps = np.array([ datetime.datetime.fromtimestamp(x, tz=datetime.timezone.utc) for x in T_gps ]) # Build arrays E = [] E_other = [] B = [] T = [] cal = []
print(root.filename) # ----------- Start the logger ------------- # log_filename = os.path.join(out_root, 'log.txt') if args.debug: logging.basicConfig(level=logging.DEBUG, format='[%(name)s]\t%(levelname)s\t%(message)s') else: logging.basicConfig(level=logging.INFO, format='[%(name)s]\t%(levelname)s\t%(message)s') logging.getLogger('matplotlib').setLevel(logging.WARNING) if os.path.exists(args.input): # Load it logging.info(f'Loading file {args.input}') dd = read_survey_XML(args.input) # Plot it # plot_survey_data(dd,args.output, args.show_plots) if args.show_metadata: logging.info(f'Plotting data and metadata') plot_survey_data_and_metadata(dd, filename=args.output, show_plots=args.show_plots) else: logging.info(f'Plotting data only') plot_survey_data(dd, filename=args.output, show_plots=args.show_plots) else: logging.warning(f'Cannot find file {args.input}')