def generate_survey_quicklooks(in_root, out_root,
start_date=None, stop_date=None, plot_length=3, last_run_time=None,
line_plots = ['Lshell','altitude','lat','lon','solution_status','daylight']):
logger = logging.getLogger('generate_survey_quicklooks')
if not last_run_time:
last_run_time=datetime.datetime.utcfromtimestamp(0)
if not start_date:
start_date = datetime.datetime(1980,1,1,0,0,0)
if not stop_date:
stop_date = datetime.datetime.now()
# edges for plot times
hour_segs = np.arange(21,23,1)
for root, dirs, files in os.walk(in_root):
for fname in files:
if fname.startswith('VPM_survey_data_') and fname.endswith('.xml'):
filetime = datetime.datetime.utcfromtimestamp(os.path.getmtime(os.path.join(root, fname)))
if filetime < last_run_time:
logger.info(f'skipping {fname} (no changes since {last_run_time})')
else:
day = datetime.datetime.strptime(fname, 'VPM_survey_data_%Y-%m-%d.xml')
if (day >= start_date) and (day <= stop_date):
filename = os.path.join(root, fname)
print(f'Loading {filename}')
S_data = read_survey_XML(filename)
formatS = S_data[0]
#for h1,h2 in zip(hour_segs[:-1], hour_segs[1:]):
d1 = day + datetime.timedelta(hours=21,minutes=15)
d2 = day + datetime.timedelta(hours=21,minutes=30)
t1 = d1.replace(tzinfo = datetime.timezone.utc).timestamp()
t2 = d2.replace(tzinfo = datetime.timezone.utc).timestamp()
S_filt = list(filter(lambda x: (get_timestamp(x) >= t1) and (get_timestamp(x) < t2), S_data))
outdir = os.path.join(out_root,f'{day.year}','{:02d}'.format(day.month))
if S_filt:
fig = plot_survey_data_and_metadata(S_filt,t1=d1, t2=d2,
line_plots = [1],
show_plots=True, lshell_file='resources/Lshell_dict.pkl')
fig.suptitle(f"VPM Survey Data: {day.strftime('%D')}\n" +\
f"{d1.strftime('%H:%M:%S')} -- {d2.strftime('%H:%M:%S')} UT \n gain = " + formatS['gain'] + ", filter = " + formatS['filter'])
if not os.path.exists(outdir):
os.makedirs(outdir)
outfile = os.path.join(outdir,
f"VPM_survey_data_{d1.strftime('%Y-%m-%d_%H%M--')}{d2.strftime('%H%M')}.png")
fig.savefig(outfile, dpi=120)
plt.close(fig)
def save_survey_to_file_tree(S_data, out_root, file_types=['xml', 'mat']):
logger = logging.getLogger('save_survey_to_file_tree')
# -------------------- Filter out invalid entries -----------------------------
S_valid = list(filter(lambda x: valid_GPS_mask(x), S_data))
S_invalid = list(filter(lambda x: not valid_GPS_mask(x), S_data))
# ---------------- Quantize timestamps into range of days -----------------------------
dates = [datetime.datetime.fromtimestamp(x['GPS'][0]['timestamp'],
tz=datetime.timezone.utc) for x in S_valid]
days_to_do = np.unique([x.replace(hour=0, minute=0, second=0) for x in dates])
logger.info(f"Days to do: {[x.strftime('%Y-%m-%d') for x in days_to_do]}")
logger.info(f'{len(S_data)} total survey products: {len(S_valid)} valid, {len(S_invalid)} rejected')
for d in days_to_do:
logger.info(f'doing {d}')
t1 = d.timestamp()
t2 = (d + datetime.timedelta(days=1)).timestamp()
S_filt = list(filter(lambda x: (get_timestamp(x) >= t1) and (get_timestamp(x) < t2), S_data))
if S_filt:
outpath = os.path.join(out_root,'xml',f'{d.year}', '{:02d}'.format(d.month))
fname = f"VPM_survey_data_{d.strftime('%Y-%m-%d')}.xml"
# Check for previous file
outfile = os.path.join(outpath, fname)
# Here we're using the XML files as the master record. Load previous
# entries and merge with the new ones; remove duplicates.
if not os.path.exists(outpath):
os.makedirs(outpath)
if os.path.exists(outfile):
logger.info('file exists! Loading previous entries')
S_previous = read_survey_XML(outfile)
logger.info(f'Joining {len(S_previous)} entries with current {len(S_filt)}')
S_filt.extend(S_previous)
len_pre_filt = len(S_filt)
S_filt = unique_entries(S_filt)
logger.info(f'rejecting {len_pre_filt - len(S_filt)} duplicate entries')
logger.info(f'saving {len(S_filt)} survey entries')
S_filt = sorted(S_filt, key=lambda x: get_timestamp(x))
# need to do short v long survey checking here so that is is reflected in all ftypes
dt_prev = 10 # set something greater than 8 to start
for si, Scheck in enumerate(S_filt):
if si == len(S_filt) - 1:
# last packet, assume same as last
survey_type = survey_type_last
else:
next_data = S_filt[si+1]
tdate = datetime.datetime.utcfromtimestamp(Scheck['header_timestamp'])
tdate_next = datetime.datetime.utcfromtimestamp(next_data['header_timestamp'])
# find tdelta
dt_total = tdate_next - tdate
dt = dt_total.total_seconds()
if dt < 8 or dt_prev < 8: # doing this catches the last short survey packet
survey_type = 'short'
else:
survey_type = 'long'
# replace survey type with correct output
Scheck['survey_type'] = survey_type
survey_type_last = survey_type
dt_prev = dt
# Can save xml, matlab, or pickle file types:
for ftype in file_types:
outpath = os.path.join(out_root,ftype,f'{d.year}', '{:02d}'.format(d.month))
fname = f"VPM_survey_data_{d.strftime('%Y-%m-%d')}.{ftype}"
outfile = os.path.join(outpath, fname)
if not os.path.exists(outpath):
os.makedirs(outpath)
if ftype =='xml':
# Write XML file
write_survey_XML(S_filt, outfile)
if ftype=='mat':
# Write MAT file
savemat(outfile, {'survey_data' : S_filt})
if ftype=='pkl':
# Write pickle file
with open(outfile,'wb') as file:
pickle.dump(S_filt, file)
# ---------------- Cache invalid packets -----------------------------
if S_invalid:
invalid_xml_file = os.path.join(out_root,'xml','invalid_entries.xml')
if os.path.exists(invalid_xml_file):
logger.info('file exists! Loading previous entries')
S_previous = read_survey_XML(invalid_xml_file)
logger.info(f'Joining {len(S_previous)} entries with current {len(S_invalid)}')
S_invalid.extend(S_previous)
len_pre_filt = len(S_invalid)
S_invalid = unique_entries(S_invalid)
logger.info(f'rejecting {len_pre_filt - len(S_invalid)} duplicate entries')
logger.info(f'saving {len(S_invalid)} rejected survey entries')
S_invalid = sorted(S_invalid, key=lambda x: get_timestamp(x))
for ftype in file_types:
invalid_file = os.path.join(out_root,ftype,f'invalid_entries.{ftype}')
if ftype=='xml':
write_survey_XML(S_invalid, invalid_file)
if ftype=='mat':
savemat(invalid_file, {'survey_data': S_invalid})
if ftype=='pkl':
with open(outfile,'wb') as file:
pickle.dump(S_invalid, file)
# -----------------------------------
return fig
def get_timestamp(x):
try:
ts = x['GPS'][0]['timestamp']
except:
print("entry is missing GPS data")
ts = x['header_timestamp']
return ts
data_root = '/Users/rileyannereid/macworkspace/VPM_data/issues/'
S_data = read_survey_XML(data_root + 'VPM_survey_data_2020-06-28.xml')
day = datetime.datetime.strptime('VPM_survey_data_2020-06-28.xml',
'VPM_survey_data_%Y-%m-%d.xml')
d1 = day + datetime.timedelta(hours=15, minutes=0)
d2 = day + datetime.timedelta(hours=18, minutes=0)
t1 = d1.replace(tzinfo=datetime.timezone.utc).timestamp()
t2 = d2.replace(tzinfo=datetime.timezone.utc).timestamp()
S_filt = list(
filter(lambda x: (get_timestamp(x) >= t1) and (get_timestamp(x) < t2),
S_data))
# Set up the frame:
# --------------- Latex Plot Beautification --------------------------
fig_width = 12
def save_survey_to_file_tree(S_data, out_root, file_types=['xml']):
logger = logging.getLogger('save_survey_to_file_tree')
# -------------------- Filter out invalid entries -----------------------------
S_valid = list(filter(lambda x: valid_GPS_mask(x), S_data))
S_invalid = list(filter(lambda x: not valid_GPS_mask(x), S_data))
# ---------------- Quantize timestamps into range of days -----------------------------
dates = [
datetime.datetime.fromtimestamp(x['GPS'][0]['timestamp'],
tz=datetime.timezone.utc)
for x in S_valid
]
days_to_do = np.unique(
[x.replace(hour=0, minute=0, second=0) for x in dates])
logger.info(f"Days to do: {[x.strftime('%Y-%m-%d') for x in days_to_do]}")
logger.info(
f'{len(S_data)} total survey products: {len(S_valid)} valid, {len(S_invalid)} rejected'
)
for d in days_to_do:
logger.info(f'doing {d}')
t1 = d.timestamp()
t2 = (d + datetime.timedelta(days=1)).timestamp()
S_filt = list(
filter(
lambda x: (get_timestamp(x) >= t1) and (get_timestamp(x) < t2),
S_data))
if S_filt:
outpath = os.path.join(out_root, 'xml', f'{d.year}',
'{:02d}'.format(d.month))
fname = f"VPM_survey_data_{d.strftime('%Y-%m-%d')}.xml"
# Check for previous file
outfile = os.path.join(outpath, fname)
# Here we're using the XML files as the master record. Load previous
# entries and merge with the new ones; remove duplicates.
if not os.path.exists(outpath):
os.makedirs(outpath)
if os.path.exists(outfile):
logger.info('file exists! Loading previous entries')
S_previous = read_survey_XML(outfile)
logger.info(
f'Joining {len(S_previous)} entries with current {len(S_filt)}'
)
S_filt.extend(S_previous)
len_pre_filt = len(S_filt)
S_filt = unique_entries(S_filt)
logger.info(
f'rejecting {len_pre_filt - len(S_filt)} duplicate entries'
)
logger.info(f'saving {len(S_filt)} survey entries')
S_filt = sorted(S_filt, key=lambda x: get_timestamp(x))
# Can save xml, matlab, or pickle file types:
for ftype in file_types:
outpath = os.path.join(out_root, ftype, f'{d.year}',
'{:02d}'.format(d.month))
fname = f"VPM_survey_data_{d.strftime('%Y-%m-%d')}.{ftype}"
outfile = os.path.join(outpath, fname)
if not os.path.exists(outpath):
os.makedirs(outpath)
if ftype == 'xml':
# Write XML file
write_survey_XML(S_filt, outfile)
if ftype == 'mat':
# Write MAT file
savemat(outfile, {'survey_data': S_filt})
if ftype == 'pkl':
# Write pickle file
with open(outfile, 'wb') as file:
pickle.dump(S_filt, file)
# ---------------- Cache invalid packets -----------------------------
if S_invalid:
invalid_xml_file = os.path.join(out_root, 'xml', 'invalid_entries.xml')
if os.path.exists(invalid_xml_file):
logger.info('file exists! Loading previous entries')
S_previous = read_survey_XML(invalid_xml_file)
logger.info(
f'Joining {len(S_previous)} entries with current {len(S_invalid)}'
)
S_invalid.extend(S_previous)
len_pre_filt = len(S_invalid)
S_invalid = unique_entries(S_invalid)
logger.info(
f'rejecting {len_pre_filt - len(S_invalid)} duplicate entries')
logger.info(f'saving {len(S_invalid)} rejected survey entries')
S_invalid = sorted(S_invalid, key=lambda x: get_timestamp(x))
for ftype in file_types:
invalid_file = os.path.join(out_root, ftype,
f'invalid_entries.{ftype}')
if ftype == 'xml':
write_survey_XML(S_invalid, invalid_file)
if ftype == 'mat':
savemat(invalid_file, {'survey_data': S_invalid})
if ftype == 'pkl':
with open(outfile, 'wb') as file:
pickle.dump(S_invalid, file)
import numpy as np
import pickle
from scipy.io import loadmat
import os
import json
import datetime
import matplotlib.pyplot as plt
from file_handlers import read_survey_XML
S_data = read_survey_XML(
'/Users/rileyannereid/macworkspace/special/VPM_survey_data_2020-05-11.xml')
S_data = sorted(S_data, key=lambda f: f['header_timestamp'])
S_with_GPS = list(
filter(
lambda x: (('GPS' in x) and ('timestamp' in x['GPS'][0]) and
('lat' in x['GPS'][0]) and ('horiz_speed' in x['GPS'][0])),
S_data))
S_with_GPS = sorted(S_with_GPS, key=lambda f: f['GPS'][0]['timestamp'])
T_gps = np.array([x['GPS'][0]['timestamp'] for x in S_with_GPS])
dts_gps = np.array([
datetime.datetime.fromtimestamp(x, tz=datetime.timezone.utc) for x in T_gps
])
# Build arrays
E = []
E_other = []
B = []
T = []
cal = []
print(root.filename)
# ----------- Start the logger -------------
# log_filename = os.path.join(out_root, 'log.txt')
if args.debug:
logging.basicConfig(level=logging.DEBUG,
format='[%(name)s]\t%(levelname)s\t%(message)s')
else:
logging.basicConfig(level=logging.INFO,
format='[%(name)s]\t%(levelname)s\t%(message)s')
logging.getLogger('matplotlib').setLevel(logging.WARNING)
if os.path.exists(args.input):
# Load it
logging.info(f'Loading file {args.input}')
dd = read_survey_XML(args.input)
# Plot it
# plot_survey_data(dd,args.output, args.show_plots)
if args.show_metadata:
logging.info(f'Plotting data and metadata')
plot_survey_data_and_metadata(dd,
filename=args.output,
show_plots=args.show_plots)
else:
logging.info(f'Plotting data only')
plot_survey_data(dd,
filename=args.output,
show_plots=args.show_plots)
else:
logging.warning(f'Cannot find file {args.input}')