def on_sub_btn(self, event):
if self.parent.user_warning(
"WARNING: This is a non-reversable function please be sure this is what you want to do. Also if you are not looking at the 0 phase profile this may have unexpected results, please view the 0 phase profile before hitting OK."
):
try:
self.deg = int(self.deg_box.GetValue())
except (ValueError, TypeError) as e:
self.parent.user_warning(
"Degree of polynomial must be a natural number")
try:
mag_path = os.path.join(self.parent.dsk_row["data_dir"],
self.parent.dsk_row["comp_name"])
mag_df = utl.open_mag_file(mag_path)
self.projected_distances = utl.calc_projected_distance(
self.parent.dsk_row["inter_lon"],
self.parent.dsk_row["inter_lat"], mag_df["lon"],
mag_df["lat"], self.parent.dsk_row["strike"])["dist"]
shifted_mag = mag_df['mag'].tolist()
self.pols = np.polyfit(self.projected_distances, shifted_mag,
self.deg)
self.poly = np.polyval(self.pols, self.projected_distances)
mag_df["mag"] = mag_df["mag"] - self.poly
utl.write_mag_file_df(mag_df, mag_path)
except AttributeError:
return
self.on_plot_btn(event)
def shipmag_preprocess(shipmag_files):
for shipmag_file in shipmag_files:
if os.path.basename(shipmag_file).split('.')[-1].startswith('c'):
shutil.copyfile(shipmag_file, shipmag_file + '.lp')
ship_df = utl.open_mag_file(shipmag_file)
latlon_df = ship_df[['lat', 'lon']]
latlon_file = shipmag_file + ".latlon"
latlon_df.to_csv(latlon_file, sep=' ', index=False, header=False)
def plot_tracks(chrons_info,
results_directory,
tracks=[],
track_dir="all_tracks",
lon_0=180,
lat_0=0,
cuts=False):
"""
plots track files in tracks with chron info on a default ortho map centered on the pacific for rough estimation of intersept and places these plots in the results directory. If no tracks provided it plots them all
"""
if tracks == []:
tracks = glob.glob('../raw_data/hi_alt/**/*.DAT') + glob.glob(
'../raw_data/ship/**/*.lp')
all_tracks_dir = os.path.join(results_directory, track_dir)
utl.check_dir(all_tracks_dir)
#Start loop making plots
for track in tracks:
#Get Track Name
track_name = os.path.basename(os.path.basename(track))
if not cuts: track_name = track_name.split('.')[0]
else: track_name = track_name.replace('.', '-')
#Create Figure
fig = plt.figure(figsize=(9, 9), dpi=80)
#Create Map
aero_track_map = create_basic_map(projection='ortho',
center_lon=lon_0,
center_lat=lat_0,
fig=fig)
#Create Chron markers
plot_chron_info(chrons_info, aero_track_map)
dfin = utl.open_mag_file(track)
lats = list(map(float, dfin['lat'].tolist()))
lons = list(map(float, dfin['lon'].tolist()))
aero_track_handle, = aero_track_map.plot(lons,
lats,
color='k',
zorder=3,
label=track_name,
transform=ccrs.PlateCarree())
#plot title and labels
plt.title(track_name)
handles = [aero_track_handle]
add_chron_info_to_legend(chrons_info, handles)
plt.legend(handles=handles, loc='best')
fig.savefig(os.path.join(all_tracks_dir, track_name + ".png"))
plt.close(fig)
def get_idx_from_plot_pick(deskew_row,plot_pick,flip=False,dist_e=None):
data_file_path = os.path.join(deskew_row["data_dir"],deskew_row["comp_name"])
data_df = utl.open_mag_file(data_file_path)
if flip: projected_distances = utl.calc_projected_distance(deskew_row['inter_lon'],deskew_row['inter_lat'],data_df['lon'].tolist(),data_df['lat'].tolist(),(180+deskew_row['strike'])%360)
else: projected_distances = utl.calc_projected_distance(deskew_row['inter_lon'],deskew_row['inter_lat'],data_df['lon'].tolist(),data_df['lat'].tolist(),deskew_row['strike'])
min_idx = (projected_distances["dist"]-plot_pick).abs().idxmin()
return min_idx,projected_distances["dist"][min_idx]
def get_track_intersects(chron_to_analyse,
tracks_or_cuts,
spreading_zone_files,
data_directory='.',
bounding_lats=(-90, 90),
bounding_lons=(0, 360),
e=1):
""" This function works in 0-360 longitude because otherwise there would be a discontinuty in the Pacific the region of interest """
chron, chron_color = chron_to_analyse
chron_name = "chron%s" % (str(chron))
bound_check_func = lambda x: bounding_lats[0] < float(x[
1]) and bounding_lats[1] > float(x[1]) and bounding_lons[0] < float(x[
0]) and bounding_lons[1] > float(x[0])
intersecting_tracks, out_string = [], ""
for track in tqdm(tracks_or_cuts):
print(track)
dft = utl.open_mag_file(track)
if dft.empty: continue
lt = [[utl.convert_to_0_360(lon),
float(lat)] for lon, lat in zip(dft['lon'], dft['lat'])]
if not list(filter(bound_check_func, lt)):
print("track out of bounds, skipping track")
continue
for spreading_zone_file in spreading_zone_files:
lsz = [[line.split()[0], line.split()[1]]
for line in open(spreading_zone_file).readlines()
if len(line.split()) > 1]
if not list(filter(bound_check_func, lsz)): continue
idx = intersect_bf(lt, lsz, e=e)
if not any(idx): continue
else:
print("-----------intersected in bounds-------------")
intersecting_tracks.append(track)
out_string += "%s\t%s\t%s\n" % (track, spreading_zone_file,
str(idx))
break
print("found %d intersecting tracks" % len(intersecting_tracks))
utl.check_dir(data_directory)
fout_name = os.path.join(
data_directory,
"usable_tracks_and_intersects_for_%s.txt" % str(chron_name))
if os.path.isfile(fout_name):
print("backing up %s to %s" % (fout_name, fout_name + '.bak'))
shutil.copyfile(fout_name, fout_name + '.bak')
fout = open(fout_name, 'w+')
print("writing to %s" % fout_name)
fout.write(out_string)
fout.close()
return intersecting_tracks, fout_name
def create_deskewed_data_file(deskew_path):
#read deskew file
deskew_df = utl.open_deskew_file(deskew_path)
#iterate mag files
for i,row in deskew_df.iterrows():
#read mag files
data_path = os.path.join(row['data_dir'],row['comp_name'])
data_df = utl.open_mag_file(data_path)
#deskew mag data
data_df['deskewed_mag'] = phase_shift_data(data_df['mag'],float(row['phase_shift']))
#save deskewed mag data as $DATAFILE.deskew
print("writing %s"%(data_path+'.deskewed'))
data_df[['lon','lat','deskewed_mag']].to_csv(data_path+'.deskewed',sep=',',header=False,index=False)
def find_fz_crossings(deskew_path,fz_directory=os.path.join('..','raw_data','fracture_zones')):
deskew_df = utl.open_deskew_file(deskew_path)
get_fz_loc = read_and_fit_fz_data(fz_directory)
fz_inter_dict = {}
for i,row in deskew_df.iterrows():
data_path = os.path.join(row['data_dir'],row['comp_name'])
data_df = utl.open_mag_file(data_path)
track_lon_lats = [[utl.convert_to_0_360(lon),lat] for lon,lat in zip(data_df['lon'],data_df['lat'])]
inters = get_fz_loc(track_lon_lats)
if inters != []: fz_inter_dict[row['comp_name']] = inters
fz_inter_df = pd.DataFrame({'inters':fz_inter_dict})
fz_inter_df.to_csv('fz_intercepts.txt',sep='\t')
def make_tabs(directory):
ship_cut_tracks = glob.glob(os.path.join(directory, '**', '**', '*.lp'))
aero_cut_tracks = glob.glob(
os.path.join(directory, '**', '**', '**', '*.DAT'))
print(
"------NOTE: all _tab files will be skipped to prevent recursion-------"
)
print(ship_cut_tracks + aero_cut_tracks)
for cut_track in (ship_cut_tracks + aero_cut_tracks):
if '_tab.' in cut_track: continue
df = open_mag_file(cut_track)
df.to_csv(cut_track.split('.')[0] + '_tab.' + cut_track.split('.')[1],
sep='\t',
index=False)
def flip_data_file(data_path):
#backup old data file
data_back = data_path+'.bak'
print('backing up %s to %s'%(data_path,data_back))
shutil.copyfile(data_path,data_back)
#flip data file
data_df = open_mag_file(data_path)
print("flipping data in %s"%data_path)
# import pdb; pdb.set_trace()
data_df = data_df.iloc[::-1]
#write out data file
print("overwriting %s"%data_path)
write_mag_file_df(data_df,data_path)
def get_shipmag_decimal_year(row,deg_e=.01):
"""
takes a row (pandas series) of a deskew file which is of type ship and returns the decimal year for the
intersection point. returns none if not found.
"""
if row['track_type']!='ship':
raise ValueError("get_shipmag_decimal_year can only run on shipmag data recieved data of type %s instead"%str(row['track_type']))
data_file_path = os.path.join(row["data_dir"],row["comp_name"])
data_df = utl.open_mag_file(data_file_path)
# data_df = pd.read_csv(data_file_path,names=["dist","decimal_year","mag","lat","lon"],delim_whitespace=True)
decimal_year=None
for j,datarow in data_df.iterrows(): #iterate to find the distance associated with the current lat lon
if (float(datarow['lat'])>=float(row['inter_lat'])-deg_e and \
float(datarow['lat'])<=float(row['inter_lat'])+deg_e) and \
(utl.convert_to_0_360(datarow['lon'])>=utl.convert_to_0_360(row['inter_lon'])-deg_e and \
utl.convert_to_0_360(datarow['lon'])<=utl.convert_to_0_360(row['inter_lon'])+deg_e):
decimal_year=float(datarow['dec_year']); break
return decimal_year
def fit_poly(self):
try:
self.deg = int(self.deg_box.GetValue())
except (ValueError, TypeError) as e:
self.parent.user_warning(
"Degree of polynomial must be a natural number")
try:
mag_path = os.path.join(self.parent.dsk_row["data_dir"],
self.parent.dsk_row["comp_name"])
mag_df = utl.open_mag_file(mag_path)
self.projected_distances = utl.calc_projected_distance(
self.parent.dsk_row["inter_lon"],
self.parent.dsk_row["inter_lat"], mag_df["lon"], mag_df["lat"],
(180 + self.parent.dsk_row["strike"]) % 360)["dist"]
shifted_mag = sk.phase_shift_data(
mag_df['mag'].tolist(), self.parent.dsk_row["phase_shift"])
self.pols = np.polyfit(self.projected_distances, shifted_mag,
self.deg)
self.poly = np.polyval(self.pols, self.projected_distances)
except AttributeError:
return
def draw_figures(self):
####################################################Get Values
dsk_row = self.parent.dsk_row
track = self.parent.track
ddis = float(self.parent.samp_dis_box.GetValue())
if ddis==0: self.parent.user_warning("Synthetic is required for comparision of phase, start by initalilzing a synthetic"); return
synth_dis = self.parent.dis_synth
synth_mag = self.parent.synth
filter_type = self.filter_type_box.GetValue()
lowcut = float(self.lowcut_box.GetValue())
highcut = float(self.highcut_box.GetValue())
order = int(self.order_box.GetValue())
left_bound = float(self.low_bound_box.GetValue())
right_bound = float(self.high_bound_box.GetValue())
aero_diff = float(self.aero_diff_box.GetValue())
left_idx = np.argmin(np.abs(synth_dis-left_bound))
right_idx = np.argmin(np.abs(synth_dis-right_bound))
left_idx,right_idx = min([left_idx,right_idx]),max([left_idx,right_idx])
bin_range,bin_num = (-180,180),120
###################################################Filter Data
data_path = os.path.join(dsk_row["data_dir"],dsk_row["comp_name"])
data_df = utl.open_mag_file(data_path)
projected_distances = utl.calc_projected_distance(dsk_row['inter_lon'],dsk_row['inter_lat'],data_df['lon'].tolist(),data_df['lat'].tolist(),(180+dsk_row['strike'])%360)
shifted_mag = sk.phase_shift_data(data_df["mag"],dsk_row["phase_shift"])
if np.any(np.diff(projected_distances["dist"])<0): itshifted_mag = np.interp(-synth_dis,-projected_distances["dist"],shifted_mag)
else: itshifted_mag = np.interp(synth_dis,projected_distances["dist"],shifted_mag)
fitshifted_mag = self.filters[filter_type](itshifted_mag,lowcut,highcut,fs=1/ddis,order=order)
###################################################Actual Plotting
outer = gridspec.GridSpec(4, 1)
###################################################Axis 0: Magnetic profiles
self.ax0 = self.fig.add_subplot(outer[0])
if self.parent.show_other_comp: #Handle Other Aeromag Component
if dsk_row["track_type"]=="aero":
if "Ed.lp" in track:
other_track = track.replace("Ed.lp","Vd.lp")
total_track = track.replace("Ed.lp","Td.lp")
other_phase = dsk_row["phase_shift"]-90
elif "Hd.lp" in track:
other_track = track.replace("Hd.lp","Vd.lp")
total_track = track.replace("Hd.lp","Td.lp")
other_phase = dsk_row["phase_shift"]-90
elif "Vd.lp" in track:
other_track = track.replace("Vd.lp","Ed.lp")
total_track = track.replace("Vd.lp","Td.lp")
if other_track not in self.parent.deskew_df["comp_name"].tolist(): other_track = track.replace("Vd.lp","Hd.lp")
other_phase = dsk_row["phase_shift"]+90
else: self.parent.user_warning("Improperly named component files should have either Ed.lp, Hd.lp, or Vd.lp got: %s"%track); return
oth_row = self.parent.deskew_df[self.parent.deskew_df["comp_name"]==other_track].iloc[0]
oth_data_path = os.path.join(oth_row["data_dir"],oth_row["comp_name"])
tot_data_path = os.path.join(oth_row["data_dir"],total_track) #Should be in same place
oth_data_df = utl.open_mag_file(oth_data_path)
oth_shifted_mag = sk.phase_shift_data(oth_data_df["mag"],other_phase)
if np.any(np.diff(projected_distances["dist"])<0): oth_itshifted_mag = np.interp(-synth_dis,-projected_distances["dist"],oth_shifted_mag)
else: oth_itshifted_mag = np.interp(synth_dis,projected_distances["dist"],oth_data_df)
oth_fitshifted_mag = self.filters[filter_type](oth_itshifted_mag,lowcut,highcut,fs=1/ddis,order=order)
if filter_type=="None": psk.plot_skewness_data(oth_row,other_phase,self.ax0,xlims=[None,None],color='darkgreen',zorder=2,picker=True,alpha=.7,return_objects=True,flip=True)
else: self.ax0.plot(synth_dis,oth_fitshifted_mag,color="#299C29",zorder=3,alpha=.6)
tot_data_df = utl.open_mag_file(tot_data_path)
if np.any(np.diff(projected_distances["dist"])<0): tot_imag = np.interp(-synth_dis,-projected_distances["dist"],tot_data_df["mag"])
else: tot_imag = np.interp(synth_dis,projected_distances["dist"],tot_data_df["mag"])
tot_fimag = self.filters[filter_type](tot_imag,lowcut,highcut,fs=1/ddis,order=order)
if filter_type=="None": psk.plot_skewness_data(dsk_row,dsk_row["phase_shift"],self.ax0,xlims=[None,None],zorder=3,picker=True,return_objects=True,flip=True)
else: self.ax0.plot(synth_dis,fitshifted_mag,color="#7F7D7D",zorder=3,alpha=.6)
self.ax0.plot(self.parent.dis_synth,self.parent.synth,'r-',alpha=.4,zorder=1)
self.ax0.set_ylabel("Magnetic Profiles")
# self.ax0.get_xaxis().set_ticklabels([])
###################################################Axis 1/2: Phase Angles and Differences
self.ax1 = self.fig.add_subplot(outer[1], sharex=self.ax0)
self.ax2 = self.fig.add_subplot(outer[2], sharex=self.ax0)
###################################################Calculate: Phase Differences
trimmed_dis = synth_dis[left_idx:right_idx]
trimmed_synth = synth_mag[left_idx:right_idx]
trimmed_fitshifted_mag = fitshifted_mag[left_idx:right_idx]
al_data = np.angle(hilbert(fitshifted_mag),deg=False)[left_idx:right_idx]
al_synth = np.angle(hilbert(np.real(synth_mag)),deg=False)[left_idx:right_idx]
data_synth_diff = phase_diff_func(al_synth,al_data)
if self.parent.show_other_comp and dsk_row["track_type"]=="aero":
trimmed_oth_fitshifted_mag = oth_fitshifted_mag[left_idx:right_idx]
al_oth = np.angle(hilbert(oth_fitshifted_mag),deg=False)[left_idx:right_idx]
oth_synth_diff = phase_diff_func(al_synth,al_oth)
oth_data_diff = phase_diff_func(al_oth,al_data)
if abs(aero_diff) > 0:
idx = ma.array(np.abs(oth_data_diff)<aero_diff)
self.ax1.plot((trimmed_dis[~idx]),(np.rad2deg(al_oth[~idx])),color="darkgreen",linestyle=":")
self.ax2.plot((trimmed_dis[~idx]),(oth_synth_diff[~idx]),color="tab:pink",alpha=.8,linestyle=":")
self.ax2.plot((trimmed_dis[~idx]),(oth_data_diff[~idx]),color="tab:grey",alpha=.8,linestyle=":")
self.ax1.plot((trimmed_dis[~idx]),(np.rad2deg(al_data[~idx])),color="k",linestyle=":")
self.ax1.plot((trimmed_dis[~idx]),(np.rad2deg(al_synth[~idx])),color="r",linestyle=":")
self.ax2.plot((trimmed_dis[~idx]),(data_synth_diff[~idx]),color="tab:red",alpha=.8,linestyle=":")
# import pdb; pdb.set_trace()
# not_trimmed_dis = (trimmed_dis[~idx])
# not_trimmed_dis[np.diff(~idx,prepend=[0])] = ma.masked
# not_al_data = (al_data[~idx])
# not_al_data[np.diff(~idx)] = ma.masked
# not_al_synth = (al_synth[~idx])
# not_al_synth[np.diff(~idx)] = ma.masked
# not_al_oth = (al_oth[~idx])
# not_al_oth[np.diff(~idx)] = ma.masked
# not_data_synth_diff = (data_synth_diff[~idx])
# not_data_synth_diff[np.diff(~idx)] = ma.masked
# not_oth_synth_diff = (oth_synth_diff[~idx])
# not_oth_synth_diff[np.diff(~idx)] = ma.masked
# not_oth_data_diff = (oth_data_diff[~idx])
# not_oth_data_diff[np.diff(~idx)] = ma.masked
trimmed_dis = (trimmed_dis[idx])
al_data = (al_data[idx])
al_synth = (al_synth[idx])
al_oth = (al_oth[idx])
data_synth_diff = (data_synth_diff[idx])
oth_synth_diff = (oth_synth_diff[idx])
oth_data_diff = (oth_data_diff[idx])
self.ax1.plot(trimmed_dis,np.rad2deg(al_oth),color="darkgreen")
self.ax2.plot(trimmed_dis,oth_synth_diff,color="tab:pink",alpha=.8)
self.ax2.plot(trimmed_dis,oth_data_diff,color="tab:grey",alpha=.8)
self.ax1.plot(trimmed_dis,np.rad2deg(al_data),color="k")
self.ax1.plot(trimmed_dis,np.rad2deg(al_synth),color="r")
self.ax2.plot(trimmed_dis,data_synth_diff,color="tab:red",alpha=.8)
# self.ax2.get_xaxis.set_ticklabels
self.ax0.set_xlim(*self.parent.ax.get_xlim())
self.ax0.set_ylim(*self.parent.ax.get_ylim())
self.ax1.set_ylabel("Phase Angles")
self.ax2.set_ylabel("Phase Differences")
###################################################Axis 2.1: Power Spectrum
# inner = gridspec.GridSpecFromSubplotSpec(1, 2, subplot_spec=outer[2])#, hspace=0.)
# self.ax2 = self.fig.add_subplot(inner[0])
###################################################Axis 2.2: Phase Statistics
self.ax3 = self.fig.add_subplot(outer[3])
if self.parent.show_other_comp and dsk_row["track_type"]=="aero":
self.ax3.hist(oth_synth_diff,range=bin_range,bins=bin_num,color="tab:pink",alpha=.5,zorder=2)
self.ax3.hist(oth_data_diff,range=bin_range,bins=bin_num,color="tab:grey",alpha=.5,zorder=1)
self.ax3.hist(data_synth_diff,range=bin_range,bins=bin_num,color="tab:red",alpha=.5,zorder=3)
self.ax3.axvline(np.median(data_synth_diff),color="k",alpha=.5,zorder=5,linestyle=":")
self.ax3.axvline(np.mean(data_synth_diff),color="k",alpha=.5,zorder=5)
self.ax3.axvspan(np.mean(data_synth_diff)-np.std(data_synth_diff),np.mean(data_synth_diff)+np.std(data_synth_diff),color="tab:grey",alpha=.3,zorder=0)
self.ax3.annotate(r"$\theta_{mean}$ = $%.1f^\circ \pm %.1f^\circ$"%(np.mean(data_synth_diff),np.std(data_synth_diff)) + "\n" + r"$\theta_{median}$ = %.1f$^\circ$"%np.median(data_synth_diff),xy=(0.02,1-0.02),xycoords="axes fraction",bbox=dict(boxstyle="round", fc="w",alpha=.5),fontsize=self.fontsize,va='top',ha='left')
self.ax3.set_ylabel(r"$\Delta \theta$ Count")
self.fig.suptitle("%s\n%s\n"%(dsk_row["sz_name"],track))
###################################################Power Figure
N = (right_idx-left_idx) #Length of signal in distance domain
NW = 3 #following Parker and O'brien '97 and HJ-Gordon '03 we use a time-bandwith product of 6 (Nw is half)
Ns = 5 #Number of points to use in running average smoothing
#Handle Distance Domain
# import pdb; pdb.set_trace()
Sk_complex, weights, eigenvalues=pmtm(itshifted_mag[left_idx:right_idx], NW=NW, NFFT=N, show=False)
Sk = np.abs(Sk_complex)**2
smoothed_tshifted_freq = (np.mean(Sk * np.transpose(weights), axis=0) * ddis)[N//2:][::-1]
# smoothed_tshifted_freq = np.convolve(smoothed_tshifted_freq, np.ones((Ns,))/Ns, mode='same') #10 point running average smoothing
tdata_freqs = np.linspace(0.0, 1.0/(2.0*ddis), N-N//2) #0 to Nyquest
self.power_ax = self.power_fig.add_subplot(111)
if self.parent.show_other_comp and dsk_row["track_type"]=="aero":
Sk_complex, weights, eigenvalues=pmtm(oth_itshifted_mag[left_idx:right_idx], NW=NW, NFFT=N, show=False)
Sk = np.abs(Sk_complex)**2
oth_smoothed_tshifted_freq = (np.mean(Sk * np.transpose(weights), axis=0) * ddis)[N//2:][::-1]
# oth_smoothed_tshifted_freq = np.convolve(oth_smoothed_tshifted_freq, np.ones((Ns,))/Ns, mode='same') #10 point running average smoothing
self.power_ax.semilogy(tdata_freqs, oth_smoothed_tshifted_freq, color="darkgreen")
# self.power_ax.semilogy(tdata_freqs, oth_smoothed_tshifted_freq+smoothed_tshifted_freq, color="grey")
Sk_complex, weights, eigenvalues=pmtm(tot_imag[left_idx:right_idx], NW=NW, NFFT=N, show=False)
Sk = np.abs(Sk_complex)**2
tot_smoothed_tshifted_freq = (np.mean(Sk * np.transpose(weights), axis=0) * ddis)[N//2:][::-1]
# tot_smoothed_tshifted_freq = np.convolve(tot_smoothed_tshifted_freq, np.ones((Ns,))/Ns, mode='same') #10 point running average smoothing
self.power_ax.semilogy(tdata_freqs, tot_smoothed_tshifted_freq, color="tab:orange")
#Old Numpy Method
# synth_freqs = np.fft.fftfreq(len(synth_dis[left_idx:right_idx]),ddis)
# tdata_freqs = np.fft.fftfreq(len(shifted_mag[left_idx:right_idx]),ddis)
# tshifted_freq = np.fft.fft(shifted_mag[left_idx:right_idx])
# fitshifted_freq = np.fft.fft(fitshifted_mag[left_idx:right_idx])
# tsynth_freq = np.fft.fft(synth_mag[left_idx:right_idx])
self.power_ax.semilogy(tdata_freqs, smoothed_tshifted_freq, color="k",zorder=100)
# self.power_ax.semilogy(tdata_freqs, np.abs(tshifted_freq), color="k")
# self.power_ax.plot(synth_freqs, np.abs(fitshifted_freq), color="#7F7D7D")
# self.power_ax.plot(synth_freqs, np.abs(tsynth_freq), color="r")
self.power_ax.set_xlim(0.0,0.4)
self.power_ax.set_ylim(1e-1,1e6)
def cut_tracks_and_flip(track_cuts, data_directory, heading="east"):
cut_tracks, flipped_data = [], []
for track, cuts in track_cuts.items():
print("Starting Track: %s" % track)
directory, path = os.path.split(track)
dfin = utl.open_mag_file(track)
# fin = open(track,'r')
# lines = fin.readlines()
# fin.close()
# lines = [line.split() for line in lines]
# dfin = pd.DataFrame(lines,columns=["time","lat","lon","n_comp","s_comp","h_comp","v_comp","mag","dec","inc","None","alt"])
lats = list(map(float, dfin['lat'].tolist()))
lons = list(map(utl.convert_to_0_360, dfin['lon'].tolist()))
df_segments = []
for cut in cuts:
try:
cut_index = [[lon, lat] for lon, lat in zip(lons, lats)
].index([utl.convert_to_0_360(cut[0]), cut[1]])
except ValueError as e:
import pdb
pdb.set_trace()
print("cutting track: %s along index: %d" % (track, cut_index))
df_segments.append(dfin.loc[:cut_index])
dfin = dfin.loc[cut_index:]
df_segments.append(dfin)
i = 1
for df_segment in df_segments:
if len(df_segment) == 0: continue
if heading == 'east':
flip_bool = (utl.convert_to_0_360(df_segment['lon'].iloc[0]) >
utl.convert_to_0_360(df_segment['lon'].iloc[-1])
) #is heading easterly
elif heading == 'west':
flip_bool = (utl.convert_to_0_360(df_segment['lon'].iloc[0]) <
utl.convert_to_0_360(df_segment['lon'].iloc[-1])
) #is heading westerly
elif heading == 'north':
flip_bool = (
df_segment['lat'].iloc[0] > df_segment['lat'].iloc[-1]
) #is heading northerly
elif heading == 'south':
flip_bool = (
df_segment['lat'].iloc[0] < df_segment['lat'].iloc[-1]
) #is heading southerly
else:
print(
"the heading provided is not a cardinal direction please rerun with this corrected"
)
return
if flip_bool:
print(
"flipping data for cut: %d track: %s such that path is %serly and thus (hopefully) oldest data first"
% (i, path, heading))
df_segment = df_segment.iloc[::-1]
flipped_data.append(track.split('.')[0] + '.c%d' % i)
if not os.path.isdir(os.path.join(directory, 'c%d' % i)):
print("making directory %s" %
os.path.join(directory, 'c%d' % i))
utl.check_dir(os.path.join(directory, 'c%d' % i))
segment_path = os.path.join(directory, 'c%d' % i,
path.split('.')[0] + '.c%d' % i)
i += 1
print("writing: %s" % segment_path)
df_segment.to_csv(segment_path,
sep='\t',
header=False,
index=False)
cut_tracks.append(segment_path)
f_flipped = open(os.path.join(data_directory, "flipped_data.txt"), 'w+')
f_flipped.write(reduce(lambda x, y: x + '\n' + y, flipped_data))
f_flipped.close()
return cut_tracks, flipped_data
def aeromag_preprocess(aeromag_files,
date_file=os.path.join('..', 'raw_data',
'dates.aeromag'),
geoid=Geodesic.WGS84):
for aeromag_file in aeromag_files: #iterate over all aeromag files
track, extension = os.path.basename(aeromag_file).split(
'.') #segment the name into parts
#read data and make a empty dataframe for output data
adf = utl.open_mag_file(aeromag_file).dropna()
ddf = pd.read_csv(date_file, sep='\t', index_col=0)
idf = pd.DataFrame(columns=[
'dis', 'lat', 'lon', 'alt', 'v_comp', 'e_comp', 'n_comp', 'h_comp',
't_comp'
])
dis = 0
decimal_year = float(ddf.loc[track]['decimal_year'])
prev_lat, prev_lon = None, None
for i, row in adf.iterrows(): #iterate over rows
row["lon"] = utl.convert_to_0_360(row["lon"])
adf.at[i, "lon"] = row["lon"]
try:
#check for data gaps
if np.isnan(row['lat']) or np.isnan(row['lon']) or np.isnan(
row['alt']) or np.isnan(row['mag']) or np.isnan(
row['v_comp']) or np.isnan(
row['e_comp']) or np.isnan(
row['n_comp']) or np.isnan(row['h_comp']):
continue
#check None
elif (row['lat'] == None) or (row['lon'] == None) or (
row['alt'] == None) or (row['mag'] == None) or (
row['v_comp'] == None) or (row['e_comp'] == None):
continue
#check for absurd values outside of the domain of the varible (this will capture null values of -99999)
elif (abs(float(row['lat'])) > 90) or (abs(
utl.convert_to_180_180(row['lon'])) > 180) or (float(
row['alt']) < 0) or (abs(float(
row['mag'])) == 99999) or (abs(
float(row['v_comp'])) == 99999) or (abs(
float(row['e_comp'])) == 99999):
continue
except ValueError as e:
continue #This implies a value which is not convertable to a float as all of these should be floats this datum must be skipped
if prev_lat != None and prev_lon != None: #calculate distance
dis += geoid.Inverse(float(row['lat']), float(row['lon']),
prev_lat, prev_lon)['s12'] / 1000
adf.at[i, 'dis'] = dis
#calculate and remove IGRF
dec, inc, mag = ipmag.igrf([
decimal_year,
float(row['alt']) * 0.3048e-3,
float(row['lat']),
float(row['lon'])
])
res_v_comp = mag * np.sin(np.deg2rad(inc))
res_e_comp = mag * np.cos(np.deg2rad(inc)) * np.sin(
np.deg2rad(dec))
res_n_comp = mag * np.cos(np.deg2rad(inc)) * np.cos(
np.deg2rad(dec))
res_h_comp = mag * np.cos(np.deg2rad(inc))
res_t_comp = mag
adf.at[i, 'res_v_comp'] = float(row['v_comp']) - res_v_comp
adf.at[i, 'res_e_comp'] = float(row['e_comp']) - res_e_comp
adf.at[i, 'res_n_comp'] = float(row['n_comp']) - res_n_comp
adf.at[i, 'res_h_comp'] = float(row['h_comp']) - res_h_comp
adf.at[i, 'res_t_comp'] = float(row['mag']) - res_t_comp
prev_lat, prev_lon = float(row['lat']), float(row['lon'])
# adf = adf[(adf['res_e_comp']<3000) & (adf['res_n_comp']<3000) & (adf['res_v_comp']<3000) & (adf['res_h_comp']<3000) & (adf['res_t_comp']<3000)]
#remove a second order polynomial fromm the magnetic data I don't know why but this is something done
for col in [
'res_e_comp', 'res_n_comp', 'res_h_comp', 'res_v_comp',
'res_t_comp'
]:
# pols = np.polyfit(adf['dis'].tolist(),adf[col].tolist(),3)
# mag_fit = np.polyval(pols,adf['dis'].tolist())
# adf['cor'+col.lstrip('res')] = adf[col].to_numpy() - mag_fit
adf['cor' + col.lstrip('res')] = adf[col].to_numpy()
#iterpolate and round data
adf = adf.dropna()
idf['dis'] = np.arange(adf['dis'].iloc[0], adf['dis'].iloc[-1] + .1,
.1) #spacing of 1 km, because I can
idf['lat'] = np.interp(idf['dis'], adf['dis'], adf['lat'])
idf['lon'] = np.interp(idf['dis'], adf['dis'], adf['lon'])
idf['alt'] = np.interp(idf['dis'], adf['dis'], .3048 * adf['alt'])
idf['v_comp'] = np.interp(idf['dis'], adf['dis'], adf['cor_v_comp'])
idf['e_comp'] = np.interp(idf['dis'], adf['dis'], adf['cor_e_comp'])
idf['n_comp'] = np.interp(idf['dis'], adf['dis'], adf['cor_n_comp'])
idf['h_comp'] = np.interp(idf['dis'], adf['dis'], adf['cor_h_comp'])
idf['t_comp'] = np.interp(idf['dis'], adf['dis'], adf['cor_t_comp'])
adf[['dis', 'alt', 'cor_v_comp', 'lat',
'lon']].to_csv(aeromag_file + '.Vd',
index=False,
header=False,
sep='\t',
float_format="%.3f")
adf[['dis', 'alt', 'cor_e_comp', 'lat',
'lon']].to_csv(aeromag_file + '.Ed',
index=False,
header=False,
sep='\t',
float_format="%.3f")
adf[['dis', 'alt', 'cor_n_comp', 'lat',
'lon']].to_csv(aeromag_file + '.Nd',
index=False,
header=False,
sep='\t',
float_format="%.3f")
adf[['dis', 'alt', 'cor_h_comp', 'lat',
'lon']].to_csv(aeromag_file + '.Hd',
index=False,
header=False,
sep='\t',
float_format="%.3f")
adf[['dis', 'alt', 'cor_t_comp', 'lat',
'lon']].to_csv(aeromag_file + '.Td',
index=False,
header=False,
sep='\t',
float_format="%.3f")
idf[['dis', 'alt', 'v_comp', 'lat',
'lon']].to_csv(aeromag_file + '.Vd.lp',
index=False,
header=False,
sep='\t',
float_format="%.3f")
idf[['dis', 'alt', 'e_comp', 'lat',
'lon']].to_csv(aeromag_file + '.Ed.lp',
index=False,
header=False,
sep='\t',
float_format="%.3f")
idf[['dis', 'alt', 'n_comp', 'lat',
'lon']].to_csv(aeromag_file + '.Nd.lp',
index=False,
header=False,
sep='\t',
float_format="%.3f")
idf[['dis', 'alt', 'h_comp', 'lat',
'lon']].to_csv(aeromag_file + '.Hd.lp',
index=False,
header=False,
sep='\t',
float_format="%.3f")
idf[['dis', 'alt', 't_comp', 'lat',
'lon']].to_csv(aeromag_file + '.Td.lp',
index=False,
header=False,
sep='\t',
float_format="%.3f")
if extension.startswith('c'):
shutil.copyfile(aeromag_file, aeromag_file + '.lp')
latlon_df = adf[['lat', 'lon']]
latlon_file = aeromag_file + ".latlon"
latlon_df.to_csv(latlon_file, sep=' ', index=False, header=False)
for dt in deskew_tracks:
dt[0].remove()
for df in deskew_fill:
df[0].remove()
except NameError:
pass
except TypeError:
pass
print("Plotting Tracks: ")
deskew, deskew_tracks, deskew_fill = utl.open_deskew_file(
dsk_path), [], []
for j, (i, row) in enumerate(deskew.iterrows()):
if remove_bad_data and row["quality"] != "g": continue
# Read in deskewed profile
# This is hard-coded now. It will be updated to take a list of profiles in the future
dskd = utl.open_mag_file(
os.path.join(row['data_dir'], row["comp_name"]))
print("\tPlotting: ", row["comp_name"])
# Define the angle along which to project
perp = row["strike"] - 180
lon = dskd["lon"].tolist()
lat = dskd["lat"].tolist()
mag = sk.phase_shift_data(dskd["mag"].tolist(), row["phase_shift"])
# print("\t\t",perp)
# print("\t\t",np.array(lon).shape,np.array(lat).shape,np.array(mag).shape)
# Find distance to project
if row["track_type"] == 'ship':
pcol = '#000000'
scle = 0.2 * 1e3
elif row["track_type"] == 'aero':
def auto_dsk(dsk_row,synth,bounds,conv_limit=0,conv_bounds=[None,None],phase_args=(0.,360.,1.),highcut=0.,order=3):
"""
Returns the maximum likelihood phase shift to deskew the data to match a provided synthetic given a bounds
on a window to match.
Parameters
----------
dsk_row : Pandas.Series
Single row of a deskew file with valid path to data file
synth : list
This should be the output of the make synthetic function it needs to contain three elements
0) an array of the synthetic magnetic anomalies
1) an array of the distance coordinates of the points in 0, should be of equal length to 0, MUST be in
the same coordinate system as the profile provided in dsk_row!!! Which it may not by default.
2) the distance resolution of the synthetic in 0 and 1
bounds : list of floats
Has two elements which corespond to the left and right bounds of the window
conv_limit : float, optional
Weather or not to realign the anomaly each phase shift using a time lagged convolution method which
increases runtime significantly but can also increase accuracy. This argument should be a positve
float which corresponds to the amount of +- shift the anomaly is allowed to move used otherwise it should
be 0 to not use the shift method (Default: 0, which implies not to use method).
conv_bounds : list of 2 floats, optional
The left and right boundary in the distance domain to use to time lag convolve the synthetic and the filtered
data signal. Thus 300 km of signal can be convolved but only the 10 km of motion allowed to pin down the
crossing location. (Default: [None,None], which implies conv_bounds=bounds)
phase_args : tuple or other unpackable sequence, optional
Arguments to np.arange which define the phases searched in the minimization. (Default: (0.,360.,1.) which
implies a search of the entire parameter space of phases at 1 degree resolution)
highcut : float, optional
The upper cutoff frequency to filter the data by in order to remove any topographic anomalies in the data.
This value should be between 0 and Nyquest of the synthetic which MUST be regularly sampled like those
returned by make_synthetic. The data is up or down sampled to the synthetic before filtering. (Default:
0 which implies not to filter the data)
order : int, optional
The order of the lowpass butterworth filter to apply to the data.
Returns
----------
best_phase : float
The maximum liklehood phase shift to match the data to the synthetic
best_shift : float
the maximum likelihood shift for the best_phase which aligned the two anomalies
phase_func : Numpy.NdArray
The summed phase asynchrony between the data and the synthetic as a function of phase shift (best_phase is
the global minimum of this function)
best_shifts : Numpy.NdArray
the maximum likelihood shift as a function of the phase shift
"""
#Unpack Arguments
dage = dsk_row["age_max"]-dsk_row["age_min"]
phases = np.arange(*phase_args)
left_bound,right_bound = bounds
synth_mag = np.array(synth[0])
synth_dis = np.array(synth[1])
ddis = synth[2]
data_path = os.path.join(dsk_row["data_dir"],dsk_row["comp_name"])
data_df = utl.open_mag_file(data_path)
projected_distances = utl.calc_projected_distance(dsk_row['inter_lon'],dsk_row['inter_lat'],data_df['lon'].tolist(),data_df['lat'].tolist(),(180+dsk_row['strike'])%360)
if conv_limit: #create the fully interpolated profile for convolution
#create the shortened synthetic for the time lagged convolution
if isinstance(conv_bounds[0],type(None)): conv_bounds[0] = bounds[0]
if isinstance(conv_bounds[1],type(None)): conv_bounds[1] = bounds[1]
left_idx = np.argmin(np.abs(synth_dis - conv_bounds[0]))
right_idx = np.argmin(np.abs(synth_dis - conv_bounds[1]))
right_idx,left_idx = max([right_idx,left_idx]),min([right_idx,left_idx])
conv_synth,conv_synth_dis = synth_mag[left_idx:right_idx],synth_dis[left_idx:right_idx]
if np.any(np.diff(projected_distances["dist"])<0): #redefine to the right because interp dumbs
mag = data_df["mag"].to_numpy()[::-1]
mag_dis = projected_distances["dist"].to_numpy()[::-1]
full_imag = np.interp(conv_synth_dis,mag_dis,mag)
if highcut: full_fimag = butter_lowpass_filter(full_imag,highcut=highcut,fs=1/ddis,order=order)
else: full_fimag = full_imag
#trim to only window of relivence
left_idx = np.argmin(np.abs(synth_dis - left_bound))
right_idx = np.argmin(np.abs(synth_dis - right_bound))
right_idx,left_idx = max([right_idx,left_idx]),min([right_idx,left_idx])
tsynth_mag = synth_mag[left_idx:right_idx]
tsynth_dis = synth_dis[left_idx:right_idx]
N = len(tsynth_mag) #because this is easier and regularly sampled plus the user can set it simply
al2 = np.angle(hilbert(np.real(tsynth_mag),N),deg=False)
best_shifts = [] #record best shifts as function of phase shift
phase_async_func = [] #record summed phase asynchrony as a function of phase shift
for i,phase in enumerate(phases):
shifted_mag = phase_shift_data(data_df["mag"],phase)
if conv_limit: #DON'T YOU KNOW WE'RE GONNAAAA DOOOOOOO THE COOONVOLUTIOOOON!!!
shifted_full_fimag = phase_shift_data(full_fimag,phase)
correlation_func = np.abs(np.convolve(shifted_full_fimag,conv_synth,"full"))
correlation_func = correlation_func[int(len(conv_synth)-conv_limit/ddis+.5):int(len(conv_synth)+conv_limit/ddis+.5)]
best_shift = ddis*(len(correlation_func)/2-np.argmax(correlation_func))/2
else: best_shift = 0.
#trim the data to the right segments
left_idx = np.argmin(np.abs(projected_distances["dist"] - left_bound + best_shift))
right_idx = np.argmin(np.abs(projected_distances["dist"]- right_bound + best_shift))
right_idx,left_idx = max([right_idx,left_idx]),min([right_idx,left_idx])
tproj_dist = projected_distances["dist"][left_idx:right_idx] + best_shift
tshifted_mag = shifted_mag[left_idx:right_idx]
#numpy.interp only works for monotonic increasing independent variable data
if np.any(np.diff(tproj_dist)<0): itshifted_mag = np.interp(-tsynth_dis,-tproj_dist,tshifted_mag)
else: itshifted_mag = np.interp(tsynth_dis,tproj_dist,tshifted_mag)
if highcut: fitshifted_mag = butter_lowpass_filter(itshifted_mag,highcut=highcut,fs=1/ddis,order=order)
else: fitshifted_mag = itshifted_mag
al1 = np.angle(hilbert(fitshifted_mag,N),deg=False)
phase_asynchrony = np.sin((al1-al2)/2) #shouldn't go negative but...just in case
best_shifts.append(best_shift)
phase_async_func.append(phase_asynchrony.sum())
best_idx = np.argmin(phase_async_func)
return phases[best_idx],best_shifts[best_idx],phase_async_func,best_shifts
def plot_tracks(self):
try:
dsk_row = self.parent.dsk_row
dsk_data = self.parent.deskew_df[self.parent.deskew_df["sz_name"]
== dsk_row["sz_name"]]
except AttributeError:
return
# projected_distances = utl.calc_projected_distance(dsk_row['inter_lon'],dsk_row['inter_lat'],mag_data['lon'].tolist(),mag_data['lat'].tolist(),dsk_row['strike'])
# dis = max(abs(projected_distances["dist"]))*1000
# geodict1 = Geodesic.WGS84.Direct(dsk_row['inter_lat'],dsk_row['inter_lon'],dsk_row['strike']-90,dis)
# geodict2 = Geodesic.WGS84.Direct(dsk_row['inter_lat'],dsk_row['inter_lon'],dsk_row['strike']+90,dis)
self.ax.plot([geodict1["lon2"], geodict2["lon2"]],
[geodict1["lat2"], geodict2["lat2"]],
transform=ccrs.Geodetic(),
color="black",
linewidth=1,
linestyle='--')
for j, (i, row) in enumerate(dsk_data.iterrows()):
# Read in deskewed profile
# This is hard-coded now. It will be updated to take a list of profiles in the future
if not os.path.isfile(infile):
self.parent.user_warning("Data file %s could not be found" %
infile)
dskd = utl.open_mag_file(
os.path.join(row['data_dir'], row["comp_name"]))
# Define the angle along which to project
perp = row["strike"] - 180
lon = dskd["lon"]
lat = dskd["lat"]
mag = sk.phase_shift_data(dskd["mag"].tolist(), row["phase_shift"])
# Find distance to project
if row["track_type"] == 'ship':
pcol = '#000000'
scle = 0.2 * 1e3
if row["track_type"] == 'aero':
if 'Ed' in row["comp_name"]:
pcol = 'purple'
else:
pcol = 'darkorchid'
scle = 0.5 * 1e3
# Project amplitude onto map
mlats, mlons = [], []
for i in range(len(mag)):
gdsc = self.geoid.Direct(lat[i], lon[i], perp, mag[i] * scle)
mlons.append(gdsc['lon2'])
mlats.append(gdsc['lat2'])
# Plot map elements
deskew_tracks.append(
self.ax.plot(utl.convert_to_0_360(lon),
lat,
'--',
linewidth=1.0,
transform=ccrs.PlateCarree(),
color=pcol,
zorder=990))
deskew_tracks.append(
self.ax.plot(utl.convert_to_0_360(mlons),
mlats,
'-',
linewidth=1.0,
transform=ccrs.PlateCarree(),
color=pcol,
zorder=1000))
deskew_fill.append(
self.ax.fill_between(utl.convert_to_0_360(
np.array(mlons)[mag > 0]),
np.array(mlats)[mag > 0],
lat[mag > 0],
transform=ccrs.PlateCarree(),
alpha=0.5,
color=pcol))
def plot_az_strike(track, spreading_zone_file, idx, az, strike, chron_color,
chron_name, results_directory, fout_name):
#Create Figure
fig = plt.figure(figsize=(9, 9), dpi=80)
#Create Chron markers
dft = utl.open_mag_file(track)
lt = [[utl.convert_to_0_360(lon), float(lat)]
for lon, lat in zip(dft['lon'], dft['lat'])]
at = np.array(lt)
lsz = [
list(map(float, line.split()))
for line in open(spreading_zone_file).readlines()
]
asz = np.array(lsz)
#Create Map
# gcm = create_basic_map() #uses defaults, hit shift-tab in parens to see what they are
llcrnrlon = min(at[:, 0]) - 20 if min(at[:, 0]) - 20 > 0 else 0
llcrnrlat = min(at[:, 1]) - 20 if min(at[:, 1]) - 20 > -89 else -89
urcrnrlon = max(at[:, 0]) + 20 if max(at[:, 0]) + 20 < 360 else 360
urcrnrlat = max(at[:, 1]) + 20 if max(at[:, 1]) + 20 < 89 else 89
gcm = create_basic_map(projection='merc',
llcrnrlat=llcrnrlat,
urcrnrlat=urcrnrlat,
llcrnrlon=llcrnrlon,
urcrnrlon=urcrnrlon,
fig=fig)
sz_handle, = gcm.plot(asz[:, 0],
asz[:, 1],
color=chron_color,
zorder=1,
label=chron_name,
transform=ccrs.PlateCarree())
gcm_handle, = gcm.plot(at[:, 0],
at[:, 1],
color='k',
zorder=2,
label=os.path.basename(track),
transform=ccrs.PlateCarree())
gcm.scatter(at[idx[1][0]][0],
at[idx[1][0]][1],
color='g',
marker='o',
s=10,
zorder=3,
label='nearest intercept',
transform=ccrs.PlateCarree())
geodict = Geodesic(6371000., 0.).Direct(float(at[idx[1][0]][1]),
float(at[idx[1][0]][0]), float(az),
1000000)
b_lon, b_lat = (360 + geodict["lon2"]) % 360, geodict["lat2"]
gcm.arrow(b_lon,
b_lat,
b_lon - at[idx[1][0]][0],
b_lat - at[idx[1][0]][1],
fc="white",
ec="r",
linewidth=1,
head_width=1,
head_length=1,
label='azimuth',
transform=ccrs.PlateCarree())
geodict = Geodesic(6371000., 0.).Direct(float(at[idx[1][0]][1]),
float(at[idx[1][0]][0]),
float(strike), 1000000)
b_lon, b_lat = (360 + geodict["lon2"]) % 360, geodict["lat2"]
gcm.arrow(b_lon,
b_lat,
b_lon - at[idx[1][0]][0],
b_lat - at[idx[1][0]][1],
fc="white",
ec="pink",
linewidth=1,
head_width=1,
head_length=1,
label='strike',
transform=ccrs.PlateCarree())
#plot title and labels
plt.title(os.path.basename(track))
plt.legend(loc='best')
az_plots_dir = os.path.join(results_directory, "azimuth_strike_plots")
utl.check_dir(az_plots_dir)
fig.savefig(
os.path.join(az_plots_dir,
os.path.basename(fout_name)[:-5] + "png"))
plt.close(fig)