def rate(self, ground_truth, prediction, label, ts_length,
shapelet_length):
"""
Computes TP, FP, TN, FN according to section IV-A.
"""
matches = defaultdict(lambda: [])
tp = []
if len(prediction) > 0:
for i, e in enumerate(ground_truth):
match_id = find_nearest(prediction, e)
match = prediction[match_id]
distance = abs(match - e)
if distance < shapelet_length:
matches[match].append((distance, e))
for p in matches.keys():
asdf = min(matches[p], key=lambda x: x[0])
tp.append((p, asdf[1]))
self.confusion_matrix[label].TP += len(tp)
self.confusion_matrix[label].FP += len(prediction) - len(tp)
self.confusion_matrix[label].FN += len(ground_truth) - len(tp)
self.confusion_matrix[label].TN += (
ts_length - len(ground_truth)) - self.confusion_matrix[label].FP
for x, y in tp:
self.confusion_matrix[label].time_differences.append(abs(x - y))
def ns_lookup(self, wavelength, tolerance=0.30):
"""Lookup set of test wavelengths closes with a given tolerance, see if night sky"""
# TODO - Possible to make this check if brightness of night sky is expected
near = utilities.find_nearest(self.ns_atlas[:, 0], wavelength)
if abs(near - wavelength) < tolerance:
return self.ns_atlas[self.ns_atlas[:, 0] == near][0]
else:
return []
def findTags(self):
for offset in self.mz:
newMZ = np.array(self.mz) - offset
for tagCandidate in self.tagCandidates:
tagPeak2 = tagCandidate['peaks'][0]
tagPeak3 = tagCandidate['peaks'][1]
tagPeak4 = tagCandidate['peaks'][2]
index2 = util.find_nearest(tagPeak2, newMZ)
index3 = util.find_nearest(tagPeak3, newMZ)
index4 = util.find_nearest(tagPeak4, newMZ)
peak1 = 0
peak2 = newMZ[index2]
peak3 = newMZ[index3]
peak4 = newMZ[index4]
#print(peak1, peak2, peak3, peak4)
base1 = 0
base2 = peak2 - tagPeak2
base3 = peak3 - tagPeak3
base4 = peak4 - tagPeak4
baseAvg = (base1 + base2 + base3 + base4)/4.0
#print(base1, base2, base3, base4, baseAvg)
ss1 = (base1 - baseAvg)**2
ss2 = (base2 - baseAvg)**2
ss3 = (base3 - baseAvg)**2
ss4 = (base4 - baseAvg)**2
sse = ss1 + ss2 + ss3 + ss4
#print(sse)
if sse<0.2:
p1 = self.getPeak(peak1 + offset)
p2 = self.getPeak(peak2 + offset)
p3 = self.getPeak(peak3 + offset)
p4 = self.getPeak(peak4 + offset)
complementScore = 0
for p in [p1, p2, p3, p4]:
if p.hasCounterpart:
complementScore += 1
rank = p1.rank + p2.rank + p3.rank + p4.rank
rankScore = self.calculateIntensityRankPValue(rank)
if rankScore < 0.1:
print(tagCandidate['sequence'], rankScore, complementScore)
def createPeaks(self):
for i in range(0, len(self.mz)):
mz = self.mz[i]
intensity = self.intensity[i]
rank = self.ranks[i]
counterpartMass = self.MH - mz + 1
counterpartIndex = util.find_nearest(counterpartMass, self.mz)
counterpartDifference = abs(self.mz[counterpartIndex] - mz)
if counterpartDifference < self.mzTolerance:
hasCounterpart = True
else:
hasCounterpart = False
self.peakList.append(Peak(mz, intensity, hasCounterpart, rank))
def create_connections(self):
for i in range(0, len(self.mz) - 1):
for j in range(i + 1, len(self.mz)):
mass_diff = self.mz[j] - self.mz[i]
if mass_diff >= self.minMass and mass_diff <= self.maxMass:
self.edges += 1
closest_aa_mass_idx = util.find_nearest(
self.aa.values(), mass_diff)
match = False
if abs(self.aa.values()[closest_aa_mass_idx] -
mass_diff) <= self.massTolerance:
match = True
self.connections.append(
MassConnection(self.mz[i], self.mz[j], mass_diff,
match))
def get_hr_radar_dap_data(dap_urls, st_list, jd_start, jd_stop):
# Use only data within 1.00 degrees
obs_df = []
obs_or_model = False
max_dist = 1.0
# Use only data where the standard deviation of the time series exceeds 0.01 m (1 cm).
# This eliminates flat line model time series that come from land points that should have had missing values.
min_var = 0.1
data_idx = []
df_list = []
for url in dap_urls:
#only look at 6km hf radar
if 'http://hfrnet.ucsd.edu/thredds/dodsC/HFRNet/USWC/' in url and "6km" in url and "GNOME" in url:
print url
#get url
nc = netCDF4.Dataset(url, 'r')
lat_dim = nc.variables['lat']
lon_dim = nc.variables['lon']
time_dim = nc.variables['time']
u_var = None
v_var = None
for key in nc.variables.iterkeys():
key_dim = nc.variables[key]
try:
if key_dim.standard_name == "surface_eastward_sea_water_velocity":
u_var = key_dim
elif key_dim.standard_name == "surface_northward_sea_water_velocity":
v_var = key_dim
elif key_dim.standard_name == "time":
time = key_dim
except:
#only if the standard name is not available
pass
#manage dates
dates = num2date(time_dim[:],
units=time_dim.units,
calendar='gregorian')
date_idx = []
date_list = []
for i, date in enumerate(dates):
if jd_start < date < jd_stop:
date_idx.append(i)
date_list.append(date)
#manage location
for st in st_list:
station = st_list[st]
f_lat = station['lat']
f_lon = station['lon']
ret = find_nearest(f_lat, f_lon, lat_dim[:], lon_dim[:])
lat_idx = ret[0]
lon_idx = ret[1]
dist_deg = ret[2]
#print "lat,lon,dist=",ret
if len(u_var.dimensions) == 3:
#3dimensions
ret = cycleAndGetData(u_var, v_var, date_idx, lat_idx,
lon_idx)
u_vals = ret[0]
v_vals = ret[1]
lat_idx = ret[2]
lon_idx = ret[3]
print "lat,lon,dist=", ret[2], ret[3]
try:
#turn vectors in the speed and direction
ws = uv2ws(u_vals, v_vals)
wd = uv2wd(u_vals, v_vals)
data_spd = []
data_dir = []
data = {}
data['sea_water_speed (cm/s)'] = np.array(ws)
data[
'direction_of_sea_water_velocity (degree)'] = np.array(
wd)
time = np.array(date_list)
df = pd.DataFrame(
data=data,
index=time,
columns=[
'sea_water_speed (cm/s)',
'direction_of_sea_water_velocity (degree)'
])
df_list.append({
"name": st,
"data": df,
"lat": lat_dim[lat_idx],
"lon": lon_dim[lon_idx],
"ws_pts": np.count_nonzero(~np.isnan(ws)),
"wd_pts": np.count_nonzero(~np.isnan(wd)),
"dist": dist_deg,
'from': url
})
except Exception, e:
print "\t\terror:", e
else:
pass
def aggregate(in_data, agg_data, res=0, pad=0, maskandnorm=False):
"""
Add the two data sets together and return the combined arrays.
Expand the horizontal dimensions as necessary to fit in_data with agg_data.
The two data sets must include the coordinate variables lon,lat, and time.
"""
# ---------------------------------------------------------------- #
# find range of coordinates
if agg_data:
lat_min = np.minimum(in_data['lat'].min(), agg_data['lat'].min())
lat_max = np.maximum(in_data['lat'].max(), agg_data['lat'].max())
lon_min = np.minimum(in_data['lon'].min(), agg_data['lon'].min())
lon_max = np.maximum(in_data['lon'].max(), agg_data['lon'].max())
tshape = in_data['unit_hydrograph'].shape[0]
else:
lat_min = in_data['lat'].min()
lat_max = in_data['lat'].max()
lon_min = in_data['lon'].min()
lon_max = in_data['lon'].max()
tshape = in_data['unit_hydrograph'].shape[0]
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# make output arrays for lons/lats and initialize fraction/unit_hydrograph
# pad output arrays so there is a space =pad around inputs
lats = np.arange((lat_min - res * pad), (lat_max + res * (pad + 1)),
res)[::-1]
lons = np.arange((lon_min - res * pad), (lon_max + res * (pad + 1)), res)
fraction = np.zeros((len(lats), len(lons)), dtype=np.float64)
unit_hydrograph = np.zeros((tshape, len(lats), len(lons)),
dtype=np.float64)
log.debug('fraction shape %s' % str(fraction.shape))
log.debug('unit_hydrograph shape %s' % str(unit_hydrograph.shape))
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# find target index locations of all corners for both datasets
# Not that the lat inds are inverted
ilat_min_ind = find_nearest(lats, np.max(in_data['lat']))
ilat_max_ind = find_nearest(lats, np.min(in_data['lat'])) + 1
ilon_min_ind = find_nearest(lons, np.min(in_data['lon']))
ilon_max_ind = find_nearest(lons, np.max(in_data['lon'])) + 1
log.debug('in_data fraction shape: %s', str(in_data['fraction'].shape))
if agg_data:
alat_min_ind = find_nearest(lats, np.max(agg_data['lat']))
alat_max_ind = find_nearest(lats, np.min(agg_data['lat'])) + 1
alon_min_ind = find_nearest(lons, np.min(agg_data['lon']))
alon_max_ind = find_nearest(lons, np.max(agg_data['lon'])) + 1
log.debug('agg_data fraction shape: %s',
str(agg_data['fraction'].shape))
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Place data
fraction[ilat_min_ind:ilat_max_ind,
ilon_min_ind:ilon_max_ind] += in_data['fraction']
unit_hydrograph[:, ilat_min_ind:ilat_max_ind,
ilon_min_ind:ilon_max_ind] += in_data['unit_hydrograph']
if agg_data:
fraction[alat_min_ind:alat_max_ind,
alon_min_ind:alon_max_ind] += agg_data['fraction']
unit_hydrograph[:, alat_min_ind:alat_max_ind,
alon_min_ind:alon_max_ind] += agg_data[
'unit_hydrograph']
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Mask and Normalize the unit unit_hydrograph
if (maskandnorm):
# Normalize the unit_hydrograph (each cell should sum to 1)
yv, xv = np.nonzero(fraction > 0.0)
unit_hydrograph[:, yv, xv] /= unit_hydrograph[:, yv, xv].sum(axis=0)
# Mask the unit_hydrograph and make sure they sum to 1 at each grid
# cell
ym, xm = np.nonzero(fraction <= 0.0)
unit_hydrograph[:, ym, xm] = FILLVALUE_F
log.info('Completed final aggregation step')
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Put all the data into agg_data variable and return to main
agg_data['timesteps'] = in_data['timesteps']
agg_data['unit_hydrograph_dt'] = in_data['unit_hydrograph_dt']
agg_data['lon'] = lons
agg_data['lat'] = lats
agg_data['fraction'] = fraction
agg_data['unit_hydrograph'] = unit_hydrograph
# ---------------------------------------------------------------- #
return agg_data
def aggregate(in_data, agg_data, res=0, pad=0, maskandnorm=False):
"""
Add the two data sets together and return the combined arrays.
Expand the horizontal dimensions as necessary to fit in_data with agg_data.
The two data sets must include the coordinate variables lon,lat, and time.
"""
# ---------------------------------------------------------------- #
# find range of coordinates
if agg_data:
lat_min = np.minimum(in_data['lat'].min(), agg_data['lat'].min())
lat_max = np.maximum(in_data['lat'].max(), agg_data['lat'].max())
lon_min = np.minimum(in_data['lon'].min(), agg_data['lon'].min())
lon_max = np.maximum(in_data['lon'].max(), agg_data['lon'].max())
tshape = in_data['unit_hydrograph'].shape[0]
else:
lat_min = in_data['lat'].min()
lat_max = in_data['lat'].max()
lon_min = in_data['lon'].min()
lon_max = in_data['lon'].max()
tshape = in_data['unit_hydrograph'].shape[0]
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# make output arrays for lons/lats and initialize fraction/unit_hydrograph
# pad output arrays so there is a space =pad around inputs
lats = np.arange((lat_min-res*pad), (lat_max+res*(pad+1)), res)[::-1]
lons = np.arange((lon_min-res*pad), (lon_max+res*(pad+1)), res)
fraction = np.zeros((len(lats), len(lons)))
unit_hydrograph = np.zeros((tshape, len(lats), len(lons)))
log.debug('fraction shape %s' % str(fraction.shape))
log.debug('unit_hydrograph shape %s' % str(unit_hydrograph.shape))
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# find target index locations of all corners for both datasets
# Not that the lat inds are inverted
ilat_min_ind = find_nearest(lats, np.max(in_data['lat']))
ilat_max_ind = find_nearest(lats, np.min(in_data['lat']))+1
ilon_min_ind = find_nearest(lons, np.min(in_data['lon']))
ilon_max_ind = find_nearest(lons, np.max(in_data['lon']))+1
log.debug('in_data fraction shape: %s' % str(in_data['fraction'].shape))
if agg_data:
alat_min_ind = find_nearest(lats, np.max(agg_data['lat']))
alat_max_ind = find_nearest(lats, np.min(agg_data['lat']))+1
alon_min_ind = find_nearest(lons, np.min(agg_data['lon']))
alon_max_ind = find_nearest(lons, np.max(agg_data['lon']))+1
log.debug('agg_data fraction shape: %s' % str(agg_data['fraction'].shape))
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Place data
fraction[ilat_min_ind:ilat_max_ind, ilon_min_ind:ilon_max_ind] += in_data['fraction']
unit_hydrograph[:, ilat_min_ind:ilat_max_ind, ilon_min_ind:ilon_max_ind] += in_data['unit_hydrograph']
if agg_data:
fraction[alat_min_ind:alat_max_ind, alon_min_ind:alon_max_ind] += agg_data['fraction']
unit_hydrograph[:, alat_min_ind:alat_max_ind, alon_min_ind:alon_max_ind] += agg_data['unit_hydrograph']
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Mask and Normalize the unit unit_hydrograph
if (maskandnorm):
# Normalize the unit_hydrograph (each cell should sum to 1)
yv, xv = np.nonzero(fraction > 0.0)
unit_hydrograph[:, yv, xv] /= unit_hydrograph[:, yv, xv].sum(axis=0)
# Mask the unit_hydrograph and make sure they sum to 1 at each grid cell
ym, xm = np.nonzero(fraction <= 0.0)
unit_hydrograph[:, ym, xm] = FILLVALUE_F
log.info('Completed final aggregation step')
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Put all the data into agg_data variable and return to main
agg_data['timesteps'] = in_data['timesteps']
agg_data['unit_hydrograph_dt'] = in_data['unit_hydrograph_dt']
agg_data['lon'] = lons
agg_data['lat'] = lats
agg_data['fraction'] = fraction
agg_data['unit_hydrograph'] = unit_hydrograph
# ---------------------------------------------------------------- #
return agg_data
print '-------------------------------'
print 'Halo ID:',id
print 'log10(M_DM):',np.round(np.log10(DM_mass),3)
print 'log10(M_*):',np.round(np.log10(stellar_mass),3)
stellar_mass_wrec=0.0 #stellar masses corrected for recycling
for si,s in enumerate(stars):
mass=s[1]
z=(1./s[2])-1
Z=s[3]
Zi=u.find_nearest(mets,Z)
Zn=mets[Zi]
age=np.interp(z,redshifts,ages)*1000.
agei=u.find_nearest(ssp[Zn]['age'],age)
agen=ssp[Zn]['age'][agei]
stellar_mass_wrec+=mass*ssp[Zn]['frac'][agei]
ste_sed=(10**10)*mass*ssp[Zn]['sed'][agei] # stellar SED for just this star!
Hbeta_flux=mass*(10**10)*ssp[Zn]['nebula_lines']['fluxes']['Hbeta'][agei]
if si==0:
stellar_sed=ste_sed
for scope, title in zip([10, 8, 11, 14], ['Maximal Transmission (10Hz)', 'Maximal Transmission (2Hz)', 'Maximal Coupling (10Hz)', 'Anti-Symmetric Cavity Reflection']):
time, channel_1_data, channel_2_data, channel_1_props, channel_2_props = read_folder(scope)
ch2_freq, ch2_linvoltage = linear_volt2freq(channel_2_data)
boot_plots(time, [channel_1_data, channel_2_data, ch2_linvoltage], ['CH1', 'CH2', 'CH2 fit'], 'Time(s)')
plt.title('%s : Raw Data' % title)
plt.savefig(str(graph_dir.joinpath('scope_%s_raw.png' % scope)))
plt.close()
axes = boot_plots(ch2_freq, [channel_1_data], ['Cavity Response'], 'Frequency(MHz)')
if scope in (8, 10):
point_labeller(axes, 'max', ch2_freq, channel_1_data)
# We also wish to find FWHM here... jenky code but its one time use
max_voltage = channel_1_data.max()
max_index = np.argmax(channel_1_data)
FWHM_index_low, FWHM_index_high = find_nearest(channel_1_data[:max_index], max_voltage/2), max_index + find_nearest(channel_1_data[max_index:], max_voltage/2)
FEHM_low, FWHM_high = ch2_freq[FWHM_index_low], ch2_freq[FWHM_index_high]
FWHM = abs(FWHM_high - FEHM_low)
title = '%s : FWHM = %.2f MHz' % (title, FWHM)
plt.title(title)
plt.savefig(str(graph_dir.joinpath('scope_%s.png' % scope)))
plt.close()
Q_l = ch2_freq[max_index]/FWHM
print('For graph %s we have %s as the quality factor' % (title, Q_l))
gamma = gamma_sq(Q_l, ch2_freq[max_index], ch2_freq)
axes = boot_plots(ch2_freq, [gamma], ['Power Reflection Coefficent '], 'Frequency(MHz)', y_axis='$|\Gamma|^2$')
print '-------------------------------'
print '-------------------------------'
print 'Halo ID:', id
print 'log10(M_DM):', np.round(np.log10(DM_mass), 3)
print 'log10(M_*):', np.round(np.log10(stellar_mass), 3)
stellar_mass_wrec = 0.0 #stellar masses corrected for recycling
for si, s in enumerate(stars):
mass = s[1]
z = (1. / s[2]) - 1
Z = s[3]
Zi = u.find_nearest(mets, Z)
Zn = mets[Zi]
age = np.interp(z, redshifts, ages) * 1000.
agei = u.find_nearest(ssp[Zn]['age'], age)
agen = ssp[Zn]['age'][agei]
stellar_mass_wrec += mass * ssp[Zn]['frac'][agei]
ste_sed = (10**10) * mass * ssp[Zn]['sed'][
agei] # stellar SED for just this star!
Hbeta_flux = mass * (
10**10) * ssp[Zn]['nebula_lines']['fluxes']['Hbeta'][agei]
if si == 0:
