def convert_data_to_common_fmt(self):
""" Common Data to Common format """
from numpy import ma
header = ""
unitheader = ""
items = self.masterparamlist.items()
items.sort()
for param, unit in items: # assumes this will be displayed alpabetically
header = header + param + ", "
unitheader = unitheader + unit + ", "
try:
idx = self.paramlist[param][1]
try:
finaldata = ma.column_stack((finaldata, self.data[:, idx]))
except:
finaldata = self.data[:, idx]
except:
tmp = self.data[:, 0].copy()
tmp.mask = True
try:
finaldata = ma.column_stack((finaldata, tmp))
except:
finaldata = tmp
self.fieldheader = header
self.unitheader = unitheader
self.finaldata = ma.column_stack((self.dates, finaldata))
def convert_data_to_common_fmt(self):
""" Common Data to Common format """
from numpy import ma
header = ''
unitheader = ''
items = self.masterparamlist.items()
items.sort()
for param, unit in items: #assumes this will be displayed alpabetically
header = header + param + ', '
unitheader = unitheader + unit + ', '
try:
idx = self.paramlist[param][1]
try:
finaldata = ma.column_stack((finaldata, self.data[:, idx]))
except:
finaldata = self.data[:, idx]
except:
tmp = self.data[:, 0].copy()
tmp.mask = True
try:
finaldata = ma.column_stack((finaldata, tmp))
except:
finaldata = tmp
self.fieldheader = header
self.unitheader = unitheader
self.finaldata = ma.column_stack((self.dates, finaldata))
def reference_phase_generator(**kwarg):
sampling_frequency = kwarg["sampling_frequency"]
communication_frequency = kwarg["communication_frequency"]
sampling_time = 1 / sampling_frequency
communication_time = 1 / communication_frequency
ratio = communication_time / sampling_time
preamble = array([1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0])
inPhase_elements = preamble[::2]
quadrature_elements = preamble[1::2]
inPhase_elements = inPhase_elements * 2 - 1
quadrature_elements = quadrature_elements * 2 - 1
# creation of pulse shaping in sin
k = arange(0, 2 * communication_time, sampling_time)
pulse_shape = sin((math.pi * k) / (2 * communication_time))
pulse_shape = repmat(pulse_shape, 16, 1)
# Oversample of I & Q and blocker order 0
Iup = repmat(inPhase_elements, int(2 * ratio), 1)
Qup = repmat(quadrature_elements, int(2 * ratio), 1)
# signal pulse shaping
Iup = Iup * pulse_shape.transpose()
Qup = Qup * pulse_shape.transpose()
# wave shaping
Iup = Iup.reshape(1, int(16 * 2 * ratio))
Qup = Qup.reshape(1, int(16 * 2 * ratio))
# repeat the matrix for 8 symbols
I8up = repmat(Iup, 1, 8)
Q8up = repmat(Qup, 1, 8)
# application of 90 degrees offset on Q for reference only
Iup = column_stack((Iup, zeros([1, 10], 'int')))
Qup = column_stack((zeros([1, 10], 'int'), Qup))
# construction of a phase cycle
signal = Iup + 1j * Qup
phase_cycle = unwrap(angle(signal))
phase_cycle = array(phase_cycle[0, 10:-10])
# applying the 90 degrees offset on Q
I8up = column_stack((I8up, zeros([1, 10], 'int')))
Q8up = column_stack((zeros([1, 10], 'int'), Q8up))
# complex signal shaping
signal = I8up + 1j * Q8up
# creation of the theoretical phase
phase_ref = unwrap(angle(signal))
phase_ref = phase_ref[0, 10:]
return phase_ref
def postProcessor_burst_collector(self):
# saving the whole burst in the data-bank
self.collected_burst = self.collected_burst[1:]
self.collected_burst = reshape(self.collected_burst,
size(self.collected_burst), 1)
if size(self.data_collection) == 0:
self.data_collection = self.collected_burst
else:
self.data_collection = column_stack(
(self.data_collection, self.collected_burst))
self.label_vector = column_stack(
(self.label_vector, self.device_label))
self.collected_burst = 0
def finger_print_burst_collector(self):
# saving the whole burst in the data-bank
self.collected_burst = self.collected_burst[1:]
self.collected_burst = reshape(self.collected_burst,
[size(self.collected_burst, 0), 1])
if size(self.data_collection) == 0:
self.data_collection = self.collected_burst
else:
self.data_collection = column_stack(
(self.data_collection, self.collected_burst))
self.label_vector = column_stack(
(self.label_vector, self.device_label))
self.collected_burst = 0
def __Read_GDHY(v_name, year, lon_min, lon_max, lat_min, lat_max):
f = netCDF4.Dataset("RawData/yields/" + v_name + "/yield_" +
str(year) + ".nc4")
lats = f.variables['lat'][:] # degrees north (360)
lons = f.variables['lon'][:] # degrees east (720)
var = f.variables["var"][:] # yield in t/ha (360,720)
f.close()
# rearrange in right format
var_lonpos = var[:, lons < 180]
var_lonneg = var[:, lons > 180]
var_changed = np_ma.column_stack([var_lonneg, var_lonpos])
lons = np.arange(-179.75, 180, 0.5)
# define longitudes/latitudes corresponding to West Africa
lons_WA = lons[(lons >= lon_min) & (lons <= lon_max)]
lats_WA = lats[(lats >= lat_min) & (lats <= lat_max)]
# reduce to region of West Africa and set missing values to NAN
# shifted by one cell in both directions
var_rel = var_changed[((lats>=(lat_min+0.5)) & (lats<=(lat_max+0.5))),:] \
[:,((lons>=(lon_min+0.5)) & (lons<=(lon_max+0.5)))]
var_rel.set_fill_value(value=np.nan)
var_rel_filled = var_rel.filled()
mask_rel = var_rel.mask
return (var_rel_filled, mask_rel, lats_WA, lons_WA)
def calc_sal_EC(self, param, val, idx):
"""
convert non normalised conductivity to salinity NOTE MUST BE DONE LAST AFTER OTHER CONVERSIONS
using seawater package
"""
print "~ calculating salinity from non normalised EC using seawater package~"
from numpy import ma
import seawater
self.paramlist[param] = ("PSU", self.paramlist[param][1])
T = val[:, self.paramlist["Temperature"][1]]
cond = val[:, self.paramlist["EC_Norm"][1]]
# absolute pressure in dbar
P = val[:, self.paramlist["WaterLevel(Non_Vented)"][1]] * 1.0197 + 10.1325
# if (self.paramlist['EC_Norm'][0]!='ms/cm':
# cond = self.conversion_factor[self.paramlist['EC_Norm'][0]][cond]
R = cond / 42.914
self.paramlist[param] = ("EC_Norm", self.paramlist[param][1])
sal = seawater.salt(R, T, P)
return ma.column_stack((val, sal))
def calc_sal_EC(self, param, val, idx):
"""
convert non normalised conductivity to salinity NOTE MUST BE DONE LAST AFTER OTHER CONVERSIONS
using seawater package
"""
print '~ calculating salinity from non normalised EC using seawater package~'
from numpy import ma
import seawater
self.paramlist[param] = ('PSU', self.paramlist[param][1])
T = val[:, self.paramlist['Temperature'][1]]
cond = val[:, self.paramlist['EC_Norm'][1]]
#absolute pressure in dbar
P = val[:,
self.paramlist['WaterLevel(Non_Vented)'][1]] * 1.0197 + 10.1325
#if (self.paramlist['EC_Norm'][0]!='ms/cm':
# cond = self.conversion_factor[self.paramlist['EC_Norm'][0]][cond]
R = cond / 42.914
self.paramlist[param] = ('EC_Norm', self.paramlist[param][1])
sal = seawater.salt(R, T, P)
return (ma.column_stack((val, sal)))
def extend_interp(datafield): # add masked values at southernmost end southernlimitmask = ma.masked_all(len(self.olon)) olat_ext = np.append(-82.1,self.olat) dfield_ext = ma.concatenate([ma.column_stack(southernlimitmask), datafield], 0) # f = interp2d(self.olon, olat_ext, dfield_ext) # return f(self.pismlon, self.pismlat) return interp(dfield_ext, self.olon, olat_ext, self.pismlon, self.pismlat)
def calc_sal_MCTD(self, param, val, idx):
"""
convert conductivity to salinity NOTE MUST BE DONE LAST AFTER OTHER CONVERSIONS
http://www.coastal-usa.com/The%20Macro%20Manual.pdf
"""
print "~ calculating MCTD salinity ~"
from numpy import ma
import seawater
self.paramlist[param] = ("PSU", self.paramlist[param][1])
T = val[:, self.paramlist["Temperature"][1]]
cond = val[:, self.paramlist["EC_Norm"][1]]
# absolute pressure in dbar
P = val[:, self.paramlist["WaterLevel(Non_Vented)"][1]] * 1.0197 + 10.1325
# if (self.paramlist['EC_Norm'][0]!='ms/cm':
# cond = self.conversion_factor[self.paramlist['EC_Norm'][0]][cond]
R = cond / 42.914
# sal2 = seawater.salt(R,T,P)
# pressure correction
F = (1.60836 * pow(10, -5)) * P - (5.4845 * pow(10, -10)) * pow(P, 2) + (6.166 * pow(10, -15)) * pow(P, 3)
F = F / (1 + (3.0786 * pow(10, -2)) * T + (3.169 * pow(10, -4)) * pow(T, 2))
R = R / (1 + F)
# Temperature Correction
T100 = T / 100.0
R = R / (
0.6765836 + 2.005294 * (T100) + 1.11099 * pow(T100, 2) - 0.726684 * pow(T100, 3) + 0.13587 * pow(T100, 4)
)
# salinity
self.paramlist[param] = ("EC_Norm", self.paramlist[param][1])
sal = (
-0.8996
+ 28.8567 * R
+ 12.18882 * pow(R, 2)
- 10.61869 * pow(R, 3)
+ 5.9862 * pow(R, 4)
- 1.32311 * pow(R, 5)
+ R
* (R - 1)
* (
0.0442 * T
- 0.46 * pow(10, -3) * pow(T, 2)
- 4 * pow(10, -3) * R * T
+ (1.25 * pow(10, -4) - 2.9 * pow(10, -6) * T) * P
)
)
return ma.column_stack((val, sal))
def calc_sal_MCTD(self, param, val, idx):
"""
convert conductivity to salinity NOTE MUST BE DONE LAST AFTER OTHER CONVERSIONS
http://www.coastal-usa.com/The%20Macro%20Manual.pdf
"""
print '~ calculating MCTD salinity ~'
from numpy import ma
import seawater
self.paramlist[param] = ('PSU', self.paramlist[param][1])
T = val[:, self.paramlist['Temperature'][1]]
cond = val[:, self.paramlist['EC_Norm'][1]]
#absolute pressure in dbar
P = val[:,
self.paramlist['WaterLevel(Non_Vented)'][1]] * 1.0197 + 10.1325
#if (self.paramlist['EC_Norm'][0]!='ms/cm':
# cond = self.conversion_factor[self.paramlist['EC_Norm'][0]][cond]
R = cond / 42.914
#sal2 = seawater.salt(R,T,P)
#pressure correction
F = (1.60836 * pow(10, -5)) * P - (5.4845 * pow(10, -10)) * pow(
P, 2) + (6.166 * pow(10, -15)) * pow(P, 3)
F = F / (1 + (3.0786 * pow(10, -2)) * T +
(3.169 * pow(10, -4)) * pow(T, 2))
R = R / (1 + F)
#Temperature Correction
T100 = T / 100.0
R = R / (0.6765836 + 2.005294 * (T100) + 1.11099 * pow(T100, 2) -
0.726684 * pow(T100, 3) + 0.13587 * pow(T100, 4))
#salinity
self.paramlist[param] = ('EC_Norm', self.paramlist[param][1])
sal = -0.8996 + 28.8567 * R + 12.18882 * pow(R, 2) - 10.61869 * pow(
R, 3) + 5.9862 * pow(R, 4) - 1.32311 * pow(R, 5) + R * (
R - 1) * (0.0442 * T - 0.46 * pow(10, -3) * pow(T, 2) -
4 * pow(10, -3) * R * T +
(1.25 * pow(10, -4) - 2.9 * pow(10, -6) * T) * P)
return (ma.column_stack((val, sal)))