def __init__(self): global transport variables = ['meridional_transport','psi'] num_cores = 6 data = np.ones((len(variables),len(scow.months),scow.latitude.shape[0],scow.longitude.shape[0]))*np.nan beta = c.beta.repeat(scow.longitude.shape[0]).reshape((c.beta.shape[0],scow.longitude.shape[0])) for i in xrange(scow.data.shape[1]): transport = scow.data[2,i,:,:]/beta psi = Parallel(n_jobs=num_cores)(delayed(integration)(lat) for lat in scow.latitude) psi = np.array(psi) D = np.array([transport.copy()/(c.rho*1.e+6),psi.copy()/(c.rho*1.e+6)]) data[:,i,:,:] = D del transport # Here I could derive psi in y and get the zonal sverdrup transport (I think I won't need it) # "Isolating" the subtropical gyre ibad = (np.abs(scow.latitude) <= 5) | (np.abs(scow.latitude) >= 50) data[:,:,ibad,:] = np.nan self.latitude = scow.latitude self.longitude = scow.longitude self.variables = variables self.data = data
def preprocess(frames, thresh=90):
'''frames is an iterable of numpy.array objects.
uses scharr filter to get cell edges from DIC channel,
scaling down to uint8 dtypes.
returns a stack of image list of frames.
preprocessing should be done with this output
to avoid normalizing intensity with respect
to variable empirical parameters.'''
md1 = np.median(frames[0])
mx1 = np.max(frames[0]) #nearly 65535 for uint16
md2 = np.median(frames[-1])
mx2 = np.max(frames[-1])
md = md1 / 2 + md2 / 2
mx = mx1 / 2 + mx2 / 2
def processInput(img):
fno = img.frame_no
img = 255 / (mx - md) * (img - md)
edges = np.sqrt(scharr_h(img)**2 + scharr_v(img)**2)
return pims.Frame(np.uint8(edges), frame_no=fno + 1)
start = time.time()
inputs = frames
edges = Parallel(n_jobs=num_cores,
verbose=.1)(delayed(processInput)(img) for img in inputs)
end = time.time()
print('{} seconds elapsed taking scharr filtration.'.format(
np.around(end - start, 1)))
#denoise the background of the scharr transform with a lower bound threshold edge intensity
imgs = edges.copy()
for img in imgs:
img[img < thresh] = 0
img[img > thresh] = 255
return imgs
def __init__(self): global transport variables = ['zonal_transport','meridional_transport','psi'] num_cores = 6 data = np.ones((len(variables),len(scow.months),scow.latitude.shape[0],scow.longitude.shape[0]))*np.nan f = c.f.repeat(scow.longitude.shape[0]).reshape((c.f.shape[0],scow.longitude.shape[0])) for i in xrange(scow.data.shape[1]): zonal_transport = scow.data[1,i,:,:]/(f*c.rho) meridional_transport = -scow.data[0,i,:,:]/(f*c.rho) transport = meridional_transport.copy() psi = Parallel(n_jobs=num_cores)(delayed(integration)(lat) for lat in scow.latitude) psi = np.array(psi) D = np.array([zonal_transport.copy()/1.e+6,meridional_transport.copy()/1.e+6,psi.copy()/1.e+6]) data[:,i,:,:] = D del transport # "Isolating" the subtropical gyre ibad = (np.abs(scow.latitude) <= 5) | (np.abs(scow.latitude) >= 50) data[:,:,ibad,:] = np.nan self.latitude = scow.latitude self.longitude = scow.longitude self.variables = variables self.data = data
def _update_filters(self, X):
if self.verbose:
last_score = self._bound(X)
start_t = time.time()
U = Parallel(n_jobs=self.n_jobs)(delayed(global_update_U)(
X[:, j], self.U[:, j], self.gamma[j], self.alpha, self.nu,
self.rho, self.EA, self.ElogA, self.verbose)
for j in xrange(self.n_feats))
U = np.vstack(U).T
self.U = U.copy()
if self.verbose:
score = self._bound(X)
print_increment('U', last_score, score)
last_score = score
self._update_gamma(X)
if self.verbose:
score = self._bound(X)
print_increment('gamma', last_score, score)
last_score = score
self._update_alpha(X)
if self.verbose:
score = self._bound(X)
print_increment('alpha', last_score, score)
if self.verbose:
t = time.time() - start_t
print('Update free parameters\ttime: %.2f' % t)
def _update_filters(self, X):
if self.verbose:
last_score = self._bound(X)
start_t = time.time()
U = Parallel(n_jobs=self.n_jobs)(
delayed(global_update_U)(
X[:, j], self.U[:, j], self.gamma[j], self.alpha,
self.nu, self.rho, self.EA, self.ElogA, self.verbose
)
for j in xrange(self.n_feats)
)
U = np.vstack(U).T
self.U = U.copy()
if self.verbose:
score = self._bound(X)
print_increment('U', last_score, score)
last_score = score
self._update_gamma(X)
if self.verbose:
score = self._bound(X)
print_increment('gamma', last_score, score)
last_score = score
self._update_alpha(X)
if self.verbose:
score = self._bound(X)
print_increment('alpha', last_score, score)
if self.verbose:
t = time.time() - start_t
print('Update free parameters\ttime: %.2f' % t)
def rollingcv(grid, dat, orizzonte, par=False, intercept=True, ensemble=False):
'''
Here I run rolling window cross validation. That is:
calling ---- the training data and **** the validation data
we iteratively run:
iter 1.) -------****
iter 2.) --------****
iter 3.) ---------****
iter 4.) ----------****
Then we take the validation error over the iterations
'''
if par:
num_cores = multiprocessing.cpu_count() - 1
print('Running in parallel')
error = Parallel(n_jobs=num_cores, backend='multiprocessing')(
delayed(fun_rolling)(i, grid, dat, orizzonte, intercept)
for i in range(len(grid)))
else:
print('Running not in parallel: safest choice at the moment')
error = [
fun_rolling(i, grid, dat, orizzonte, intercept)
for i in range(len(grid))
]
if ensemble:
lam = []
tht = []
er = []
full_error_path = error.copy()
min_error = min(error)
max_error = max(error)
prog = True
model = 0
max_models = 100
while prog:
idx, val = min(enumerate(error), key=itemgetter(1))
lam.append(np.round(grid[idx]['lambda'], 5))
tht.append(np.round(grid[idx]['theta'], 5))
er.append(np.round(error[idx], 4))
## Set the current minimum value to the max
## so that the second smallest value is the new minimum
error[idx] = max_error
model += 1
if val > min_error * 1.2 or model == max_models:
prog = False
return (full_error_path, er, lam, tht)
else:
idx = min(enumerate(error), key=itemgetter(1))[0]
return (error, grid[idx]['lambda'], grid[idx]['theta'])
def sim_matrix(inchis):
"""Computes pairwise similarity matrix between all compounds in the `inchis` list.
Parameters
----------
inchis : list
A list of inchi strings
Returns
-------
np.ndarray
"""
n_total = len(inchis)
sims = Parallel(n_jobs=-1, verbose=100, backend="multiprocessing")(
delayed(parallel_wrapper)(MolFromInchi(inchi), inchis[(idx +
1):], n_total)
for idx, inchi in enumerate(inchis))
sims = np.stack(sims)
sims += sims.copy().T
sims += np.eye(n_total)
return sims
def __init__(self):
global transport
variables = ['zonal_transport', 'meridional_transport', 'psi']
num_cores = 6
data = np.ones(
(len(variables), len(scow.months), scow.latitude.shape[0],
scow.longitude.shape[0])) * np.nan
f = c.f.repeat(scow.longitude.shape[0]).reshape(
(c.f.shape[0], scow.longitude.shape[0]))
for i in xrange(scow.data.shape[1]):
zonal_transport = scow.data[1, i, :, :] / (f * c.rho)
meridional_transport = -scow.data[0, i, :, :] / (f * c.rho)
transport = meridional_transport.copy()
psi = Parallel(n_jobs=num_cores)(delayed(integration)(lat)
for lat in scow.latitude)
psi = np.array(psi)
D = np.array([
zonal_transport.copy() / 1.e+6,
meridional_transport.copy() / 1.e+6,
psi.copy() / 1.e+6
])
data[:, i, :, :] = D
del transport
# "Isolating" the subtropical gyre
ibad = (np.abs(scow.latitude) <= 5) | (np.abs(scow.latitude) >= 50)
data[:, :, ibad, :] = np.nan
self.latitude = scow.latitude
self.longitude = scow.longitude
self.variables = variables
self.data = data
def loocv(grid, dat, par=False, intercept=True, ensemble=False):
'''
Leave one out cross validation (automatically implemented in the fit)
'''
if par:
num_cores = multiprocessing.cpu_count() - 1
print('Running in parallel')
error = Parallel(n_jobs=num_cores, backend='multiprocessing')(
delayed(fun_loocv)(i, grid, dat, intercept)
for i in range(len(grid)))
else:
print('Running not in parallel: safest choice at the moment')
error = [fun_loocv(i, grid, dat, intercept) for i in range(len(grid))]
### Ensemble method or note
if ensemble:
lam = []
tht = []
er = []
full_error_path = error.copy()
min_error = min(error)
max_error = max(error)
prog = True
model = 0
max_models = 100
while prog:
idx, val = min(enumerate(error), key=itemgetter(1))
lam.append(np.round(grid[idx]['lambda'], 5))
tht.append(np.round(grid[idx]['theta'], 5))
er.append(np.round(error[idx], 4))
## Set the current minimum value to the max
## so that the second smallest value is the new minimum
error[idx] = max_error
model += 1
if val > min_error * 1.2 or model == max_models:
prog = False
return (full_error_path, er, lam, tht)
else:
idx = min(enumerate(error), key=itemgetter(1))[0]
return (error, grid[idx]['lambda'], grid[idx]['theta'])
def __init__(self):
global transport
variables = ['meridional_transport', 'psi']
num_cores = 6
data = np.ones(
(len(variables), len(scow.months), scow.latitude.shape[0],
scow.longitude.shape[0])) * np.nan
beta = c.beta.repeat(scow.longitude.shape[0]).reshape(
(c.beta.shape[0], scow.longitude.shape[0]))
for i in xrange(scow.data.shape[1]):
transport = scow.data[2, i, :, :] / beta
psi = Parallel(n_jobs=num_cores)(delayed(integration)(lat)
for lat in scow.latitude)
psi = np.array(psi)
D = np.array([
transport.copy() / (c.rho * 1.e+6),
psi.copy() / (c.rho * 1.e+6)
])
data[:, i, :, :] = D
del transport
# Here I could derive psi in y and get the zonal sverdrup transport (I think I won't need it)
# "Isolating" the subtropical gyre
ibad = (np.abs(scow.latitude) <= 5) | (np.abs(scow.latitude) >= 50)
data[:, :, ibad, :] = np.nan
self.latitude = scow.latitude
self.longitude = scow.longitude
self.variables = variables
self.data = data
time_binary.reset_index(inplace=True)
time_binary.drop('index', axis=1, inplace=True)
# if observation took place before spray, zero out time
# else return elapsed time between spray and observation
print('negating sprays after traps...')
for col in time_binary.columns:
time_binary[col] = time_binary[col].map(lambda x: 0 if x < 0 else x)
print('done')
# https://chrisalbon.com/python/data_wrangling/pandas_rename_multiple_columns/
time_binary.columns = distance_binary.columns
time_binary_backup = time_binary.copy()
distance_binary_backup = distance_binary.copy()
time_tp = time_binary.transpose()
distance_tp = distance_binary.transpose()
time_tp = time_binary.transpose()
distance_tp = distance_binary.transpose()
def CalculateDistance(i):
distances = []
if i % 500 == 0:
print('evaluating time binaries ' + str(i))
d = i
cols = distance_tp[[d]]
# Now lets try and work out how many unique events we have just to compare
# with the GeoNet catalog of 20 events on this day in this sequence
for master in detections:
keep = True
for slave in detections:
if not master == slave and\
abs(master.detect_time - slave.detect_time) <= 6.0:
# If the events are within 6s of each other then test which
# was the 'best' match, strongest detection
if not master.detect_val > slave.detect_val:
keep = False
break
if keep:
unique_detections.append(master)
print('We made a total of ' + str(len(unique_detections)) + ' detections')
for detection in unique_detections:
print('Detection at :' + str(detection.detect_time) + ' for template ' +
detection.template_name + ' with a cross-correlation sum of: ' +
str(detection.detect_val))
# We can plot these too
stplot = st.copy()
template = templates[template_names.index(detection.template_name)]
lags = sorted([tr.stats.starttime for tr in template])
maxlag = lags[-1] - lags[0]
stplot.trim(starttime=detection.detect_time - 10,
endtime=detection.detect_time + maxlag + 10)
plotting.detection_multiplot(stplot, template,
[detection.detect_time.datetime])
# Now lets try and work out how many unique events we have just to compare
# with the GeoNet catalog of 20 events on this day in this sequence
for master in detections:
keep = True
for slave in detections:
if not master == slave and\
abs(master.detect_time - slave.detect_time) <= 6.0:
# If the events are within 6s of each other then test which
# was the 'best' match, strongest detection
if not master.detect_val > slave.detect_val:
keep = False
break
if keep:
unique_detections.append(master)
print('We made a total of ' + str(len(unique_detections)) + ' detections')
for detection in unique_detections:
print('Detection at :' + str(detection.detect_time) + ' for template ' +
detection.template_name + ' with a cross-correlation sum of: ' +
str(detection.detect_val))
# We can plot these too
stplot = st.copy()
template = templates[template_names.index(detection.template_name)]
lags = sorted([tr.stats.starttime for tr in template])
maxlag = lags[-1] - lags[0]
stplot.trim(starttime=detection.detect_time - 10,
endtime=detection.detect_time + maxlag + 10)
plotting.detection_multiplot(stplot, template,
[detection.detect_time.datetime])
