def block_average(timeseries, new_freq=''):
"""
Reduce size of timeseries by taking averages of larger block size.
Input: timeseries, new_freq (str) See scikits.timeseries doc
Output: block averaged timeseries obj. in new frequency
"""
# Label timeseries data with new frequency
# ie: [5.5, 4.5] | [13-May-2009 11:40 13-May-2009 11:50] becomes
# [5.5, 4.5] | [13-May-2009 13-May-2009]
timeseries = timeseries.asfreq(new_freq)
# Create empty arrays, set first block_time
current_block_values = []
averages = []
timesteps = []
current_block_time = timeseries.dates[0]
# For each index in timeseries, if the block of time has changed,
# average the previous block values. Otherwise keep adding
# values to be averaged.
for index in range(0, len(timeseries)):
if current_block_time != timeseries.dates[index]:
averages.append(npmean(current_block_values))
timesteps.append(current_block_time)
current_block_values = []
current_block_time = timeseries.dates[index]
current_block_values.append(timeseries[index])
# Take average for last (or only) time block
if current_block_values:
averages.append(npmean(current_block_values))
timesteps.append(current_block_time)
# Return new block averages and timesteps as timeseries object
return ts.time_series(averages, dates=timesteps)
def block_average(timeseries, new_freq=''):
"""
Reduce size of timeseries by taking averages of larger block size.
Input: timeseries, new_freq (str) See scikits.timeseries doc
Output: block averaged timeseries obj. in new frequency
"""
# Label timeseries data with new frequency
# ie: [5.5, 4.5] | [13-May-2009 11:40 13-May-2009 11:50] becomes
# [5.5, 4.5] | [13-May-2009 13-May-2009]
timeseries = timeseries.asfreq(new_freq)
# Create empty arrays, set first block_time
current_block_values = []
averages = []
timesteps = []
current_block_time = timeseries.dates[0]
# For each index in timeseries, if the block of time has changed,
# average the previous block values. Otherwise keep adding
# values to be averaged.
for index in range(0,len(timeseries)):
if current_block_time != timeseries.dates[index]:
averages.append(npmean(current_block_values))
timesteps.append(current_block_time)
current_block_values = []
current_block_time = timeseries.dates[index]
current_block_values.append(timeseries[index])
# Take average for last (or only) time block
if current_block_values:
averages.append(npmean(current_block_values))
timesteps.append(current_block_time)
# Return new block averages and timesteps as timeseries object
return ts.time_series(averages,dates=timesteps)
def scale_X(x_train, x_test, image=False):
"""
Function to scale the sequences from cost
Depending if the image arg is activated they are scaled
as sequences or images
Args
----
x_train:
x_test
image: boolean, to standardize as an image or as a sequence
Returns
-----
x_train_esc
x_test_esc
scaler: can be a Scaler object from sklearn or a list of [mean, std]
depending on the arg image
"""
if image:
mean = npmean(vstack(x_train).reshape(-1, 8, 8), axis=0)
std = npstd(vstack(x_train).reshape(-1, 8, 8), axis=0)
x_train_esc = [(asarray(i).reshape(-1, 8, 8) - mean) / std
for i in x_train]
x_test_esc = [(asarray(i).reshape(-1, 8, 8) - mean) / std
for i in x_test]
scaler = [mean, std]
else:
scaler = StandardScaler()
scaler.fit(vstack(x_train))
x_train_esc = [scaler.transform(i).tolist() for i in x_train]
x_test_esc = [scaler.transform(i).tolist() for i in x_test]
return x_train_esc, x_test_esc, scaler
def ClusterSummary(i, j, MaskedCluster):
# print(str(i) + "," + str(j))
# print(MaskedCluster.shape)
Temp = MaskedCluster
FinalClusterMean = 0
FinalClusterSd = 0
NonZeroIndex = npnonzero(Temp)
if npsum(NonZeroIndex) == 0:
return([FinalClusterMean,FinalClusterSd,i,j])
#print(npsum(Temp > 0))
TempNonZero = Temp[NonZeroIndex]
#TempNonzeronan = TempNonZero[npisnan(TempNonZero, where=False)]
TempNonzeronan = TempNonZero[~npisnan(TempNonZero)]
if(TempNonzeronan.size > 0):
# FinalClusterMean = npmean(TempNonzeronan)
# FinalClusterSd = npstd(TempNonzeronan)
with warnings.catch_warnings():
warnings.filterwarnings('error')
try:
FinalClusterMean = npmean(TempNonZero)
FinalClusterSd = npstd(TempNonZero)
except RuntimeWarning:
FinalClusterMean = 0
FinalClusterSd = 0
return([FinalClusterMean,FinalClusterSd,i,j])
def ave_power(self, mode="lin"):
"""
事前にスライスされた帯域のスペクトルパワーの平均を算出
:param mode:
:return: 平均パワー[db]
"""
from numpy import mean as npmean
from numpy import log10 as nplog10
power = self.get_pow()
if mode == "lin":
average_power = npmean(power)
return 10 * nplog10(average_power)
elif mode == "log":
average_power = npmean(10 * nplog10(power))
return average_power
def ave_power(self, mode="lin"):
"""
事前にスライスされた帯域のスペクトルパワーの平均を算出
:param mode:
:return: 平均パワー[db]
"""
from numpy import mean as npmean
from numpy import log10 as nplog10
power = self.get_pow()
if mode == "lin":
average_power = npmean(power)
return 10*nplog10(average_power)
elif mode == "log":
average_power = npmean(10 * nplog10(power))
return average_power
def rigid_transform_3D(MatA, MatB):
'''
Pass in 2 numpy arrays to get the R and t
'''
assert len(MatA) == len(MatB)
N = MatA.shape[0]
comA = npmean(MatA, axis=0)
comB = npmean(MatB, axis=0)
A = MatA - nptile(comA, (N, 1))
B = MatB - nptile(comB, (N, 1))
H = npdot(nptranspose(A), B)
U, S, V = nplinalgsvd(H)
R = npdot(nptranspose(V), nptranspose(U))
if nplinalgdet(R) < 0:
V[2, :] *= -1
R = npdot(nptranspose(V), nptranspose(U))
t = -npdot(R, nptranspose(comA)) + nptranspose(comB)
return R, t
def silhouette(dataset, centers_coord, inst_cluster_id): # Silhouette is a way of summarize the BSS and SSE # metrics into a single measure value. So, obvious enough, # it is also a INNER CRITERION for unsupervised quality metrics. inst_num = dataset.shape[0] inst_sil = array([0.0] * inst_num) center_num = len(centers_coord) for center_id in range(center_num): this_cluster_insts = \ where(inst_cluster_id == center_id)[0] for inst_id in this_cluster_insts: # Calculate average distance of this # instance with every other instance # of the SAME cluster inst_avg_dist_intracluster = \ max(abs(npmean(dataset[this_cluster_insts,:] -\ dataset[inst_id,:], axis=0))) # Calculate the MINIMAL average distance # of this instance with every other ins- # tance of DIFFERENT clusters for each # other cluster inst_min_avg_dist_intercluster = inf for i in range(center_num): if i != center_id: inst_min_avg_dist_intercluster =\ min(inst_min_avg_dist_intercluster, \ max(abs(npmean(dataset[inst_cluster_id == i,:] -\ dataset[inst_id,:], axis=0)))) # Calculate this instance silhouette inst_sil[inst_id] = \ (inst_min_avg_dist_intercluster - \ inst_avg_dist_intracluster)/\ max(inst_min_avg_dist_intercluster, \ inst_avg_dist_intracluster) return npmean(inst_sil)
def OLDClusterSummary(i, MaskedClusterAllBands, NumberOfBands): FinalClusterMean = zeros(NumberOfBands) FinalClusterSd = zeros(NumberOfBands) for j in range(0, NumberOfBands): #Mean = MaskedClusterAllBands(:,:,j) Temp = MaskedClusterAllBands[:, :, j] TempNonZero = Temp[npnonzero(Temp)] TempNonzeronan = TempNonZero[~npisnan(TempNonZero)] #TempNonan = Temp[!np.npisnan(Temp)] FinalClusterMean[j] = npmean(TempNonzeronan) FinalClusterSd[j] = npstd(TempNonzeronan) return([FinalClusterMean,FinalClusterSd,i])
def _calc_accel(jack_dist):
from numpy import mean as npmean
from numpy import sum as npsum
from numpy import errstate
jack_mean = npmean(jack_dist)
numer = npsum((jack_mean - jack_dist)**3)
denom = 6.0 * (npsum((jack_mean - jack_dist)**2)**1.5)
with errstate(invalid='ignore'):
# does not raise warning if invalid division encountered.
return numer / denom
def sse(dataset, centers_coord, inst_cluster_id): # SEE : Sum of Squared Errors # It measures Cluster's Cohesion quality # It is a INNER CRITERION for unsupervised quality metrics center_num = len(centers_coord) sse_value = 0.0 for center_id in range(center_num): this_cluster_insts = where(inst_cluster_id == center_id)[0] cluster_mean = npmean(dataset[this_cluster_insts,:], axis=0) sse_value += max(sum((dataset[this_cluster_insts,:] \ - cluster_mean)**2.0)) return sse_value
def bss(dataset, centers_coord, inst_cluster_id): # BSS : Between Sum of Squares # It measures Separation between clusters # It is a INNER CRITERION for unsupervised quality metrics center_num = len(centers_coord) dataset_mean = dataset.mean(axis=0) bss_value = 0.0 for center_id in range(center_num): this_cluster_insts = where(inst_cluster_id == center_id)[0] cluster_mean = npmean(dataset[this_cluster_insts,:], axis=0) sqr_coord_diffs = (dataset_mean - cluster_mean)**2.0 bss_value += len(this_cluster_insts) * max(sqr_coord_diffs) return bss_value
def visit_polygon(self,polygon):
""" Adds the fields to edit a polygon's basic qualities, like position """
# grab the coordinates from the polygon
coords = polygon.coordinates
# back up its coordinates, if they are not already backed up
try: polygon.pure_coordinates
except: polygon.pure_coordinates = polygon.coordinates.copy()
# define their center as their mean along each axis
mcoords = npround(npmean(coords,0),0)
coordview = tk.Label(self.window,text='Origin')
coord_setter = tk.Frame(self.window)
text_vars = tuple(StringVar() for i in range(4))
# loops don't create new scopes but methods do
# also it's pretty cool that i gets implied by the loop variable
def add_entry(event=None):
""" Adds an entry field to modify a given axis """
textVar = text_vars[i]
textVar.set(str(mcoords[i]))
e = tk.Entry(coord_setter,textvariable=textVar,width=12)
e.grid(column=i,row=0)
def shift_poly(event = None):
""" Applies the transformation to shift the polygon, does not redraw canvas """
try: # it could be that they entered a non-float
dx = tuple(mcoords[j] - float(text_vars[j].get()) for j in range(4))
except Exception as e: # in which case, just ignore it
dx = 0
# shift each coordinate by the displacement implied by the entry field
coords = [ [el + dx[j] for j,el in \
enumerate(coord)] for coord in polygon.pure_coordinates]
# update the polygon's coordinates (it expects a numpy object)
polygon.coordinates = nparray(coords)
polygon._dirty()
# bind to the entry field update-on-entry
e.bind('<Return>', shift_poly )
# add all four entry widgets to update 3 axes plus homogeneous
for i in range(4): add_entry(i)
self.fields.append(coordview)
self.fields.append(coord_setter)
def apply_dilution(mdl_indexes, dilution_factors, calculated_concentrations):
"""
Calculates the concentration before dilution
:param mdl_indexes:
:param dilution_factors:
:param calculated_concentrations:
:return:
"""
cc = nparray(calculated_concentrations)
df = nparray(dilution_factors[:-1])
mdl_concs = cc[mdl_indexes]
mdl = npmean(mdl_concs)
cc_2 = cc * df - (df - 1) * mdl
calculated_concentrations = list(cc_2)
return calculated_concentrations
def summarizeValues(self, dictSelectedRegions):
selectedRegions = [
region.values()[0] for region in dictSelectedRegions
]
if (self.summarizationMethod == "mean"):
from numpy import mean as npmean
return [{'mean': npmean(selectedRegions, axis=0).tolist()}]
elif (self.summarizationMethod == "max"):
import numpy as np
#1. CALCULATE THE SUM OF THE ABS VALUES FOR EACH REGION
valuesSum = np.sum(np.abs(selectedRegions), axis=1)
#2. GET THE MAX
maxSum = np.max(valuesSum)
#3. FIND THE POSITION OF MAX VALUE
#TODO: WHAT IF MORE THAN 1 MAX??
indices = [i for i, x in enumerate(valuesSum) if x == maxSum]
return [{'max': selectedRegions[indices[0]]}]
else:
return dictSelectedRegions
def tem_match(dat, pars=(), template=None):
try:
tem=template.getElements("Data", "template", depth=1)[0]
except:
print "Matching requires a template. Defaulting to Mean"
return None
td=tem.getData()
disc=zeros(dat.shape[0], dat.dtype)
for i in range(dat.shape[1]):
dc=dat[:,i]
tem=td[:,2*i]
err=match(dc, tem)
md=sum(tem**2)
err=minimum(err, md)
disc+=err
lead, length = getLL(pars, template)
mid = length - lead
tempoff = npmean(npargmin(td[mid-15:mid+15,range(0,dat.shape[1],2)]*2, axis=0))#since the template tends to be offest a touch
xtralen = length - (mid -(15-round(tempoff)))
disc = concatenate([disc[xtralen:], max(disc)*ones(xtralen)])
return disc
def pdw_period_find(times,
mags,
errs,
autofreq=True,
init_p=None,
end_p=None,
f_step=1.0e-4,
phasebinsize=None,
sigclip=10.0,
nworkers=None,
verbose=False):
'''This is the parallel version of the function above.
Uses the string length method in Dworetsky 1983 to calculate the period of a
time-series of magnitude measurements and associated magnitude errors. This
can optionally bin in phase to try to speed up the calculation.
PARAMETERS:
time: series of times at which mags were measured (usually some form of JD)
mag: timeseries of magnitudes (np.array)
err: associated errs per magnitude measurement (np.array)
init_p, end_p: interval to search for periods between (both ends inclusive)
f_step: step in frequency [days^-1] to use
RETURNS:
tuple of the following form:
(periods (np.array),
string_lengths (np.array),
good_period_mask (boolean array))
'''
# remove nans
find = npisfinite(times) & npisfinite(mags) & npisfinite(errs)
ftimes, fmags, ferrs = times[find], mags[find], errs[find]
mod_mags = (fmags - npmin(fmags)) / (2.0 *
(npmax(fmags) - npmin(fmags))) - 0.25
if len(ftimes) > 9 and len(fmags) > 9 and len(ferrs) > 9:
# get the median and stdev = 1.483 x MAD
median_mag = np.median(fmags)
stddev_mag = (np.median(np.abs(fmags - median_mag))) * 1.483
# sigclip next
if sigclip:
sigind = (np.abs(fmags - median_mag)) < (sigclip * stddev_mag)
stimes = ftimes[sigind]
smags = fmags[sigind]
serrs = ferrs[sigind]
LOGINFO('sigclip = %s: before = %s observations, '
'after = %s observations' %
(sigclip, len(times), len(stimes)))
else:
stimes = ftimes
smags = fmags
serrs = ferrs
# make sure there are enough points to calculate a spectrum
if len(stimes) > 9 and len(smags) > 9 and len(serrs) > 9:
# get the frequencies to use
if init_p:
endf = 1.0 / init_p
else:
# default start period is 0.1 day
endf = 1.0 / 0.1
if end_p:
startf = 1.0 / end_p
else:
# default end period is length of time series
startf = 1.0 / (stimes.max() - stimes.min())
# if we're not using autofreq, then use the provided frequencies
if not autofreq:
frequencies = np.arange(startf, endf, stepsize)
LOGINFO(
'using %s frequency points, start P = %.3f, end P = %.3f' %
(frequencies.size, 1.0 / endf, 1.0 / startf))
else:
# this gets an automatic grid of frequencies to use
frequencies = get_frequency_grid(stimes,
minfreq=startf,
maxfreq=endf)
LOGINFO('using autofreq with %s frequency points, '
'start P = %.3f, end P = %.3f' %
(frequencies.size, 1.0 / frequencies.max(),
1.0 / frequencies.min()))
# set up some internal stuff
fold_time = npmin(ftimes) # fold at the first time element
j_range = len(fmags) - 1
epsilon = 2.0 * npmean(ferrs)
delta_l = 0.34 * (epsilon - 0.5 *
(epsilon**2)) * (len(ftimes) -
npsqrt(10.0 / epsilon))
keep_threshold_1 = 1.6 + 1.2 * delta_l
l = 0.212 * len(ftimes)
sig_l = len(ftimes) / 37.5
keep_threshold_2 = l + 4.0 * sig_l
# generate the tasks
tasks = [(x, ftimes, mod_mags, fold_time, j_range,
keep_threshold_1, keep_threshold_2, phasebinsize)
for x in frequencies]
# fire up the pool and farm out the tasks
if (not nworkers) or (nworkers > NCPUS):
nworkers = NCPUS
LOGINFO('using %s workers...' % nworkers)
pool = Pool(nworkers)
strlen_results = pool.map(pdw_worker, tasks)
pool.close()
pool.join()
del pool
periods, strlens, goodflags = zip(*strlen_results)
periods, strlens, goodflags = (np.array(periods),
np.array(strlens),
np.array(goodflags))
strlensort = npargsort(strlens)
nbeststrlens = strlens[strlensort[:5]]
nbestperiods = periods[strlensort[:5]]
nbestflags = goodflags[strlensort[:5]]
bestperiod = nbestperiods[0]
beststrlen = nbeststrlens[0]
bestflag = nbestflags[0]
return {
'bestperiod': bestperiod,
'beststrlen': beststrlen,
'bestflag': bestflag,
'nbeststrlens': nbeststrlens,
'nbestperiods': nbestperiods,
'nbestflags': nbestflags,
'strlens': strlens,
'periods': periods,
'goodflags': goodflags
}
else:
LOGERROR(
'no good detections for these times and mags, skipping...')
return {
'bestperiod': npnan,
'beststrlen': npnan,
'bestflag': npnan,
'nbeststrlens': None,
'nbestperiods': None,
'nbestflags': None,
'strlens': None,
'periods': None,
'goodflags': None
}
else:
LOGERROR('no good detections for these times and mags, skipping...')
return {
'bestperiod': npnan,
'beststrlen': npnan,
'bestflag': npnan,
'nbeststrlens': None,
'nbestperiods': None,
'nbestflags': None,
'strlens': None,
'periods': None,
'goodflags': None
}
def mean(lst):
if len(lst) == 0:
return 0
return npmean(lst)
def run(dataset,
k,
it_max=1000,
min_variation=1.0e-4,
labels=None,
full_output=True):
# Init centers_coord
centers_id = random.randint(dataset.shape[0], size=k)
centers_coord = dataset[centers_id, :]
# Each instance will be put on a initial random cluster
inst_cluster_id = random.randint(k, size=dataset.shape[0])
# Auxiliary vectors to keep code cleaner
auxvec_cur_distances = array([0.0] * k)
prev_variation = 1.0 + min_variation
it = 0
while it < it_max and prev_variation >= min_variation:
it += 1
for inst_id in range(dataset.shape[0]):
nearest_center = inst_cluster_id[inst_id]
for center_id in range(k):
# For each instance, calculate the distance between
# each center
auxvec_cur_distances[center_id] = \
Kmeans.__euclideandist__(\
dataset[inst_id,:],
centers_coord[center_id,:])
# For each instance, let it be part of the nearest
# cluster
inst_cluster_id[inst_id] = argmin(auxvec_cur_distances)
# For each cluster, calculate the new center coordinates
for center_id in range(k):
part_aux = dataset[inst_cluster_id == center_id, :]
if part_aux.shape[0] > 0:
new_cur_cluster_coords = npmean(part_aux, axis=0)
# Calculate variation between previous centers_coord and
# new ones (using infinite norm)
prev_variation = max(prev_variation, \
max(abs(centers_coord[center_id] - \
new_cur_cluster_coords)))
centers_coord[center_id] = new_cur_cluster_coords
# Build up answer
ans = {
"inst_cluster_id": inst_cluster_id,
"centers_coord": centers_coord,
}
if full_output:
ans = {
"clustering_method" : "K-Means",
"k" : k,
**ans,
**ClusterMetrics.runall(
dataset=dataset,
centers_coord=centers_coord,
inst_cluster_id=inst_cluster_id,
labels=labels),
}
return ans
def hit_rate((rule, support)): return npmean( map(lambda p: hit_rate_pair(rule, support, p)[0], support) )
def lightcurve_ptp_measures(ftimes, fmags, ferrs):
'''
This calculates various point-to-point measures (eta in Kim+ 2014).
'''
ndet = len(fmags)
if ndet > 9:
timediffs = npdiff(ftimes)
# get rid of stuff with time diff = 0.0
nzind = npnonzero(timediffs)
ftimes, fmags, ferrs = ftimes[nzind], fmags[nzind], ferrs[nzind]
# recalculate ndet and diffs
ndet = ftimes.size
timediffs = npdiff(ftimes)
# calculate the point to point measures
p2p_abs_magdiffs = npabs(npdiff(fmags))
p2p_squared_magdiffs = npdiff(fmags) * npdiff(fmags)
robstd = npmedian(npabs(fmags - npmedian(fmags))) * 1.483
robvar = robstd * robstd
# these are eta from the Kim+ 2014 paper - ratio of point-to-point
# difference to the variance of the entire series
# this is the robust version
eta_robust = npmedian(p2p_abs_magdiffs) / robvar
eta_robust = eta_robust / (ndet - 1.0)
# this is the usual version
eta_normal = npsum(p2p_squared_magdiffs) / npvar(fmags)
eta_normal = eta_normal / (ndet - 1.0)
timeweights = 1.0 / (timediffs * timediffs)
# this is eta_e modified for uneven sampling from the Kim+ 2014 paper
eta_uneven_normal = ((npsum(timeweights * p2p_squared_magdiffs) /
(npvar(fmags) * npsum(timeweights))) *
npmean(timeweights) *
(ftimes.max() - ftimes.min()) *
(ftimes.max() - ftimes.min()))
# this is robust eta_e modified for uneven sampling from the Kim+ 2014
# paper
eta_uneven_robust = ((npsum(timeweights * p2p_abs_magdiffs) /
(robvar * npsum(timeweights))) *
npmedian(timeweights) * (ftimes[-1] - ftimes[0]) *
(ftimes[-1] - ftimes[0]))
return {
'eta_normal': eta_normal,
'eta_robust': eta_robust,
'eta_uneven_normal': eta_uneven_normal,
'eta_uneven_robust': eta_uneven_robust
}
else:
return None
def mean(numbers):
if numpy:
return npmean(numbers)
elif py3statistics:
return p3mean(numbers)
return float(sum(numbers)) / max(len(numbers), 1)
# log-returns
ret = diff(logprice_weekly)
# y = log(squared returns)
y = log(ret**2)
# -
# ## Fit the stochastic volatility model
# +
# initial parameters
phi0 = 0
phi1 = .99
sQ = 0.14
alpha = npmean(y)
sR0 = 0.9
mu1 = -2
sR1 = 2
initpar = [phi0, phi1, sQ, alpha, sR0, mu1, sR1]
param, fval, exitflag, output = FitStochasticVolatilityModel(y, initpar)
phi = param[0]
phi1 = param[1]
sQ = param[2]
alpha = param[3]
sR0 = param[4]
mu1 = param[5]
sR1 = param[6]
_, log_hiddenvol2 = FilterStochasticVolatility(y, phi0, phi1, sQ, alpha, sR0,
mu1, sR1)
def log_progress(sequence: list, every=None, size=None, name='Items', userProgress=None):
'''Creates a progress bar in jupyter notebooks.
Automatically detects the size of a list and estimates the best step
size for progress bar updates. This function also automatically estimates
the total time to completion of the iterations, updating the estimate using
the time that every step takes.
If the sequence argument is an iterator, the total number of elements cannot
determined. In this case, the user must define the `every` parameter to indicate
the update frequency of the progress bar.
If the progress bar is used in a nested loop, passing a list to the `userProgress`
argument will force the re-utilization of `ipywidgets` objects, preventing the
creation of a new progress bar at every iteration of the inner loop.
This progress bar was based on https://github.com/alexanderkuk/log-progress.
Args:
sequence : An iterable object.
every (int): The update frequency.
size (int): The number of elements in the sequence.
name (str): The name of the progress bar.
userProgress (list): List for creation of nested progress bars.
'''
from ipywidgets import IntProgress, HTML, HBox, Label
from IPython.display import display
from numpy import mean as npmean
from collections import deque
from math import floor
from datetime import datetime
from string import Template
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = floor(float(size)*0.005) # every 0.5%, minimum is 1
else:
assert every is not None, 'sequence is iterator, set every'
# For elapsed time
initTime = datetime.now()
totTime = "?"
labTempl = Template(" (~ $min total time (min) ; $ell minutes elapsed)")
# If provided, we use the objects already created.
# If not provided, we create from scratch.
if userProgress is None or userProgress == []:
progress = IntProgress(min=0, max=1, value=1)
label = HTML()
labelTime = Label("")
box = HBox(children=[label, progress, labelTime])
if userProgress == []:
userProgress.append(box)
display(box)
else:
box = userProgress[0]
if is_iterator:
#progress = IntProgress(min=0, max=1, value=1)
box.children[1].min = 0
box.children[1].max = 1
box.children[1].value = 1
box.children[1].bar_style = 'info'
else:
#progress = IntProgress(min=0, max=size, value=0)
box.children[1].min = 0
box.children[1].max = size
box.children[1].value = 0
# For remaining time estimation
deltas = deque()
lastTime = None
meandelta = 0
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
box.children[0].value = '{name}: {index} / ?'.format(
name=name,
index=index
)
else:
box.children[1].value = index
box.children[0].value = u'{name}: {index} / {size}'.format(
name=name,
index=index,
size=size
)
# Estimates remaining time with average delta per iteration
# Uses (at most) the last 30 iterations
if len(deltas) == 101:
deltas.popleft()
if lastTime:
deltas.append( (datetime.now() - lastTime).total_seconds() )
meandelta = npmean(deltas)/60.0 # From seconds to minute
totTime = round(meandelta*size/float(every), 3) # Mean iteration for all iterations
else:
totTime = "?" # First iteration has no time
lastTime = datetime.now()
# All ellapsed time in minutes
elapsed = round( (datetime.now() - initTime).total_seconds()/60.0, 3)
box.children[2].value = labTempl.safe_substitute({"min":totTime,
"ell":elapsed})
yield record
except:
box.children[1].bar_style = 'danger'
raise
else:
box.children[1].bar_style = 'success'
box.children[1].value = index
box.children[0].value = "{name}: {index}".format(
name=name,
index=str(index or '?')
)
plt.ylabel("Rate (Hz)")
plt.subplot(413)
plt.plot(re1[500:], ri1[500:], label='phase plane', color='k')
plt.legend(loc='best')
plt.xlabel("r_e (Hz)")
plt.ylabel("r_i (Hz)")
plt.subplot(414)
fs, psd = create_psd(re1[500:] + ri1[500:], 10000)
plt.plot(fs[:100], psd[:100], label='r_e')
plt.legend(loc='best')
plt.xlabel("Freq (Hz)")
plt.ylabel("PSD")
print("Mean re1 : {0}".format(npmean(re1)))
print("Mean ri1: {0}".format(npmean(ri1)))
print("Max STIM F (Hz) {0}".format(fs[argmax(psd[:100])]))
# 1
plt.figure(figsize=(14, 10))
plt.subplot(311)
plt.plot(t, re2, label='1: E')
plt.plot(t, ri2, label='1: I')
plt.legend(loc='best')
plt.xlabel("Time (ms)")
plt.ylabel("Rate (Hz)")
plt.subplot(312)
plt.plot(re2[500:], ri2[500:], label='phase plane', color='k')
plt.legend(loc='best')
def mean(lst):
if len(lst) == 0:
return 0
return npmean(lst)
def lightcurve_ptp_measures(ftimes, fmags, ferrs):
'''This calculates various point-to-point measures (`eta` in Kim+ 2014).
Parameters
----------
ftimes,fmags,ferrs : np.array
The input mag/flux time-series with all non-finite elements removed.
Returns
-------
dict
A dict with values of the point-to-point measures, including the `eta`
variability index (often used as its inverse `inveta` to have the same
sense as increasing variability index -> more likely a variable star).
'''
ndet = len(fmags)
if ndet > 9:
timediffs = npdiff(ftimes)
# get rid of stuff with time diff = 0.0
nzind = npnonzero(timediffs)
ftimes, fmags, ferrs = ftimes[nzind], fmags[nzind], ferrs[nzind]
# recalculate ndet and diffs
ndet = ftimes.size
timediffs = npdiff(ftimes)
# calculate the point to point measures
p2p_abs_magdiffs = npabs(npdiff(fmags))
p2p_squared_magdiffs = npdiff(fmags) * npdiff(fmags)
robstd = npmedian(npabs(fmags - npmedian(fmags))) * 1.483
robvar = robstd * robstd
# these are eta from the Kim+ 2014 paper - ratio of point-to-point
# difference to the variance of the entire series
# this is the robust version
eta_robust = npmedian(p2p_abs_magdiffs) / robvar
eta_robust = eta_robust / (ndet - 1.0)
# this is the usual version
eta_normal = npsum(p2p_squared_magdiffs) / npvar(fmags)
eta_normal = eta_normal / (ndet - 1.0)
timeweights = 1.0 / (timediffs * timediffs)
# this is eta_e modified for uneven sampling from the Kim+ 2014 paper
eta_uneven_normal = ((npsum(timeweights * p2p_squared_magdiffs) /
(npvar(fmags) * npsum(timeweights))) *
npmean(timeweights) *
(ftimes.max() - ftimes.min()) *
(ftimes.max() - ftimes.min()))
# this is robust eta_e modified for uneven sampling from the Kim+ 2014
# paper
eta_uneven_robust = ((npsum(timeweights * p2p_abs_magdiffs) /
(robvar * npsum(timeweights))) *
npmedian(timeweights) * (ftimes[-1] - ftimes[0]) *
(ftimes[-1] - ftimes[0]))
return {
'eta_normal': eta_normal,
'eta_robust': eta_robust,
'eta_uneven_normal': eta_uneven_normal,
'eta_uneven_robust': eta_uneven_robust
}
else:
return None
def Chip_Classify(ImageLocation,SaveLocation,ImageFile,NumberOfClusters,InitialCluster):
ticOverall = time.time()
#sleep(random.beta(1,1)*30)
# Reshape InitialCluster
InitialCluster = array(InitialCluster).reshape((NumberOfClusters,-1))
ImageIn = imread(ImageFile)
with rio.open(ImageFile) as gtf_img:
Info = gtf_img.profile
Info.update(dtype=rio.int8)
#print(time.time()-tic)
ImageRow, ImageColumn, NumberOfBands = ImageIn.shape
if NumberOfBands > 8:
NumberOfBands = NumberOfBands - 1
# prealocate
Cluster = zeros((ImageRow, ImageColumn, NumberOfClusters))
CountClusterPixels = zeros((NumberOfClusters, 1))
MeanCluster = zeros((NumberOfClusters, NumberOfBands))
EuclideanDistanceResultant = zeros((ImageRow, ImageColumn, NumberOfClusters))
#os.mkdir('local/larry.leigh.temp/')
directory = '/tmp/ChipS'
if not os.path.exists(directory):
os.makedirs(directory)
print('starting big loop')
tic = time.time()
for j in range(0,ImageRow):
# if(j % 10 == 0):
# progbar(j, ImageRow)
for k in range(0, ImageColumn):
temp = ImageIn[j, k, 0:NumberOfBands]
#EuclideanDistanceResultant[j, k, :] = np.npsqrt(np.npsum(np.nppower(np.subtract(np.matlib.repmat(temp, NumberOfClusters, 1), InitialCluster[: ,:]), 2), axis = 1))
EuclideanDistanceResultant[j, k, :] = npsqrt(npsum(nppower((matlib.repmat(temp, NumberOfClusters, 1)) - InitialCluster, 2), axis=1))
DistanceNearestCluster = min(EuclideanDistanceResultant[j, k, :])
#print(str(j) +" "+ str(k))
for l in range(0, NumberOfClusters):
if DistanceNearestCluster != 0:
if DistanceNearestCluster == EuclideanDistanceResultant[j, k, l]:
CountClusterPixels[l] = CountClusterPixels[l] + 1
for m in range(0, NumberOfBands):
MeanCluster[l, m] = MeanCluster[l, m] + ImageIn[j, k, m]
Cluster[j, k, l] = l
# progbar(ImageRow, ImageRow)
print('\n')
# print(Cluster.shape)
# print(CountClusterPixels.shape)
# print(EuclideanDistanceResultant.shape)
# print(MeanCluster.shape)
print('\nfinished big loop')
ImageDisplay = npsum(Cluster, axis = 2)
print("Execution time: " + str(time.time() - tic))
#print(globals())
#shelver("big.loop",['Cluster','CountClusterPixels','EuclideanDistanceResultant','MeanCluster'])
savez("big.loop.serial",Cluster=Cluster,
CountClusterPixels=CountClusterPixels,
EuclideanDistanceResultant=EuclideanDistanceResultant,
MeanCluster=MeanCluster)
ClusterPixelCount = count_nonzero(Cluster, axis = 2)
print("Non-zero cluster pixels: " + str(ClusterPixelCount))
#Calculate TSSE within clusters
TsseCluster = zeros((1, NumberOfClusters))
CountTemporalUnstablePixel = 0
# TSSECluster Serial
print("Starting TSSE Cluster computation (Serial version)\n")
tic = time.time()
for j in range(0, ImageRow):
for k in range(0, ImageColumn):
FlagSwitch = int(max(Cluster[j, k, :]))
#print(Cluster[j, k, :]) #This prints to the log
#store SSE of related to each pixel
if FlagSwitch == 0:
CountTemporalUnstablePixel = CountTemporalUnstablePixel + 1
else:
#Might be TsseCluster[0,FlagSwitch-1]
#TsseCluster[0,FlagSwitch - 1] = TsseCluster[0,FlagSwitch - 1] + np.sum(np.power(np.subtract(np.squeeze(ImageIn[j, k, 0:NumberOfBands - 1]), np.transpose(InitialCluster[FlagSwitch - 1, :])),2), axis = 0)
TsseCluster[0,FlagSwitch] = TsseCluster[0,FlagSwitch] + npsum(nppower((squeeze(ImageIn[j, k, 0:NumberOfBands]) - transpose(InitialCluster[FlagSwitch, :])),2))
#count the number of pixels in each cluster
#Collected_ClusterPixelCount[FlagSwitch] = Collected_ClusterPixelCount[FlagSwitch] + 1
Totalsse = npsum(TsseCluster)
print("Execution time: " + str(time.time() - tic))
savez("small.loop.serial",CountTemporalUnstablePixel=CountTemporalUnstablePixel,TsseCluster=TsseCluster)
#get data for final stats....
#calculate the spatial mean and standard deviation of each cluster
ClusterMeanAllBands = zeros((NumberOfClusters, NumberOfBands))
ClusterSdAllBands = zeros((NumberOfClusters, NumberOfBands))
print('finished small loop')
#print(time.time()-tic)
# Cluster Summary Serial
tic = time.time()
FinalClusterMean = zeros(NumberOfBands)
FinalClusterSd = zeros(NumberOfBands)
for i in range(0, NumberOfClusters):
Temp = Cluster[:, :, i]
Temp[Temp == i] = 1
MaskedClusterAllBands = Temp[:,:,None]*ImageIn[:, :, 0:NumberOfBands]
for j in range(0, NumberOfBands):
#Mean = MaskedClusterAllBands(:,:,j)
Temp = MaskedClusterAllBands[:, :, j]
TempNonZero = Temp[npnonzero(Temp)]
TempNonzeronan = TempNonZero[~npisnan(TempNonZero)]
#TempNonan = Temp[!np.isnan(Temp)]
with warnings.catch_warnings():
warnings.filterwarnings('error')
try:
FinalClusterMean[j] = npmean(TempNonZero)
FinalClusterSd[j] = npstd(TempNonZero)
except RuntimeWarning:
FinalClusterMean[j] = 0
FinalClusterSd[j] = 0
ClusterMeanAllBands[i, :] = FinalClusterMean[:]
ClusterSdAllBands[i, :] = FinalClusterSd[:]
print("Execution time: " + str(time.time() - tic))
savez("cluster.summary.serial",ClusterMeanAllBands=ClusterMeanAllBands,ClusterSdAllBands=ClusterSdAllBands)
filename = str(SaveLocation) + 'ImageDisplay_' + ImageFile[len(ImageFile)-32:len(ImageFile)-3] + 'mat'
print('Got filename. Now save the data')
print(filename)
save(filename, ImageDisplay)
filename = str(SaveLocation) + 'ClusterCount' + str(NumberOfClusters) + '_' + ImageFile[len(ImageFile)-32:len(ImageFile)-4] + '.tif'
#geotiffwrite(filename, int8(ImageDisplay), Info.RefMatrix);
with rio.open(filename, 'w', **Info) as dst:
dst.write(int8(ImageDisplay), 1)
filename = str(SaveLocation) + 'Stats_' + ImageFile[len(ImageFile)-32:len(ImageFile)-3] + 'mat'
savez(filename, [MeanCluster, CountClusterPixels, ClusterPixelCount, ClusterMeanAllBands, ClusterSdAllBands, Totalsse])
print('done!')
print(time.time()-ticOverall)
def dworetsky_period_find(time,
mag,
err,
init_p,
end_p,
f_step,
verbose=False):
'''
This is the super-slow naive version taken from my thesis work.
Uses the string length method in Dworetsky 1983 to calculate the period of a
time-series of magnitude measurements and associated magnitude
errors. Searches in linear frequency space (which obviously doesn't
correspond to a linear period space).
PARAMETERS:
time: series of times at which mags were measured (usually some form of JD)
mag: timeseries of magnitudes (np.array)
err: associated errs per magnitude measurement (np.array)
init_p, end_p: interval to search for periods between (both ends inclusive)
f_step: step in frequency [days^-1] to use
RETURNS:
tuple of the following form:
(periods (np.array),
string_lengths (np.array),
good_period_mask (boolean array))
'''
mod_mag = (mag - npmin(mag)) / (2.0 * (npmax(mag) - npmin(mag))) - 0.25
fold_time = npmin(time) # fold at the first time element
init_f = 1.0 / end_p
end_f = 1.0 / init_p
n_freqs = npceil((end_f - init_f) / f_step)
if verbose:
print('searching %s frequencies between %s and %s days^-1...' %
(n_freqs, init_f, end_f))
out_periods = npempty(n_freqs, dtype=np.float64)
out_strlens = npempty(n_freqs, dtype=np.float64)
p_goodflags = npempty(n_freqs, dtype=bool)
j_range = len(mag) - 1
for i in range(int(n_freqs)):
period = 1.0 / init_f
# print('P: %s, f: %s, i: %s, n_freqs: %s, maxf: %s' %
# (period, init_f, i, n_freqs, end_f))
phase = (time - fold_time) / period - npfloor(
(time - fold_time) / period)
phase_sort_ind = npargsort(phase)
phase_sorted = phase[phase_sort_ind]
mod_mag_sorted = mod_mag[phase_sort_ind]
strlen = 0.0
epsilon = 2.0 * npmean(err)
delta_l = 0.34 * (epsilon - 0.5 *
(epsilon**2)) * (len(time) - npsqrt(10.0 / epsilon))
keep_threshold_1 = 1.6 + 1.2 * delta_l
l = 0.212 * len(time)
sig_l = len(time) / 37.5
keep_threshold_2 = l + 4.0 * sig_l
# now calculate the string length
for j in range(j_range):
strlen += npsqrt((mod_mag_sorted[j + 1] - mod_mag_sorted[j])**2 +
(phase_sorted[j + 1] - phase_sorted[j])**2)
strlen += npsqrt((mod_mag_sorted[0] - mod_mag_sorted[-1])**2 +
(phase_sorted[0] - phase_sorted[-1] + 1)**2)
if ((strlen < keep_threshold_1) or (strlen < keep_threshold_2)):
p_goodflags[i] = True
out_periods[i] = period
out_strlens[i] = strlen
init_f += f_step
return (out_periods, out_strlens, p_goodflags)
def _get_map_values(self,
structure_instance,
emmap,
res_map,
sim_sigma_coeff=0.187,
win=5):
"""
Returns avg map density from voxels occupied by overlapping residue fragments.
Arguments:
*structure_instance*
Structure instance of the model.
*emmap*
Map instance the model is to be placed on.
*res_map*
Resolution of the map
*sim_sigma_coeff*
Sigma factor used for blurring
*win*
Fragment length (odd values)
"""
dict_chain_indices, dict_chain_res, dict_res_dist = self.get_indices(
structure_instance, emmap, res_map, sim_sigma_coeff)
origin = emmap.origin
apix = emmap.apix
box_size = emmap.box_size()
nz, ny, nx = emmap.fullMap.shape
zg, yg, xg = mgrid[0:nz, 0:ny, 0:nx]
indi = zip(xg.ravel(), yg.ravel(), zg.ravel())
#for residues not in rigid bodies: consider pentapeptides
dict_chain_scores = {}
for ch in dict_chain_indices:
dict_res_scores = {}
dict_res_indices = dict_chain_indices[ch]
for res in dict_res_indices:
if not dict_res_scores.has_key(res):
indices = dict_res_indices[res][:]
#consider residues on both sides. NOTE: wont work for insertion codes!
#need to rewite res numbers to avoid insertion codes
for ii in range(1, int(round((win + 1) / 2))):
try:
#get prev residue indices
indices.extend(dict_res_indices[dict_chain_res[ch][
dict_chain_res[ch].index(res) - ii]])
except:
pass
for ii in range(1, int(round((win + 1) / 2))):
try:
indices.extend(dict_res_indices[dict_chain_res[ch][
dict_chain_res[ch].index(res) + ii]])
except:
pass
tmplist = indices[:]
setlist = set(tmplist)
indices = list(setlist)
sc_indices = []
for ii in indices:
sc_indices.append(indi[ii])
'''
if len(indices) < 10:
try:
dict_res_scores[res] = dict_res_scores[dict_chain_res[ch][dict_chain_res[ch].index(res)-1]]
try: dict_res_scores[res] = (dict_res_scores[res]+dict_res_scores[dict_chain_res[ch][dict_chain_res[ch].index(res)+1]])/2.0
except (IndexError,KeyError): pass
except (IndexError,KeyError):
try: dict_res_scores[res] = dict_res_scores[dict_chain_res[ch][dict_chain_res[ch].index(res)+1]]
except (IndexError,KeyError): dict_res_scores[res] = 0.0
continue
'''
array_indices = array(sc_indices)
ind_arrxyz = transpose(array_indices)
ind_arrzyx = (ind_arrxyz[2], ind_arrxyz[1], ind_arrxyz[0])
dict_res_scores[res] = npmean(emmap.fullMap[ind_arrzyx])
dict_chain_scores[ch] = dict_res_scores.copy()
return dict_chain_scores
def sigclip_magseries(times,
mags,
errs,
sigclip=None,
iterative=False,
niterations=None,
meanormedian='median',
magsarefluxes=False):
'''
Select the finite times, magnitudes (or fluxes), and errors from the
passed values, and apply symmetric or asymmetric sigma clipping to them.
Returns sigma-clipped times, mags, and errs.
Args:
times (np.array): ...
mags (np.array): numpy array to sigma-clip. Does not assume all values
are finite. Does not assume anything about whether they're
positive/negative.
errs (np.array): ...
iterative (bool): True if you want iterative sigma-clipping. If
niterations is not set and this is True, sigma-clipping is iterated
until no more points are removed.
niterations (int): maximum number of iterations to perform for
sigma-clipping. If None, the iterative arg takes precedence, and
iterative=True will sigma-clip until no more points are removed.
If niterations is not None and iterative is False, niterations takes
precedence and iteration will occur.
meanormedian (string): either 'mean' for sigma-clipping based on the
mean value, or 'median' for sigma-clipping based on the median value.
Default is 'median'.
magsarefluxes (bool): True if your "mags" are in fact fluxes, i.e. if
"dimming" corresponds to your "mags" getting smaller.
sigclip (float or list): If float, apply symmetric sigma clipping. If
list, e.g., [10., 3.], will sigclip out greater than 10-sigma dimmings
and greater than 3-sigma brightenings. Here the meaning of "dimming"
and "brightening" is set by *physics* (not the magnitude system), which
is why the `magsarefluxes` kwarg must be correctly set.
Returns:
stimes, smags, serrs: (sigmaclipped values of each).
'''
returnerrs = True
# fake the errors if they don't exist
# this is inconsequential to sigma-clipping
# we don't return these dummy values if the input errs are None
if errs is None:
# assume 0.1% errors if not given
# this should work for mags and fluxes
errs = 0.001 * mags
returnerrs = False
# filter the input times, mags, errs; do sigclipping and normalization
find = npisfinite(times) & npisfinite(mags) & npisfinite(errs)
ftimes, fmags, ferrs = times[find], mags[find], errs[find]
# get the center value and stdev
if meanormedian == 'median': # stddev = 1.483 x MAD
center_mag = npmedian(fmags)
stddev_mag = (npmedian(npabs(fmags - center_mag))) * 1.483
elif meanormedian == 'mean':
center_mag = npmean(fmags)
stddev_mag = npstddev(fmags)
else:
LOGWARNING("unrecognized meanormedian value given to "
"sigclip_magseries: %s, defaulting to 'median'" %
meanormedian)
meanormedian = 'median'
center_mag = npmedian(fmags)
stddev_mag = (npmedian(npabs(fmags - center_mag))) * 1.483
# sigclip next for a single sigclip value
if sigclip and isinstance(sigclip, (float, int)):
if not iterative and niterations is None:
sigind = (npabs(fmags - center_mag)) < (sigclip * stddev_mag)
stimes = ftimes[sigind]
smags = fmags[sigind]
serrs = ferrs[sigind]
else:
#
# iterative version adapted from scipy.stats.sigmaclip
#
# First, if niterations is not set, iterate until covergence
if niterations is None:
delta = 1
this_times = ftimes
this_mags = fmags
this_errs = ferrs
while delta:
if meanormedian == 'mean':
this_center = npmean(this_mags)
this_stdev = npstddev(this_mags)
elif meanormedian == 'median':
this_center = npmedian(this_mags)
this_stdev = (npmedian(
npabs(this_mags - this_center))) * 1.483
this_size = this_mags.size
# apply the sigclip
tsi = ((npabs(this_mags - this_center)) <
(sigclip * this_stdev))
# update the arrays
this_times = this_times[tsi]
this_mags = this_mags[tsi]
this_errs = this_errs[tsi]
# update delta and go to the top of the loop
delta = this_size - this_mags.size
else: # If iterating only a certain number of times
this_times = ftimes
this_mags = fmags
this_errs = ferrs
iter_num = 0
delta = 1
while iter_num < niterations and delta:
if meanormedian == 'mean':
this_center = npmean(this_mags)
this_stdev = npstddev(this_mags)
elif meanormedian == 'median':
this_center = npmedian(this_mags)
this_stdev = (npmedian(
npabs(this_mags - this_center))) * 1.483
this_size = this_mags.size
# apply the sigclip
tsi = ((npabs(this_mags - this_center)) <
(sigclip * this_stdev))
# update the arrays
this_times = this_times[tsi]
this_mags = this_mags[tsi]
this_errs = this_errs[tsi]
# update the number of iterations and delta and
# go to the top of the loop
delta = this_size - this_mags.size
iter_num += 1
# final sigclipped versions
stimes, smags, serrs = this_times, this_mags, this_errs
# this handles sigclipping for asymmetric +ve and -ve clip values
elif sigclip and isinstance(sigclip, list) and len(sigclip) == 2:
# sigclip is passed as [dimmingclip, brighteningclip]
dimmingclip = sigclip[0]
brighteningclip = sigclip[1]
if not iterative and niterations is None:
if magsarefluxes:
nottoodimind = ((fmags - center_mag) >
(-dimmingclip * stddev_mag))
nottoobrightind = ((fmags - center_mag) <
(brighteningclip * stddev_mag))
else:
nottoodimind = ((fmags - center_mag) <
(dimmingclip * stddev_mag))
nottoobrightind = ((fmags - center_mag) >
(-brighteningclip * stddev_mag))
sigind = nottoodimind & nottoobrightind
stimes = ftimes[sigind]
smags = fmags[sigind]
serrs = ferrs[sigind]
else:
#
# iterative version adapted from scipy.stats.sigmaclip
#
if niterations is None:
delta = 1
this_times = ftimes
this_mags = fmags
this_errs = ferrs
while delta:
if meanormedian == 'mean':
this_center = npmean(this_mags)
this_stdev = npstddev(this_mags)
elif meanormedian == 'median':
this_center = npmedian(this_mags)
this_stdev = (npmedian(
npabs(this_mags - this_center))) * 1.483
this_size = this_mags.size
if magsarefluxes:
nottoodimind = ((this_mags - this_center) >
(-dimmingclip * this_stdev))
nottoobrightind = ((this_mags - this_center) <
(brighteningclip * this_stdev))
else:
nottoodimind = ((this_mags - this_center) <
(dimmingclip * this_stdev))
nottoobrightind = ((this_mags - this_center) >
(-brighteningclip * this_stdev))
# apply the sigclip
tsi = nottoodimind & nottoobrightind
# update the arrays
this_times = this_times[tsi]
this_mags = this_mags[tsi]
this_errs = this_errs[tsi]
# update delta and go to top of the loop
delta = this_size - this_mags.size
else: # If iterating only a certain number of times
this_times = ftimes
this_mags = fmags
this_errs = ferrs
iter_num = 0
delta = 1
while iter_num < niterations and delta:
if meanormedian == 'mean':
this_center = npmean(this_mags)
this_stdev = npstddev(this_mags)
elif meanormedian == 'median':
this_center = npmedian(this_mags)
this_stdev = (npmedian(
npabs(this_mags - this_center))) * 1.483
this_size = this_mags.size
if magsarefluxes:
nottoodimind = ((this_mags - this_center) >
(-dimmingclip * this_stdev))
nottoobrightind = ((this_mags - this_center) <
(brighteningclip * this_stdev))
else:
nottoodimind = ((this_mags - this_center) <
(dimmingclip * this_stdev))
nottoobrightind = ((this_mags - this_center) >
(-brighteningclip * this_stdev))
# apply the sigclip
tsi = nottoodimind & nottoobrightind
# update the arrays
this_times = this_times[tsi]
this_mags = this_mags[tsi]
this_errs = this_errs[tsi]
# update the number of iterations and delta
# and go to top of the loop
delta = this_size - this_mags.size
iter_num += 1
# final sigclipped versions
stimes, smags, serrs = this_times, this_mags, this_errs
else:
stimes = ftimes
smags = fmags
serrs = ferrs
if returnerrs:
return stimes, smags, serrs
else:
return stimes, smags, None
def L2(input_path_L1, input_path_Compensate, output_path, E0_const):
try:
input_fp_Compensate = open(input_path_Compensate, 'r')
except IOError:
print "IO error;Check the input File: ", input_path_Compensate
except:
print "Unexpected Open Error: ", input_path_Compensate
input_fp_Compensate.close()
try:
input_fp_L1 = open(input_path_L1, 'r')
except IOError:
print "IO error;Check the input File: ", input_path_L1
except Error:
print "Unexpected Open Error: ", input_path_L1
input_fp_L1.close()
#output file path
output_file_path = os.path.join(output_path, 'ResultL2.csv')
try:
output_fp = open(output_file_path, 'w+')
except IOError:
print "IO error;Check the output File: ", output_file_path
return 'L2 failed'
#output_plot_1�� Reynold-Taylor Equation����
output_plot_2_file_path = os.path.join(output_path, 'Plot_L2_2.csv')
try:
output_plot_2_fp = open(output_plot_2_file_path, 'w+')
except IOError:
print "IO error;Check the output File: ", output_plot_2_file_path
return 'L2 failed'
try:
Compensate_csv = csv.reader(input_fp_Compensate, delimiter = ',')
except csv.Error:
print "Parse ErrorCheck the input File: ", input_path_Compensate
except StandardError:
print "Unexpected Read Error: ", input_path_Compensate
try:
L1_csv = csv.reader(input_fp_L1, delimiter = ',')
except csv.Error:
print "Parse ErrorCheck the input File: ", input_path_L1
except StandardError:
print "Unexpected Read Error: ", input_path_L1
n_Compensate = 0
n_L1 = 0
data_Compensate = []
data_L1 = []
for row in Compensate_csv:
data_Compensate.append(row)
n_Compensate = n_Compensate + 1
for row in L1_csv:
data_L1.append(row)
n_L1 = n_L1 + 1
#Data count check
if(n_Compensate != n_L1):
print 'Count Error;Process count dismatch between Compensate and L1'
return 'L2 failed'
#initialize
date = []
rsdn = npzeros(n_Compensate)
Ta = npzeros(n_Compensate)
h2o = npzeros(n_Compensate)
#press = npzeros(n_Compensate)
#Read Input Data
i = 0
for row in data_Compensate:
rsdn[i] = float(row[0])
Ta[i] = float(row[1])
h2o[i] = float(row[2])
i = i + 1
press = 998.0
#initialize
Fs = npzeros(n_L1)
Fc = npzeros(n_L1)
Fsc = npzeros(n_L1)
Hs = npzeros(n_L1)
Hc = npzeros(n_L1)
Hsc = npzeros(n_L1)
LEs = npzeros(n_L1)
LEc = npzeros(n_L1)
LEsc = npzeros(n_L1)
co2 = npzeros(n_L1)
ustar = npzeros(n_L1)
itime = npzeros(n_L1)
iustar = npzeros(n_L1)
date = []
i = 0
for row in data_L1:
date.append(row[0])
Fs[i] = float(row[1])
Fc[i] = float(row[2])
Fsc[i] = float(row[3])
Hs[i] = float(row[4])
Hc[i] = float(row[5])
Hsc[i] = float(row[6])
LEs[i] = float(row[7])
LEc[i] = float(row[8])
LEsc[i] = float(row[9])
co2[i] = float(row[10])
ustar[i] = float(row[14])
itime[i] = float(row[15])
iustar[i] = float(row[16])
i = i + 1
# Define constants and parameters for gap filling
#--------------------------------------------------------------------------
num_day = 28
ni = 36
nd = 10
n1 = 2 # how many the largest points are considered for respiration
# DO NOT Modify!
#num_point_per_day = 24 # number of data points per day (48 -> 30 min avg time)
#avgtime = 30
#determine num_point_per_day automatically . using datetime module
date_1st = datetime.datetime.strptime(date[0], "%Y-%m-%d %H:%M")
date_2nd = datetime.datetime.strptime(date[1], "%Y-%m-%d %H:%M")
date_diff = date_2nd - date_1st
avgtime = int(date_diff.seconds / 60) # averaging time (minutes)
num_point_per_day = 1440 / avgtime # number of data points per day (1440 : minutes of a day)
num_segment = num_point_per_day * num_day
num_avg = int(n_L1 / num_segment)
num_day_2 = 7
# nday_re = 20
# noverlap = 5
num_day_re = 20
noverlap = 5
ni = int(num_point_per_day * 3 / 4) # the data point that night starts
nd = 300 / avgtime # how many the largest points are considered for respiration (300 : minitues of 5 hours)
#--------------------------------------------------------------------------
#E0_const = True # Do you want to use constant E0 for one year? Y/N
beta0 = nparray([2, 200])
Tref = 10.0
T0 = -46.02
gap_limit = 0.025
ustar_limit = 0.5
upper_Fc = 0.35 # upper limit of nighttime CO2 flux (mg/m2/s)
Fc_limit = 0.005
## Information for MLT
drsdn = 50.0 # W/m2
dta = 2.5 # oC
dvpd = 5.0 # 5 hPa
rv = 461.51
#--------------------------------------------------------------------------
upper_co2 = 1000.0 # upper limit of CO2 concent.(mg/m3)
upper_h2o = 60.0 # upper limit of H2O concent. (g/m3)
upper_Ta = 60.0 # upper limit of air temperature (oC)
lower_Fc = -3.0 # lower limit of daytime CO2 flux (mg/m2/s)
lower_LE = -200 # lower limit of LE (W/m2)
lower_H = -300 # lower limit of H (W/m2)
upper_Fc = 3 # upper limit of nighttime CO2 flux (mg/m2/s)
upper_LE = 800 # upper limit of LE (W/m2)
upper_H = 800 # upper limit of H (W/m2)
upper_agc = 95.0 # upper limit of AGC value
ustar_limit = 0.03 # minimum ustar for filtering out nighttime fluxes
Fc_limit = 0.005 # lower limit of Re (ecosystem respiration) (mg/m2/s)
gap_limit = 0.025 # 0.025 --> 95# confidence interval
Tak = npzeros(len(Ta))
tr = npzeros(len(Ta))
ea = npzeros(len(Ta))
es = npzeros(len(Ta))
vpd = npzeros(len(Ta))
#--------------------------------------------------------------------------
# calculation of vapor pressure deficit
a = [13.3185, 1.9760, 0.6445, 0.1299]
for i in range(n_Compensate):
Tak[i] = Ta[i] + 273.15
tr[i] = 1.0-(373.15/Tak[i])
es[i] = 1013.25*exp(a[0]*tr[i]-a[1]*(tr[i]**2)-(a[2]*(tr[i]**3))-a[3]*(tr[i]**4)) # hPa
for i in range(n_L1):
ea[i] = h2o[i]
vpd[i]= float(es[i]) - float(ea[i]) #unit is hPa
Fc_filled = copy.deepcopy(Fsc)
print 'Gap Filling Process'
print 'Before running this program, '
print ' please make sure that you correctly set all parameters'
#print 'E0_const'. E0_const
#print 'nn', nn
#print 'num_point_per_day', num_point_per_day
#print 'num_day_2', num_day_2
#print 'num_day_re', num_day_re
#print 'noverlap', noverlap
#print 'drsdn', drsdn
#print 'dta', dta
#print 'dvpd', dvpd
#print '-------------------------------------------------------------------'
index = []
for main_j in range(num_avg): # loop for gap-filling of CO2 fluxes
seg_start_i = main_j * num_segment
seg_fin_i = seg_start_i + num_segment
if((seg_start_i + 2 * num_segment) > n_L1):
seg_fin_i = n_L1
x2 = []
x3 = []
#--------------------------------------------------------------------------
if(main_j == 0):
print 'Application of modified lookup table method'
#--------------------------------------------------------------------------
for i in range(seg_start_i, seg_fin_i):
if(itime[i] == 1):
ii = 0
if(isnan(Fsc[i]) == True):
jj = 0
while ((ii < 1) and (jj <= 4)):
ta_f = Ta[i]
rsdn_f = rsdn[i]
vpd_f = vpd[i]
i0 = i - jj * num_day_2 * num_point_per_day
i1 = i + jj * num_day_2 * num_point_per_day+1
if(i0 < 1):
i0 = 0
i1 = 2 * jj * num_day_2 * num_point_per_day+1
if(i1 >= n_L1):
i0 = n_L1 - 2 * jj * num_day_2 * num_point_per_day
i1 = n_L1
if(i0 < 1):
i0 = 0
ks = 0
for j in range(i0, i1):
if((fabs(vpd_f - vpd[j]) < dvpd) and \
(fabs(rsdn_f - rsdn[j]) < drsdn) and \
(fabs(ta_f - Ta[j]) < dta) and \
(isnan(Fsc[j]) == False)):
ks = ks + 1
x3_temp = []
x2.append(Fsc[j])
x3_temp.append(j)
x3_temp.append(vpd[j])
x3_temp.append(rsdn[j])
x3_temp.append(Ta[j])
x3_temp.append(ks)
x3.append(x3_temp)
ii = ks
#index_temp = []
#index_temp.append(i)
#index_temp.append(i0)
#index_temp.append(i1)
#index_temp.append(ks)
#index_temp.append(main_j)
#index.append(index_temp)
if(ks >= 1):
Fc_filled[i] = npmedian(nparray(x2))
jj = jj + 1
x2 = []
x3 = []
if(ii < 1):
jj = 0
while(ii < 1):
rsdn_f = rsdn[i]
i0 = i - jj * num_day_2 * num_point_per_day
i1 = i + jj * num_day_2 * num_point_per_day+1
if(i0 < 1):
i0 = 0
i1 = 2 * jj * num_day_2 * num_point_per_day+1
if(i1 >= n_L1):
i0 = n_L1 - 2 * jj * num_day_2 * num_point_per_day
i1 = n_L1
if(i0 < 0):
i0 = 0
ks = 0
for j in range(i0, i1):
if((fabs(rsdn_f - rsdn[j]) < drsdn) and \
(isnan(Fsc[j]) == False)):
ks = ks + 1
x3_temp = []
x2.append(Fsc[j])
x3_temp.append(j)
x3_temp.append(vpd[j])
x3_temp.append(rsdn[j])
x3_temp.append(Ta[j])
x3_temp.append(ks)
x3.append(x3_temp)
ii = ks
#index_temp = []
#index_temp.append(i)
#index_temp.append(i0)
#index_temp.append(i1)
#index_temp.append(ks)
#index_temp.append(main_j)
#index.append(index_temp)
if(ks >= 1):
Fc_filled[i] = npmedian(nparray(x2))
jj = jj + 1
x2 = []
x3 = []
x2 = []
x3 = []
ks = 0
d = npzeros((n_L1, 5))
d2 = npzeros((n_L1, 5))
dd = npzeros((n_L1, 5))
x4 = []
#Regression to Lloyd-Taylor equation
print 'Regression to Lloyd-Taylor equation'
if(E0_const == True):
for i in range(ni-1, n_Compensate, num_point_per_day):
t1 = npzeros(nd)
for j in range(nd):
t1[j] = Fsc[i + j]
#Set to 'descend'
t2, IX = Common.matlab_sort(t1)
k2 = 0
for k in range(nd-1):
if((isnan(t2[k]) == False) and \
(t2[k] < upper_Fc) and \
(t2[k+1] > Fc_limit)):
k2 = k2 + 1
if(k2 >= 2):
for j in range(nd-1):
if((itime[i+1 + IX[j]] == 0) and \
(isnan(t2[j]) == False) and \
(isnan(Ta[i+1 + IX[j]]) == False) and \
(t2[j] < upper_Fc) and \
(t2[j + 1] > Fc_limit) and \
(iustar[i + IX[j]] == 0) and \
(iustar[i + IX[j+1]] == 0)):
x3.append(t2[j])
x3.append(t2[j+1])
x2.append(Ta[i + IX[j]])
x2.append(Ta[i + IX[j+1]])
x4.append(date[i + IX[j]])
x4.append(date[i + IX[j+1]])
ks = ks + n1
break
TC = copy.deepcopy(nparray(x2))
PV = copy.deepcopy(nparray(x3))
betafit = spfmin(Common.Reco, beta0, args = (TC, PV), disp=False)
A = betafit[0]
B = betafit[1]
yfit = npzeros(len(TC))
for i in range(len(TC)):
yfit[i] = A * exp(B * (1 / (10 + 46.02) - 1 / (TC[i] + 46.02)))
E0 = betafit[1]
E0l = copy.deepcopy(E0)
# figure(1)
# plot(TC][PV,'ko',TC][yfit,'or')
# grid
# xlabel('air temperature (^oC)')
# ylabel('Ecosystem respiration(mgm^{-2}s^{-1})')
# TC = x2'
# PV = x3'
#
# [beta][resnorm] = lsqcurvefit(@myfun][beta0][TC][PV)
#
# A=beta(1)
# B=beta(2)
# yfit=A.*exp(B.*(1./(10.+46.02)-1./(TC+46.02)))
# E0 = betafit(2)
# E0l = E0
#
#
# figure(5)
# plot(TC][PV,'ko',TC][yfit,'or')
# grid
# xlabel('air temperature (^oC)')
# ylabel('Ecosystem respiration(mgm^{-2}s^{-1})')
x2 = []
x3 = []
t1 = []
t2 = []
TC = []
PV = []
yfit = npzeros(len(TC))
#num_day_re = 20
#noverlap = 5
#avgtime = 30
delta = (60 / avgtime) * 24 * num_day_re
dnoverlap = (60 / avgtime) * 24 * noverlap
jj = 0
sday = []
Rref = []
RE_limit = []
stdev_E0 = []
E0v = []
REs = []
Taylor_date = []
yfit_array = []
for i in range(0, n_L1, dnoverlap):
i0 = int(i - delta / 2)
i1 = int(i + delta / 2)
if(i0 < 1):
i0 = 0
i1 = int(i0 + delta)
if(i1 >= n_L1):
i0 = int(n_L1 - delta) - 1
i1 = n_L1
ks = 1
for j in range(i0+ni-1, i1, num_point_per_day):
t1 = npzeros(nd)
for k in range(nd):
t1[k] = Fsc[j + k]
#Set to 'descend'
t2, IX = Common.matlab_sort(t1)
k2 = 1
for k in range(nd-1):
if((isnan(t2[k]) == False) and \
(t2[k] < upper_Fc) and \
(t2[k+1] > Fc_limit)):
k2 = k2 + 1
if(k2 >= n1):
for k in range(nd-1):
if((itime[j+1 +IX[k]] == 0) and \
(isnan(t2[k]) == False) and \
(isnan(Ta[j + IX[k]]) == False) and \
(t2[k] < upper_Fc) and \
(t2[k+1] > Fc_limit) and \
(iustar[j +IX[k]] == 0) and \
(iustar[j +IX[k+1]] == 0)):
x3.append(t2[k])
x3.append(t2[k+1])
x2.append(Ta[j + IX[k]])
x2.append(Ta[j + IX[k+1]])
Taylor_date.append(str(date[j + IX[k]]))
Taylor_date.append(str(date[j + IX[k+1]]))
ks = ks + n1
break
ks = ks - 1
if(ks < 6):
if(E0_const == True):
Rref.append(float('NaN'))
RE_limit.append(float('NaN'))
jj = jj + 1
else:
Rref.append(float('NaN'))
E0v.append(float('NaN'))
stdev_E0.append(float('NaN'))
RE_limit.append(float('NaN'))
jj = jj + 1
else:
TC = copy.deepcopy(nparray(x2))
PV = copy.deepcopy(nparray(x3))
if(E0_const == True):
betafit = spfmin(Common.Reco2, beta0, args = (TC, PV, E0l), disp=False)
A = betafit[0]
Rref.append(A)
for j in range(len(TC)):
yfit = A * exp(E0 * (1.0/(10.0+46.02) - 1.0/(TC[j] + 46.02)))
yfit_array.append(yfit)
REs.append(PV[j] - yfit)
sz = nparray(REs).shape
upper = fabs(Common.tq(gap_limit, sz[0]-1 ) )
RE_limit.append(upper*Common.stdn1(nparray(REs))/sqrt(sz[0]))
jj = jj + 1
else:
betafit=spfmin(Common.Reco2, beta0, args = (TC, PV, E0))
A=betafit[0]
B=betafit[1]
Rref.append(A)
E0v.append(B)
if((B < 0) or (B > 450)):
E0v.append(float('NaN'))
for j in range(len(TC)):
yfit = A * exp(E0v[jj] * (1.0 / (10.0 + 46.02) - 1.0 / (TC[j] + 46.02)))
yfit_array.append(yfit)
REs.append(PV[j] - yfit)
sz = nparray(REs).shape
upper = abs(Common.tq(gap_limit, sz[0]-1))
stdev_E0.append(Common.stdn1(REs) / sqrt(sz[0]))
RE_limit.append(upper * Common.stdn1(nparray(REs)) / sqrt(sz[0]))
jj = jj + 1
#Regression to Lloyd-Taylor equation with 28-day segmentation
date_extracted = re.search('^(\d{4}[-]\d{2}[-]\d{2})',str(Taylor_date[0]))
#print date_extracted.group(0)
if(date_extracted != None):
fname = 'Plot_L2_1_'+str(date_extracted.group(0))+'.csv'
output_plot_1_file_path = os.path.join(output_path, fname)
try:
output_plot_1_fp = open(output_plot_1_file_path, 'w+')
except IOError:
print "IO error;Check the output File: ", output_plot_1_file_path
return 'L2 failed'
for i in range(len(TC)):
file_plot_str = StringIO()
file_plot_str.write(Taylor_date[i] + ',') #1
file_plot_str.write(str(A) + ',') #2
file_plot_str.write(str(B) + ',') #3
file_plot_str.write(str(TC[i]) + ',') #4
file_plot_str.write(str(PV[i]) + ',' ) #5
file_plot_str.write(str(yfit_array[i]) + '\n' ) #6
output_plot_string = file_plot_str.getvalue()
output_plot_1_fp.write(output_plot_string)
output_plot_1_fp.close()
sday_temp = []
sday_temp.append(i)
sday_temp.append(i0)
sday_temp.append(i1)
sday.append(sday_temp)
x2 = []
x3 = []
t1 = []
t2 = []
TC = []
PV = []
Taylor_date = []
REs = []
yfit = []
sday = nparray(sday)
if(E0_const == True):
print 'Long-term E0 '
E0s = copy.deepcopy(E0l)
else:
E0v_s = []
stdev_E0_s = []
for k in range(len(E0v)):
E0v_s.append(E0v[k]/stdev_E0[k])
stdev_E0_s.append(1/stdev_E0[k])
print 'Short-term E0 '
E0s = npnansum(E0v_s)/npnansum(stdev_E0_s)
Rref = []
#REs = []
#RE_limit = []
jj = 0
for i in range(0, n_L1, dnoverlap):
i0 = i - delta / 2
i1 = i + delta / 2
if(i0 < 1):
i0 = 0
i1 = i0 + delta
if(i1 >= n_L1):
i0 = n_L1 - delta - 1
i1 = n_L1
ks = 1
for j in range(i0+ni-1, i1, num_point_per_day):
t1 = npzeros(nd)
for k in range(nd):
t1[k] = Fsc[j + k]
#Set to 'descend'
t2, IX = Common.matlab_sort(t1)
k2 = 1
for k in range(nd-1):
if((isnan(t2[k]) == False) and \
(t2[k] < upper_Fc) and \
(t2[k+1] > Fc_limit)):
k2 = k2 + 1
if(k2 >= n1):
for k in range(nd-1):
if((itime[j+1 + IX[k]] == 0) and \
(isnan(t2[k]) == False) and \
(isnan(Ta[j + IX[k]]) == False) and \
(t2[k] < upper_Fc) and \
(t2[k+1] > Fc_limit) and \
(iustar[j + IX[k]] == 0) and \
(iustar[j + IX[k+1]] == 0)):
x3.append(t2[k])
x3.append(t2[k + 1])
x2.append(Ta[j + IX[k]])
x2.append(Ta[j + IX[k+1]])
ks = ks + n1
break
ks = ks - 1
if(ks < 6):
# Rref.append(Rref[jj])
# RE_limit.append(RE_limit[jj])
Rref.append(Rref[-1])
RE_limit.append(RE_limit[-1])
if(E0_const != True):
stdev_E0.append(stdev_E0[jj])
jj = jj + 1
else:
TC = nparray(x2)
PV = nparray(x3)
betafit = spfmin(Common.Reco2, beta0, args = (TC, PV, E0s), disp=False)
A=betafit[0]
Rref.append(A)
for j in range(len(TC)):
yfit = Rref[jj] * exp(E0s * (1 / (10 + 46.02) - 1 / (TC[j] + 46.02)))
REs.append(PV[j]-yfit)
sz = nparray(REs).shape
upper = abs(Common.tq(gap_limit, sz[0]-1))
RE_limit.append(upper*Common.stdn1(REs)/sqrt(sz[0]))
jj = jj + 1
x2 = []
x3 = []
t1 = []
t2 = []
TC = []
PV = []
#for k in REs:
# print k
REs = []
#for k in Rref:
# print k
##
ks = 0
nsp2 = npzeros((n_L1 / num_point_per_day))
RE = npzeros(n_L1)
GPP = npzeros(n_L1)
for i in range(n_L1):
RE[i] = float('NaN')
GPP[i] = float('NaN')
for i in range(0, n_L1, num_point_per_day):
i0 = i
i1 = i + num_point_per_day
if(i0 >=sday[ks][1]):
ks = ks + 1
if(i0 >= sday[len(sday)-1][1]):
ks = len(sday)-1
for j in range(i0, i1):
if(E0_const == True):
yfit=Rref[ks-1] * exp(E0l * (1.0 / (10 + 46.02) - 1.0 / (Ta[j] + 46.02)))
else:
yfit=Rref[ks-1] * exp(E0s * (1.0 / (10 + 46.02) - 1.0 / (Ta[j] + 46.02)))
RE[j] = yfit
if(itime[j]==0): # nighttime condition
RE[j] = Fc_filled[j]
if((isnan(Fsc[j]) == True) or \
((Fsc[j]-yfit) < RE_limit[ks-1]) or \
((Fsc[j]-yfit) > 1.0 * RE_limit[ks-1]) or \
(iustar[j] == 1)):
nsp2[ks-1] = nsp2[ks-1] + 1
Fc_filled[j] = yfit
RE[j] = Fc_filled[j]
GPP[j] = RE[j]- Fc_filled[j]
#figure(2)
#plot(time][Fsc][time][Fc_filled[:],'or')
#set(gca,'XTick',[year0:1/12:year0+0.9999999])
#set(gca,'xticklabel',xticks)
#ylim = [-1.5][1.5]
#set(gca,'xLim',xlim(:))
#ylabel('F_c (mgm^{-2}s^{-1})')
#--------------------------------------------------------------------------
#--------------------------------------------------------------------------
print 'Gap-filling of LE'
#--------------------------------------------------------------------------
x2 = []
x3 = []
index = []
LE_filled = copy.deepcopy(LEsc)
for main_j in range(num_avg): # loop for gap-filling of H2O fluxes
seg_start_i = main_j * num_segment
seg_fin_i = seg_start_i + num_segment
if((seg_start_i + 2 * num_segment) > n_L1):
seg_fin_i = n_L1
x2 = []
x3 = []
for i in range(seg_start_i, seg_fin_i):
ii = 0
if(isnan(LEsc[i]) == True):
jj = 0
while((ii < 1) and (jj <= 4)):
ta_f = Ta[i]
rsdn_f = rsdn[i]
vpd_f = vpd[i]
i0 = i - jj * num_day_2 * num_point_per_day
i1 = i + jj * num_day_2 * num_point_per_day+1
if(i0 < 1):
i0 = 0
i1 = 2 * jj * num_day_2 * num_point_per_day+1
if(i1 >= n_L1):
i0 = n_L1 - 2 * jj * num_day_2 * num_point_per_day - 1
i1 = n_L1
if(i0 < 1):
i0 = 0
ks = 0
for j in range(i0, i1):
if((fabs(vpd_f-vpd[j]) < dvpd) and \
(fabs(rsdn_f-rsdn[j]) < drsdn) and \
(fabs(ta_f-Ta[j]) < dta) and \
(isnan(LEsc[j]) == False)):
x3_temp = []
x2.append(LEsc[j])
x3_temp.append(j)
x3_temp.append(vpd[j])
x3_temp.append(rsdn[j])
x3_temp.append(Ta[j])
x3_temp.append(ks)
x3.append(x3_temp)
ks = ks + 1
ii = ks
#index_temp = []
#index_temp.append(i)
#index_temp.append(i0)
#index_temp.append(i1)
#index_temp.append(ks)
#index_temp.append(main_j)
#index.append(index_temp)
if(ks >= 1):
LE_filled[i] = npmedian(nparray(x2))
jj = jj + 1
x2 = []
x3 = []
if(ii < 1):
jj = 0
while(ii < 1):
rsdn_f = rsdn[i]
i0 = i - jj * num_day_2 * num_point_per_day
i1 = i + jj * num_day_2 * num_point_per_day + 1
if(i0 < 1):
i0 = 0
i1 = 2 * jj * num_day_2 * num_point_per_day + 1
if(i1 >= n_L1):
i0 = n_L1 - 2 * jj * num_day_2 * num_point_per_day - 1
i1 = n_L1
if(i0 < 1):
i0 = 0
ks = 0
for j in range(i0, i1):
if((fabs(rsdn_f-rsdn[j]) < drsdn) and \
(isnan(LEsc[j]) == False)):
x3_temp = []
x2.append(LEsc[j])
x3_temp.append(j)
x3_temp.append(vpd[j])
x3_temp.append(rsdn[j])
x3_temp.append(Ta[j])
x3_temp.append(ks)
x3.append(x3_temp)
ks = ks + 1
ii = ks
#index_temp = []
#index_temp.append(i)
#index_temp.append(i0)
#index_temp.append(i1)
#index_temp.append(ks)
#index_temp.append(main_j)
#index.append(index_temp)
if(ks >= 1):
LE_filled[i] = npmedian(nparray(x2))
jj = jj + 1
x2 = []
x3 = []
x2 = []
x3 = []
ks = 0
#figure(3)
#plot(time][LEsc][time][LE_filled[:],'or')
#set(gca,'XTick',[year0:1/12:year0+0.9999999])
#set(gca,'xticklabel',xticks)
#ylim = [-100][600]
#set(gca,'xLim',xlim(:))
#ylabel('LE (Wm^{-2})')
##
#--------------------------------------------------------------------------
print 'Gap-filling of H (sensible heat flux)'
#--------------------------------------------------------------------------
x2 = []
x3 = []
index = []
H_filled = copy.deepcopy(Hsc)
for main_j in range(num_avg): # loop for gap-filling of H2O fluxes
seg_start_i = main_j * num_segment
seg_fin_i = seg_start_i + num_segment
if((seg_start_i + 2 * num_segment) >= n_L1):
seg_fin_i = n_L1
x2 = []
x3 = []
for i in range(seg_start_i, seg_fin_i):
ii = 0
if(isnan(Hsc[i]) == True):
jj = 0
while ((ii < 1) and (jj <= 4)):
ta_f = Ta[i]
rsdn_f = rsdn[i]
vpd_f = vpd[i]
i0 = i - jj * num_day_2 * num_point_per_day
i1 = i + jj * num_day_2 * num_point_per_day+1
if(i0 < 1):
i0 = 0
i1 = 2 * jj * num_day_2 * num_point_per_day+1
if(i1 >= n_L1):
i0 = n_L1 - 2 * jj * num_day_2 * num_point_per_day - 1
i1 = n_L1
if(i0 < 1):
i0 = 0
ks = 0
for j in range(i0, i1):
if((fabs(vpd_f-vpd[j]) < dvpd) and \
(fabs(rsdn_f-rsdn[j]) < drsdn) and \
(fabs(ta_f-Ta[j]) < dta) and \
(isnan(Hsc[j]) == False)):
x3_temp = []
x2.append(Hsc[j])
x3_temp.append(j)
x3_temp.append(vpd[j])
x3_temp.append(rsdn[j])
x3_temp.append(Ta[j])
x3_temp.append(ks)
ks = ks + 1
x3.append(x3_temp)
ii = ks
#index_temp = []
#index_temp.append(i)
#index_temp.append(i0)
#index_temp.append(i1)
#index_temp.append(ks)
#index_temp.append(main_j)
#index.append(index_temp)
if(ks >= 1):
H_filled[i] = npmedian(nparray(x2))
jj = jj + 1
x2 = []
x3 = []
if(ii < 1):
jj = 0
while(ii < 1):
rsdn_f = rsdn[i]
i0 = i - jj * num_day_2 * num_point_per_day
i1 = i + jj * num_day_2 * num_point_per_day+1
if(i0 < 1):
i0 = 0
i1 = 2 * jj * num_day_2 * num_point_per_day+1
if(i1 >= n_L1):
i0 = n_L1 - 2 * jj * num_day_2 * num_point_per_day - 1
i1 = n_L1
if(i0 < 1):
i0 = 0
ks = 0
for j in range(i0, i1):
if((fabs(rsdn_f-rsdn[j]) < drsdn) and \
(isnan(Hsc[j]) == False)):
ks = ks + 1
x3_temp = []
x2.append(Hsc[j])
x3_temp.append(j)
x3_temp.append(vpd[j])
x3_temp.append(rsdn[j])
x3_temp.append(Ta[j])
x3_temp.append(ks)
x3.append(x3_temp)
ii = ks
#index_temp = []
#index_temp.append(i)
#index_temp.append(i0)
#index_temp.append(i1)
#index_temp.append(ks)
#index_temp.append(main_j)
#index.append(index_temp)
if(ks >= 1):
H_filled[i] = npmedian(nparray(x2))
jj = jj + 1
x2 = []
x3 = []
x2 = []
x3 = []
ks = 0
#figure(4)
#plot(time,Hsc,time,H_filled[:],'or')
#set(gca,'XTick',[year0:1/12:year0+0.9999999])
#set(gca,'xticklabel',xticks)
#ylim = [-100,600]
#set(gca,'xLim',xlim(:))
#ylabel('H (Wm^{-2})')
##
print '-------------------------------------------------------------------'
print 'Calculating daily mean values'
print '-------------------------------------------------------------------'
# disp('Press any key to calculate daily mean values')
# pause
print '-------------------------------------------------------------------'
print 'calculation of daily mean. Unit seg_start_i [C g/m2/day].'
print '-------------------------------------------------------------------'
Fsc_daily = npzeros(n_L1/num_point_per_day)
GPP_daily = npzeros(n_L1/num_point_per_day)
RE_daily = npzeros(n_L1/num_point_per_day)
ET_daily = npzeros(n_L1/num_point_per_day)
H_daily = npzeros(n_L1/num_point_per_day)
LE_daily = npzeros(n_L1/num_point_per_day)
H_daily = npzeros(n_L1/num_point_per_day)
k = 0
for i in range(0, n_L1, num_point_per_day):
for j in range(i,i + num_point_per_day):
Fsc_daily[k] = Fsc_daily[k] + Fc_filled[j]
GPP_daily[k] = GPP_daily[k] + GPP[j]
RE_daily[k] = RE_daily[k] + RE[j]
ET_daily[k] = ET_daily[k] + LE_filled[j]
Fsc_daily[k] = Fsc_daily[k] * (60*float(avgtime)/1000*12/44)
GPP_daily[k] = GPP_daily[k] * (60*float(avgtime)/1000*12/44)
RE_daily[k] = RE_daily[k] * (60*float(avgtime)/1000*12/44)
ET_daily[k] = ET_daily[k] * (60*float(avgtime)/(2440)/1000)
k = k + 1
NEE_annual= npmean(Fc_filled)*float((1800*48*(n_L1/(60.0/avgtime*24))*12/44/1000.0))
GPP_annual = npmean(GPP)*float((1800*48*(n_L1/(60.0/avgtime*24))*12/44/1000.0))
RE_annual = npmean(RE)*float((1800*48*(n_L1/(60.0/avgtime*24))*12/44/1000.0))
NEE_std_annual = Common.stdn1(Fsc_daily)/sqrt(n_L1/(60.0/avgtime*24))*(n_L1/(60.0/avgtime*24))
GPP_std_annual = Common.stdn1(GPP_daily)/sqrt(n_L1/(60.0/avgtime*24))*(n_L1/(60.0/avgtime*24))
RE_std_annual = Common.stdn1(RE_daily)/sqrt(n_L1/(60.0/avgtime*24))*(n_L1/(60.0/avgtime*24))
print 'NEE_annual', NEE_annual
print 'GPP_annual', GPP_annual
print 'RE_annual', RE_annual
print 'NEE_std_annual', NEE_std_annual
print 'GPP_std_annual', GPP_std_annual
print 'RE_std_annual', RE_std_annual
print '-------------------------------------------------------------------'
print 'Calculating daily mean ETs'
print '-------------------------------------------------------------------'
print 'calculation of daily mean. Unit seg_start_i [C g/m2/day].'
print '-------------------------------------------------------------------'
k = 0
for i in range(0, n_L1, num_point_per_day):
for j in range(i,i + num_point_per_day):
LE_daily[k] = LE_daily[k] \
+ LE_filled[j]*(60*float(avgtime)/(2440*1000))
k = k + 1
# npmean(Fc_filled)
LE_annual = npmean(LE_filled)*(1800.0*48.0*float(n_L1/(60.0/avgtime*24))/2440.0/1000.0)
LE_std_annual = Common.stdn1(LE_daily)/sqrt(n_L1/float(60.0/avgtime*24.0))*float(n_L1/(60.0/avgtime*24.0))
print 'LE_annaul', LE_annual
print 'LE_std_annaul', LE_std_annual
print '-------------------------------------------------------------------'
print 'Calculating daily npmean heating rate'
print '-------------------------------------------------------------------'
print 'calculation of daily npmean heating rate. Unit seg_start_i [MJ/m2/day].'
print '-------------------------------------------------------------------'
k = 0
for i in range(0, n_L1, num_point_per_day):
for j in range(i,i + num_point_per_day):
H_daily[k] = H_daily[k] \
+ H_filled[j]*(60*float(avgtime)/(1004*1.0))/(10E6)
k = k + 1
H_annual = sum(H_daily)
H_std_annual = Common.stdn1(H_daily)
print 'H_annaul', H_annual
print 'H_std_annaul', H_std_annual
for i in range(len(Fsc)):
file_plot_str = StringIO()
file_plot_str.write(str(date[i]) + ',') #1
file_plot_str.write(str(Fsc[i]) + ',') #2
file_plot_str.write(str(Fc_filled[i]) + ',') #3
file_plot_str.write(str(LEsc[i]) + ',') #4
file_plot_str.write(str(LE_filled[i]) + ',') #5
file_plot_str.write(str(Hsc[i]) + ',') #6
file_plot_str.write(str(H_filled[i]) + '\n') #7
output_plot_string = file_plot_str.getvalue()
output_plot_2_fp.write(output_plot_string)
output_plot_2_fp.close()
#For output
#Assume data start from 0:00
output_Fsc_daily = npzeros(n_L1)
output_GPP_daily = npzeros(n_L1)
output_RE_daily = npzeros(n_L1)
output_ET_daily = npzeros(n_L1)
output_LE_daily = npzeros(n_L1)
output_H_daily = npzeros(n_L1)
j = 0
for i in range(n_L1):
output_Fsc_daily[i] = Fsc_daily[j]
output_GPP_daily[i] = GPP_daily[j]
output_RE_daily[i] = RE_daily[j]
output_ET_daily[i] = ET_daily[j]
output_H_daily[i] = H_daily[j]
output_LE_daily[i] = LE_daily[j]
if((i+1) % num_point_per_day == 0):
j = j + 1
for i in range(n_L1):
file_str = StringIO()
file_str.write(str(output_ET_daily[i]) + ',') #1
file_str.write(str(Fc_filled[i]) + ',' ) #2
file_str.write(str(output_Fsc_daily[i]) + ',' ) #3
file_str.write(str(GPP[i]) + ',' ) #4
file_str.write(str(output_GPP_daily[i]) + ',') #5
file_str.write(str(GPP_annual) + ',') #6
file_str.write(str(GPP_std_annual) + ',') #7
file_str.write(str(H_filled[i]) + ',') #8
file_str.write(str(output_H_daily[i]) + ',') #9
file_str.write(str(H_annual) + ',') #10
file_str.write(str(H_std_annual) + ',') #11
file_str.write(str(LE_filled[i]) + ',') #12
file_str.write(str(output_LE_daily[i]) + ',') #13
file_str.write(str(LE_annual) + ',') #14
file_str.write(str(LE_std_annual) + ',') #15
file_str.write(str(NEE_annual) + ',') #16
file_str.write(str(NEE_std_annual) + ',') #17
file_str.write(str(output_RE_daily[i]) + ',') #18
file_str.write(str(RE_annual) + ',') #19
file_str.write(str(RE_std_annual) + ',') #20
file_str.write(str(co2[i]) + ',') #21
file_str.write(str(rsdn[i]) + ',') #22
file_str.write(str(ea[i]) + ',') #23
file_str.write(str(h2o[i]) + ',') #24
file_str.write(str(Ta[i]) + ',') #25
file_str.write(str(vpd[i]) + '\n' ) #26
output_string = file_str.getvalue()
output_fp.write(output_string)
output_fp.close()
return 'L2 Done'
