def sepseg(tm0,tsep=1):
"""
Seperate Segments
Assign a label to segments. Segments can have gaps no longer than
tsep days.
Parameters
----------
tm : masked array with time. Gaps are labeled with mask.
tsep : Segments can have gaps no longer than tsep.
Returns
-------
label : Masked array. Label identifies which segment a given point
belongs to. Masked elements do not belong to any segment.
Useage
------
>>> label = sepseg(tm)
>>> sL = ma.notmasked_contiguous(label)
`sL` is the list of slices corresponding to unique labels.
"""
# Copy to prevent problems from changing the mask in place
tm = tm0.copy()
label = np.zeros(tm.size)
label[:] = np.nan
nsep = tsep / keptoy.lc
gap = ma.masked_array(tm.data,mask=~tm.mask,copy=True)
nsep = tsep / keptoy.lc
gapSlice = ma.notmasked_contiguous(gap)
if gapSlice is None:
label[:] = 0
return label
for g in gapSlice:
if g.stop-g.start < nsep:
tm.mask[g] = False
sL = ma.notmasked_contiguous(tm)
nseg = len(sL)
for i in range(nseg):
s = sL[i]
label[s] = i
label = ma.masked_invalid(label)
return label
def cbvseg(tm,segsep=1):
"""
CBV Segmentation
Split time series into segments for CBV fitting. Sections are
considered distinct if they are seperated by more than segsep days
"""
nsep = segsep / keptoy.lc
gap = ma.masked_array(tm.data,mask=~tm.mask,copy=True)
nsep = segsep / keptoy.lc
gapSlice = ma.notmasked_contiguous(gap)
for g in gapSlice:
if g.stop-g.start < nsep:
tm.mask[g] = False
return ma.notmasked_contiguous(tm)
def dt(t0):
t = copy.deepcopy(t0)
fm = ma.masked_array(t.f,mask=t.fmask)
tm = ma.masked_array(t.TIME,mask=t.fmask)
label = sepseg(tm)
sL = ma.notmasked_contiguous(label)
# If there is only one slice.
if type(sL) == slice:
sL = [sL]
id = sL2id(sL)
tnd = fm.copy()
temp = [spldtm(tm[s],fm[s]) for s in sL]
temp = ma.hstack(temp)
tnd[id] = temp
return fm-tnd
def noiseyReg(t, dM, thresh=2):
"""
Noisey Region
If certain regions are much noisier than others, we should remove
them. A typical value of the noise is computed using a median
absolute deviation (MAD) on individual regions. If certain regions are
noiser by thresh, we mask them out.
Parameters
----------
dM : Single event statistic (masked array)
thresh : If region has MAD > thresh * typical MAD throw it out.
"""
tm = ma.masked_array(t, mask=dM.mask)
label = detrend.sepseg(tm)
sL = ma.notmasked_contiguous(label)
madseg = np.array([ma.median(ma.abs(dM[s])) for s in sL])
madLC = np.median(madseg) # Typical MAD of the light curve.
isNoisey = np.zeros(tm.size).astype(bool)
sNL = [] # List to keep track of the noisey segments
for s, mads in zip(sL, madseg):
if mads > thresh * madLC:
isNoisey[s] = True
sNL.append(s)
if len(sNL) != 0:
print "Removed following time ranges because data is noisey"
print "----------------------------------------------------"
for s in sNL:
print "%.2f %.2f " % (t[s.start], t[s.stop - 1])
return isNoisey
def maskIntrp(x0,y0,nContig=None):
"""
Use linear interpolation to fill in masked pointss
Parameters
----------
x0 : Independent var
y0 : Masked array.
nContig : Skip if masked region contains more than a certain
number of Contiguous masked points. nContig = None means
interpolate through everything.
"""
x,y = x0.copy(),y0.copy()
x = ma.masked_array(x,mask=y.mask,copy=True)
xc,yc = x.compressed(),y.compressed()
sp = UnivariateSpline(xc,yc,k=1,s=0)
x.mask = ~x.mask
sL = ma.notmasked_contiguous(x)
if sL is None:
return x.data,y
if nContig is None:
for s in sL:
y[s] = sp(x[s])
else:
for s in sL:
nNans = s.stop-s.start
if nNans <= nContig:
y[s] = sp(x[s])
return x.data,y
def nanIntrp(x0,y0,nContig=3):
"""
Use linear interpolation to fill in the nan-points.
nContig - Don't fill in nan values if there are more than a
certain number of contiguous ones.
"""
x,y = x0.copy(),y0.copy()
y = ma.masked_invalid(y)
x = ma.masked_array(x,mask=y.mask)
xc,yc = x.compressed(),y.compressed()
sp = UnivariateSpline(xc,yc,k=1,s=0)
x.mask = ~x.mask
sL = ma.notmasked_contiguous(x)
for s in sL:
nNans = s.stop-s.start
if nNans <= nContig:
y[s] = sp(x[s])
return x.data,y.data
def get_phidp_unf(radar, ncp_lev=0.4, rhohv_lev=0.6, debug=False, ncpts=20,
doc=-10, overide_sys_phase=False, sys_phase=-135,
nowrap=None, refl_field=None, ncp_field=None,
rhv_field=None, phidp_field=None):
"""
Get Unfolded Phi differential phase
Parameters
----------
radar : Radar
The input radar.
ncp_lev :
Miminum normal coherent power level. Regions below this value will
not be included in the calculation.
rhohv_lev :
Miminum copolar coefficient level. Regions below this value will not
be included in the calculation.
debug : bool, optioanl
True to print debugging information, False to supress printing.
ncpts : int
Minimum number of points in a ray. Regions within a ray smaller than
this or beginning before this gate number are excluded from
calculations.
doc : int or None.
Index of first gate not to include in field data, None include all.
overide_sys_phase : bool, optional
True to use `sys_phase` as the system phase. False will determine a
value automatically.
sys_phase : float, optional
System phase, not used if overide_sys_phase is False.
nowrap : or None
Gate number where unwrapping should begin. `None` will unwrap all
gates.
refl_field ncp_field, rhv_field, phidp_field : str
Field names within the radar object which represent the horizonal
reflectivity, normal coherent power, the copolar coefficient, and the
differential phase shift. A value of None for any of these parameters
will use the default field name as defined in the Py-ART
configuration file.
Returns
-------
cordata : array
Unwrapped phi differential phase.
"""
# parse the field parameters
if refl_field is None:
refl_field = get_field_name('reflectivity')
if ncp_field is None:
ncp_field = get_field_name('normalized_coherent_power')
if rhv_field is None:
rhv_field = get_field_name('cross_correlation_ratio')
if phidp_field is None:
phidp_field = get_field_name('differential_phase')
if doc is not None:
my_phidp = radar.fields[phidp_field]['data'][:, 0:doc]
my_rhv = radar.fields[rhv_field]['data'][:, 0:doc]
my_ncp = radar.fields[ncp_field]['data'][:, 0:doc]
my_z = radar.fields[refl_field]['data'][:, 0:doc]
else:
my_phidp = radar.fields[phidp_field]['data']
my_rhv = radar.fields[rhv_field]['data']
my_ncp = radar.fields[ncp_field]['data']
my_z = radar.fields[refl_field]['data']
t = time()
if overide_sys_phase:
system_zero = sys_phase
else:
system_zero = det_sys_phase(
radar, ncp_field=ncp_field, rhv_field=rhv_field,
phidp_field=phidp_field)
if system_zero is None:
system_zero = sys_phase
cordata = np.zeros(my_rhv.shape, dtype=float)
for radial in range(my_rhv.shape[0]):
my_snr = snr(my_z[radial, :])
notmeteo = np.logical_or(np.logical_or(
my_ncp[radial, :] < ncp_lev,
my_rhv[radial, :] < rhohv_lev), my_snr < 10.0)
x_ma = ma.masked_where(notmeteo, my_phidp[radial, :])
try:
ma.notmasked_contiguous(x_ma)
for slc in ma.notmasked_contiguous(x_ma):
# so trying to get rid of clutter and small things that
# should not add to phidp anyway
if slc.stop - slc.start < ncpts or slc.start < ncpts:
x_ma.mask[slc.start - 1:slc.stop + 1] = True
c = 0
except TypeError: # non sequence, no valid regions
c = 1 # ie do nothing
x_ma.mask[:] = True
except AttributeError:
# sys.stderr.write('No Valid Regions, ATTERR \n ')
# sys.stderr.write(myfile.times['time_end'].isoformat() + '\n')
# print x_ma
# print x_ma.mask
c = 1 # also do nothing
x_ma.mask = True
if 'nowrap' is not None:
# Start the unfolding a bit later in order to avoid false
# jumps based on clutter
unwrapped = copy.deepcopy(x_ma)
end_unwrap = unwrap_masked(x_ma[nowrap::], centered=False)
unwrapped[nowrap::] = end_unwrap
else:
unwrapped = unwrap_masked(x_ma, centered=False)
#end so no clutter expected
system_max = unwrapped[np.where(np.logical_not(
notmeteo))][-10:-1].mean() - system_zero
unwrapped_fixed = np.zeros(len(x_ma), dtype=float)
based = unwrapped-system_zero
based[0] = 0.0
notmeteo[0] = False
based[-1] = system_max
notmeteo[-1] = False
unwrapped_fixed[np.where(np.logical_not(based.mask))[0]] = \
based[np.where(np.logical_not(based.mask))[0]]
if len(based[np.where(np.logical_not(based.mask))[0]]) > 11:
unwrapped_fixed[np.where(based.mask)[0]] = \
np.interp(np.where(based.mask)[0],
np.where(np.logical_not(based.mask))[0],
smooth_and_trim(based[np.where(
np.logical_not(based.mask))[0]]))
else:
unwrapped_fixed[np.where(based.mask)[0]] = \
np.interp(np.where(based.mask)[0],
np.where(np.logical_not(based.mask))[0],
based[np.where(np.logical_not(based.mask))[0]])
if c != 1:
cordata[radial, :] = unwrapped_fixed
else:
cordata[radial, :] = np.zeros(my_rhv.shape[1])
if debug:
print("Exec time: ", time() - t)
return cordata
def get_phidp_unf(radar,
ncp_lev=0.4,
rhohv_lev=0.6,
debug=False,
ncpts=20,
doc=-10,
overide_sys_phase=False,
sys_phase=-135,
nowrap=None,
refl_field=None,
ncp_field=None,
rhv_field=None,
phidp_field=None):
"""
Get Unfolded Phi differential phase
Parameters
----------
radar : Radar
The input radar.
ncp_lev :
Miminum normal coherent power level. Regions below this value will
not be included in the calculation.
rhohv_lev :
Miminum copolar coefficient level. Regions below this value will not
be included in the calculation.
debug : bool, optioanl
True to print debugging information, False to supress printing.
ncpts : int
Minimum number of points in a ray. Regions within a ray smaller than
this or beginning before this gate number are excluded from
calculations.
doc : int or None.
Index of first gate not to include in field data, None include all.
overide_sys_phase : bool, optional
True to use `sys_phase` as the system phase. False will determine a
value automatically.
sys_phase : float, optional
System phase, not used if overide_sys_phase is False.
nowrap : or None
Gate number where unwrapping should begin. `None` will unwrap all
gates.
refl_field ncp_field, rhv_field, phidp_field : str
Field names within the radar object which represent the horizonal
reflectivity, normal coherent power, the copolar coefficient, and the
differential phase shift. A value of None for any of these parameters
will use the default field name as defined in the Py-ART
configuration file.
Returns
-------
cordata : array
Unwrapped phi differential phase.
"""
# parse the field parameters
if refl_field is None:
refl_field = get_field_name('reflectivity')
if ncp_field is None:
ncp_field = get_field_name('normalized_coherent_power')
if rhv_field is None:
rhv_field = get_field_name('cross_correlation_ratio')
if phidp_field is None:
phidp_field = get_field_name('differential_phase')
if doc is not None:
my_phidp = radar.fields[phidp_field]['data'][:, 0:doc]
my_rhv = radar.fields[rhv_field]['data'][:, 0:doc]
my_ncp = radar.fields[ncp_field]['data'][:, 0:doc]
my_z = radar.fields[refl_field]['data'][:, 0:doc]
else:
my_phidp = radar.fields[phidp_field]['data']
my_rhv = radar.fields[rhv_field]['data']
my_ncp = radar.fields[ncp_field]['data']
my_z = radar.fields[refl_field]['data']
t = time()
if overide_sys_phase:
system_zero = sys_phase
else:
system_zero = det_sys_phase(radar,
ncp_field=ncp_field,
rhv_field=rhv_field,
phidp_field=phidp_field)
if system_zero is None:
system_zero = sys_phase
cordata = np.zeros(my_rhv.shape, dtype=float)
for radial in range(my_rhv.shape[0]):
my_snr = snr(my_z[radial, :])
notmeteo = np.logical_or(
np.logical_or(my_ncp[radial, :] < ncp_lev,
my_rhv[radial, :] < rhohv_lev), my_snr < 10.0)
x_ma = ma.masked_where(notmeteo, my_phidp[radial, :])
try:
ma.notmasked_contiguous(x_ma)
for slc in ma.notmasked_contiguous(x_ma):
# so trying to get rid of clutter and small things that
# should not add to phidp anyway
if slc.stop - slc.start < ncpts or slc.start < ncpts:
x_ma.mask[slc.start - 1:slc.stop + 1] = True
c = 0
except TypeError: # non sequence, no valid regions
c = 1 # ie do nothing
x_ma.mask[:] = True
except AttributeError:
# sys.stderr.write('No Valid Regions, ATTERR \n ')
# sys.stderr.write(myfile.times['time_end'].isoformat() + '\n')
# print x_ma
# print x_ma.mask
c = 1 # also do nothing
x_ma.mask = True
if 'nowrap' is not None:
# Start the unfolding a bit later in order to avoid false
# jumps based on clutter
unwrapped = copy.deepcopy(x_ma)
end_unwrap = unwrap_masked(x_ma[nowrap::], centered=False)
unwrapped[nowrap::] = end_unwrap
else:
unwrapped = unwrap_masked(x_ma, centered=False)
#end so no clutter expected
system_max = unwrapped[np.where(
np.logical_not(notmeteo))][-10:-1].mean() - system_zero
unwrapped_fixed = np.zeros(len(x_ma), dtype=float)
based = unwrapped - system_zero
based[0] = 0.0
notmeteo[0] = False
based[-1] = system_max
notmeteo[-1] = False
unwrapped_fixed[np.where(np.logical_not(based.mask))[0]] = \
based[np.where(np.logical_not(based.mask))[0]]
if len(based[np.where(np.logical_not(based.mask))[0]]) > 11:
unwrapped_fixed[np.where(based.mask)[0]] = \
np.interp(np.where(based.mask)[0],
np.where(np.logical_not(based.mask))[0],
smooth_and_trim(based[np.where(
np.logical_not(based.mask))[0]]))
else:
unwrapped_fixed[np.where(based.mask)[0]] = \
np.interp(np.where(based.mask)[0],
np.where(np.logical_not(based.mask))[0],
based[np.where(np.logical_not(based.mask))[0]])
if c != 1:
cordata[radial, :] = unwrapped_fixed
else:
cordata[radial, :] = np.zeros(my_rhv.shape[1])
if debug:
print("Exec time: ", time() - t)
return cordata
def repairMap(self, indicemap, map, xoff, yoff, ghz):
self.map = map
self.indicemap = indicemap
self.xoff = xoff
self.yoff = yoff
flag = False
if (flag == True):
###
print("original map modified - ini")
print(datetime.datetime.now())
###
originalmapmodified = (self.map - np.min(self.map)).astype(FLOAT)
originalmapmodified = originalmapmodified / np.max(originalmapmodified)
if (flag == True):
###
print("original map modified - end")
print(datetime.datetime.now())
###
if (flag == True):
###
print("constructor artificial map - ini")
print(datetime.datetime.now())
###
artificialMap = ArtificialMap(self.xoff, self.yoff, self.pathBpos,
self.pathstbeams)
if (flag == True):
###
print("constructor artificial map - end")
print(datetime.datetime.now())
###
if (flag == True):
###
print("getArtificialMap - ini")
print(datetime.datetime.now())
###
normcs1 = artificialMap.getArtificialMap(self.indicemap)
if (flag == True):
###
print("getArtificialMap - end")
print(datetime.datetime.now())
###
if (flag == True):
###
print("getCoordinates - ini")
print(datetime.datetime.now())
###
nxoff, nyoff = artificialMap.getCoordinates()
if (flag == True):
###
print("getCoordinates - end")
print(datetime.datetime.now())
###
if (flag == True):
###
print("getCoordinates ghz- ini")
print(datetime.datetime.now())
###
xcal1, ycal1, xcal, ycal = artificialMap.getCalCoordinates(ghz)
if (flag == True):
###
print("getCoordinates ghz- end")
print(datetime.datetime.now())
###
repairedMap = np.zeros(originalmapmodified.shape, dtype='float')
dify = np.ediff1d(ycal1, to_end=ZERO)
msk = (dify <> ZERO)
one = ma.array(xcal1, mask=msk)
onecont = ma.notmasked_contiguous(one)
if (flag == True):
###
print("loop de correcao delta - ini")
print(datetime.datetime.now())
###
for k in range(len(onecont)): #tiras:
minnxoff = np.min(nxoff[onecont[k]])
maxnxoff = np.max(nxoff[onecont[k]])
lennxoff = len(nxoff[onecont[k]])
#xx = np.linspace(minnxoff,maxnxoff,lennxoff)
tiempo = np.arange(1 - lennxoff, lennxoff)
lineOriginalMod = originalmapmodified[onecont[k]]
lineNormcs1 = normcs1[onecont[k]]
cross_correlation = correlate(lineOriginalMod, lineNormcs1)
shift_calculated = tiempo[
cross_correlation.argmax()] * 1.0 * maxnxoff / lennxoff
corregido = np.roll(map[onecont[k]],
-np.int(shift_calculated / 2.))
repairedMap[onecont[k]] = corregido
if (flag == True):
###
print("loop de correcao delta - end")
print(datetime.datetime.now())
###
if (flag == True):
###
print("adjusting final antes de sair da rotina - ini")
print(datetime.datetime.now())
###
# adjusting the map to originals values
for i in range(len(repairedMap)):
if (repairedMap[i] == ZEROVALUE):
repairedMap[i] = (np.min(map)).astype(float)
if (flag == True):
###
print("adjusting final antes de sair da rotina - end")
print(datetime.datetime.now())
###
return repairedMap
def Stochastic_Process(j):
Profile, num_profiles = Initialise_model(j)
peak_enlarg, mu_peak, s_peak, Year_behaviour, User_list = Initialise_inputs(
j)
'''
Calculation of the peak time range, which is used to discriminate between off-peak and on-peak coincident switch-on probability
Calculates first the overall Peak Window (taking into account all User classes).
The peak window is just a time window in which coincident switch-on of multiple appliances assumes a higher probability than off-peak
Within the peak window, a random peak time is calculated and then enlarged into a peak_time_range following again a random procedure
'''
windows_curve = np.zeros(1440) #creates an empty daily profile
Tot_curve = np.zeros(1440) #creates another empty daily profile
for Us in User_list:
App_count = 0
for App in Us.App_list:
#Calculate windows curve, i.e. the theoretical maximum curve that can be obtained, for each app, by switching-on always all the 'n' apps altogether in any time-step of the functioning windows
single_wcurve = Us.App_list[App_count].daily_use * np.mean(
Us.App_list[App_count].POWER
) * Us.App_list[
App_count].number #this computes the curve for the specific App
windows_curve = np.vstack(
[windows_curve, single_wcurve]
) #this stacks the specific App curve in an overall curve comprising all the Apps within a User class
App_count += 1
Us.windows_curve = windows_curve #after having iterated for all the Apps within a User class, saves the overall User class theoretical maximum curve
Us.windows_curve = np.transpose(np.sum(Us.windows_curve,
axis=0)) * Us.num_users
Tot_curve = Tot_curve + Us.windows_curve #adds the User's theoretical max profile to the total theoretical max comprising all classes
peak_window = np.transpose(np.argwhere(Tot_curve == np.amax(
Tot_curve))) #Find the peak window within the theoretical max profile
peak_time = round(
random.normalvariate(round(np.average(peak_window)),
1 / 3 * (peak_window[0, -1] - peak_window[0, 0]))
) #Within the peak_window, randomly calculate the peak_time using a gaussian distribution
peak_time_range = np.arange(
(peak_time - round(
math.fabs(peak_time -
(random.gauss(peak_time, (peak_enlarg * peak_time)))))),
(peak_time + round(
math.fabs(peak_time -
random.gauss(peak_time, (peak_enlarg * peak_time)))))
) #the peak_time is randomly enlarged based on the calibration parameter peak_enlarg
'''
The core stochastic process starts here. For each profile requested by the software user,
each Appliance instance within each User instance is separately and stochastically generated
'''
for prof_i in range(
num_profiles
): #the whole code is repeated for each profile that needs to be generated
Tot_Classes = np.zeros(
1440
) #initialise an empty daily profile that will be filled with the sum of the hourly profiles of each User instance
for Us in User_list: #iterates for each User instance (i.e. for each user class)
Us.load = np.zeros(1440) #initialise empty load for User instance
for i in range(
Us.num_users
): #iterates for every single user within a User class. Each single user has its own separate randomisation
if Us.user_preference == 0:
rand_daily_pref = 0
pass
else:
rand_daily_pref = random.randint(1, Us.user_preference)
for App in Us.App_list: #iterates for all the App types in the given User class
#initialises variables for the cycle
tot_time = 0
App.daily_use = np.zeros(1440)
if random.uniform(
0, 1
) > App.occasional_use: #evaluates if occasional use happens or not
continue
else:
pass
if App.Pref_index == 0:
pass
else:
if rand_daily_pref == App.Pref_index: #evaluates if daily preference coincides with the randomised daily preference number
pass
else:
continue
if App.wd_we == Year_behaviour[
prof_i] or App.wd_we == 2: #checks if the app is allowed in the given yearly behaviour pattern
pass
else:
continue
#recalculate windows start and ending times randomly, based on the inputs
rand_window_1 = np.array([
int(
random.uniform(
(App.window_1[0] - App.random_var_1),
(App.window_1[0] + App.random_var_1))),
int(
random.uniform(
(App.window_1[1] - App.random_var_1),
(App.window_1[1] + App.random_var_1)))
])
if rand_window_1[0] < 0:
rand_window_1[0] = 0
if rand_window_1[1] > 1440:
rand_window_1[1] = 1440
rand_window_2 = np.array([
int(
random.uniform(
(App.window_2[0] - App.random_var_2),
(App.window_2[0] + App.random_var_2))),
int(
random.uniform(
(App.window_2[1] - App.random_var_2),
(App.window_2[1] + App.random_var_2)))
])
if rand_window_2[0] < 0:
rand_window_2[0] = 0
if rand_window_2[1] > 1440:
rand_window_2[1] = 1440
rand_window_3 = np.array([
int(
random.uniform(
(App.window_3[0] - App.random_var_3),
(App.window_3[0] + App.random_var_3))),
int(
random.uniform(
(App.window_3[1] - App.random_var_3),
(App.window_3[1] + App.random_var_3)))
])
if rand_window_3[0] < 0:
rand_window_3[0] = 0
if rand_window_3[1] > 1440:
rand_window_3[1] = 1440
#redefines functioning windows based on the previous randomisation of the boundaries
if App.flat == 'yes': #if the app is "flat" the code stops right after filling the newly created windows without applying any further stochasticity
App.daily_use[
rand_window_1[0]:rand_window_1[1]] = np.full(
np.diff(rand_window_1),
App.POWER[prof_i] * App.number)
App.daily_use[
rand_window_2[0]:rand_window_2[1]] = np.full(
np.diff(rand_window_2),
App.POWER[prof_i] * App.number)
App.daily_use[
rand_window_3[0]:rand_window_3[1]] = np.full(
np.diff(rand_window_3),
App.POWER[prof_i] * App.number)
Us.load = Us.load + App.daily_use
continue
else: #otherwise, for "non-flat" apps it puts a mask on the newly defined windows and continues
App.daily_use[
rand_window_1[0]:rand_window_1[1]] = np.full(
np.diff(rand_window_1), 0.001)
App.daily_use[
rand_window_2[0]:rand_window_2[1]] = np.full(
np.diff(rand_window_2), 0.001)
App.daily_use[
rand_window_3[0]:rand_window_3[1]] = np.full(
np.diff(rand_window_3), 0.001)
App.daily_use_masked = np.zeros_like(
ma.masked_not_equal(App.daily_use, 0.001))
App.power = App.POWER[prof_i]
#random variability is applied to the total functioning time and to the duration of the duty cycles, if they have been specified
random_var_t = random.uniform((1 - App.r_t), (1 + App.r_t))
if App.activate == 1:
App.p_11 = App.P_11 * (
random.uniform((1 - App.Thermal_P_var),
(1 + App.Thermal_P_var))
) #randomly variates the power of thermal apps, otherwise variability is 0
App.p_12 = App.P_12 * (
random.uniform((1 - App.Thermal_P_var),
(1 + App.Thermal_P_var))
) #randomly variates the power of thermal apps, otherwise variability is 0
random_cycle1 = np.concatenate(
((np.ones(
int(App.t_11 * (random.uniform(
(1 + App.r_c1),
(1 - App.r_c1))))) * App.p_11),
(np.ones(
int(App.t_12 * (random.uniform(
(1 + App.r_c1), (1 - App.r_c1))))) *
App.p_12))) #randomise also the fixed cycle
random_cycle2 = random_cycle1
random_cycle3 = random_cycle1
elif App.activate == 2:
App.p_11 = App.P_11 * (
random.uniform((1 - App.Thermal_P_var),
(1 + App.Thermal_P_var))
) #randomly variates the power of thermal apps, otherwise variability is 0
App.p_12 = App.P_12 * (
random.uniform((1 - App.Thermal_P_var),
(1 + App.Thermal_P_var))
) #randomly variates the power of thermal apps, otherwise variability is 0
App.p_21 = App.P_21 * (
random.uniform((1 - App.Thermal_P_var),
(1 + App.Thermal_P_var))
) #randomly variates the power of thermal apps, otherwise variability is 0
App.p_22 = App.P_22 * (
random.uniform((1 - App.Thermal_P_var),
(1 + App.Thermal_P_var))
) #randomly variates the power of thermal apps, otherwise variability is 0
random_cycle1 = np.concatenate(
((np.ones(
int(App.t_11 * (random.uniform(
(1 + App.r_c1),
(1 - App.r_c1))))) * App.p_11),
(np.ones(
int(App.t_12 * (random.uniform(
(1 + App.r_c1), (1 - App.r_c1))))) *
App.p_12))) #randomise also the fixed cycle
random_cycle2 = np.concatenate(
((np.ones(
int(App.t_21 * (random.uniform(
(1 + App.r_c2),
(1 - App.r_c2))))) * App.p_21),
(np.ones(
int(App.t_22 * (random.uniform(
(1 + App.r_c2), (1 - App.r_c2))))) *
App.p_22))) #randomise also the fixed cycle
random_cycle3 = random_cycle1
elif App.activate == 3:
App.p_11 = App.P_11 * (
random.uniform((1 - App.Thermal_P_var),
(1 + App.Thermal_P_var))
) #randomly variates the power of thermal apps, otherwise variability is 0
App.p_12 = App.P_12 * (
random.uniform((1 - App.Thermal_P_var),
(1 + App.Thermal_P_var))
) #randomly variates the power of thermal apps, otherwise variability is 0
App.p_21 = App.P_12 * (
random.uniform((1 - App.Thermal_P_var),
(1 + App.Thermal_P_var))
) #randomly variates the power of thermal apps, otherwise variability is 0
App.p_22 = App.P_22 * (
random.uniform((1 - App.Thermal_P_var),
(1 + App.Thermal_P_var))
) #randomly variates the power of thermal apps, otherwise variability is 0
App.p_31 = App.P_31 * (
random.uniform((1 - App.Thermal_P_var),
(1 + App.Thermal_P_var))
) #randomly variates the power of thermal apps, otherwise variability is 0
App.p_32 = App.P_32 * (
random.uniform((1 - App.Thermal_P_var),
(1 + App.Thermal_P_var))
) #randomly variates the power of thermal apps, otherwise variability is 0
random_cycle1 = random.choice([
np.concatenate(((np.ones(
int(App.t_11 * (random.uniform(
(1 + App.r_c1),
(1 - App.r_c1))))) * App.p_11), (np.ones(
int(App.t_12 * (random.uniform(
(1 + App.r_c1),
(1 - App.r_c1))))) * App.p_12))),
np.concatenate(((np.ones(
int(App.t_12 * (random.uniform(
(1 + App.r_c1),
(1 - App.r_c1))))) * App.p_12), (np.ones(
int(App.t_11 * (random.uniform(
(1 + App.r_c1),
(1 - App.r_c1))))) * App.p_11)))
]) #randomise also the fixed cycle
random_cycle2 = random.choice([
np.concatenate(((np.ones(
int(App.t_21 * (random.uniform(
(1 + App.r_c2),
(1 - App.r_c2))))) * App.p_21), (np.ones(
int(App.t_22 * (random.uniform(
(1 + App.r_c2),
(1 - App.r_c2))))) * App.p_22))),
np.concatenate(((np.ones(
int(App.t_22 * (random.uniform(
(1 + App.r_c2),
(1 - App.r_c2))))) * App.p_22), (np.ones(
int(App.t_21 * (random.uniform(
(1 + App.r_c2),
(1 - App.r_c2))))) * App.p_21)))
])
random_cycle3 = random.choice([
np.concatenate(((np.ones(
int(App.t_31 * (random.uniform(
(1 + App.r_c3),
(1 - App.r_c3))))) * App.p_31), (np.ones(
int(App.t_32 * (random.uniform(
(1 + App.r_c3),
(1 - App.r_c3))))) * App.p_32))),
np.concatenate(((np.ones(
int(App.t_32 * (random.uniform(
(1 + App.r_c3),
(1 - App.r_c3))))) * App.p_32), (np.ones(
int(App.t_31 * (random.uniform(
(1 + App.r_c3),
(1 - App.r_c3))))) * App.p_31)))
]) #this is to avoid that all cycles are sincronous
else:
pass
rand_time = round(
random.uniform(App.func_time,
int(App.func_time * random_var_t)))
#control to check that the total randomised time of use does not exceed the total space available in the windows
if rand_time > 0.99 * (np.diff(rand_window_1) +
np.diff(rand_window_2) +
np.diff(rand_window_3)):
rand_time = int(
0.99 *
(np.diff(rand_window_1) + np.diff(rand_window_2) +
np.diff(rand_window_3)))
max_free_spot = rand_time #free spots are used to detect if there's still space for switch_ons. Before calculating actual free spots, the max free spot is set equal to the entire randomised func_time
while tot_time <= rand_time: #this is the key cycle, which runs for each App until the switch_ons and their duration equals the randomised total time of use of the App
#check how many windows to consider
if App.num_windows == 1:
switch_on = int(
random.choice([
random.uniform(rand_window_1[0],
(rand_window_1[1]))
]))
elif App.num_windows == 2:
switch_on = int(
random.choice([
random.uniform(rand_window_1[0],
(rand_window_1[1])),
random.uniform(rand_window_2[0],
(rand_window_2[1]))
]))
else:
switch_on = int(
random.choice([
random.uniform(rand_window_1[0],
(rand_window_1[1])),
random.uniform(rand_window_2[0],
(rand_window_2[1])),
random.uniform(rand_window_3[0],
(rand_window_3[1]))
]))
#Identifies a random switch on time within the available functioning windows
if App.daily_use[
switch_on] == 0.001: #control to check if the app is not already on at the randomly selected switch-on time
if switch_on in range(rand_window_1[0],
rand_window_1[1]):
if np.any(
App.
daily_use[switch_on:rand_window_1[1]]
!= 0.001
): #control to check if there are any other switch on times after the current one
next_switch = [
switch_on + k[0] for k in np.where(
App.daily_use[switch_on:] != 0.001)
] #identifies the position of next switch on time and sets it as a limit for the duration of the current switch on
if (
next_switch[0] - switch_on
) >= App.func_cycle and max_free_spot >= App.func_cycle:
upper_limit = min(
(next_switch[0] - switch_on),
min(rand_time,
rand_window_1[1] - switch_on))
elif (
next_switch[0] - switch_on
) < App.func_cycle and max_free_spot >= App.func_cycle: #if next switch_on event does not allow for a minimum functioning cycle without overlapping, but there are other larger free spots, the cycle tries again from the beginning
continue
else:
upper_limit = next_switch[
0] - switch_on #if there are no other options to reach the total time of use, empty spaces are filled without minimum cycle restrictions until reaching the limit
else:
upper_limit = min(
rand_time, rand_window_1[1] - switch_on
) #if there are no other switch-on events after the current one, the upper duration limit is set this way
if upper_limit >= App.func_cycle: #if the upper limit is higher than minimum functioning time, an array of indexes is created to be later put in the profile
indexes = np.arange(
switch_on, switch_on + (int(
random.uniform(
App.func_cycle, upper_limit)))
) #a random duration is chosen between the upper limit and the minimum cycle
else:
indexes = np.arange(
switch_on, switch_on + upper_limit
) #this is the case in which empty spaces need to be filled without constraints to reach the total time goal
elif switch_on in range(
rand_window_2[0], rand_window_2[1]
): #if random switch_on happens in windows2, same code as above is repeated for windows2
if np.any(App.
daily_use[switch_on:rand_window_2[1]]
!= 0.001):
next_switch = [
switch_on + k[0] for k in np.where(
App.daily_use[switch_on:] != 0.001)
]
if (
next_switch[0] - switch_on
) >= App.func_cycle and max_free_spot >= App.func_cycle:
upper_limit = min(
(next_switch[0] - switch_on),
min(rand_time,
rand_window_2[1] - switch_on))
elif (
next_switch[0] - switch_on
) < App.func_cycle and max_free_spot >= App.func_cycle:
continue
else:
upper_limit = next_switch[0] - switch_on
else:
upper_limit = min(
rand_time,
rand_window_2[1] - switch_on)
if upper_limit >= App.func_cycle:
indexes = np.arange(
switch_on, switch_on + (int(
random.uniform(
App.func_cycle, upper_limit))))
else:
indexes = np.arange(
switch_on, switch_on + upper_limit)
else: #if switch_on is not in window1 nor in window2, it shall be in window3. Same code is repreated
if np.any(App.
daily_use[switch_on:rand_window_3[1]]
!= 0.001):
next_switch = [
switch_on + k[0] for k in np.where(
App.daily_use[switch_on:] != 0.001)
]
if (
next_switch[0] - switch_on
) >= App.func_cycle and max_free_spot >= App.func_cycle:
upper_limit = min(
(next_switch[0] - switch_on),
min(rand_time,
rand_window_3[1] - switch_on))
elif (
next_switch[0] - switch_on
) < App.func_cycle and max_free_spot >= App.func_cycle:
continue
else:
upper_limit = next_switch[0] - switch_on
else:
upper_limit = min(
rand_time,
rand_window_3[1] - switch_on)
if upper_limit >= App.func_cycle:
indexes = np.arange(
switch_on, switch_on + (int(
random.uniform(
App.func_cycle, upper_limit))))
else:
indexes = np.arange(
switch_on, switch_on + upper_limit)
tot_time = tot_time + indexes.size #the count of total time is updated with the size of the indexes array
if tot_time > rand_time: #control to check when the total functioning time is reached. It will be typically overcome, so a correction is applied to avoid this
indexes_adj = indexes[:-(
tot_time - rand_time
)] #correctes indexes size to avoid overcoming total time
if np.in1d(peak_time_range, indexes_adj).any(
) and App.fixed == 'no': #check if indexes are in peak window and if the coincident behaviour is locked by the "fixed" attribute
coincidence = min(
App.number,
max(
1,
math.ceil(
random.gauss(
math.ceil(App.number *
mu_peak),
(s_peak * App.number *
mu_peak))))
) #calculates coincident behaviour within the peak time range
elif np.in1d(peak_time_range, indexes_adj).any(
) == False and App.fixed == 'no': #check if indexes are off-peak and if coincident behaviour is locked or not
Prob = random.uniform(
0, (App.number - 1) / App.number
) #calculates probability of coincident switch_ons off-peak
array = np.arange(0,
App.number) / App.number
try:
on_number = np.max(
np.where(Prob >= array)) + 1
except ValueError:
on_number = 1
coincidence = on_number #randomly selects how many apps are on at the same time for each app type based on the above probabilistic algorithm
else:
coincidence = App.number #this is the case when App.fixed is activated. All 'n' apps of an App instance are switched_on altogether
if App.activate > 0: #evaluates if the app has some duty cycles to be considered
if indexes_adj.size > 0:
evaluate = round(
np.mean(indexes_adj)
) #calculates the mean time position of the current switch_on event, to later select the proper duty cycle
else:
evaluate = 0
#based on the evaluate value, selects the proper duty cycle and puts the corresponding power values in the indexes range
if evaluate in range(
App.cw11[0],
App.cw11[1]) or evaluate in range(
App.cw12[0], App.cw12[1]):
np.put(App.daily_use, indexes_adj,
(random_cycle1 * coincidence))
np.put(App.daily_use_masked,
indexes_adj,
(random_cycle1 * coincidence),
mode='clip')
elif evaluate in range(
App.cw21[0],
App.cw21[1]) or evaluate in range(
App.cw22[0], App.cw22[1]):
np.put(App.daily_use, indexes_adj,
(random_cycle2 * coincidence))
np.put(App.daily_use_masked,
indexes_adj,
(random_cycle2 * coincidence),
mode='clip')
else:
np.put(App.daily_use, indexes_adj,
(random_cycle3 * coincidence))
np.put(App.daily_use_masked,
indexes_adj,
(random_cycle3 * coincidence),
mode='clip')
else: #if no duty cycles are specififed, a regular switch_on event is modelled
np.put(
App.daily_use, indexes_adj,
(App.power * (random.uniform(
(1 - App.Thermal_P_var),
(1 + App.Thermal_P_var))) *
coincidence)
) #randomises also the App Power if Thermal_P_var is on
np.put(App.daily_use_masked,
indexes_adj,
(App.power * (random.uniform(
(1 - App.Thermal_P_var),
(1 + App.Thermal_P_var))) *
coincidence),
mode='clip')
App.daily_use_masked = np.zeros_like(
ma.masked_greater_equal(
App.daily_use_masked, 0.001)
) #updates the mask excluding the current switch_on event to identify the free_spots for the next iteration
tot_time = (
tot_time - indexes.size
) + indexes_adj.size #updates the total time correcting the previous value
break #exit cycle and go to next App
else: #if the tot_time has not yet exceeded the App total functioning time, the cycle does the same without applying corrections to indexes size
if np.in1d(
peak_time_range,
indexes).any() and App.fixed == 'no':
coincidence = min(
App.number,
max(
1,
math.ceil(
random.gauss(
math.ceil(App.number *
mu_peak),
(s_peak * App.number *
mu_peak)))))
elif np.in1d(peak_time_range, indexes).any(
) == False and App.fixed == 'no':
Prob = random.uniform(
0, (App.number - 1) / App.number)
array = np.arange(0,
App.number) / App.number
try:
on_number = np.max(
np.where(Prob >= array)) + 1
except ValueError:
on_number = 1
coincidence = on_number
else:
coincidence = App.number
if App.activate > 0:
if indexes.size > 0:
evaluate = round(np.mean(indexes))
else:
evaluate = 0
if evaluate in range(
App.cw11[0],
App.cw11[1]) or evaluate in range(
App.cw12[0], App.cw12[1]):
np.put(App.daily_use, indexes,
(random_cycle1 * coincidence))
np.put(App.daily_use_masked,
indexes,
(random_cycle1 * coincidence),
mode='clip')
elif evaluate in range(
App.cw21[0],
App.cw21[1]) or evaluate in range(
App.cw22[0], App.cw22[1]):
np.put(App.daily_use, indexes,
(random_cycle2 * coincidence))
np.put(App.daily_use_masked,
indexes,
(random_cycle2 * coincidence),
mode='clip')
else:
np.put(App.daily_use, indexes,
(random_cycle3 * coincidence))
np.put(App.daily_use_masked,
indexes,
(random_cycle3 * coincidence),
mode='clip')
else:
np.put(App.daily_use, indexes,
(App.power * (random.uniform(
(1 - App.Thermal_P_var),
(1 + App.Thermal_P_var))) *
coincidence))
np.put(App.daily_use_masked,
indexes,
(App.power * (random.uniform(
(1 - App.Thermal_P_var),
(1 + App.Thermal_P_var))) *
coincidence),
mode='clip')
App.daily_use_masked = np.zeros_like(
ma.masked_greater_equal(
App.daily_use_masked, 0.001))
tot_time = tot_time #no correction applied to previously calculated value
free_spots = [
] #calculate how many free spots remain for further switch_ons
try:
for j in ma.notmasked_contiguous(
App.daily_use_masked):
free_spots.append(j.stop - j.start)
except TypeError:
free_spots = [0]
max_free_spot = max(free_spots)
else:
continue #if the random switch_on falls somewhere where the App has been already turned on, tries again from beginning of the while cycle
Us.load = Us.load + App.daily_use #adds the App profile to the User load
Tot_Classes = Tot_Classes + Us.load #adds the User load to the total load of all User classes
Profile.append(
Tot_Classes
) #appends the total load to the list that will contain all the generated profiles
print('Profile', prof_i + 1, '/', num_profiles,
'completed') #screen update about progress of computation
return (Profile)
def append_phidp_unf(myfile, **kwargs):
ncp_lev = kwargs.get('ncp_lev', 0.4)
rhohv_lev = kwargs.get('rhohv_lev', 0.6)
debug = kwargs.get('debug', False)
ncpts = kwargs('ncpts', 20)
d = myfile.read_a_field(myfile.fields.index('PHIDP_F'))
d = myfile.read_a_field(myfile.fields.index('RHOHV_F'))
d = myfile.read_a_field(myfile.fields.index('NCP_F'))
d = myfile.read_a_field(myfile.fields.index('DBZ_F'))
t = time()
system_zero = det_sys_phase(myfile, -141.097702627)
cordata = np.zeros(myfile.RHOHV_F.shape, dtype=float)
for sweep in range(myfile.RHOHV_F.shape[0]):
if debug:
print "sweep :: ", sweep
for radial in range(myfile.RHOHV_F.shape[1]):
my_snr = snr(myfile.DBZ_F[sweep, radial, :])
notmeteo = np.logical_or(
np.logical_or(myfile.NCP_F[sweep, radial, :] <
ncp_lev, myfile.RHOHV_F[sweep, radial, :] <
rhohv_lev), my_snr < 10.0)
x_ma = ma.masked_where(notmeteo, myfile.PHIDP_F[sweep, radial, :])
try:
ma.notmasked_contiguous(x_ma)
for slc in ma.notmasked_contiguous(x_ma):
# so trying to get rid of clutter and small things that
# should not add to phidp anyway
if slc.stop - slc.start < ncpts or slc.start < ncpts:
x_ma.mask[slc.start - 1:slc.stop + 1] = True
c = 0
except TypeError: # non sequence, no valid regions
c = 1 # ie do nothing
x_ma.mask[:] = True
except AttributeError:
sys.stderr.write('No Valid Regions, ATTERR \n ')
sys.stderr.write(myfile.times['time_end'].isoformat()+'\n')
c = 1 # also do nothing
x_ma.mask = True
unwrapped = unwrap_masked(x_ma, centered=False)
# end so no clutter expected
system_max = unwrapped[np.where(np.logical_not(
notmeteo))][-10:-1].mean()-system_zero
unwrapped_fixed = np.zeros(len(x_ma), dtype=float)
based = unwrapped - system_zero
based[0] = 0.0
notmeteo[0] = False
based[-1] = system_max
notmeteo[-1] = False
unwrapped_fixed[np.where(np.logical_not(based.mask))[0]] = \
based[np.where(np.logical_not(based.mask))[0]]
if len(based[np.where(np.logical_not(based.mask))[0]]) > 11:
unwrapped_fixed[np.where(based.mask)[0]] = \
np.interp(np.where(based.mask)[0],
np.where(np.logical_not(based.mask))[0],
smooth_and_trim(based[np.where(
np.logical_not(based.mask))[0]]))
else:
unwrapped_fixed[np.where(based.mask)[0]] = \
np.interp(np.where(based.mask)[0],
np.where(np.logical_not(based.mask))[0],
based[np.where(np.logical_not(based.mask))[0]])
if c != 1:
cordata[sweep, radial, :] = unwrapped_fixed
else:
cordata[sweep, radial, :] = np.zeros(myfile.RHOHV_F.shape[2])
if debug:
print "Exec time: ", time()-t
old_len = len(myfile.field_headers)
hdr = myfile.field_headers[myfile.fields.index('PHIDP_F')].copy()
newf = 'PHIDP_UNF'
hdr['field_name'] = newf
myfile.field_headers.append(hdr)
setattr(myfile, newf, cordata)
myfile.fields.append(newf)
return 1