def run(self, process=-9999, process_or_thread='p', **kwargs):
'''
# 并行计算加进度条
:param func: input a kenel_function
:param params: para1,para2,para3... = params
:param process: number of cpu
:param thread_or_process: multi-thread or multi-process,'p' or 't'
:param kwargs: tqdm kwargs
:return:
'''
if process > 0:
if process_or_thread == 'p':
pool = multiprocessing.Pool(process)
elif process_or_thread == 't':
pool = TPool(process)
else:
raise IOError('process_or_thread key error, input keyword such as "p" or "t"')
results = list(tqdm(pool.imap(self.func, self.params), total=len(self.params), **kwargs))
pool.close()
pool.join()
return results
else:
if process_or_thread == 'p':
pool = multiprocessing.Pool()
elif process_or_thread == 't':
pool = TPool()
else:
raise IOError('process_or_thread key error, input keyword such as "p" or "t"')
results = list(tqdm(pool.imap(self.func, self.params), total=len(self.params), **kwargs))
pool.close()
pool.join()
return results
def optimize_children(raw_children: Population, multiprocess: bool = True) -> Population:
_optimize = lambda idx_child: idx_child[1].optimize(pos=idx_child[0])
if multiprocess:
with TPool() as p:
children = list(tqdm(
p.imap(_optimize, enumerate(raw_children)), total=len(raw_children), desc="Optimizing children"
))
else:
children = []
for idx, child in tqdm(enumerate(raw_children), total=len(raw_children), desc="Optimizing children"):
children.append(_optimize((idx, child)))
return children
def process(i):
def thread(j):
from model import User
try:
User.get_all(i, j)
User.add_user(i, j)
User.get_all(i, j)
except Exception as exc:
logging.error(f'{i}-{j} {type(exc).__name__}:{exc.args}')
try:
tpool = TPool(TPOOL_SIZE)
list(
tpool.apply_async(thread, (j, ))
for j in range(1, THREAD_NUMBER + 1))
tpool.close()
tpool.join()
except Exception as exc:
logging.error(f'{i} {type(exc).__name__}:{exc.args}')
def main():
manager = mp.Manager()
results = manager.dict()
pool = TPool(12)
jobs = []
startDateTime = datetime.datetime.strptime(args.convTime, '%Y%m%d%H%M')
intervalDateTime = datetime.timedelta(
hours=2, minutes=0
) #hours = int(args.convInterval[:2]), minutes=int([args.convInterval[2:]]))
station = args.sensor
# Query all L2 files for the sensor
totalRadarObjects = []
totalSweepDateTimes = []
hrIter = datetime.timedelta(hours=0)
while True: # grab a specific interval of files
radarObjects, sweepDateTimes = pull_data(startDateTime=(startDateTime+hrIter),\
station=station)
totalRadarObjects.extend(radarObjects[:-1])
totalSweepDateTimes.extend(
sweepDateTimes[:-1]) # remove trailing *_MDM file
if totalSweepDateTimes[-1] - startDateTime >= intervalDateTime:
break
else:
hrIter += datetime.timedelta(hours=1)
fileDict = {'L2File': totalRadarObjects, 'Time': totalSweepDateTimes}
fileDF = pd.DataFrame(fileDict)
print(
f'Start time: {startDateTime}, Interval: {intervalDateTime}, End Time: {startDateTime + intervalDateTime}'
)
filesToStream = fileDF[((fileDF['Time'] >= startDateTime) \
& (fileDF['Time'] <= startDateTime + \
intervalDateTime))]['L2File'].tolist() # Bitwise operators, conditions double wrapped in perentheses to handle overriding
logging.info(f'files: {[obj.key for obj in filesToStream]}')
if len(filesToStream) < 8:
warnings.warn("n of radar inputs is not sufficent for curve smoothing",
UserWarning)
# --- Stream files ahead of time to avoid error with multiprocessing and file handles ---
filesToWorkers = []
for L2FileStream in tqdm(filesToStream, desc="Streaming L2 Files"):
try:
if datetime.datetime.strptime(
L2FileStream.key[20:35],
'%Y%m%d_%H%M%S') >= datetime.datetime(2016, 1, 1):
filesToWorkers.append(Level2File(L2FileStream.get()['Body']))
else:
bytestream = BytesIO(L2FileStream.get()['Body'].read())
with gzip.open(bytestream, 'rb') as f:
filesToWorkers.append(Level2File(f))
except:
print("value Error, Most likely in parsing header")
# --- Create pool for workers ---
for file in filesToWorkers:
job = pool.apply_async(calculate_radar_stats, (results, file))
jobs.append(job)
# --- Commit pool to workers ---
for job in tqdm(jobs, desc="Bounding & Searching Data"):
job.get()
pool.close()
pool.join()
columns = [
'sweepDateTime', 'metadata', 'sensorData', 'indices', 'xlocs', 'ylocs',
'data', 'polyVerts', 'offset', 'areaValue', 'refValue', 'varRefValue'
]
print(
'Creating Dataframe... (This may take a while if plotting significant data)'
)
resultsDF = pd.DataFrame.from_dict(results,
orient='index',
columns=columns) #SUPER slow
print('Converting datetimes...')
resultsDF['sweepDateTime'] = pd.to_datetime(resultsDF.sweepDateTime)
print('Sorting...')
resultsDF.sort_values(by='sweepDateTime', inplace=True)
#resultsDF.to_csv(args.output + '.csv', index = False)
print(resultsDF[['areaValue', 'refValue']].head(5))
# --- Plot time series---
fig, axes = plt.subplots(8, 8, figsize=(30, 30))
date_format = mpl_dates.DateFormatter('%H:%Mz')
for i, (dt, record) in tqdm(enumerate(resultsDF.iterrows()),
desc='Plotting Slices'):
plotx = i % 8
ploty = int(i / 8)
negXLim = -.5
posXLim = 1.5
negYLim = -1.0
posYLim = 1.0
norm, cmap = ctables.registry.get_with_steps('NWSReflectivity', 5, 5)
tempdata = record[
'data'] # create a deep copy of data to maipulate for plotting
tempdata[tempdata == 0] = np.ma.masked # mask out 0s for plotting
axes[ploty][plotx].pcolormesh(record['xlocs'],
record['ylocs'],
tempdata,
norm=norm,
cmap=cmap,
shading='auto')
axes[ploty][plotx].set_aspect(aspect='equal')
axes[ploty][plotx].set_xlim(negXLim, posXLim)
axes[ploty][plotx].set_ylim(negYLim, posYLim)
pVXs, pVYs = zip(
*record['polyVerts']
) # create lists of x and y values for transformed polyVerts
axes[ploty][plotx].plot(pVXs, pVYs)
if negXLim < record['offset'][1] < posXLim and \
negYLim < record['offset'][0] < posYLim:
axes[ploty][plotx].plot(record['offset'][1], record['offset'][0],
'o') # Location of the radar
axes[ploty][plotx].text(record['offset'][1], record['offset'][0],
record['sensorData']['siteID'])
axes[ploty][plotx].plot(0.0, 0.0, 'bx') # Location of the convection
axes[ploty][plotx].text(0.0, 0.0, str(args.convLatLon))
add_timestamp(axes[ploty][plotx],
record['sweepDateTime'],
y=0.02,
high_contrast=True)
axes[ploty][plotx].tick_params(axis='both', which='both')
print('Calculating Statistics...')
# pull data out of DF to make code cleaner
datetimes = resultsDF['sweepDateTime'].tolist()
#elapsedtimes = list(map(lambda x: x - min(datetimes), datetimes)) # not currently used, need to get this working
areaValues = resultsDF['areaValue'].tolist() # area ≥ 35dbz within ROI
refValues = np.array(
resultsDF['refValue'].tolist()
) # mean reflectivity ≥ 35dbz within ROI (conversion: (val-65)*0.5) [https://mesonet.agron.iastate.edu/GIS/rasters.php?rid=2]
if np.nan in refValues:
warnings.warn(
"Radar inputs contains instance with no ref values >= thresh",
UserWarning)
varValues = resultsDF['varRefValue'].tolist(
) # variance of mean reflectivity ≥ 35dbz within ROI
cvValues = np.array([
a / b for a, b in zip(varValues, refValues)
]) * 0.5 # coeff. of variation for mean reflectivity ≥ 35dbz within ROI
# Frequency
N = len(refValues)
T = 1.0 / N
yf = fft(refValues)
w = blackman(N)
ywf = fft(refValues * w)
# Normalization
areaNorm = areaValues / np.max(areaValues)
xf = np.linspace(0, 1.0 / (2.0 * T), N // 2)
cvNorm = cvValues / np.max(cvValues)
areaCVValuesNormalized = np.multiply(areaNorm, cvNorm)
# Curve Smoothing
window = len(
resultsDF.index
) // 8 # ~2 hours/8 = ~15 mins ----> number of samples in moving average ( helps counteract more visible noise in higher temporal resolution data)
yAreaAvg = movingaverage(
areaValues, window)[window // 2:-window //
2] # create moving averages for time series'
yRefAvg = movingaverage(refValues, window)[window // 2:-window // 2]
yCVAvg = movingaverage(cvValues, window)[window // 2:-window // 2]
yAreaCVNormAvg = movingaverage(areaCVValuesNormalized,
window)[window // 2:-window // 2]
# local minima & maxima on smoothed curves
minTemporalWindow = window * 2
areaLocalMax = argrelmax(yAreaAvg)
areaLocalMin = argrelmin(yAreaAvg)
endpoints = []
if yAreaAvg[0] <= np.all(yAreaAvg[1:window+1]) or\
yAreaAvg[0] >= np.all(yAreaAvg[1:window+1]):
endpoints.append(0)
if yAreaAvg[-1] <= np.all(yAreaAvg[len(yAreaAvg-1)-window+1:-2]) or\
yAreaAvg[-1] >= np.all(yAreaAvg[len(yAreaAvg-1)-window+1:-2]):
endpoints.append(len(yAreaAvg) - 1)
#print(f'Area: Endpoints: {yAreaAvg[endpoints]}, Local Maxes: {yAreaAvg[areaLocalMax]}, Local Mins: {yAreaAvg[areaLocalMin]}')
areaExtremaRaw = sorted(
areaLocalMax[0].tolist() + areaLocalMin[0].tolist() + endpoints
) # combine mins, maxes, and endpoints (if endpoints are an extreme) then sort
areaExtrema = [
x for x in areaExtremaRaw[1:]
if x - areaExtremaRaw[0] >= minTemporalWindow
] # remove maxima that are within threshold of first one
areaExtrema = [areaExtremaRaw[0]
] + areaExtrema # add back in forst one to begining
logging.info(f'Area Values: {yAreaAvg}')
logging.info(f'Area Extrema: {yAreaAvg[areaExtrema]}')
refLocalMax = argrelmax(yRefAvg)
refLocalMin = argrelmin(yRefAvg)
endpoints = []
if yRefAvg[0] <= np.all(yRefAvg[1:window+1]) or\
yRefAvg[0] >= np.all(yRefAvg[1:window+1]):
endpoints.append(0)
if yRefAvg[-1] <= np.all(yRefAvg[len(yRefAvg-1)-window+1:-2]) or\
yRefAvg[-1] >= np.all(yRefAvg[len(yRefAvg-1)-window+1:-2]):
endpoints.append(len(yRefAvg) - 1)
refExtremaRaw = sorted(refLocalMax[0].tolist() + refLocalMin[0].tolist() +
endpoints)
refExtrema = [
x for x in refExtremaRaw[1:]
if x - refExtremaRaw[0] >= minTemporalWindow
]
refExtrema = [refExtremaRaw[0]] + refExtrema
logging.info(f'Ref Values: {yRefAvg}')
logging.info(f'Ref Extrema: {yRefAvg[refExtrema]}')
#cvLocalMax = argrelmax(yCVAvg)
#cvLocalMin = argrelmin(yCVAvg)
#endpoints = []
#if yCVAvg[0] <= np.all(yCVAvg[1:window+1]) or\
# yCVAvg[0] >= np.all(yCVAvg[1:window+1]):
# endpoints.append(0)
#if yCVAvg[-1] <= np.all(yCVAvg[len(yCVAvg-1)-window+1:-2]) or\
# yCVAvg[-1] >= np.all(yCVAvg[len(yCVAvg-1)-window+1:-2]):
# endpoints.append(len(yCVAvg)-1)
#cvExtremaRaw = sorted(cvLocalMax[0].tolist()+cvLocalMin[0].tolist()+endpoints)
#cvExtrema = [x for x in cvExtremaRaw[1:] if x-cvExtremaRaw[0]>=minTemporalWindow]
#cvExtrema = [cvExtremaRaw[0]]+cvExtrema
#logging.info((f'CV Values: {yCVAvg}')
#logging.info((f'CV Extrema: {yCVAvg[cvExtrema]}')
yAreaCVNormLocalMax = argrelmax(yAreaCVNormAvg)
yAreaCVNormLocalMin = argrelmin(yAreaCVNormAvg)
endpoints = []
if yAreaCVNormAvg[0] <= np.all(yAreaCVNormAvg[1:window+1]) or\
yAreaCVNormAvg[0] >= np.all(yAreaCVNormAvg[1:window+1]):
endpoints.append(0)
if yAreaCVNormAvg[-1] <= np.all(yAreaCVNormAvg[len(yAreaCVNormAvg-1)-window+1:-2]) or\
yAreaCVNormAvg[-1] >= np.all(yAreaCVNormAvg[len(yAreaCVNormAvg-1)-window+1:-2]):
endpoints.append(len(yAreaCVNormAvg) - 1)
yAreaCVNormExtremaRaw = sorted(yAreaCVNormLocalMax[0].tolist() +
yAreaCVNormLocalMin[0].tolist() + endpoints)
yAreaCVNormExtrema = [
x for x in yAreaCVNormExtremaRaw[1:]
if x - yAreaCVNormExtremaRaw[0] >= minTemporalWindow
]
yAreaCVNormExtrema = [yAreaCVNormExtremaRaw[0]] + yAreaCVNormExtrema
logging.info(f'AreaCVNorm Extrema: {yAreaCVNormAvg[yAreaCVNormExtrema]}')
# Find slopes of Build-up Lines
# Area
xArea = np.array(datetimes[window // 2:-window // 2])[np.array(
[areaExtrema[0], areaExtrema[1]]
)] # grab datetime (x component) of the leftmost bounds (determined by window size), and the first extreme on the smoothed curve (sm curve is already bound by window, we need to apply bounds to datetimes)
xAreaDiff = xArea[1] - xArea[
0] # subtract the later value from the former to get our delta x
yArea = yAreaAvg[np.array(
[areaExtrema[0], areaExtrema[1]]
)] # grab the values (y component) of the sm curve at the begining and at the first extreme
yAreaDiff = yArea[1] - yArea[0] # subtract to find delta y
slopeArea = np.arctan(yAreaDiff /
xAreaDiff.seconds) # calc the slope angle
logging.info(f'Slope of Area: {slopeArea}')
# Reflectivity
xRef = np.array(datetimes[window // 2:-window // 2])[np.array(
[refExtrema[0], refExtrema[1]])]
xRefDiff = xRef[1] - xRef[0]
yRef = yRefAvg[np.array([refExtrema[0], refExtrema[1]])]
yRefDiff = yRef[1] - yRef[0]
slopeRef = np.arctan(yRefDiff / xRefDiff.seconds)
print(f'Slope of Reflectivity: {slopeRef}')
# Product of Area and Coefficent of Variation of Reflectivity
xProduct = np.array(datetimes[window // 2:-window // 2])[np.array(
[yAreaCVNormExtrema[0], yAreaCVNormExtrema[1]])]
XProductDiff = xProduct[1] - xProduct[0]
yProduct = yAreaCVNormAvg[np.array(
[yAreaCVNormExtrema[0], yAreaCVNormExtrema[1]])]
yProductDiff = yProduct[1] - yProduct[0]
slopeProduct = np.arctan(yProductDiff / XProductDiff.seconds)
print(f'Slope of Product: {slopeProduct}')
print('Plotting Additional Data and Saving Output...')
# Area for Reflectivity ≥ 35dbz
axes[-1][-5].plot_date(datetimes, areaValues, linestyle='solid', ms=2)
axes[-1][-5].plot_date(datetimes[window // 2:-window // 2],
yAreaAvg,
linestyle='solid',
ms=2)
axes[-1][-5].plot_date(
np.array(datetimes[window // 2:-window // 2])[np.array(
[areaExtrema[0], areaExtrema[1]])],
yAreaAvg[np.array([areaExtrema[0], areaExtrema[1]])],
linestyle="solid",
ms=2)
axes[-1][-5].legend(['Area Delta', 'Sm. Area Delta', 'Build-up Rate'])
axes[-1][-5].xaxis.set_major_formatter(date_format)
plt.setp(axes[-1][-5].xaxis.get_majorticklabels(),
rotation=45,
ha="right",
rotation_mode="anchor")
axes[-1][-5].set_title('Area of Reflectivity ≥ 35dbz (km^2)')
# Mean of Reflectivity ≥ 35dbz
axes[-1][-4].plot_date(datetimes, refValues, linestyle='solid', ms=2)
#axes[-1][-4].plot_date(datetimes[window//2:-window//2], yRefAvg, linestyle='solid', ms=2)
#axes[-1][-4].plot_date(np.array(datetimes[window//2:-window//2])[np.array([0,refLocalMax[0][0]])], yRefAvg[np.array([0,refLocalMax[0][0]])], linestyle="solid", ms=2)
axes[-1][-4].plot_date(datetimes[window // 2:-window // 2],
yRefAvg,
linestyle='solid',
ms=2)
axes[-1][-4].plot_date(np.array(
datetimes[window // 2:-window // 2])[np.array(
[refExtrema[0], refExtrema[1]])],
yRefAvg[np.array([refExtrema[0], refExtrema[1]])],
linestyle="solid",
ms=2)
axes[-1][-4].legend(['Ref Delta', 'Sm. Ref Delta', 'Build-up Rate'])
axes[-1][-4].xaxis.set_major_formatter(date_format)
plt.setp(axes[-1][-4].xaxis.get_majorticklabels(),
rotation=45,
ha="right",
rotation_mode="anchor")
axes[-1][-4].set_title('Mean of Reflectivity ≥ 35dbz')
# Product of cv reflectivity and area
axes[-1][-3].plot_date(datetimes,
areaCVValuesNormalized,
linestyle='solid',
ms=2)
axes[-1][-3].plot_date(datetimes[window // 2:-window // 2],
yAreaCVNormAvg,
linestyle='solid',
ms=2)
axes[-1][-3].plot_date(
np.array(datetimes[window // 2:-window // 2])[np.array(
[yAreaCVNormExtrema[0], yAreaCVNormExtrema[1]])],
yAreaCVNormAvg[np.array([yAreaCVNormExtrema[0],
yAreaCVNormExtrema[1]])],
linestyle="solid",
ms=2)
axes[-1][-3].legend(
['Area*cv_Ref Delta', 'Sm. Area*cv_Ref Delta', 'Build-up Rate'])
axes[-1][-3].xaxis.set_major_formatter(date_format)
plt.setp(axes[-1][-3].xaxis.get_majorticklabels(),
rotation=45,
ha="right",
rotation_mode="anchor")
axes[-1][-3].set_title('Norm Product:\nCV Reflectivity * Area ≥ 35dbz')
# Coeff. of Variance of Reflectivity ≥ 35dbz
axes[-1][-2].plot_date(datetimes, cvValues, linestyle='solid', ms=2)
axes[-1][-2].plot_date(datetimes[window // 2:-window // 2],
yCVAvg,
linestyle='solid',
ms=2)
axes[-1][-2].legend(['CV Delta', 'Sm. CV Delta'])
axes[-1][-2].xaxis.set_major_formatter(date_format)
plt.setp(axes[-1][-2].xaxis.get_majorticklabels(),
rotation=45,
ha="right",
rotation_mode="anchor")
axes[-1][-2].set_title('CV of Reflectivity ≥ 35dbz')
# Testing plot
axes[-1][-1].semilogy(xf[1:N // 2], 2.0 / N * np.abs(yf[1:N // 2]), '-b')
axes[-1][-1].semilogy(xf[1:N // 2], 2.0 / N * np.abs(ywf[1:N // 2]), '-r')
axes[-1][-1].legend(['FFT', 'FFT w. Window'])
#axes[-1][-1].plot(xf, 2.0/N * np.abs(yf[0:N//2]),linestyle='solid', ms=2)
#axes[-1][-1].plot_date(datetimes, yCVAvg, linestyle='solid')
#axes[-1][-1].xaxis.set_major_formatter(date_format)
plt.setp(axes[-1][-1].xaxis.get_majorticklabels(),
rotation=45,
ha="right",
rotation_mode="anchor")
axes[-1][-1].set_title('Testing Plot (Frequency)')
plt.tight_layout()
plt.savefig(args.output + 'Nexrad.png') # Set the output file name
#plt.show()
f_o = open(args.output + 'log_stats_area_nexrad.txt', 'a')
f_o.write(datetimes[0].strftime("%Y%m%d%H%M%S") + '\t' +
str(args.convLatLon) + '\t' + str(args.convBearing) + '\t' +
str(args.scaleFactor) + '\t' + str(np.max(areaValues)) + '\t' +
str(np.max(refValues)) + '\t' +
str(slopeArea) # std dev of LIS aligned data
+ '\t' + str(slopeRef) + '\t' + str(slopeProduct) + '\n')
f_o.close()