def run(self, process=-9999, process_or_thread='p', **kwargs): ''' # 并行计算加进度条 :param func: input a kenel_function :param params: para1,para2,para3... = params :param process: number of cpu :param thread_or_process: multi-thread or multi-process,'p' or 't' :param kwargs: tqdm kwargs :return: ''' if process > 0: if process_or_thread == 'p': pool = multiprocessing.Pool(process) elif process_or_thread == 't': pool = TPool(process) else: raise IOError('process_or_thread key error, input keyword such as "p" or "t"') results = list(tqdm(pool.imap(self.func, self.params), total=len(self.params), **kwargs)) pool.close() pool.join() return results else: if process_or_thread == 'p': pool = multiprocessing.Pool() elif process_or_thread == 't': pool = TPool() else: raise IOError('process_or_thread key error, input keyword such as "p" or "t"') results = list(tqdm(pool.imap(self.func, self.params), total=len(self.params), **kwargs)) pool.close() pool.join() return results
def optimize_children(raw_children: Population, multiprocess: bool = True) -> Population: _optimize = lambda idx_child: idx_child[1].optimize(pos=idx_child[0]) if multiprocess: with TPool() as p: children = list(tqdm( p.imap(_optimize, enumerate(raw_children)), total=len(raw_children), desc="Optimizing children" )) else: children = [] for idx, child in tqdm(enumerate(raw_children), total=len(raw_children), desc="Optimizing children"): children.append(_optimize((idx, child))) return children
def process(i): def thread(j): from model import User try: User.get_all(i, j) User.add_user(i, j) User.get_all(i, j) except Exception as exc: logging.error(f'{i}-{j} {type(exc).__name__}:{exc.args}') try: tpool = TPool(TPOOL_SIZE) list( tpool.apply_async(thread, (j, )) for j in range(1, THREAD_NUMBER + 1)) tpool.close() tpool.join() except Exception as exc: logging.error(f'{i} {type(exc).__name__}:{exc.args}')
def main(): manager = mp.Manager() results = manager.dict() pool = TPool(12) jobs = [] startDateTime = datetime.datetime.strptime(args.convTime, '%Y%m%d%H%M') intervalDateTime = datetime.timedelta( hours=2, minutes=0 ) #hours = int(args.convInterval[:2]), minutes=int([args.convInterval[2:]])) station = args.sensor # Query all L2 files for the sensor totalRadarObjects = [] totalSweepDateTimes = [] hrIter = datetime.timedelta(hours=0) while True: # grab a specific interval of files radarObjects, sweepDateTimes = pull_data(startDateTime=(startDateTime+hrIter),\ station=station) totalRadarObjects.extend(radarObjects[:-1]) totalSweepDateTimes.extend( sweepDateTimes[:-1]) # remove trailing *_MDM file if totalSweepDateTimes[-1] - startDateTime >= intervalDateTime: break else: hrIter += datetime.timedelta(hours=1) fileDict = {'L2File': totalRadarObjects, 'Time': totalSweepDateTimes} fileDF = pd.DataFrame(fileDict) print( f'Start time: {startDateTime}, Interval: {intervalDateTime}, End Time: {startDateTime + intervalDateTime}' ) filesToStream = fileDF[((fileDF['Time'] >= startDateTime) \ & (fileDF['Time'] <= startDateTime + \ intervalDateTime))]['L2File'].tolist() # Bitwise operators, conditions double wrapped in perentheses to handle overriding logging.info(f'files: {[obj.key for obj in filesToStream]}') if len(filesToStream) < 8: warnings.warn("n of radar inputs is not sufficent for curve smoothing", UserWarning) # --- Stream files ahead of time to avoid error with multiprocessing and file handles --- filesToWorkers = [] for L2FileStream in tqdm(filesToStream, desc="Streaming L2 Files"): try: if datetime.datetime.strptime( L2FileStream.key[20:35], '%Y%m%d_%H%M%S') >= datetime.datetime(2016, 1, 1): filesToWorkers.append(Level2File(L2FileStream.get()['Body'])) else: bytestream = BytesIO(L2FileStream.get()['Body'].read()) with gzip.open(bytestream, 'rb') as f: filesToWorkers.append(Level2File(f)) except: print("value Error, Most likely in parsing header") # --- Create pool for workers --- for file in filesToWorkers: job = pool.apply_async(calculate_radar_stats, (results, file)) jobs.append(job) # --- Commit pool to workers --- for job in tqdm(jobs, desc="Bounding & Searching Data"): job.get() pool.close() pool.join() columns = [ 'sweepDateTime', 'metadata', 'sensorData', 'indices', 'xlocs', 'ylocs', 'data', 'polyVerts', 'offset', 'areaValue', 'refValue', 'varRefValue' ] print( 'Creating Dataframe... (This may take a while if plotting significant data)' ) resultsDF = pd.DataFrame.from_dict(results, orient='index', columns=columns) #SUPER slow print('Converting datetimes...') resultsDF['sweepDateTime'] = pd.to_datetime(resultsDF.sweepDateTime) print('Sorting...') resultsDF.sort_values(by='sweepDateTime', inplace=True) #resultsDF.to_csv(args.output + '.csv', index = False) print(resultsDF[['areaValue', 'refValue']].head(5)) # --- Plot time series--- fig, axes = plt.subplots(8, 8, figsize=(30, 30)) date_format = mpl_dates.DateFormatter('%H:%Mz') for i, (dt, record) in tqdm(enumerate(resultsDF.iterrows()), desc='Plotting Slices'): plotx = i % 8 ploty = int(i / 8) negXLim = -.5 posXLim = 1.5 negYLim = -1.0 posYLim = 1.0 norm, cmap = ctables.registry.get_with_steps('NWSReflectivity', 5, 5) tempdata = record[ 'data'] # create a deep copy of data to maipulate for plotting tempdata[tempdata == 0] = np.ma.masked # mask out 0s for plotting axes[ploty][plotx].pcolormesh(record['xlocs'], record['ylocs'], tempdata, norm=norm, cmap=cmap, shading='auto') axes[ploty][plotx].set_aspect(aspect='equal') axes[ploty][plotx].set_xlim(negXLim, posXLim) axes[ploty][plotx].set_ylim(negYLim, posYLim) pVXs, pVYs = zip( *record['polyVerts'] ) # create lists of x and y values for transformed polyVerts axes[ploty][plotx].plot(pVXs, pVYs) if negXLim < record['offset'][1] < posXLim and \ negYLim < record['offset'][0] < posYLim: axes[ploty][plotx].plot(record['offset'][1], record['offset'][0], 'o') # Location of the radar axes[ploty][plotx].text(record['offset'][1], record['offset'][0], record['sensorData']['siteID']) axes[ploty][plotx].plot(0.0, 0.0, 'bx') # Location of the convection axes[ploty][plotx].text(0.0, 0.0, str(args.convLatLon)) add_timestamp(axes[ploty][plotx], record['sweepDateTime'], y=0.02, high_contrast=True) axes[ploty][plotx].tick_params(axis='both', which='both') print('Calculating Statistics...') # pull data out of DF to make code cleaner datetimes = resultsDF['sweepDateTime'].tolist() #elapsedtimes = list(map(lambda x: x - min(datetimes), datetimes)) # not currently used, need to get this working areaValues = resultsDF['areaValue'].tolist() # area ≥ 35dbz within ROI refValues = np.array( resultsDF['refValue'].tolist() ) # mean reflectivity ≥ 35dbz within ROI (conversion: (val-65)*0.5) [https://mesonet.agron.iastate.edu/GIS/rasters.php?rid=2] if np.nan in refValues: warnings.warn( "Radar inputs contains instance with no ref values >= thresh", UserWarning) varValues = resultsDF['varRefValue'].tolist( ) # variance of mean reflectivity ≥ 35dbz within ROI cvValues = np.array([ a / b for a, b in zip(varValues, refValues) ]) * 0.5 # coeff. of variation for mean reflectivity ≥ 35dbz within ROI # Frequency N = len(refValues) T = 1.0 / N yf = fft(refValues) w = blackman(N) ywf = fft(refValues * w) # Normalization areaNorm = areaValues / np.max(areaValues) xf = np.linspace(0, 1.0 / (2.0 * T), N // 2) cvNorm = cvValues / np.max(cvValues) areaCVValuesNormalized = np.multiply(areaNorm, cvNorm) # Curve Smoothing window = len( resultsDF.index ) // 8 # ~2 hours/8 = ~15 mins ----> number of samples in moving average ( helps counteract more visible noise in higher temporal resolution data) yAreaAvg = movingaverage( areaValues, window)[window // 2:-window // 2] # create moving averages for time series' yRefAvg = movingaverage(refValues, window)[window // 2:-window // 2] yCVAvg = movingaverage(cvValues, window)[window // 2:-window // 2] yAreaCVNormAvg = movingaverage(areaCVValuesNormalized, window)[window // 2:-window // 2] # local minima & maxima on smoothed curves minTemporalWindow = window * 2 areaLocalMax = argrelmax(yAreaAvg) areaLocalMin = argrelmin(yAreaAvg) endpoints = [] if yAreaAvg[0] <= np.all(yAreaAvg[1:window+1]) or\ yAreaAvg[0] >= np.all(yAreaAvg[1:window+1]): endpoints.append(0) if yAreaAvg[-1] <= np.all(yAreaAvg[len(yAreaAvg-1)-window+1:-2]) or\ yAreaAvg[-1] >= np.all(yAreaAvg[len(yAreaAvg-1)-window+1:-2]): endpoints.append(len(yAreaAvg) - 1) #print(f'Area: Endpoints: {yAreaAvg[endpoints]}, Local Maxes: {yAreaAvg[areaLocalMax]}, Local Mins: {yAreaAvg[areaLocalMin]}') areaExtremaRaw = sorted( areaLocalMax[0].tolist() + areaLocalMin[0].tolist() + endpoints ) # combine mins, maxes, and endpoints (if endpoints are an extreme) then sort areaExtrema = [ x for x in areaExtremaRaw[1:] if x - areaExtremaRaw[0] >= minTemporalWindow ] # remove maxima that are within threshold of first one areaExtrema = [areaExtremaRaw[0] ] + areaExtrema # add back in forst one to begining logging.info(f'Area Values: {yAreaAvg}') logging.info(f'Area Extrema: {yAreaAvg[areaExtrema]}') refLocalMax = argrelmax(yRefAvg) refLocalMin = argrelmin(yRefAvg) endpoints = [] if yRefAvg[0] <= np.all(yRefAvg[1:window+1]) or\ yRefAvg[0] >= np.all(yRefAvg[1:window+1]): endpoints.append(0) if yRefAvg[-1] <= np.all(yRefAvg[len(yRefAvg-1)-window+1:-2]) or\ yRefAvg[-1] >= np.all(yRefAvg[len(yRefAvg-1)-window+1:-2]): endpoints.append(len(yRefAvg) - 1) refExtremaRaw = sorted(refLocalMax[0].tolist() + refLocalMin[0].tolist() + endpoints) refExtrema = [ x for x in refExtremaRaw[1:] if x - refExtremaRaw[0] >= minTemporalWindow ] refExtrema = [refExtremaRaw[0]] + refExtrema logging.info(f'Ref Values: {yRefAvg}') logging.info(f'Ref Extrema: {yRefAvg[refExtrema]}') #cvLocalMax = argrelmax(yCVAvg) #cvLocalMin = argrelmin(yCVAvg) #endpoints = [] #if yCVAvg[0] <= np.all(yCVAvg[1:window+1]) or\ # yCVAvg[0] >= np.all(yCVAvg[1:window+1]): # endpoints.append(0) #if yCVAvg[-1] <= np.all(yCVAvg[len(yCVAvg-1)-window+1:-2]) or\ # yCVAvg[-1] >= np.all(yCVAvg[len(yCVAvg-1)-window+1:-2]): # endpoints.append(len(yCVAvg)-1) #cvExtremaRaw = sorted(cvLocalMax[0].tolist()+cvLocalMin[0].tolist()+endpoints) #cvExtrema = [x for x in cvExtremaRaw[1:] if x-cvExtremaRaw[0]>=minTemporalWindow] #cvExtrema = [cvExtremaRaw[0]]+cvExtrema #logging.info((f'CV Values: {yCVAvg}') #logging.info((f'CV Extrema: {yCVAvg[cvExtrema]}') yAreaCVNormLocalMax = argrelmax(yAreaCVNormAvg) yAreaCVNormLocalMin = argrelmin(yAreaCVNormAvg) endpoints = [] if yAreaCVNormAvg[0] <= np.all(yAreaCVNormAvg[1:window+1]) or\ yAreaCVNormAvg[0] >= np.all(yAreaCVNormAvg[1:window+1]): endpoints.append(0) if yAreaCVNormAvg[-1] <= np.all(yAreaCVNormAvg[len(yAreaCVNormAvg-1)-window+1:-2]) or\ yAreaCVNormAvg[-1] >= np.all(yAreaCVNormAvg[len(yAreaCVNormAvg-1)-window+1:-2]): endpoints.append(len(yAreaCVNormAvg) - 1) yAreaCVNormExtremaRaw = sorted(yAreaCVNormLocalMax[0].tolist() + yAreaCVNormLocalMin[0].tolist() + endpoints) yAreaCVNormExtrema = [ x for x in yAreaCVNormExtremaRaw[1:] if x - yAreaCVNormExtremaRaw[0] >= minTemporalWindow ] yAreaCVNormExtrema = [yAreaCVNormExtremaRaw[0]] + yAreaCVNormExtrema logging.info(f'AreaCVNorm Extrema: {yAreaCVNormAvg[yAreaCVNormExtrema]}') # Find slopes of Build-up Lines # Area xArea = np.array(datetimes[window // 2:-window // 2])[np.array( [areaExtrema[0], areaExtrema[1]] )] # grab datetime (x component) of the leftmost bounds (determined by window size), and the first extreme on the smoothed curve (sm curve is already bound by window, we need to apply bounds to datetimes) xAreaDiff = xArea[1] - xArea[ 0] # subtract the later value from the former to get our delta x yArea = yAreaAvg[np.array( [areaExtrema[0], areaExtrema[1]] )] # grab the values (y component) of the sm curve at the begining and at the first extreme yAreaDiff = yArea[1] - yArea[0] # subtract to find delta y slopeArea = np.arctan(yAreaDiff / xAreaDiff.seconds) # calc the slope angle logging.info(f'Slope of Area: {slopeArea}') # Reflectivity xRef = np.array(datetimes[window // 2:-window // 2])[np.array( [refExtrema[0], refExtrema[1]])] xRefDiff = xRef[1] - xRef[0] yRef = yRefAvg[np.array([refExtrema[0], refExtrema[1]])] yRefDiff = yRef[1] - yRef[0] slopeRef = np.arctan(yRefDiff / xRefDiff.seconds) print(f'Slope of Reflectivity: {slopeRef}') # Product of Area and Coefficent of Variation of Reflectivity xProduct = np.array(datetimes[window // 2:-window // 2])[np.array( [yAreaCVNormExtrema[0], yAreaCVNormExtrema[1]])] XProductDiff = xProduct[1] - xProduct[0] yProduct = yAreaCVNormAvg[np.array( [yAreaCVNormExtrema[0], yAreaCVNormExtrema[1]])] yProductDiff = yProduct[1] - yProduct[0] slopeProduct = np.arctan(yProductDiff / XProductDiff.seconds) print(f'Slope of Product: {slopeProduct}') print('Plotting Additional Data and Saving Output...') # Area for Reflectivity ≥ 35dbz axes[-1][-5].plot_date(datetimes, areaValues, linestyle='solid', ms=2) axes[-1][-5].plot_date(datetimes[window // 2:-window // 2], yAreaAvg, linestyle='solid', ms=2) axes[-1][-5].plot_date( np.array(datetimes[window // 2:-window // 2])[np.array( [areaExtrema[0], areaExtrema[1]])], yAreaAvg[np.array([areaExtrema[0], areaExtrema[1]])], linestyle="solid", ms=2) axes[-1][-5].legend(['Area Delta', 'Sm. Area Delta', 'Build-up Rate']) axes[-1][-5].xaxis.set_major_formatter(date_format) plt.setp(axes[-1][-5].xaxis.get_majorticklabels(), rotation=45, ha="right", rotation_mode="anchor") axes[-1][-5].set_title('Area of Reflectivity ≥ 35dbz (km^2)') # Mean of Reflectivity ≥ 35dbz axes[-1][-4].plot_date(datetimes, refValues, linestyle='solid', ms=2) #axes[-1][-4].plot_date(datetimes[window//2:-window//2], yRefAvg, linestyle='solid', ms=2) #axes[-1][-4].plot_date(np.array(datetimes[window//2:-window//2])[np.array([0,refLocalMax[0][0]])], yRefAvg[np.array([0,refLocalMax[0][0]])], linestyle="solid", ms=2) axes[-1][-4].plot_date(datetimes[window // 2:-window // 2], yRefAvg, linestyle='solid', ms=2) axes[-1][-4].plot_date(np.array( datetimes[window // 2:-window // 2])[np.array( [refExtrema[0], refExtrema[1]])], yRefAvg[np.array([refExtrema[0], refExtrema[1]])], linestyle="solid", ms=2) axes[-1][-4].legend(['Ref Delta', 'Sm. Ref Delta', 'Build-up Rate']) axes[-1][-4].xaxis.set_major_formatter(date_format) plt.setp(axes[-1][-4].xaxis.get_majorticklabels(), rotation=45, ha="right", rotation_mode="anchor") axes[-1][-4].set_title('Mean of Reflectivity ≥ 35dbz') # Product of cv reflectivity and area axes[-1][-3].plot_date(datetimes, areaCVValuesNormalized, linestyle='solid', ms=2) axes[-1][-3].plot_date(datetimes[window // 2:-window // 2], yAreaCVNormAvg, linestyle='solid', ms=2) axes[-1][-3].plot_date( np.array(datetimes[window // 2:-window // 2])[np.array( [yAreaCVNormExtrema[0], yAreaCVNormExtrema[1]])], yAreaCVNormAvg[np.array([yAreaCVNormExtrema[0], yAreaCVNormExtrema[1]])], linestyle="solid", ms=2) axes[-1][-3].legend( ['Area*cv_Ref Delta', 'Sm. Area*cv_Ref Delta', 'Build-up Rate']) axes[-1][-3].xaxis.set_major_formatter(date_format) plt.setp(axes[-1][-3].xaxis.get_majorticklabels(), rotation=45, ha="right", rotation_mode="anchor") axes[-1][-3].set_title('Norm Product:\nCV Reflectivity * Area ≥ 35dbz') # Coeff. of Variance of Reflectivity ≥ 35dbz axes[-1][-2].plot_date(datetimes, cvValues, linestyle='solid', ms=2) axes[-1][-2].plot_date(datetimes[window // 2:-window // 2], yCVAvg, linestyle='solid', ms=2) axes[-1][-2].legend(['CV Delta', 'Sm. CV Delta']) axes[-1][-2].xaxis.set_major_formatter(date_format) plt.setp(axes[-1][-2].xaxis.get_majorticklabels(), rotation=45, ha="right", rotation_mode="anchor") axes[-1][-2].set_title('CV of Reflectivity ≥ 35dbz') # Testing plot axes[-1][-1].semilogy(xf[1:N // 2], 2.0 / N * np.abs(yf[1:N // 2]), '-b') axes[-1][-1].semilogy(xf[1:N // 2], 2.0 / N * np.abs(ywf[1:N // 2]), '-r') axes[-1][-1].legend(['FFT', 'FFT w. Window']) #axes[-1][-1].plot(xf, 2.0/N * np.abs(yf[0:N//2]),linestyle='solid', ms=2) #axes[-1][-1].plot_date(datetimes, yCVAvg, linestyle='solid') #axes[-1][-1].xaxis.set_major_formatter(date_format) plt.setp(axes[-1][-1].xaxis.get_majorticklabels(), rotation=45, ha="right", rotation_mode="anchor") axes[-1][-1].set_title('Testing Plot (Frequency)') plt.tight_layout() plt.savefig(args.output + 'Nexrad.png') # Set the output file name #plt.show() f_o = open(args.output + 'log_stats_area_nexrad.txt', 'a') f_o.write(datetimes[0].strftime("%Y%m%d%H%M%S") + '\t' + str(args.convLatLon) + '\t' + str(args.convBearing) + '\t' + str(args.scaleFactor) + '\t' + str(np.max(areaValues)) + '\t' + str(np.max(refValues)) + '\t' + str(slopeArea) # std dev of LIS aligned data + '\t' + str(slopeRef) + '\t' + str(slopeProduct) + '\n') f_o.close()