year_upper = 2012 month_lower = 6 month_upper = 6 day_lower = 1 day_upper = 30 hour_lower = 8 hour_upper = 23 minute_lower = 0 minute_upper = 45 time_params = np.array([ year_lower, year_upper, month_lower, month_upper, day_lower, day_upper, hour_lower, hour_upper, minute_lower, minute_upper ]) datetimes = utilities.get_datetime_objects(time_params) datestrings = [j.strftime("%Y%m%d%H%M") for j in datetimes] lonlats = Dataset( '/ouce-home/data/satellite/meteosat/seviri/15-min/native/' 'lonlats.NA_MiddleEast.nc') # These need to be regridded to regular for consistency with cloud mask lons = lonlats.variables['longitude'][:] lats = lonlats.variables['latitude'][:] lonmask = lons > 360 latmask = lats > 90 lons = np.ma.array(lons, mask=lonmask) lats = np.ma.array(lats, mask=latmask) sdf_previous = None clouds_previous = None
month_lower = 6 month_upper = 8 day_lower = 1 day_upper = 31 hour_lower = 0 hour_upper = 23 minute_lower = 0 minute_upper = 45 # Generate the full set of datetimes for the entire period time_params = np.array([year_lower, year_upper, month_lower, month_upper, day_lower, day_upper, hour_lower, hour_upper, minute_lower, minute_upper]) SDF_datetimes = utilities.get_datetime_objects(time_params) SDF_datestrings = [j.strftime("%Y%m%d%H%M") for j in SDF_datetimes] # Get Ian's lats and lons sdf_test = Dataset( '/soge-home/data_not_backed_up/satellite/meteosat/seviri' '/15-min/0.03x0.03/sdf/nc/JUNE2010/SDF_v2/SDF_v2.' '201006031500.nc') ianlons = sdf_test.variables['longitude'][:] ianlats = sdf_test.variables['latitude'][:] # Get my lats and lons tch_lonlats = Dataset('/soge-home/projects/seviri_dust/raw_seviri_data' '/bt_nc/native_bt_lonlats.nc') tchlons = tch_lonlats.variables['longitude'][:]
def wrapper(yearmonth): year_lower = yearmonth[0] year_upper = yearmonth[0] month_lower = yearmonth[-1][0] month_upper = yearmonth[-1][-1] day_lower = 1 day_upper = 31 hour_lower = 0 hour_upper = 23 minute_lower = 0 minute_upper = 45 time_params = np.array([ year_lower, year_upper, month_lower, month_upper, day_lower, day_upper, hour_lower, hour_upper, minute_lower, minute_upper ]) # Get datetime objects between the above bounds window_datetime_lower = datetime.datetime(year_lower, 6, 1, 0, 0) \ - datetime.timedelta(days=7) window_datetime_upper = datetime.datetime(year_upper, 8, 31, 23, 45) \ + datetime.timedelta(days=7) datetimes = utilities.get_datetime_objects(time_params) datestrings = [j.strftime("%Y%m%d%H%M") for j in datetimes] # Get lats and lons sdf_test = Dataset( '/soge-home/data_not_backed_up/satellite/meteosat/seviri' '/15-min/0.03x0.03/sdf/nc/JUNE2010/SDF_v2/SDF_v2.' '201006031500.nc') lons, lats = np.meshgrid(sdf_test.variables['longitude'][:], sdf_test.variables['latitude'][:]) lonmask = lons > 360 latmask = lats > 90 lons = np.ma.array(lons, mask=lonmask) lats = np.ma.array(lats, mask=latmask) sdf_previous = None raw_sdf_prev = [] clouds_previous = None ids_previous = [] cloud_ids_previous = [] deep_conv_IDs_prev = None LLJ_plumes_IDs_prev = [] k = 0 available_colours = np.arange(0, 41) # To pick back up where you left off, simply add all the keys in plume # archive to used IDs here - perhaps include an option to do that used_ids = [] used_cloud_ids = [] used_colour_IDs = {} plume_objects = [] flicker_ids = [] reintroduced_ids = [] last_10_ids = [] last_10_ids = np.asarray(last_10_ids) prev_dust_assoc_clouds = None # Restrict the lons and lats to the CWS alone lonbool = np.asarray([j >= -20 and j <= 10 for j in lons[0]]) if run: plume_objects = {} plume_archive = shelve.open( '/soge-home/projects/seviri_dust/plumetracker/' 'plume_archive_flicker_v4_prob_v4_' '' + str(yearmonth[0])) if pickup: archived_ids = np.asarray([j for j in plume_archive]) for i in archived_ids: used_ids.append(int(i)) with open('date_i_' + str(yearmonth[0]) + '.txt', 'r') as f: pickup_date_i = f.read() datestrings = datestrings[int(pickup_date_i):] datetimes = datetimes[int(pickup_date_i):] for date_i in np.arange(0, len(datestrings)): runtime = datetimes[date_i] - datetimes[0] print '\n' + datestrings[date_i] + '\n' totaltest = datetime.datetime.now() if tch_sdfs == False: sdf_root = '/soge-home/data_not_backed_up/satellite/meteosat' \ '/seviri/15' \ '-min/0.03x0.03/sdf/nc/'+datetimes[date_i].strftime( '%B').upper()+str(datetimes[date_i].year)+'/SDF_v2/' else: sdf_root = '/soge-home/projects/seviri_dust/sdf/'\ +datetimes[date_i].strftime('%B')\ +str(datetimes[date_i].year)+'/' if os.path.isfile(sdf_root+'SDF_v2.' + \ datestrings[date_i] + '.nc'): sdf = Dataset( sdf_root + 'SDF_v2.' + \ datestrings[date_i] + '.nc') found_file = True #print sdf else: print 'No SDF file found for this date' found_file = False if daily_bt_files: # Pull BTs from daily nc files bt = Dataset(daily_bt_root + datetimes[date_i].strftime("%B%Y") + '/BT_' + datetimes[date_i].strftime("%Y%m%d") + '.nc') found_file = True else: try: bt = Dataset( '/ouce-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/bt' '/nc/' + datetimes[date_i].strftime("%B").upper() + str(datetimes[date_i].year) + '/H-000-MSG2__' '-MSG2________-' 'IR_BrightnessTemperatures___' '-000005___-' + datestrings[date_i] + '-__.nc') found_file = True #print bt12.shape #print clouds_now.shape #cloud_lons, cloud_lats = np.meshgrid(cloud_lons, cloud_lats) except: if os.path.isfile( '/ouce-home/data/satellite/meteosat/seviri/' '15-min/' '0.03x0.03/bt' '/nc/' + datetimes[date_i].strftime("%B").upper() + str(datetimes[date_i].year) + '/H-000-MSG1__' '-MSG1________-' 'IR_BrightnessTemperatures___' '-000005___-' + datestrings[date_i] + '-__.nc'): bt = Dataset( '/ouce-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/bt' '/nc/' + datetimes[date_i].strftime("%B").upper() + str(datetimes[date_i].year) + '/H-000-MSG1__' '-MSG1________-' 'IR_BrightnessTemperatures___' '-000005___-' + datestrings[date_i] + '-__.nc') found_file = True else: found_file = False if found_file: # If the SDF file has its own longitudes and latitudes, # use these rather than the preset if 'longitude' in sdf.variables: lons = sdf.variables['longitude'][:] lonmask = lons > 360 lons = np.ma.array(lons, mask=lonmask) if 'latitude' in sdf.variables: lats = sdf.variables['latitude'][:] latmask = lats > 90 lats = np.ma.array(lats, mask=latmask) lons, lats = np.meshgrid(lons, lats) # clouds_now = np.load('/ouce-home/students/hert4173/' # 'cloud_mask_numpy_files/cloudmask' # ''+datetimes[date_i] # .strftime( # "%Y%m%d%H%M%S")+'.npy') # Some SDF files have a time dimension. For these index it out. if tch_sdfs: sdf_now = sdf.variables['SDF'][:] elif 'time' in sdf.variables: sdf_now = sdf.variables['bt108'][0] else: sdf_now = sdf.variables['bt108'][:] if daily_bt_files: # We should get data from the HDF files instead time_params_7dayw = np.array([ window_datetime_lower.year, window_datetime_upper.year, window_datetime_lower.month, window_datetime_upper.month, window_datetime_lower.day, window_datetime_upper.day, datetimes[date_i].hour, datetimes[date_i].hour, datetimes[date_i].minute, datetimes[date_i].minute ]) datetimes_7dayw = utilities.get_daily_datetime_objects( time_params_7dayw) indices = np.arange(0, len(datetimes_7dayw)) current_ind = datetimes_7dayw == datetimes[date_i] current_ind = indices[current_ind][0] f = tables.open_file( '/soge-home/projects/seviri_dust/sdf/intermediary_files/bt_15d_' + datetimes[date_i].strftime("%Y_%H_%M") + '.hdf') bt_15day = f.root.data[current_ind] f.close() # We need to extract the right timestep from the daily # nc file bt_108 = bt_15day[1] # Instead get BTs from the intermediary hdf files ( # already regridded) elif 'time' in bt.variables: bt_108 = bt.variables['bt108'][:][0] else: bt_108 = bt.variables['bt108'][:] clouds = bt_108 < 270 # Add the reintroduced plumes back into the running for i in np.arange(0, len(reintroduced_ids)): #print 'Reintroducing plume', reintroduced_ids[i] sdf_previous[sdf_previous == flicker_ids[i]] = \ reintroduced_ids[i] ids_previous = np.append(ids_previous, reintroduced_ids[i]) # Get plumes first by scanning for them sdf_plumes, new_ids, plume_ids, merge_ids = plumes.\ scan_for_plumes( sdf_now, sdf_previous, used_ids, clouds) for i in np.arange(0, len(reintroduced_ids)): if reintroduced_ids[i] not in plume_ids: #print 'But plume '+str(reintroduced_ids[i])+' sadly ' \ # 'died.' pass # Clear flicker IDs and reintroduced IDs flicker_ids = [] reintroduced_ids = [] # We could here do infilling, then at least you'll have it for # the next iteration. But if you've labelled them already you # don't have a binary. # Get clouds by scanning for them # clouds, new_cloud_ids, cloud_ids, merge_cloud_ids = \ # plumes.scan_for_plumes(clouds_now, # clouds_previous, # used_cloud_ids) for i in new_ids: used_ids.append(i) # for i in new_cloud_ids: # used_cloud_ids.append(i) old_bool = np.asarray([j in ids_previous for j in plume_ids]) if len(old_bool) > 0: old_ids = plume_ids[old_bool] else: old_ids = [] # old_cloud_bool = np.asarray([j in clouds_previous for # j in cloud_ids]) # old_cloud_ids = cloud_ids[old_cloud_bool] # Then, for each new ID, we initialise plume objects for i in np.arange(0, len(new_ids)): if debug: print 'Creating new plume', new_ids[i] plume = plumes.Plume(new_ids[i], datetimes[date_i]) plume.update_position(lats, lons, sdf_plumes, new_ids[i]) plume.update_duration(datetimes[date_i]) #if plume.plume_id == 18: # print plume.dates_observed # print plume.dates_observed plume.update_bbox() plume.update_majorminor_axes(lons, lats) plume.update_area() plume.update_max_extent() plume.update_centroid_speed() plume.update_centroid_direction() plume.check_conv_distance(lats, lons, clouds) plume.update_most_likely_source() # plume.update_leading_edge_4(sdf_plumes, lons, lats) plume_objects[str(new_ids[i])] = plume if flicker: missing_plume, missing_id, flickered = \ plume.chain_flickerchecker( raw_sdf_prev) else: flickered = False missing_plume = [] if flickered: raise ValueError('Found an overlaping plume in the ' 'previous timestep larger than size ' '250 - a new plume should not be ' 'initiated here') steps_back = 1 # As long as there is an overlapping previous plume, # keep updating it back in time while len(missing_plume) > 0: if debug: print 'Rolling back plume', new_ids[i] # We can only step back to the first timestep and no # earlier if (date_i - steps_back) < 0: missing_plume = [] break else: missing_date = datetimes[date_i - steps_back] missing_sdf_plumes = np.zeros(missing_plume.shape) missing_plume = missing_plume == 1 missing_sdf_plumes[missing_plume] = missing_id # Run all the updates that would be used for a # new plume plume.update_position(lats, lons, missing_sdf_plumes, missing_id) plume.update_duration(missing_date) #if plume.plume_id == 18: # print plume.dates_observed plume.update_bbox() plume.update_majorminor_axes(lons, lats) plume.update_area() plume.update_max_extent() plume.update_centroid_speed() plume.update_centroid_direction() plume.check_conv_distance(lats, lons, clouds) plume.update_most_likely_source() plume.process_missing_plume() #print 'Updated missing plume back '+str( # steps_back)+' steps' steps_back += 1 if (date_i - steps_back) < 0: missing_plume = [] break # Pull out data from the timestep before to # continue the chain try: if tch_sdfs: raw_sdf_prev_prev_data = Dataset(sdf_root+'SDF_v2.' + \ datestrings[date_i-steps_back] + '.nc') else: raw_sdf_prev_prev_data = Dataset( '/soge-home/data_not_backed_up/satellite/' 'meteosat/' 'seviri/' '15-min/0.03x0.03/sdf/nc/' + datetimes[date_i-steps_back].strftime( "%B").upper( ) + str(datetimes[date_i-steps_back].year) + '/SDF_v2/SDF_v2.' + \ datestrings[date_i-steps_back] + '.nc') except: print 'Adding date to list of missing dates' with open('missing_dates.txt', 'a') as my_file: my_file.write('\n' + datestrings[date_i - steps_back]) break if tch_sdfs: raw_sdf_prev_prev = \ raw_sdf_prev_prev_data.variables['SDF'][:] elif 'time' in raw_sdf_prev_prev_data.variables: raw_sdf_prev_prev = \ raw_sdf_prev_prev_data.variables[ 'bt108'][0] else: raw_sdf_prev_prev = \ raw_sdf_prev_prev_data.variables[ 'bt108'][:] missing_plume, missing_id, flickered = \ plume.chain_flickerchecker( raw_sdf_prev_prev) if flickered: #print 'Found a flickered plume. Searching ' \ # 'for the corresponding archived plume.' # We have a plume in a previous timestep # which flickered plume_archive_keys = last_10_ids.astype(int) # Sort the keys in reverse order as the # plume we want is most likely to have a # high ID plume_archive_keys[::-1].sort() missing_plume = missing_plume == 1 missing_sdf_plumes = np.zeros( missing_plume.shape) missing_sdf_plumes[missing_plume] = missing_id plume_bool = missing_sdf_plumes == missing_id search_centroid_lon = \ np.nanmean(lons[plume_bool]) search_centroid_lat = \ np.nanmean(lats[plume_bool]) plume_lons = lons[plume_bool] plume_lats = lats[plume_bool] search_date = datetimes[date_i - steps_back] #print search_date found_plume = False for key in plume_archive_keys: #print 'Searching in plume archive' if search_centroid_lon == \ plume_archive[\ str(key)].centroid_lon and \ search_centroid_lat ==\ plume_archive[str( key)].centroid_lat and \ plume_archive[str( key)].dates_observed[-1] == \ search_date: #print 'Found it in plume archive. ' \ # 'ID is', \ # plume_archive[str(key)].plume_id found_plume = True correct_plume = plume_archive[str(key)] plume_to_append = plume # Append the flickered plume to the # old one which was archived correct_plume.append_missing_plume( plume_to_append) # Add it to plume objects and remove # it from archives plume_objects[str(key)] = correct_plume del plume_archive[str(key)] # Add it to old IDs, replacing the # ID of the plume which was found to be # flickered flicker_ids.append(plume.plume_id) reintroduced_ids.append(key) missing_plume = [] # Reintroduced plumes also get removed # from the record of the last 10 ids index = np.argwhere(last_10_ids == key) last_10_ids = np.delete( last_10_ids, index) break # If we didn't find the plume in the plume # archive, it must still be active if found_plume == False: plume_object_keys = np.asarray( [int(i) for i in plume_objects]) # Sort the keys in reverse order as the # plume we want is most likely to have a # high ID plume_object_keys[::-1].sort() for key in plume_object_keys: #print 'Searching in plume objects' if search_centroid_lon == \ plume_objects[ \ str( key)].centroid_lon and \ search_centroid_lat == \ plume_objects[str( key)].centroid_lat and \ plume_objects[str( key)].dates_observed[ -1] == \ search_date: #print 'Found it in plume ' \ # 'objects. ID is', \ # plume_archive[ # str(key)].plume_id found_plume = True correct_plume = plume_archive[str( key)] plume_to_append = plume # Append the flickered plume to the # old one which was archived correct_plume.append_missing_plume( plume_to_append) # Add it to plume objects and # remove it from archives plume_objects[str( key)] = correct_plume del plume_archive[str(key)] # Add it to old IDs, replacing the # ID of the plume which was found # to be flickered flicker_ids.append(plume.plume_id) reintroduced_ids.append(key) missing_plume = [] index = np.argwhere( last_10_ids == key) last_10_ids = np.delete( last_10_ids, index) break break # Remove any IDs which were actually flickers for i in np.arange(0, len(flicker_ids)): index = np.argwhere(new_ids == flicker_ids[i]) new_ids = np.delete(new_ids, index) index = np.argwhere(ids_previous == flicker_ids[i]) ids_previous = np.delete(ids_previous, index) index = np.argwhere(plume_ids == flicker_ids[i]) plume_ids = np.delete(plume_ids, index) del plume_objects[str(flicker_ids[i])] #For merged IDs, we move the tracks to pre-merge tracks for i in np.arange(0, len(merge_ids)): plume = plume_objects[str(merge_ids[i])] plume.merge() # For old IDs, we just run an update. for i in np.arange(0, len(old_ids)): if debug: print 'Updating plume', old_ids[i] plume = plume_objects[str(old_ids[i])] #if plume.plume_id == 2: # print plume.dates_observed plume.update_position(lats, lons, sdf_plumes, old_ids[i]) plume.update_duration(datetimes[date_i]) #if plume.plume_id == 18: # print plume.dates_observed plume.update_bbox() plume.update_majorminor_axes(lons, lats) plume.update_area() plume.update_centroid_speed() plume.update_centroid_direction() plume.update_max_extent() #plume_assoc_clouds, plume_check_bool \ # = plume.flag_dust_associated_convection( # clouds, prev_dust_assoc_clouds) #dust_assoc_clouds += plume_assoc_clouds #check_bool += plume_check_bool.astype(int) # plume.update_leading_edge_4(sdf_plumes, lons, lats) plume_objects[str(old_ids[i])] = plume # Plumes which no longer exist are removed and archived if len(ids_previous) == 0: removed_ids = [] else: removed_bool = np.asarray( [j not in plume_ids for j in ids_previous]) removed_ids = ids_previous[removed_bool] for i in np.arange(0, len(removed_ids)): if debug: print 'Archiving plume', removed_ids[i] plume = plume_objects[str(removed_ids[i])] plume.update_GPE_speed() # plume.update_mechanism_likelihood() plume.update_mean_axis_offset() plume.update_llj_probability(trace) plume_archive[str(removed_ids[i])] = plume del plume_objects[str(removed_ids[i])] last_10_ids = np.append(last_10_ids, removed_ids[i]) if len(last_10_ids) > 10: last_10_ids = last_10_ids[0:10] if len(np.unique(sdf_plumes)) < 2: sdf_previous = None ids_previous = [] raw_sdf_prev = [] else: sdf_previous = sdf_plumes ids_previous = plume_ids raw_sdf_prev = sdf_now else: print 'Adding date to list of missing dates' with open('missing_dates.txt', 'a') as my_file: my_file.write('\n' + datestrings[date_i]) # Print date_i so we know where we got up to in case we have to # restart with open('date_i_' + str(yearmonth[0]) + '.txt', 'w') as f: f.write('%d' % date_i) # After the script has finished, add remaining plumes to the plume archive for i in plume_objects: plume_archive[i] = plume_objects[i] plume_archive.close()
def wrapper(yearmonth): year_lower = yearmonth[0] year_upper = yearmonth[0] month_lower = yearmonth[-1][0] #month_lower = 7 month_upper = yearmonth[-1][-1] day_lower = 1 day_upper = 31 hour_lower = 0 hour_upper = 23 minute_lower = 0 minute_upper = 45 time_params = np.array([ year_lower, year_upper, month_lower, month_upper, day_lower, day_upper, hour_lower, hour_upper, minute_lower, minute_upper ]) datetimes = utilities.get_datetime_objects(time_params) datestrings = [j.strftime("%Y%m%d%H%M") for j in datetimes] # Get lats and lons sdf_test = Dataset( '/soge-home/data_not_backed_up/satellite/meteosat/seviri' '/15-min/0.03x0.03/sdf/nc/JUNE2010/SDF_v2/SDF_v2.' '201006031500.nc') lons, lats = np.meshgrid(sdf_test.variables['longitude'][:], sdf_test.variables['latitude'][:]) lonmask = lons > 360 latmask = lats > 90 lons = np.ma.array(lons, mask=lonmask) lats = np.ma.array(lats, mask=latmask) cpo_previous = None raw_cpo_prev = [] clouds_previous = None ids_previous = [] cloud_ids_previous = [] deep_conv_IDs_prev = None LLJ_plumes_IDs_prev = [] k = 0 available_colours = np.arange(0, 41) # To pick back up where you left off, simply add all the keys in plume # archive to used IDs here - perhaps include an option to do that used_ids = [] used_cloud_ids = [] used_colour_IDs = {} flicker_ids = [] reintroduced_ids = [] last_10_ids = [] last_10_ids = np.asarray(last_10_ids) prev_dust_assoc_clouds = None found_file = True # Restrict the lons and lats to the CWS alone lonbool = np.asarray([j >= -20 and j <= 10 for j in lons[0]]) cpo_objects = {} cpo_archive = shelve.open('/soge-home/projects/seviri_dust/plumetracker/' 'cpo_archive_v4' '' + str(yearmonth[0])) if pickup: archived_ids = np.asarray([j for j in cpo_archive]) for i in archived_ids: used_ids.append(int(i)) with open('date_i_' + str(yearmonth[0]) + '.txt', 'r') as f: pickup_date_i = f.read() print 'Picking up from timestep', pickup_date_i datestrings = datestrings[int(pickup_date_i):] datetimes = datetimes[int(pickup_date_i):] cpo_archive.close() year_lower = datetimes[0].year year_upper = datetimes[-1].year month_lower = datetimes[0].month month_upper = datetimes[-1].month day_lower = datetimes[0].day day_upper = datetimes[-1].day hour_lower = 0 hour_upper = 23 minute_lower = 0 minute_upper = 45 time_params_oneday = np.array([ year_lower, year_lower, month_lower, month_lower, day_lower, day_lower, hour_lower, hour_upper, minute_lower, minute_upper ]) oneday_datetimes = utilities.get_datetime_objects(time_params_oneday) if moving_window: # Here run the moving window for our selected time period print 'Generating cloud masks for all timesteps' # Loop through each time in a single day pool = multiprocessing.Pool() for i in np.arange(0, len(oneday_datetimes)): pool.apply_async(cpo_detection.cloud_mask_mw, args=(i, datetimes, oneday_datetimes, ianlons, ianlats)) # Pull a netCDF dataset object out so the projection coordinates can # be obtained pool.close() pool.join() btdiff_2_anom_prev = None btdiff_2_anom_prev_2 = None btdiff_2_anom_prev_3 = None for date_i in np.arange(0, len(datestrings)): cpo_archive = shelve.open( '/soge-home/projects/seviri_dust/plumetracker/' 'cpo_archive_v4_' '' + str(yearmonth[0])) runtime = datetimes[date_i] - datetimes[0] print '\n' + datestrings[date_i] + '\n' totaltest = datetime.datetime.now() if found_file: #if 'time' in bt.variables: # bt_108 = bt.variables['bt108'][:][0] #else: # bt_108 = bt.variables['bt108'][:] #clouds = bt_108 < 270 # Here run CPO detection # IF A RAW DETECTED CPO FILE EXISTS, USE THAT (assuming the # algorithm hasn't changed) if calculate_cpof: cpo_now, btdiff_2_anom_prev, btdiff_2_anom_prev_2, \ btdiff_2_anom_prev_3 = cpo_detection.detect_cpo( btdiff_2_anom_prev, btdiff_2_anom_prev_2, btdiff_2_anom_prev_3, datetimes, datestrings, date_i, lons, lats, cloud_lons, cloud_lats, daily_cloudmask, double_digits, mesh, daily_bt) cpo_now[cpo_now > 0] = 1 else: cpo_data = Dataset('/ouce-home/projects/seviri_dust/cpof/' + datetimes[date_i].strftime("%B%Y") + '/CPOF_' + datetimes[date_i].strftime("%Y%m%d%H%M") + '.nc') cpo_now = cpo_data.variables['CPOF'][:] # Save the raw CPO detection to file # These should be stored in the project directory, not home #np.save('raw_detected_cpo_'+datestrings[date_i], cpo_now) # Add the reintroduced plumes back into the running for i in np.arange(0, len(reintroduced_ids)): #print 'Reintroducing CPO', reintroduced_ids[i] cpo_previous[cpo_previous == flicker_ids[i]] = \ reintroduced_ids[i] ids_previous = np.append(ids_previous, reintroduced_ids[i]) # Get plumes first by scanning for them cpo_blobs, new_ids, blob_ids, merge_ids = cpo.\ scan_for_blobs( cpo_now, cpo_previous, used_ids) #print 'New IDs', new_ids for i in np.arange(0, len(reintroduced_ids)): if reintroduced_ids[i] not in blob_ids: #print 'But plume '+str(reintroduced_ids[i])+' sadly ' \ # 'died.' pass # Clear flicker IDs and reintroduced IDs flicker_ids = [] reintroduced_ids = [] for i in new_ids: used_ids.append(i) old_bool = np.asarray([j in ids_previous for j in blob_ids]) if len(old_bool) > 0: old_ids = blob_ids[old_bool] else: old_ids = [] # Then, for each new ID, we initialise plume objects for i in np.arange(0, len(new_ids)): if debug: print 'Creating new CPO', new_ids[i] cold_pool = cpo.CPO(new_ids[i], datetimes[date_i]) cold_pool.update_position(lats, lons, cpo_blobs, new_ids[i]) cold_pool.update_duration(datetimes[date_i]) #if plume.plume_id == 18: # print plume.dates_observed # print plume.dates_observed cold_pool.update_bbox() cold_pool.update_majorminor_axes(lons, lats) cold_pool.update_area() cold_pool.update_max_extent() cold_pool.update_centroid_speed() cold_pool.update_centroid_direction() #cold_pool.check_conv_distance(lats, lons, clouds) cold_pool.update_most_likely_source() # plume.update_leading_edge_4(sdf_plumes, lons, lats) cpo_objects[str(new_ids[i])] = cold_pool missing_cold_pool, missing_id, flickered = \ cold_pool.chain_flickerchecker( raw_cpo_prev) if flickered: raise ValueError('Found an overlapping CPO in the ' 'previous timestep larger than size ' '50 - a new CPO should not be ' 'initiated here') steps_back = 1 # As long as there is an overlapping previous plume, # keep updating it back in time while len(missing_cold_pool) > 0: if debug: print 'Rolling back CPO', new_ids[i] # We can only step back to the first timestep and no # earlier if (date_i - steps_back) < 0: missing_cold_pool = [] break else: missing_date = datetimes[date_i - steps_back] missing_cpo_blobs = np.zeros(missing_cold_pool.shape) missing_cold_pool = missing_cold_pool == 1 missing_cpo_blobs[missing_cold_pool] = missing_id # Run all the updates that would be used for a # new plume cold_pool.update_position(lats, lons, missing_cpo_blobs, missing_id) cold_pool.update_duration(missing_date) #if plume.plume_id == 18: # print plume.dates_observed cold_pool.update_bbox() cold_pool.update_majorminor_axes(lons, lats) cold_pool.update_area() cold_pool.update_max_extent() cold_pool.update_centroid_speed() cold_pool.update_centroid_direction() #cold_pool.check_conv_distance(lats, lons, clouds) cold_pool.update_most_likely_source() cold_pool.process_missing_plume() #print 'Updated missing plume back '+str( # steps_back)+' steps' steps_back += 1 if (date_i - steps_back) < 0: missing_cpo = [] break # Pull out data from the timestep before to # continue the chain try: raw_cpo_prev_prev = \ np.load('raw_detected_cpo_'+datestrings[ date_i]+'.npy') except: #print 'Adding date to list of missing dates' with open('missing_dates.txt', 'a') as my_file: my_file.write('\n' + datestrings[date_i - steps_back]) break missing_cold_pool, missing_id, flickered = \ cold_pool.chain_flickerchecker( raw_cpo_prev_prev) if flickered: #print 'Found a flickered plume. Searching ' \ # 'for the corresponding archived plume.' # We have a plume in a previous timestep # which flickered cold_pool_archive_keys = last_10_ids.astype(int) # Sort the keys in reverse order as the # plume we want is most likely to have a # high ID cold_pool_archive_keys[::-1].sort() missing_cold_pool = missing_cold_pool == 1 missing_cpo_blobs = np.zeros( missing_cold_pool.shape) missing_cpo_blobs[missing_cold_pool] = missing_id cold_pool_bool = missing_cpo_blobs == missing_id search_centroid_lon = \ np.nanmean(lons[cold_pool_bool]) search_centroid_lat = \ np.nanmean(lats[cold_pool_bool]) cold_pool_lons = lons[cold_pool_bool] cold_pool_lats = lats[cold_pool_bool] search_date = datetimes[date_i - steps_back] #print search_date found_cold_pool = False for key in cold_pool_archive_keys: #print 'Searching in plume archive' if search_centroid_lon == \ cpo_archive[\ str(key)].centroid_lon and \ search_centroid_lat ==\ cpo_archive[str( key)].centroid_lat and \ cpo_archive[str( key)].dates_observed[-1] == \ search_date: #print 'Found it in plume archive. ' \ # 'ID is', \ # plume_archive[str(key)].plume_id found_cold_pool = True correct_cold_pool = cpo_archive[str(key)] cold_pool_to_append = cold_pool # Append the flickered plume to the # old one which was archived correct_cold_pool.append_missing_cold_pool( cold_pool_to_append) # Add it to plume objects and remove # it from archives cpo_objects[str(key)] = correct_cold_pool del cpo_archive[str(key)] # Add it to old IDs, replacing the # ID of the plume which was found to be # flickered flicker_ids.append(cold_pool.plume_id) reintroduced_ids.append(key) missing_cold_pool = [] # Reintroduced plumes also get removed # from the record of the last 10 ids index = np.argwhere(last_10_ids == key) last_10_ids = np.delete(last_10_ids, index) break # If we didn't find the plume in the plume # archive, it must still be active if found_cold_pool == False: cold_pool_object_keys = np.asarray( [int(i) for i in cpo_objects]) # Sort the keys in reverse order as the # plume we want is most likely to have a # high ID cold_pool_object_keys[::-1].sort() for key in cold_pool_object_keys: #print 'Searching in plume objects' if search_centroid_lon == \ cpo_objects[ \ str( key)].centroid_lon and \ search_centroid_lat == \ cpo_objects[str( key)].centroid_lat and \ cpo_objects[str( key)].dates_observed[ -1] == \ search_date: found_cold_pool = True correct_cold_pool = cpo_archive[str( key)] cold_pool_to_append = cold_pool # Append the flickered plume to the # old one which was archived correct_cold_pool.append_missing_cold_pool( cold_pool_to_append) # Add it to plume objects and # remove it from archives cpo_objects[str( key)] = correct_cold_pool del cpo_archive[str(key)] # Add it to old IDs, replacing the # ID of the plume which was found # to be flickered flicker_ids.append(cold_pool.plume_id) reintroduced_ids.append(key) missing_cold_pool = [] index = np.argwhere(last_10_ids == key) last_10_ids = np.delete( last_10_ids, index) break break # Remove any IDs which were actually flickers for i in np.arange(0, len(flicker_ids)): index = np.argwhere(new_ids == flicker_ids[i]) new_ids = np.delete(new_ids, index) index = np.argwhere(ids_previous == flicker_ids[i]) ids_previous = np.delete(ids_previous, index) index = np.argwhere(blob_ids == flicker_ids[i]) blob_ids = np.delete(blob_ids, index) del cpo_objects[str(flicker_ids[i])] #For merged IDs, we move the tracks to pre-merge tracks for i in np.arange(0, len(merge_ids)): cold_pool = cpo_objects[str(merge_ids[i])] cold_pool.merge() # For old IDs, we just run an update. for i in np.arange(0, len(old_ids)): if debug: print 'Updating CPO', old_ids[i] cold_pool = cpo_objects[str(old_ids[i])] #if plume.plume_id == 2: # print plume.dates_observed cold_pool.update_position(lats, lons, cpo_blobs, old_ids[i]) cold_pool.update_duration(datetimes[date_i]) #if plume.plume_id == 18: # print plume.dates_observed cold_pool.update_bbox() cold_pool.update_majorminor_axes(lons, lats) cold_pool.update_area() cold_pool.update_centroid_speed() cold_pool.update_centroid_direction() cold_pool.update_max_extent() #plume_assoc_clouds, plume_check_bool \ # = plume.flag_dust_associated_convection( # clouds, prev_dust_assoc_clouds) #dust_assoc_clouds += plume_assoc_clouds #check_bool += plume_check_bool.astype(int) # plume.update_leading_edge_4(sdf_plumes, lons, lats) cpo_objects[str(old_ids[i])] = cold_pool # Plumes which no longer exist are removed and archived if len(ids_previous) == 0: removed_ids = [] else: removed_bool = np.asarray( [j not in blob_ids for j in ids_previous]) removed_ids = ids_previous[removed_bool] for i in np.arange(0, len(removed_ids)): if debug: print 'Archiving CPO', removed_ids[i] cold_pool = cpo_objects[str(removed_ids[i])] cold_pool.update_GPE_speed() # plume.update_mechanism_likelihood() cold_pool.update_mean_axis_offset() if cold_pool.duration.total_seconds() >= 3600: cpo_archive[str(removed_ids[i])] = cold_pool del cpo_objects[str(removed_ids[i])] print '== CPO detected ==' print 'CPO emission time:', cold_pool.emission_time print 'CPO emission lat:', cold_pool.track_centroid_lat[0] print 'CPO emission lon:', cold_pool.track_centroid_lon[0] last_10_ids = np.append(last_10_ids, removed_ids[i]) else: #print 'Too short duration - not archived' del cpo_objects[str(removed_ids[i])] #print 'Plume', plume.plume_id,'removed. Final lons and \ # lats:' #print plume.centroid_lon #print plume.centroid_lat if len(last_10_ids) > 10: last_10_ids = last_10_ids[0:10] if len(np.unique(cpo_blobs)) < 2: cpo_previous = None ids_previous = [] raw_cpo_prev = [] else: cpo_previous = cpo_blobs ids_previous = blob_ids raw_cpo_prev = cpo_now else: print 'Adding date to list of missing dates' with open('missing_dates.txt', 'a') as my_file: my_file.write('\n' + datestrings[date_i]) with open('date_i_' + str(yearmonth[0]) + '.txt', 'w') as f: f.write('%d' % date_i) cpo_archive.close()
def wrapper(y): Year_lower = years_list[y] Year_upper = years_list[y] Month_lower = 5 Month_upper = 9 Day_lower = 24 Day_upper = 8 Hour_lower = 0 Hour_upper = 23 Minute_lower = 0 Minute_upper = 45 # Generate datetime objects corresponding to these time bounds time_params = np.array([ Year_lower, Year_upper, Month_lower, Month_upper, Day_lower, Day_upper, Hour_lower, Hour_upper, Minute_lower, Minute_upper ]) datetimes = utilities.get_datetime_objects(time_params) years = np.unique(np.asarray(([j.year for j in datetimes]))) months = np.unique(np.asarray(([j.month for j in datetimes]))) days = np.unique(np.asarray(([j.day for j in datetimes]))) for m in np.arange(0, len(years)): for k in np.arange(0, len(months)): for i in np.arange(0, len(days)): day_datetimes_bool = np.asarray([ j.day == days[i] and j.month == months[k] and j.year == years[m] for j in datetimes ]) day_datetimes = datetimes[day_datetimes_bool] if len(day_datetimes) == 0: continue ncfile = pinkdust.create_time_cloudmask_nc_file( '/soge-home/projects/seviri_dust/raw_seviri_data' '/cloudmask_nc/' + day_datetimes[0].strftime('%B%Y') + '/cloudmask_' + day_datetimes[0].strftime('%Y%m%d') + '.nc', day_datetimes, lats, lons) #sub_wrapper(day_datetimes, 0) pool = multiprocessing.Pool() for j in np.arange(0, len(day_datetimes)): pool.apply_async(sub_wrapper, args=( day_datetimes, j, )) pool.close() pool.join() for j in np.arange(0, len(day_datetimes)): f = tables.open_file( '/soge-home/projects/seviri_dust/raw_seviri_data' '/intermediary_files/cloudmask_' + day_datetimes[j].strftime('%Y%m%d%H%M%S.hdf')) arrobj = f.get_node('/data') cloudmask = arrobj.read() f.close() ncfile = pinkdust.save_cloudmask_to_existing_nc( ncfile, cloudmask, day_datetimes[j]) print 'Day ' + str(days[i]) + ' done' ncfile.close() print 'Month ' + str(months[k]) + ' done'
def wrapper(i): year_lower = i year_upper = i month_lower = 6 month_upper = 8 day_lower = 1 day_upper = 31 hour_lower = 0 hour_upper = 23 minute_lower = 0 minute_upper = 45 time_params = np.array([year_lower, year_upper, month_lower, month_upper, day_lower, day_upper, hour_lower, hour_upper, minute_lower, minute_upper]) datetimes = utilities.get_datetime_objects(time_params) for date_i in np.arange(0, len(datetimes)): date = datetimes[date_i] print date year_bool = np.asarray(years == date.year) root = np.asarray(year_cm_roots)[year_bool][0] if root == root2: # We're using TCH's daily cloudmask files. These are values from # 0-2 (I think), and need selecting and regridding. try: clouddata = Dataset(root+date.strftime("%B%Y")+'/cloudmask_' + datetimes[ date_i].strftime("%Y%m%d") + '.nc') cloudmask_times = num2date(clouddata.variables['time'][:], clouddata.variables['time'].units) cloudmask_times = np.asarray([dt.datetime(j.year, j.month, j.day, j.hour, j.minute) for j in cloudmask_times]) cloudmask_bool = cloudmask_times == date clouds_data = clouddata.variables['cloud_mask'][cloudmask_bool] clouds_now = clouds_data#[cloudmask_bool] clouds_now_regridded = pinkdust.regrid_data(root2_lons, root2_lats, target_lons, target_lats, clouds_now, mesh=True) data_array[clouds_now_regridded > 1] += 1 except: print 'No cloud data for', date else: # We're using Ian's original cloud mask files. These are just a # binary one or zero and won't be regridded. try: clouddata = Dataset(root+ datetimes[date_i].strftime("%B").upper( ) + str(datetimes[date_i].year) + '_CLOUDS/eumetsat.cloud.' + datetimes[ date_i].strftime("%Y%m%d%H%M") + '.nc') clouds_now = clouddata.variables['cmask'][:][0] data_array[clouds_now == 1] += 1 except: print 'No cloud data for', date if date_i == 100: print data_array np.save('cloudcover_array', data_array) extent = ( np.min(target_lons), np.max(target_lons), np.min(target_lats), np.max(target_lats)) m = Basemap(projection='cyl', llcrnrlon=extent[0], urcrnrlon=extent[1], llcrnrlat=extent[2], urcrnrlat=extent[3], resolution='i') m.drawcoastlines(linewidth=0.5) m.drawcountries(linewidth=0.5) parallels = np.arange(10., 40, 2.) # labels = [left,right,top,bottom] m.drawparallels(parallels, labels=[False, True, True, False], linewidth=0.5) meridians = np.arange(-20., 17., 4.) m.drawmeridians(meridians, labels=[True, False, False, True], linewidth=0.5) min = 0 max = 70000 levels = MaxNLocator(nbins=15).tick_values(min, max) # discrete_cmap = utilities.cmap_discretize(cm.RdYlBu_r, 10) cmap = cm.get_cmap('RdYlBu_r') norm = BoundaryNorm(levels, ncolors=cmap.N, clip=True) data_array = np.ma.masked_where(np.isnan(data_array), data_array) m.pcolormesh(target_lons, target_lats, data_array, cmap=cmap, norm=norm, vmin=min, vmax=max) cbar = plt.colorbar(orientation='horizontal', fraction=0.056, pad=0.06) cbar.ax.set_xlabel('Counts of cloud occurrence') plt.tight_layout() plt.savefig('CPO_multiyear_cloud_frequency_2004_2012.png', bbox_inches='tight') plt.close()