def wrapper(date): print date test1 = datetime.datetime.now() # Pull out cloud mask data to be regridded and added to the nc file # glob is used for a wildcard try: clouddata = pygrib.open( glob.glob( '/soge-home/projects/seviri_dust/raw_seviri_data/cloudmask_grib/MSG*-' 'SEVI-MSGCLMK-0100-0100-' + date.strftime("%Y%m%d%H%M%S") + '*')[0]) grb = clouddata.select()[0] cloudmaskarray = grb.values[:, ::-1] cloudlats, cloudlons = grb.latlons() cloudmaskarray[cloudmaskarray >= 3] = np.nan cloudlats[cloudlats > 90] = np.nan cloudlons[cloudlons > 180] = np.nan target_area = utils.load_area( '/soge-home/projects/seviri_dust/areas.def', 'NorthAfrica') lons, lats = target_area.get_lonlats() # Generate a regular lat/lon grid for the cloud mask #regular_lons = np.linspace(np.min(lons), np.max(lons), lons.shape[1]) #regular_lats = np.linspace(np.min(lats), np.max(lats), lats.shape[0]) # Regrid cloud mask data to the same grid as the BT data cloudmask_regridded = pinkdust.regrid_data(cloudlons, cloudlats, lons, lats, cloudmaskarray, mesh=True) f = tables.open_file( '/soge-home/projects/seviri_dust/raw_seviri_data' '/intermediary_files/cloudmask/cloudmask_' + date.strftime('%Y%m%d%H%M%S.hdf'), 'w') atom = tables.Atom.from_dtype(cloudmask_regridded.dtype) filters = tables.Filters(complib='blosc', complevel=5) ds = f.create_carray(f.root, 'data', atom, cloudmask_regridded.shape, filters=filters) ds[:] = cloudmask_regridded f.close() except: with open( '/soge-home/projects/seviri_dust' '/raw_seviri_data/bt_native/missing_msgnative_dates' '.txt', 'a') as my_file: my_file.write('\n') my_file.write(date.strftime('%Y%m%d%H%M%S') + ' clouds')
def sub_wrapper(day_datetimes, j): print day_datetimes[j] try: clouddata = Dataset('/soge-home/data/satellite/meteosat/seviri/15-min/' 'native/cloudmask' '/nc/' + day_datetimes[j].strftime("%B").upper() + str(day_datetimes[j].year) + '_CLOUDS/eumetsat.cloud.' + day_datetimes[j].strftime("%Y%m%d%H%M") + '.nc') cloudmaskarray = clouddata.variables['cmask'][:] cloudmask_regridded = pinkdust.regrid_data(cloudlons, cloudlats, lons, lats, cloudmaskarray, mesh=True) except: print 'Missing cloud mask data for', \ day_datetimes[j] cloudmask_regridded = np.zeros((lats.shape[0], lons.shape[1])) cloudmask_regridded[:] = np.nan # Now, instead of writing to day array, you write to hdf f = tables.open_file( '/soge-home/projects/seviri_dust/raw_seviri_data' '/intermediary_files/cloudmask_' + day_datetimes[j].strftime('%Y%m%d%H%M%S.hdf'), 'w') atom = tables.Atom.from_dtype(cloudmask_regridded.dtype) filters = tables.Filters(complib='blosc', complevel=5) ds = f.create_carray(f.root, 'data', atom, cloudmask_regridded.shape, filters=filters) ds[:] = cloudmask_regridded f.close() print 'Wrote', day_datetimes[j]
def sub_wrapper(day_datetimes, j): print day_datetimes[j] try: clouddata = pygrib.open( glob.glob( '/soge-home/projects/seviri_dust/raw_seviri_data/cloudmask_grib/MSG*-' 'SEVI-MSGCLMK-0100-0100-' + day_datetimes[j].strftime("%Y%m%d%H%M%S") + '*')[0]) grb = clouddata.select()[0] cloudmaskarray = grb.values[:, ::-1] cloudmask_regridded = pinkdust.regrid_data(cloudlons, cloudlats, lons, lats, cloudmaskarray, mesh=True) except: print 'Missing cloud mask data for', \ day_datetimes[j] cloudmask_regridded = np.zeros((lats.shape[0], lons.shape[1])) cloudmask_regridded[:] = np.nan # Now, instead of writing to day array, you write to hdf f = tables.open_file( '/soge-home/projects/seviri_dust/raw_seviri_data' '/intermediary_files/cloudmask_' + day_datetimes[j].strftime('%Y%m%d%H%M%S.hdf'), 'w') atom = tables.Atom.from_dtype(cloudmask_regridded.dtype) filters = tables.Filters(complib='blosc', complevel=5) ds = f.create_carray(f.root, 'data', atom, cloudmask_regridded.shape, filters=filters) ds[:] = cloudmask_regridded f.close() print 'Wrote', day_datetimes[j]
def cloud_mask_mw(i): """ Moving window cloud masking to be used by multiprocessing :return: """ #print '1' date = oneday_datetimes[i] window_datetime_lower = datetime.datetime(year_lower, month_lower, day_lower, hour_lower, minute_lower) \ - datetime.timedelta(days=7) window_datetime_upper = datetime.datetime(year_upper, month_upper, day_upper, hour_upper, minute_upper) \ + datetime.timedelta(days=7) # Get datetime objects between the above bounds time_params_7dayw = np.array([window_datetime_lower.year, window_datetime_upper.year, window_datetime_lower.month, window_datetime_upper.month, window_datetime_lower.day, window_datetime_upper.day, date.hour, date.hour, date.minute, date.minute]) datetimes_7dayw = utilities.get_daily_datetime_objects( time_params_7dayw) bt_15day = np.zeros((datetimes_7dayw.shape[0], 3, ianlats.shape[0], ianlons.shape[0])) #print '2' # bt_15day[:] = np.nan cloudscreenedbt_15day = np.zeros((datetimes_7dayw.shape[0], 3, ianlats.shape[0], ianlons.shape[0])) # cloudscreenedbt_15day[:] = np.nan f = tables.open_file( '/soge-home/projects/seviri_dust/sdf/intermediary_files/' 'cloud_masked_bt_15d_' + oneday_datetimes[i].strftime( "%Y_%H_%M") + '.hdf', 'w') atom = tables.Atom.from_dtype(cloudscreenedbt_15day.dtype) filters = tables.Filters(complib='blosc', complevel=5) cs = f.create_carray(f.root, 'data', atom, cloudscreenedbt_15day.shape, filters=filters) pickup = False pickup_i = 0 #print '3' if pickup == False: g = tables.open_file( '/soge-home/projects/seviri_dust/sdf/intermediary_files/bt_15d_' + oneday_datetimes[i].strftime( "%Y_%H_%M") + '.hdf', 'w') atom = tables.Atom.from_dtype(bt_15day.dtype) filters = tables.Filters(complib='blosc', complevel=5) bts = g.create_carray(g.root, 'data', atom, bt_15day.shape, filters=filters) else: g = tables.open_file( '/soge-home/projects/seviri_dust/sdf/intermediary_files/bt_15d_' + oneday_datetimes[i].strftime( "%Y_%H_%M") + '.hdf', 'a') #print '4 - loop starting' # Loop through each day of the time window for this time of day for j in np.arange(pickup_i, len(datetimes_7dayw)): #for j in np.arange(pickup_i, pickup_i+1): #for j in np.arange(pickup_i, 1): print str(oneday_datetimes[i].hour) + str( oneday_datetimes[i].minute), 'day ', j+1 date_w = datetimes_7dayw[j] #print '5' # Extract BT data for this timestep filename = '/ouce-home/projects/seviri_dust/raw_seviri_data/bt_nc/' \ + str(date_w.strftime( '%B')) + str(date_w.year) + \ '/BT_' + str( date_w.strftime('%Y')) + str(date_w.strftime('%m')) + \ str(date_w.strftime('%d'))+ '.nc' try: btdata = Dataset(filename, 'r') except: print 'Found no BT data for ' + filename continue #print '6' # Extract cloud mask data for this timestep cloudfilename = \ '/ouce-home/projects/seviri_dust/raw_seviri_data/cloudmask_nc/' \ + str(date_w.strftime('%B')) + str(date_w.year) +\ '/cloudmask_' \ + str(date_w.strftime('%Y')) \ + str(date_w.strftime('%m')) \ + str(date_w.strftime('%d')) + '.nc' try: clouddata = Dataset(cloudfilename, 'r') except: print 'Found no cloud data for ' + cloudfilename continue #print '7' # Apply cloud screening cloudscreenedbt_15day_array, bt087, bt108, \ bt120 = sdf.cloud_screen_daily(btdata, clouddata, date_w) regcs_array = np.zeros((len(cloudscreenedbt_15day_array), ianlats.shape[ 0], ianlons.shape[0])) #print '8' for k in np.arange(0, len(cloudscreenedbt_15day_array)): regcs = pinkdust.regrid_data(tchlons, tchlats, ianlons, ianlats, cloudscreenedbt_15day_array[k], mesh=False) regcs_array[k] = regcs #print '9' regbt087 = pinkdust.regrid_data(tchlons, tchlats, ianlons, ianlats, bt087, mesh=False) regbt108 = pinkdust.regrid_data(tchlons, tchlats, ianlons, ianlats, bt108, mesh=False) regbt120 = pinkdust.regrid_data(tchlons, tchlats, ianlons, ianlats, bt120, mesh=False) #print '10' cs[j] = regcs_array bts[j, 0] = regbt087 bts[j, 1] = regbt108 bts[j, 2] = regbt120 btdata.close() #print g.root.data[0] #print f.root.data[0] #print '11' # Save cloud masked data for this time of day to file f.close() # Save cloud masked data for this time of day to file g.close()
def cloud_mask_mw(i): """ Moving window cloud masking to be used by multiprocessing :return: """ date = oneday_datetimes[i] window_datetime_lower = datetime.datetime(year_lower, month_lower, day_lower, hour_lower, minute_lower) \ - datetime.timedelta(days=7) window_datetime_upper = datetime.datetime(year_upper, month_upper, day_upper, hour_upper, minute_upper) \ + datetime.timedelta(days=7) # Get datetime objects between the above bounds time_params_7dayw = np.array([ window_datetime_lower.year, window_datetime_upper.year, window_datetime_lower.month, window_datetime_upper.month, window_datetime_lower.day, window_datetime_upper.day, date.hour, date.hour, date.minute, date.minute ]) datetimes_7dayw = utilities.get_daily_datetime_objects(time_params_7dayw) bt_15day = np.zeros( (datetimes_7dayw.shape[0], 3, ianlats.shape[0], ianlons.shape[0])) #bt_15day[:] = np.nan cloudscreenedbt_15day = np.zeros( (datetimes_7dayw.shape[0], 3, ianlats.shape[0], ianlons.shape[0])) #cloudscreenedbt_15day[:] = np.nan f = tables.open_file( '/soge-home/projects/seviri_dust/sdf/intermediary_files/' 'cloud_masked_bt_15d_' + oneday_datetimes[i].strftime("%Y_%H_%M") + '.hdf', 'w') atom = tables.Atom.from_dtype(cloudscreenedbt_15day.dtype) filters = tables.Filters(complib='blosc', complevel=5) cs = f.create_carray(f.root, 'data', atom, cloudscreenedbt_15day.shape, filters=filters) g = tables.open_file( '/soge-home/projects/seviri_dust/sdf/intermediary_files/bt_15d_' + oneday_datetimes[i].strftime("%Y_%H_%M") + '.hdf', 'w') atom = tables.Atom.from_dtype(bt_15day.dtype) filters = tables.Filters(complib='blosc', complevel=5) bts = g.create_carray(g.root, 'data', atom, bt_15day.shape, filters=filters) # Loop through each day of the time window for this time of day for j in np.arange(0, len(datetimes_7dayw)): print str(oneday_datetimes[i].hour) + str( oneday_datetimes[i].minute), 'day ', j + 1 date_w = datetimes_7dayw[j] # Extract BT data for this timestep filename = '/ouce-home/data/satellite/meteosat/seviri/15-min/' \ 'native/bt/nc' \ '/' + str(date_w.strftime( '%B').upper()) + str(date_w.year) + \ '/H-000-MSG2__-MSG2________-' \ 'IR_BrightnessTemperatures___-000005___-' + str( date_w.strftime('%Y')) + str(date_w.strftime('%m')) + \ str(date_w.strftime('%d')) + str(date_w.strftime('%H')) \ + str(date_w.strftime('%M')) + '-__.nc' try: btdata = Dataset(filename, 'r') except: print 'Found no BT data for ' + filename continue # Extract cloud mask data for this timestep cloudfilename = '/ouce-home/data/satellite/meteosat/' \ 'seviri/15-min/' \ 'native/cloudmask/nc' \ '/' + str(date_w.strftime('%B').upper()) \ + str(date_w.year) + '_CLOUDS/eumetsat.cloud.' \ + str(date_w.strftime('%Y')) \ + str(date_w.strftime('%m')) \ + str(date_w.strftime('%d')) + str( date_w.strftime('%H')) \ + str(date_w.strftime('%M')) + '.nc' try: clouddata = Dataset(cloudfilename, 'r') except: print 'Found no cloud data for ' + cloudfilename continue # Apply cloud screening cloudscreenedbt_15day_array, bt087, bt108, \ bt120 = sdf.cloud_screen(btdata, clouddata) regcs_array = np.zeros((len(cloudscreenedbt_15day_array), ianlats.shape[0], ianlons.shape[0])) for k in np.arange(0, len(cloudscreenedbt_15day_array)): regcs = pinkdust.regrid_data(lons, lats, ianlons, ianlats, cloudscreenedbt_15day_array[k], mesh=False) regcs_array[k] = regcs regbt087 = pinkdust.regrid_data(lons, lats, ianlons, ianlats, bt087, mesh=False) regbt108 = pinkdust.regrid_data(lons, lats, ianlons, ianlats, bt108, mesh=False) regbt120 = pinkdust.regrid_data(lons, lats, ianlons, ianlats, bt120, mesh=False) cs[j] = regcs_array bts[j, 0] = regbt087 bts[j, 1] = regbt108 bts[j, 2] = regbt120 btdata.close() # Save cloud masked data for this time of day to file f.close() # Save cloud masked data for this time of day to file g.close()
def detect_cpo(btdiff_2_anom_prev, btdiff_2_anom_prev_2, btdiff_2_anom_prev_3, datetimes, datestrings, date_i, lons, lats, cloud_lons, cloud_lats, daily_clouds=False, double_digits=False, mesh=False, daily_bt=False): used_ids = [] runtime = datetimes[date_i] - datetimes[0] #print '\n' + datestrings[date_i] + '\n' totaltest = datetime.datetime.now() found_file = True if daily_bt == False: if os.path.isfile('/ouce-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/bt' '/nc/' + datetimes[date_i].strftime("%B").upper() + str(datetimes[date_i].year) + '/H-000-MSG2__' '-MSG2________-' 'IR_BrightnessTemperatures___' '-000005___-' + datestrings[date_i] + '-__.nc'): bt = Dataset('/ouce-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/bt' '/nc/' + datetimes[date_i].strftime("%B").upper() + str(datetimes[date_i].year) + '/H-000-MSG2__' '-MSG2________-' 'IR_BrightnessTemperatures___' '-000005___-' + datestrings[date_i] + '-__.nc') found_file = True elif os.path.isfile('/ouce-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/bt' '/nc/' + datetimes[date_i].strftime("%B").upper() + str(datetimes[date_i].year) + '/H-000-MSG1__' '-MSG1________-' 'IR_BrightnessTemperatures___' '-000005___-' + datestrings[date_i] + '-__.nc'): bt = Dataset('/ouce-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/bt' '/nc/' + datetimes[date_i].strftime("%B").upper() + str(datetimes[date_i].year) + '/H-000-MSG1__' '-MSG1________-' 'IR_BrightnessTemperatures___' '-000005___-' + datestrings[date_i] + '-__.nc') found_file = True else: found_file = False if daily_clouds: try: cloudmask = Dataset('/soge-home/projects/seviri_dust/raw_seviri_' 'data/cloudmask_nc/' + datetimes[date_i].strftime("%B%Y") + '/cloudmask_' + datetimes[date_i].strftime("%Y%m%d") + '.nc') cloudmask_times = num2date(cloudmask.variables['time'][:], cloudmask.variables['time'].units) cloudmask_times = np.asarray([ datetime.datetime(j.year, j.month, j.day, j.hour, j.minute) for j in cloudmask_times ]) cloudmask_bool = cloudmask_times == datetimes[date_i] print np.all(cloudmask.variables['cloud_mask'][0] == \ cloudmask.variables['cloud_mask'][30]) clouds_now = cloudmask.variables['cloud_mask'][cloudmask_bool][0] found_file = True except: print 'Found no cloud mask file!' clouds_now = np.zeros(cloud_lons.shape) found_file = False else: try: cloudmask = Dataset( '/soge-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/cloudmask' '/nc/' + datetimes[date_i].strftime("%B").upper() + str(datetimes[date_i].year) + '_CLOUDS/eumetsat.cloud.' + datestrings[date_i] + '.nc') clouds_now = cloudmask.variables['cmask'][:][0] cloud_lons = cloudmask.variables['lon'][:] cloud_lats = cloudmask.variables['lat'][:] except: clouds_now = np.zeros(cloud_lons.shape) found_file = False print 'Found no cloud mask file!' if found_file: if daily_bt == False: bt087 = bt.variables['bt087'][:][0] bt12 = bt.variables['bt120'][:][0] orig_lons = bt.variables['longitude'][:] orig_lats = bt.variables['latitude'][:] orig_lons, orig_lats = np.meshgrid(orig_lons, orig_lats) else: orig_lons = lons orig_lats = lats # print bt12.shape # print clouds_now.shape window_datetime_lower = datetime.datetime(datetimes[0].year, datetimes[0].month, datetimes[0].day, datetimes[0].hour, datetimes[0].minute) \ - datetime.timedelta(days=7) window_datetime_upper = datetime.datetime(datetimes[-1].year, datetimes[-1].month, datetimes[-1].day, datetimes[-1].hour, datetimes[-1].minute) \ + datetime.timedelta(days=7) BT_15_day_lower_bound = datetimes[date_i] - datetime.timedelta(days=7) BT_15_day_upper_bound = datetimes[date_i] + datetime.timedelta(days=7) # Get datetime objects between the above bounds time_params_7dayw = np.array([ window_datetime_lower.year, window_datetime_upper.year, window_datetime_lower.month, window_datetime_upper.month, window_datetime_lower.day, window_datetime_upper.day, datetimes[date_i].hour, datetimes[date_i].hour, datetimes[date_i].minute, datetimes[date_i].minute ]) datetimes_7dayw = utilities.get_daily_datetime_objects( time_params_7dayw) indices = np.arange(0, len(datetimes_7dayw)) lower_ind = datetimes_7dayw == BT_15_day_lower_bound lower_ind = indices[lower_ind][0] upper_ind = datetimes_7dayw == BT_15_day_upper_bound upper_ind = indices[upper_ind][0] current_ind = datetimes_7dayw == datetimes[date_i] current_ind = indices[current_ind][0] if double_digits: f = tables.open_file( '/soge-home/projects/seviri_dust/sdf/intermediary_files' '/bt_15d_' + datetimes[date_i].strftime('%Y_%H_%M') + '.hdf') BT_15_days = f.root.data[lower_ind:upper_ind] bt_data = f.root.data[current_ind] f.close() else: f = tables.open_file( '/soge-home/projects/seviri_dust/sdf/intermediary_files' '/bt_15d_' + str(datetimes[date_i].year) + '_' + str(datetimes[date_i].hour) + '_' + str(datetimes[date_i].minute) + '.hdf') BT_15_days = f.root.data[lower_ind:upper_ind] bt_data = f.root.data[current_ind] f.close() if daily_bt: bt087 = bt_data[0] bt12 = bt_data[2] bt_15day_087 = BT_15_days[:, 0] #bt_15day_108 = BT_15_days[:, 1] bt_15day_120 = BT_15_days[:, 2] bt_15day_087_mean = np.nanmean(bt_15day_087, axis=0) #bt_15day_108_mean = np.nanmean(bt_15day_108, axis=0) bt_15day_120_mean = np.nanmean(bt_15day_120, axis=0) btdiff_2_15daymean = bt_15day_120_mean - bt_15day_087_mean if mesh: cloud_lons, cloud_lats = np.meshgrid(cloud_lons, cloud_lats) clouds_now_regridded = pinkdust.regrid_data(cloud_lons, cloud_lats, orig_lons, orig_lats, clouds_now, mesh=True) """ bt087_regridded = pinkdust.regrid_data(orig_lons, orig_lats, cloud_lons, cloud_lats, bt087, mesh=True) bt12_regridded = pinkdust.regrid_data(orig_lons, orig_lats, cloud_lons, cloud_lats, bt12, mesh=True) btdiff_2_15daymean_regridded = pinkdust.regrid_data(orig_lons, orig_lats, cloud_lons, cloud_lats, btdiff_2_15daymean, mesh=True) """ btdiff_2 = bt12 - bt087 btdiff_2_anom = btdiff_2 - btdiff_2_15daymean if btdiff_2_anom_prev_3 != None: btdiff_2_anom_diff = btdiff_2_anom - btdiff_2_anom_prev_3 btdiff_2_anom_diff += \ (btdiff_2_anom - btdiff_2_anom_prev_2) btdiff_2_anom_diff += \ (btdiff_2_anom - btdiff_2_anom_prev) else: btdiff_2_anom_diff = np.zeros((btdiff_2_anom.shape)) if date_i == 0: btdiff_2_anom_prev = deepcopy(btdiff_2_anom) elif date_i == 1: btdiff_2_anom_prev_2 = deepcopy(btdiff_2_anom_prev) btdiff_2_anom_prev = deepcopy(btdiff_2_anom) elif date_i == 2: btdiff_2_anom_prev_3 = deepcopy(btdiff_2_anom_prev_2) btdiff_2_anom_prev_2 = deepcopy(btdiff_2_anom_prev) btdiff_2_anom_prev = deepcopy(btdiff_2_anom) elif date_i > 2: btdiff_2_anom_prev_3 = deepcopy(btdiff_2_anom_prev_2) btdiff_2_anom_prev_2 = deepcopy(btdiff_2_anom_prev) btdiff_2_anom_prev = deepcopy(btdiff_2_anom) if daily_clouds: clouds_now_regridded = clouds_now_regridded > 1 lat_grad, lon_grad = np.gradient(btdiff_2_anom) total_grad = np.sqrt(lat_grad**2 + lon_grad**2) convolution = scipy.signal.convolve2d(clouds_now_regridded, np.ones((5, 5)), mode='same') clouds_now = convolution > 0 total_grad[clouds_now == 1] = np.nan ### PASS I ### # In the FIRST PASS the LORD sayeth unto the image, 'Let all # whose BTD is below -10K be classified as CPOs, and remove the # tiny ones' # And those who fulfilled this condition were classified, # and it was good convolution = scipy.signal.convolve2d(clouds_now, np.ones((5, 5)), mode='same') clouds_now = convolution > 0 btdiff_2_anom_diff_um = deepcopy(btdiff_2_anom_diff) btdiff_2_anom_diff[clouds_now > 0] = np.nan cpo_mask_pass_1 = btdiff_2_anom_diff < -7 label_objects, nb_labels = ndi.label(cpo_mask_pass_1) sizes = np.bincount(label_objects.ravel()) # Set clusters smaller than size 20 to zero mask_sizes = sizes > 20 mask_sizes[0] = 0 cpo_mask_pass_1 = mask_sizes[label_objects] ### PASS II ### # In the SECOND PASS the LORD sayeth unto the image, 'Let all # those included in the first pass which contain pixels which # are below -15K be classified' # And those who fulfilled this condition were classified, # and it was better cpo_mask_pass_2 = deepcopy(cpo_mask_pass_1) # Label the image and get all connected elements cpo_mask_pass_1, num = measurements.label(cpo_mask_pass_1) # Then loop through each labeled blob and find if one of the # pixels has a -15K. If it does, the whole blob is a freaking CPO. blob_ids = np.unique(cpo_mask_pass_1) blob_ids = blob_ids[blob_ids != 0] for i in np.arange(0, len(blob_ids)): target_region = cpo_mask_pass_1 == blob_ids[i] # First check if this region is within the original # time-gradient identified region (i.e. not one introduced # with the new generous gradient checker) if np.any(btdiff_2_anom_diff[target_region == 1] < -15): # Next check if there is a generous gradient contained # within this region somewhere pass else: cpo_mask_pass_2[target_region == 1] = 0 # For identified CPO regions, undo the convolution on the cloud # mask cpo_mask_um = btdiff_2_anom_diff_um < -7 # Label the image and get all connected elements cpo_mask_um, num = measurements.label(cpo_mask_um) # Then loop through each labeled blob and find if one of the # pixels has a -15K. If it does, the whole blob is a freaking CPO. blob_ids = np.unique(cpo_mask_um) blob_ids = blob_ids[blob_ids != 0] if 1 in cpo_mask_pass_2: for i in np.arange(0, len(blob_ids)): target_region = cpo_mask_um == blob_ids[i] if 1 in cpo_mask_pass_2[target_region]: #if np.any(cpo_mask_pass_2[target_region == 1] == 1): cpo_mask_pass_2[target_region] = 1 extent = (np.min(orig_lons), np.max(orig_lons), np.min(orig_lats), np.max(orig_lats)) m = Basemap(projection='cyl', llcrnrlon=extent[0], urcrnrlon=extent[1], llcrnrlat=extent[2], urcrnrlat=extent[3], resolution='i') m.drawcoastlines(linewidth=0.5) m.drawcountries(linewidth=0.5) parallels = np.arange(10., 40, 2.) # labels = [left,right,top,bottom] m.drawparallels(parallels, labels=[False, True, True, False], linewidth=0.5) meridians = np.arange(-20., 17., 4.) m.drawmeridians(meridians, labels=[True, False, False, True], linewidth=0.5) min = 5 max = -15 levels = MaxNLocator(nbins=15).tick_values(min, max) cmap = cm.get_cmap('Blues_r') norm = BoundaryNorm(levels, ncolors=cmap.N, clip=True) #print np.unique(btdiff_2_anom_diff[np.isfinite(btdiff_2_anom_diff)]) m.pcolormesh(orig_lons, orig_lats, btdiff_2_anom_diff, cmap=cmap, vmin=min, vmax=max, norm=norm) cbar = plt.colorbar(orientation='horizontal', fraction=0.056, pad=0.06) cbar.ax.set_xlabel('CPO mask pass 2') plt.tight_layout() plt.savefig('btdiff_' + datestrings[date_i] + '.png', bbox_inches='tight') plt.close() return cpo_mask_pass_2, btdiff_2_anom_prev, btdiff_2_anom_prev_2,\ btdiff_2_anom_prev_3 else: if mesh: empty_arr = np.zeros((lats.shape[0], lons.shape[1])) else: empty_arr = np.zeros((lats.shape[0], lons.shape[0])) empty_arr[:] = np.nan return empty_arr, None, None, None
for j in datetimes_7dayw])] bt_15day_087 = BT_15_days[:, 0] bt_15day_108 = BT_15_days[:, 1] bt_15day_120 = BT_15_days[:, 2] bt_15day_087_mean = np.nanmean(bt_15day_087, axis=0) bt_15day_108_mean = np.nanmean(bt_15day_108, axis=0) bt_15day_120_mean = np.nanmean(bt_15day_120, axis=0) btdiff_2_15daymean = bt_15day_120_mean - bt_15day_087_mean orig_lons, orig_lats = np.meshgrid(orig_lons, orig_lats) bt087_regridded = pinkdust.regrid_data(orig_lons, orig_lats, cloud_lons, cloud_lats, bt087) bt12_regridded = pinkdust.regrid_data(orig_lons, orig_lats, cloud_lons, cloud_lats, bt12) btdiff_2_15daymean_regridded = pinkdust.regrid_data( orig_lons, orig_lats, cloud_lons, cloud_lats, btdiff_2_15daymean) btdiff_2 = bt12_regridded - bt087_regridded btdiff_2_anom = btdiff_2 - btdiff_2_15daymean_regridded if btdiff_2_anom_prev_3 != None: btdiff_2_anom_diff = btdiff_2_anom - btdiff_2_anom_prev_3
def wrapper(bt_120_108_anom_m_prev_1, bt_120_108_anom_m_prev_2, bt_120_108_anom_m_prev_3, bt_108_087_anom_m_prev_1, bt_108_087_anom_m_prev_2, bt_108_087_anom_m_prev_3, bt_120_087_anom_m_prev_1, bt_120_087_anom_m_prev_2, bt_120_087_anom_m_prev_3, datetimes, datestrings, date_i, lons, lats, cloud_lons, cloud_lats, daily_clouds=False, double_digits=False, mesh=False, daily_bt=False): used_ids = [] runtime = datetimes[date_i] - datetimes[0] # print '\n' + datestrings[date_i] + '\n' totaltest = datetime.datetime.now() found_file = True if daily_bt == False: if os.path.isfile('/ouce-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/bt' '/nc/' + datetimes[date_i].strftime("%B").upper() + str(datetimes[date_i].year) + '/H-000-MSG2__' '-MSG2________-' 'IR_BrightnessTemperatures___' '-000005___-' + datestrings[date_i] + '-__.nc'): bt = Dataset('/ouce-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/bt' '/nc/' + datetimes[date_i].strftime("%B").upper() + str(datetimes[date_i].year) + '/H-000-MSG2__' '-MSG2________-' 'IR_BrightnessTemperatures___' '-000005___-' + datestrings[date_i] + '-__.nc') found_file = True elif os.path.isfile('/ouce-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/bt' '/nc/' + datetimes[date_i].strftime("%B").upper() + str(datetimes[date_i].year) + '/H-000-MSG1__' '-MSG1________-' 'IR_BrightnessTemperatures___' '-000005___-' + datestrings[date_i] + '-__.nc'): bt = Dataset('/ouce-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/bt' '/nc/' + datetimes[date_i].strftime("%B").upper() + str(datetimes[date_i].year) + '/H-000-MSG1__' '-MSG1________-' 'IR_BrightnessTemperatures___' '-000005___-' + datestrings[date_i] + '-__.nc') found_file = True else: found_file = False if daily_clouds: try: cloudmask = Dataset('/soge-home/projects/seviri_dust/raw_seviri_' 'data/cloudmask_nc/' + datetimes[date_i].strftime("%B%Y") + '/cloudmask_' + datetimes[date_i].strftime("%Y%m%d") + '.nc') cloudmask_times = num2date(cloudmask.variables['time'][:], cloudmask.variables['time'].units) cloudmask_times = np.asarray([ datetime.datetime(j.year, j.month, j.day, j.hour, j.minute) for j in cloudmask_times ]) cloudmask_bool = cloudmask_times == datetimes[date_i] clouds_now = cloudmask.variables['cloud_mask'][cloudmask_bool][0] found_file = True except: print 'Found no cloud mask file!' clouds_now = np.zeros(cloud_lons.shape) found_file = False sdf_root = '/soge-home/projects/seviri_dust/sdf/' \ + datetimes[date_i].strftime('%B') \ + str(datetimes[date_i].year) + '/' if os.path.isfile(sdf_root + 'SDF_v2.' + \ datestrings[date_i] + '.nc'): sdf = Dataset( sdf_root + 'SDF_v2.' + \ datestrings[date_i] + '.nc') found_file = True # print sdf sdf_now = sdf.variables['SDF'][:] else: print 'No SDF file found for this date' found_file = False else: try: cloudmask = Dataset( '/soge-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/cloudmask' '/nc/' + datetimes[date_i].strftime("%B").upper() + str(datetimes[date_i].year) + '_CLOUDS/eumetsat.cloud.' + datestrings[date_i] + '.nc') clouds_now = cloudmask.variables['cmask'][:][0] cloud_lons = cloudmask.variables['lon'][:] cloud_lats = cloudmask.variables['lat'][:] except: clouds_now = np.zeros(cloud_lons.shape) found_file = False print 'Found no cloud mask file!' sdf_root = '/soge-home/data_not_backed_up/satellite/meteosat' \ '/seviri/15' \ '-min/0.03x0.03/sdf/nc/' + datetimes[date_i].strftime( '%B').upper() + str(datetimes[date_i].year) + '/SDF_v2/' if os.path.isfile(sdf_root + 'SDF_v2.' + \ datestrings[date_i] + '.nc'): sdf = Dataset( sdf_root + 'SDF_v2.' + \ datestrings[date_i] + '.nc') found_file = True # print sdf if 'time' in sdf.variables: sdf_now = sdf.variables['bt108'][0] else: sdf_now = sdf.variables['bt108'][:] else: print 'No SDF file found for this date' found_file = False if found_file: if daily_bt == False: bt087 = bt.variables['bt087'][:][0] bt108 = bt.variables['bt108'][:][0] bt12 = bt.variables['bt120'][:][0] orig_lons = bt.variables['longitude'][:] orig_lats = bt.variables['latitude'][:] orig_lons, orig_lats = np.meshgrid(orig_lons, orig_lats) else: orig_lons = lons orig_lats = lats # print bt12.shape # print clouds_now.shape window_datetime_lower = datetime.datetime(datetimes[0].year, 6, 1, 0, 45) \ - datetime.timedelta(days=7) window_datetime_upper = datetime.datetime(datetimes[-1].year, 8, 31, 23, 45) \ + datetime.timedelta(days=7) BT_15_day_lower_bound = datetimes[date_i] - datetime.timedelta(days=7) BT_15_day_upper_bound = datetimes[date_i] + datetime.timedelta(days=7) # Get datetime objects between the above bounds time_params_7dayw = np.array([ window_datetime_lower.year, window_datetime_upper.year, window_datetime_lower.month, window_datetime_upper.month, window_datetime_lower.day, window_datetime_upper.day, datetimes[date_i].hour, datetimes[date_i].hour, datetimes[date_i].minute, datetimes[date_i].minute ]) datetimes_7dayw = utilities.get_daily_datetime_objects( time_params_7dayw) indices = np.arange(0, len(datetimes_7dayw)) lower_ind = datetimes_7dayw == BT_15_day_lower_bound lower_ind = indices[lower_ind][0] upper_ind = datetimes_7dayw == BT_15_day_upper_bound upper_ind = indices[upper_ind][0] current_ind = datetimes_7dayw == datetimes[date_i] current_ind = indices[current_ind][0] if double_digits: f = tables.open_file( '/soge-home/projects/seviri_dust/sdf/intermediary_files' '/bt_15d_' + datetimes[date_i].strftime('%Y_%H_%M') + '.hdf') BT_15_days = f.root.data[lower_ind:upper_ind] bt_data = f.root.data[current_ind] f.close() else: f = tables.open_file( '/soge-home/projects/seviri_dust/sdf/intermediary_files' '/bt_15d_' + str(datetimes[date_i].year) + '_' + str(datetimes[date_i].hour) + '_' + str(datetimes[date_i].minute) + '.hdf') BT_15_days = f.root.data[lower_ind:upper_ind] bt_data = f.root.data[current_ind] f.close() if daily_bt: bt087 = bt_data[0] bt108 = bt_data[1] bt12 = bt_data[2] bt_15day_087 = BT_15_days[:, 0] bt_15day_108 = BT_15_days[:, 1] bt_15day_120 = BT_15_days[:, 2] bt_15day_087_mean = np.nanmean(bt_15day_087, axis=0) bt_15day_108_mean = np.nanmean(bt_15day_108, axis=0) bt_15day_120_mean = np.nanmean(bt_15day_120, axis=0) btdiff_2_15daymean = bt_15day_120_mean - bt_15day_087_mean bt_108_087 = bt108 - bt087 bt_120_108 = bt12 - bt108 bt_120_087 = bt12 - bt087 bt_108_087_mean = bt_15day_108_mean - bt_15day_087_mean bt_120_108_mean = bt_15day_120_mean - bt_15day_108_mean bt_120_087_mean = bt_15day_120_mean - bt_15day_087_mean bt_108_087_anom = bt_108_087 - bt_108_087_mean bt_120_108_anom = bt_120_108 - bt_120_108_mean bt_120_087_anom = bt_120_087 - bt_120_087_mean if mesh: cloud_lons, cloud_lats = np.meshgrid(cloud_lons, cloud_lats) clouds_now_regridded = pinkdust.regrid_data(cloud_lons, cloud_lats, orig_lons, orig_lats, clouds_now, mesh=True) bt_108_087_anom_m = deepcopy(bt_108_087_anom) bt_120_087_anom_m = deepcopy(bt_120_087_anom) bt_120_108_anom_m = deepcopy(bt_120_108_anom) clouds_now_regridded[sdf_now == 1] = 0 bt_108_087_anom_m[clouds_now_regridded == 1] = np.nan bt_120_087_anom_m[clouds_now_regridded == 1] = np.nan bt_120_108_anom_m[clouds_now_regridded == 1] = np.nan #btdiff_2 = bt12 - bt087 #btdiff_2_anom = btdiff_2 - btdiff_2_15daymean if bt_108_087_anom_m_prev_1 != None: arra = (bt_120_087_anom_m - bt_120_087_anom_m_prev_3) + ( bt_120_087_anom_m - bt_120_087_anom_m_prev_2) + ( bt_120_087_anom_m - bt_120_087_anom_m_prev_1) arrb = (bt_120_108_anom_m - bt_120_108_anom_m_prev_3) + ( bt_120_108_anom_m - bt_120_108_anom_m_prev_2) + ( bt_120_108_anom_m - bt_120_108_anom_m_prev_1) arrc = (bt_108_087_anom_m - bt_108_087_anom_m_prev_3) + ( bt_108_087_anom_m - bt_108_087_anom_m_prev_2) + ( bt_108_087_anom_m - bt_108_087_anom_m_prev_1) detected_bt = (arrc - arrb) + (arra - arrb) else: detected_bt = np.zeros((bt_108_087_anom.shape)) if date_i == 0: bt_120_087_anom_m_prev_1 = deepcopy(bt_120_087_anom_m) bt_120_108_anom_m_prev_1 = deepcopy(bt_120_108_anom_m) bt_108_087_anom_m_prev_1 = deepcopy(bt_108_087_anom_m) elif date_i == 1: bt_120_087_anom_m_prev_2 = deepcopy(bt_120_087_anom_m_prev_1) bt_120_108_anom_m_prev_2 = deepcopy(bt_120_108_anom_m_prev_1) bt_108_087_anom_m_prev_2 = deepcopy(bt_108_087_anom_m_prev_1) bt_120_087_anom_m_prev_1 = deepcopy(bt_120_087_anom_m) bt_120_108_anom_m_prev_1 = deepcopy(bt_120_108_anom_m) bt_108_087_anom_m_prev_1 = deepcopy(bt_108_087_anom_m) elif date_i == 2: bt_120_087_anom_m_prev_3 = deepcopy(bt_120_087_anom_m_prev_2) bt_120_108_anom_m_prev_3 = deepcopy(bt_120_108_anom_m_prev_2) bt_108_087_anom_m_prev_3 = deepcopy(bt_108_087_anom_m_prev_2) bt_120_087_anom_m_prev_2 = deepcopy(bt_120_087_anom_m_prev_1) bt_120_108_anom_m_prev_2 = deepcopy(bt_120_108_anom_m_prev_1) bt_108_087_anom_m_prev_2 = deepcopy(bt_108_087_anom_m_prev_1) bt_120_087_anom_m_prev_1 = deepcopy(bt_120_087_anom_m) bt_120_108_anom_m_prev_1 = deepcopy(bt_120_108_anom_m) bt_108_087_anom_m_prev_1 = deepcopy(bt_108_087_anom_m) elif date_i > 2: bt_120_087_anom_m_prev_3 = deepcopy(bt_120_087_anom_m_prev_2) bt_120_108_anom_m_prev_3 = deepcopy(bt_120_108_anom_m_prev_2) bt_108_087_anom_m_prev_3 = deepcopy(bt_108_087_anom_m_prev_2) bt_120_087_anom_m_prev_2 = deepcopy(bt_120_087_anom_m_prev_1) bt_120_108_anom_m_prev_2 = deepcopy(bt_120_108_anom_m_prev_1) bt_108_087_anom_m_prev_2 = deepcopy(bt_108_087_anom_m_prev_1) bt_120_087_anom_m_prev_1 = deepcopy(bt_120_087_anom_m) bt_120_108_anom_m_prev_1 = deepcopy(bt_120_108_anom_m) bt_108_087_anom_m_prev_1 = deepcopy(bt_108_087_anom_m) if daily_clouds: clouds_now_regridded = clouds_now_regridded > 1 ### PASS I ### # In the FIRST PASS the LORD sayeth unto the image, 'Let all # whose BTD is below -10K be classified as CPOs, and remove the # tiny ones' # And those who fulfilled this condition were classified, # and it was good # NOTE: Why cloud mask here? What if there was a strong cloud # gradient in the previous three timesteps which disappeared in this # one but still pushed us over the threshold? It wouldn't be cloud # masked detected_bt_um = deepcopy(detected_bt) #if cloud_mask: # detected_bt[clouds_now > 0] = np.nan levels = np.arange(-24, 40, 8) cpo_mask_pass_1 = detected_bt < -6 label_objects, nb_labels = ndi.label(cpo_mask_pass_1) sizes = np.bincount(label_objects.ravel()) # Set clusters smaller than size 20 to zero mask_sizes = sizes > 20 mask_sizes[0] = 0 cpo_mask_pass_1 = mask_sizes[label_objects] ### PASS II ### # In the SECOND PASS the LORD sayeth unto the image, 'Let all # those included in the first pass which contain pixels which # are below -15K be classified' # And those who fulfilled this condition were classified, # and it was better cpo_mask_pass_2 = deepcopy(cpo_mask_pass_1) # Label the image and get all connected elements cpo_mask_pass_1, num = measurements.label(cpo_mask_pass_1) # Then loop through each labeled blob and find if one of the # pixels has a -15K. If it does, the whole blob is a freaking CPO. blob_ids = np.unique(cpo_mask_pass_1) blob_ids = blob_ids[blob_ids != 0] for i in np.arange(0, len(blob_ids)): target_region = cpo_mask_pass_1 == blob_ids[i] # First check if this region is within the original # time-gradient identified region (i.e. not one introduced # with the new generous gradient checker) if np.any(detected_bt[target_region == 1] < -20): # Next check if there is a generous gradient contained # within this region somewhere pass else: cpo_mask_pass_2[target_region == 1] = 0 # For identified CPO regions, undo the convolution on the cloud # mask cpo_mask_um = detected_bt_um < -6 # Label the image and get all connected elements cpo_mask_um, num = measurements.label(cpo_mask_um) # Then loop through each labeled blob and find if one of the # pixels has a -15K. If it does, the whole blob is a freaking CPO. blob_ids = np.unique(cpo_mask_um) blob_ids = blob_ids[blob_ids != 0] if 1 in cpo_mask_pass_2: for i in np.arange(0, len(blob_ids)): target_region = cpo_mask_um == blob_ids[i] if 1 in cpo_mask_pass_2[target_region]: # if np.any(cpo_mask_pass_2[target_region == 1] == 1): cpo_mask_pass_2[target_region] = 1 if debug: extent = (np.min(orig_lons), np.max(orig_lons), np.min(orig_lats), np.max(orig_lats)) m = Basemap(projection='cyl', llcrnrlon=extent[0], urcrnrlon=extent[1], llcrnrlat=extent[2], urcrnrlat=extent[3], resolution='i') m.drawcoastlines(linewidth=0.5) m.drawcountries(linewidth=0.5) parallels = np.arange(10., 40, 2.) # labels = [left,right,top,bottom] m.drawparallels(parallels, labels=[False, True, True, False], linewidth=0.5) meridians = np.arange(-20., 17., 4.) m.drawmeridians(meridians, labels=[True, False, False, True], linewidth=0.5) min = -20 max = 5 levels = MaxNLocator(nbins=15).tick_values(min, max) cmap = cm.get_cmap('Blues_r') norm = BoundaryNorm(levels, ncolors=cmap.N, clip=True) m.pcolormesh(orig_lons, orig_lats, detected_bt, cmap=cmap, vmin=min, vmax=max, norm=norm) cbar = plt.colorbar(orientation='horizontal', fraction=0.056, pad=0.06) cbar.ax.set_xlabel('BTdiff 2 anom diff') plt.tight_layout() plt.savefig('BTdiff_2_anom_diff' + datestrings[date_i] + '.png', bbox_inches='tight') plt.close() extent = (np.min(orig_lons), np.max(orig_lons), np.min(orig_lats), np.max(orig_lats)) m = Basemap(projection='cyl', llcrnrlon=extent[0], urcrnrlon=extent[1], llcrnrlat=extent[2], urcrnrlat=extent[3], resolution='i') m.drawcoastlines(linewidth=0.5) m.drawcountries(linewidth=0.5) parallels = np.arange(10., 40, 2.) # labels = [left,right,top,bottom] m.drawparallels(parallels, labels=[False, True, True, False], linewidth=0.5) meridians = np.arange(-20., 17., 4.) m.drawmeridians(meridians, labels=[True, False, False, True], linewidth=0.5) min = 180 max = 320 levels = MaxNLocator(nbins=15).tick_values(min, max) cmap = cm.get_cmap('Blues_r') norm = BoundaryNorm(levels, ncolors=cmap.N, clip=True) m.pcolormesh(orig_lons, orig_lats, bt108, cmap=cmap, vmin=min, vmax=max, norm=norm) cbar = plt.colorbar(orientation='horizontal', fraction=0.056, pad=0.06) cbar.ax.set_xlabel('BT 10.8') plt.tight_layout() plt.savefig('BT108_' + datestrings[date_i] + '.png', bbox_inches='tight') plt.close() if show_all_indicators: bt_108_087 = bt108 - bt087 bt_120_108 = bt12 - bt108 bt_120_087 = bt12 - bt087 bt_108_087_mean = bt_15day_108_mean - bt_15day_087_mean bt_120_108_mean = bt_15day_120_mean - bt_15day_108_mean bt_120_087_mean = bt_15day_120_mean - bt_15day_087_mean bt_108_087_anom = bt_108_087 - bt_108_087_mean bt_120_108_anom = bt_120_108 - bt_120_108_mean bt_120_087_anom = bt_120_087 - bt_120_087_mean bt_087_anom = bt087 - bt_15day_087_mean bt_108_anom = bt108 - bt_15day_108_mean bt_120_anom = bt12 - bt_15day_120_mean lat_grad, lon_grad = np.gradient(bt_108_087_anom) total_grad = np.sqrt(lat_grad**2 + lon_grad**2) grad_108_087_anom = deepcopy(total_grad) lat_grad, lon_grad = np.gradient(bt_120_108_anom) total_grad = np.sqrt(lat_grad**2 + lon_grad**2) grad_120_108_anom = deepcopy(total_grad) lat_grad, lon_grad = np.gradient(bt_120_087_anom) total_grad = np.sqrt(lat_grad**2 + lon_grad**2) grad_120_087_anom = deepcopy(total_grad) lat_grad, lon_grad = np.gradient(bt_087_anom) total_grad = np.sqrt(lat_grad**2 + lon_grad**2) grad_087_anom = deepcopy(total_grad) lat_grad, lon_grad = np.gradient(bt_108_anom) total_grad = np.sqrt(lat_grad**2 + lon_grad**2) grad_108_anom = deepcopy(total_grad) lat_grad, lon_grad = np.gradient(bt_120_anom) total_grad = np.sqrt(lat_grad**2 + lon_grad**2) grad_120_anom = deepcopy(total_grad) indicators = [ bt087, bt108, bt12, bt_108_087_anom, bt_120_108_anom, bt_120_087_anom, grad_108_087_anom, grad_120_108_anom, grad_120_087_anom, grad_087_anom, grad_108_anom, grad_120_anom ] extent = (np.min(orig_lons), np.max(orig_lons), np.min(orig_lats), np.max(orig_lats)) m = Basemap(projection='cyl', llcrnrlon=extent[0], urcrnrlon=extent[1], llcrnrlat=extent[2], urcrnrlat=extent[3], resolution='i') m.drawcoastlines(linewidth=0.5) m.drawcountries(linewidth=0.5) parallels = np.arange(10., 40, 2.) m.drawparallels(parallels, labels=[False, True, True, False], linewidth=0.5) meridians = np.arange(-20., 17., 4.) m.drawmeridians(meridians, labels=[True, False, False, True], linewidth=0.5) mins = [180, 180, 180, -13, -13, -13, 0, 0, 0, 0, 0, 0] maxs = [320, 320, 320, 6, 6, 6, 9, 9, 9, 50, 50, 50] labels = [ 'bt087', 'bt108', 'bt120', 'bt_108_087_anom', 'bt_120_108_anom', 'bt_120_087_anom', 'grad_108_087_anom', 'grad_120_108_anom', 'grad_120_087_anom', 'grad_087_anom', 'grad_108_anom', 'grad_120_anom' ] for i in np.arange(0, len(indicators)): min = mins[i] max = maxs[i] levels = MaxNLocator(nbins=15).tick_values(min, max) cmap = cm.get_cmap('Blues_r') norm = BoundaryNorm(levels, ncolors=cmap.N, clip=True) plot = m.pcolormesh(orig_lons, orig_lats, indicators[i], cmap=cmap, vmin=min, vmax=max, norm=norm) cbar = plt.colorbar(orientation='horizontal', fraction=0.056, pad=0.06) cbar.ax.set_xlabel(labels[i]) plt.tight_layout() plt.savefig(labels[i] + '_' + datestrings[date_i] + '.png', bbox_inches='tight') cbar.remove() plot.remove() return cpo_mask_pass_2, bt_120_108_anom_m, \ bt_120_108_anom_m_prev_1, bt_120_108_anom_m_prev_2, \ bt_120_108_anom_m_prev_3, bt_108_087_anom_m,\ bt_108_087_anom_m_prev_1, bt_108_087_anom_m_prev_2, \ bt_108_087_anom_m_prev_3, bt_120_087_anom_m,\ bt_120_087_anom_m_prev_1, bt_120_087_anom_m_prev_2, \ bt_120_087_anom_m_prev_3 else: if mesh: empty_arr = np.zeros((lats.shape[0], lons.shape[1])) else: empty_arr = np.zeros((lats.shape[0], lons.shape[0])) empty_arr[:] = np.nan return empty_arr, None, None, None
def wrapper(i): year_lower = i year_upper = i month_lower = 6 month_upper = 8 day_lower = 1 day_upper = 31 hour_lower = 0 hour_upper = 23 minute_lower = 0 minute_upper = 45 time_params = np.array([year_lower, year_upper, month_lower, month_upper, day_lower, day_upper, hour_lower, hour_upper, minute_lower, minute_upper]) datetimes = utilities.get_datetime_objects(time_params) for date_i in np.arange(0, len(datetimes)): date = datetimes[date_i] print date year_bool = np.asarray(years == date.year) root = np.asarray(year_cm_roots)[year_bool][0] if root == root2: # We're using TCH's daily cloudmask files. These are values from # 0-2 (I think), and need selecting and regridding. try: clouddata = Dataset(root+date.strftime("%B%Y")+'/cloudmask_' + datetimes[ date_i].strftime("%Y%m%d") + '.nc') cloudmask_times = num2date(clouddata.variables['time'][:], clouddata.variables['time'].units) cloudmask_times = np.asarray([dt.datetime(j.year, j.month, j.day, j.hour, j.minute) for j in cloudmask_times]) cloudmask_bool = cloudmask_times == date clouds_data = clouddata.variables['cloud_mask'][cloudmask_bool] clouds_now = clouds_data#[cloudmask_bool] clouds_now_regridded = pinkdust.regrid_data(root2_lons, root2_lats, target_lons, target_lats, clouds_now, mesh=True) data_array[clouds_now_regridded > 1] += 1 except: print 'No cloud data for', date else: # We're using Ian's original cloud mask files. These are just a # binary one or zero and won't be regridded. try: clouddata = Dataset(root+ datetimes[date_i].strftime("%B").upper( ) + str(datetimes[date_i].year) + '_CLOUDS/eumetsat.cloud.' + datetimes[ date_i].strftime("%Y%m%d%H%M") + '.nc') clouds_now = clouddata.variables['cmask'][:][0] data_array[clouds_now == 1] += 1 except: print 'No cloud data for', date if date_i == 100: print data_array np.save('cloudcover_array', data_array) extent = ( np.min(target_lons), np.max(target_lons), np.min(target_lats), np.max(target_lats)) m = Basemap(projection='cyl', llcrnrlon=extent[0], urcrnrlon=extent[1], llcrnrlat=extent[2], urcrnrlat=extent[3], resolution='i') m.drawcoastlines(linewidth=0.5) m.drawcountries(linewidth=0.5) parallels = np.arange(10., 40, 2.) # labels = [left,right,top,bottom] m.drawparallels(parallels, labels=[False, True, True, False], linewidth=0.5) meridians = np.arange(-20., 17., 4.) m.drawmeridians(meridians, labels=[True, False, False, True], linewidth=0.5) min = 0 max = 70000 levels = MaxNLocator(nbins=15).tick_values(min, max) # discrete_cmap = utilities.cmap_discretize(cm.RdYlBu_r, 10) cmap = cm.get_cmap('RdYlBu_r') norm = BoundaryNorm(levels, ncolors=cmap.N, clip=True) data_array = np.ma.masked_where(np.isnan(data_array), data_array) m.pcolormesh(target_lons, target_lats, data_array, cmap=cmap, norm=norm, vmin=min, vmax=max) cbar = plt.colorbar(orientation='horizontal', fraction=0.056, pad=0.06) cbar.ax.set_xlabel('Counts of cloud occurrence') plt.tight_layout() plt.savefig('CPO_multiyear_cloud_frequency_2004_2012.png', bbox_inches='tight') plt.close()
def wrapper(datetimes, datestrings, date_i, lons, lats, cloud_lons, cloud_lats, daily_clouds=False, double_digits=False, mesh=False, daily_bt=False): used_ids = [] runtime = datetimes[date_i] - datetimes[0] # print '\n' + datestrings[date_i] + '\n' totaltest = datetime.datetime.now() found_file = True if daily_bt == False: if os.path.isfile('/ouce-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/bt' '/nc/' + datetimes[date_i].strftime("%B").upper() + str(datetimes[date_i].year) + '/H-000-MSG2__' '-MSG2________-' 'IR_BrightnessTemperatures___' '-000005___-' + datestrings[date_i] + '-__.nc'): bt = Dataset('/ouce-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/bt' '/nc/' + datetimes[date_i].strftime("%B").upper() + str(datetimes[date_i].year) + '/H-000-MSG2__' '-MSG2________-' 'IR_BrightnessTemperatures___' '-000005___-' + datestrings[date_i] + '-__.nc') found_file = True elif os.path.isfile('/ouce-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/bt' '/nc/' + datetimes[date_i].strftime("%B").upper() + str(datetimes[date_i].year) + '/H-000-MSG1__' '-MSG1________-' 'IR_BrightnessTemperatures___' '-000005___-' + datestrings[date_i] + '-__.nc'): bt = Dataset('/ouce-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/bt' '/nc/' + datetimes[date_i].strftime("%B").upper() + str(datetimes[date_i].year) + '/H-000-MSG1__' '-MSG1________-' 'IR_BrightnessTemperatures___' '-000005___-' + datestrings[date_i] + '-__.nc') found_file = True else: found_file = False if daily_clouds: try: cloudmask = Dataset('/soge-home/projects/seviri_dust/raw_seviri_' 'data/cloudmask_nc/' + datetimes[date_i].strftime("%B%Y") + '/cloudmask_' + datetimes[date_i].strftime("%Y%m%d") + '.nc') cloudmask_times = num2date(cloudmask.variables['time'][:], cloudmask.variables['time'].units) cloudmask_times = np.asarray([ datetime.datetime(j.year, j.month, j.day, j.hour, j.minute) for j in cloudmask_times ]) cloudmask_bool = cloudmask_times == datetimes[date_i] clouds_now = cloudmask.variables['cloud_mask'][cloudmask_bool][0] found_file = True except: print 'Found no cloud mask file!' clouds_now = np.zeros(cloud_lons.shape) found_file = False sdf_root = '/soge-home/projects/seviri_dust/sdf/' \ + datetimes[date_i].strftime('%B') \ + str(datetimes[date_i].year) + '/' if os.path.isfile(sdf_root + 'SDF_v2.' + \ datestrings[date_i] + '.nc'): sdf = Dataset( sdf_root + 'SDF_v2.' + \ datestrings[date_i] + '.nc') found_file = True # print sdf sdf_now = sdf.variables['SDF'][:] else: print 'No SDF file found for this date' found_file = False else: try: cloudmask = Dataset( '/soge-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/cloudmask' '/nc/' + datetimes[date_i].strftime("%B").upper() + str(datetimes[date_i].year) + '_CLOUDS/eumetsat.cloud.' + datestrings[date_i] + '.nc') clouds_now = cloudmask.variables['cmask'][:][0] cloud_lons = cloudmask.variables['lon'][:] cloud_lats = cloudmask.variables['lat'][:] except: clouds_now = np.zeros(cloud_lons.shape) found_file = False print 'Found no cloud mask file!' sdf_root = '/soge-home/data_not_backed_up/satellite/meteosat' \ '/seviri/15' \ '-min/0.03x0.03/sdf/nc/' + datetimes[date_i].strftime( '%B').upper() + str(datetimes[date_i].year) + '/SDF_v2/' if os.path.isfile(sdf_root + 'SDF_v2.' + \ datestrings[date_i] + '.nc'): sdf = Dataset( sdf_root + 'SDF_v2.' + \ datestrings[date_i] + '.nc') found_file = True # print sdf if 'time' in sdf.variables: sdf_now = sdf.variables['bt108'][0] else: sdf_now = sdf.variables['bt108'][:] else: print 'No SDF file found for this date' found_file = False if found_file: if daily_bt == False: bt087 = bt.variables['bt087'][:][0] bt108 = bt.variables['bt108'][:][0] bt12 = bt.variables['bt120'][:][0] orig_lons = bt.variables['longitude'][:] orig_lats = bt.variables['latitude'][:] orig_lons, orig_lats = np.meshgrid(orig_lons, orig_lats) else: orig_lons = lons orig_lats = lats # print bt12.shape # print clouds_now.shape window_datetime_lower = datetime.datetime(datetimes[0].year, 6, 1, 0, 45) \ - datetime.timedelta(days=7) window_datetime_upper = datetime.datetime(datetimes[-1].year, 8, 31, 23, 45) \ + datetime.timedelta(days=7) BT_15_day_lower_bound = datetimes[date_i] - datetime.timedelta(days=7) BT_15_day_upper_bound = datetimes[date_i] + datetime.timedelta(days=7) # Get datetime objects between the above bounds time_params_7dayw = np.array([ window_datetime_lower.year, window_datetime_upper.year, window_datetime_lower.month, window_datetime_upper.month, window_datetime_lower.day, window_datetime_upper.day, datetimes[date_i].hour, datetimes[date_i].hour, datetimes[date_i].minute, datetimes[date_i].minute ]) datetimes_7dayw = utilities.get_daily_datetime_objects( time_params_7dayw) indices = np.arange(0, len(datetimes_7dayw)) lower_ind = datetimes_7dayw == BT_15_day_lower_bound lower_ind = indices[lower_ind][0] upper_ind = datetimes_7dayw == BT_15_day_upper_bound upper_ind = indices[upper_ind][0] current_ind = datetimes_7dayw == datetimes[date_i] current_ind = indices[current_ind][0] if double_digits: f = tables.open_file( '/soge-home/projects/seviri_dust/sdf/intermediary_files' '/bt_15d_' + datetimes[date_i].strftime('%Y_%H_%M') + '.hdf') BT_15_days = f.root.data[lower_ind:upper_ind] bt_data = f.root.data[current_ind] f.close() else: f = tables.open_file( '/soge-home/projects/seviri_dust/sdf/intermediary_files' '/bt_15d_' + str(datetimes[date_i].year) + '_' + str(datetimes[date_i].hour) + '_' + str(datetimes[date_i].minute) + '.hdf') BT_15_days = f.root.data[lower_ind:upper_ind] bt_data = f.root.data[current_ind] f.close() if daily_bt: bt087 = bt_data[0] bt108 = bt_data[1] bt12 = bt_data[2] bt_15day_087 = BT_15_days[:, 0] bt_15day_108 = BT_15_days[:, 1] bt_15day_120 = BT_15_days[:, 2] bt_15day_087_mean = np.nanmean(bt_15day_087, axis=0) bt_15day_108_mean = np.nanmean(bt_15day_108, axis=0) bt_15day_120_mean = np.nanmean(bt_15day_120, axis=0) if mesh: cloud_lons, cloud_lats = np.meshgrid(cloud_lons, cloud_lats) clouds_now_regridded = pinkdust.regrid_data(cloud_lons, cloud_lats, orig_lons, orig_lats, clouds_now, mesh=True) if daily_clouds: clouds_now_regridded = clouds_now_regridded > 1 if show_all_indicators: bt_108_087 = bt108 - bt087 bt_120_108 = bt12 - bt108 bt_120_087 = bt12 - bt087 bt_108_087_mean = bt_15day_108_mean - bt_15day_087_mean bt_120_108_mean = bt_15day_120_mean - bt_15day_108_mean bt_120_087_mean = bt_15day_120_mean - bt_15day_087_mean bt_108_087_anom = bt_108_087 - bt_108_087_mean bt_120_108_anom = bt_120_108 - bt_120_108_mean bt_120_087_anom = bt_120_087 - bt_120_087_mean bt_087_anom = bt087 - bt_15day_087_mean bt_108_anom = bt108 - bt_15day_108_mean bt_120_anom = bt12 - bt_15day_120_mean lat_grad, lon_grad = np.gradient(bt_108_087_anom) total_grad = np.sqrt(lat_grad**2 + lon_grad**2) grad_108_087_anom = deepcopy(total_grad) lat_grad, lon_grad = np.gradient(bt_120_108_anom) total_grad = np.sqrt(lat_grad**2 + lon_grad**2) grad_120_108_anom = deepcopy(total_grad) lat_grad, lon_grad = np.gradient(bt_120_087_anom) total_grad = np.sqrt(lat_grad**2 + lon_grad**2) grad_120_087_anom = deepcopy(total_grad) lat_grad, lon_grad = np.gradient(bt_087_anom) total_grad = np.sqrt(lat_grad**2 + lon_grad**2) grad_087_anom = deepcopy(total_grad) lat_grad, lon_grad = np.gradient(bt_108_anom) total_grad = np.sqrt(lat_grad**2 + lon_grad**2) grad_108_anom = deepcopy(total_grad) lat_grad, lon_grad = np.gradient(bt_120_anom) total_grad = np.sqrt(lat_grad**2 + lon_grad**2) grad_120_anom = deepcopy(total_grad) bt_108_087_anom_m = deepcopy(bt_108_087_anom) bt_120_087_anom_m = deepcopy(bt_120_087_anom) bt_120_108_anom_m = deepcopy(bt_120_108_anom) clouds_now_regridded[sdf_now == 1] = 0 bt_108_087_anom_m[clouds_now_regridded == 1] = np.nan bt_120_087_anom_m[clouds_now_regridded == 1] = np.nan bt_120_108_anom_m[clouds_now_regridded == 1] = np.nan return bt087, bt108, bt12, bt_108_087_anom, bt_120_087_anom, \ bt_120_108_anom, grad_087_anom, grad_108_anom, grad_120_anom,\ grad_108_087_anom, grad_120_108_anom, grad_120_087_anom, \ bt_108_087_anom_m, bt_120_087_anom_m, bt_120_108_anom_m, \ orig_lons, orig_lats, clouds_now_regridded, sdf_now
def detect_cpo(btdiff_2_anom_prev, btdiff_2_anom_prev_2, btdiff_2_anom_prev_3, datetimes, datestrings, date_i): # Get lats and lons sdf_test = Dataset( '/soge-home/data_not_backed_up/satellite/meteosat/seviri' '/15-min/0.03x0.03/sdf/nc/JUNE2010/SDF_v2/SDF_v2.' '201006031500.nc') lons, lats = np.meshgrid(sdf_test.variables['longitude'][:], sdf_test.variables['latitude'][:]) # Get cloud lats and lons cloud_test = Dataset( '/soge-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/cloudmask' '/nc/' + 'JUNE2010_CLOUDS/eumetsat.cloud.' + '201006031500.nc') cloud_lons = cloud_test.variables['lon'][:] cloud_lats = cloud_test.variables['lat'][:] cloud_lons, cloud_lats = np.meshgrid(cloud_lons, cloud_lats) lonmask = lons > 360 latmask = lats > 90 lons = np.ma.array(lons, mask=lonmask) lats = np.ma.array(lats, mask=latmask) used_ids = [] runtime = datetimes[date_i] - datetimes[0] #print '\n' + datestrings[date_i] + '\n' totaltest = datetime.datetime.now() found_file = True if os.path.isfile('/ouce-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/bt' '/nc/' + datetimes[date_i].strftime("%B").upper( ) + str(datetimes[date_i].year) + '/H-000-MSG2__' '-MSG2________-' 'IR_BrightnessTemperatures___' '-000005___-' + datestrings[date_i] + '-__.nc'): bt = Dataset( '/ouce-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/bt' '/nc/' + datetimes[date_i].strftime("%B").upper( ) + str(datetimes[date_i].year) + '/H-000-MSG2__' '-MSG2________-' 'IR_BrightnessTemperatures___' '-000005___-' + datestrings[date_i] + '-__.nc') elif os.path.isfile('/ouce-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/bt' '/nc/' + datetimes[date_i].strftime("%B").upper( ) + str(datetimes[date_i].year) + '/H-000-MSG1__' '-MSG1________-' 'IR_BrightnessTemperatures___' '-000005___-' + datestrings[date_i] + '-__.nc'): bt = Dataset( '/ouce-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/bt' '/nc/' + datetimes[date_i].strftime("%B").upper( ) + str(datetimes[date_i].year) + '/H-000-MSG1__' '-MSG1________-' 'IR_BrightnessTemperatures___' '-000005___-' + datestrings[date_i] + '-__.nc') try: cloudmask = Dataset( '/soge-home/data/satellite/meteosat/seviri/15-min/' '0.03x0.03/cloudmask' '/nc/' + datetimes[date_i].strftime("%B").upper( ) + str(datetimes[date_i].year) + '_CLOUDS/eumetsat.cloud.' + datestrings[date_i] + '.nc') clouds_now = cloudmask.variables['cmask'][:][0] cloud_lons = cloudmask.variables['lon'][:] cloud_lats = cloudmask.variables['lat'][:] except: clouds_now = np.zeros(cloud_lons.shape) found_file = False if found_file: # Produce 12-10.8 imagery bt087 = bt.variables['bt087'][:][0] bt12 = bt.variables['bt120'][:][0] bt108 = bt.variables['bt108'][:][0] orig_lons = bt.variables['longitude'][:] orig_lats = bt.variables['latitude'][:] # print bt12.shape # print clouds_now.shape f = tables.open_file( '/soge-home/projects/seviri_dust/sdf/intermediary_files/bt_15d_' + str(datetimes[date_i].year) + '_' + str( datetimes[date_i].month) + '_' + str(datetimes[date_i].hour) + '_' + str( datetimes[date_i].minute) + '.hdf') arrobj = f.get_node('/data') bt_15day = arrobj.read() f.close() window_datetime_lower = datetime.datetime(datetimes[0].year, datetimes[0].month, datetimes[0].day, datetimes[0].hour, datetimes[0].minute) \ - datetime.timedelta(days=7) window_datetime_upper = datetime.datetime(datetimes[-1].year, datetimes[-1].month, datetimes[-1].day, datetimes[-1].hour, datetimes[-1].minute) \ + datetime.timedelta(days=7) BT_15_day_lower_bound = datetimes[date_i] - datetime.timedelta( days=7) BT_15_day_upper_bound = datetimes[date_i] + datetime.timedelta( days=7) # Get datetime objects between the above bounds time_params_7dayw = np.array([window_datetime_lower.year, window_datetime_upper.year, window_datetime_lower.month, window_datetime_upper.month, window_datetime_lower.day, window_datetime_upper.day, datetimes[date_i].hour, datetimes[date_i].hour, datetimes[date_i].minute, datetimes[date_i].minute]) datetimes_7dayw = utilities.get_daily_datetime_objects( time_params_7dayw) BT_15_days = \ bt_15day[np.asarray([j >= BT_15_day_lower_bound and j <= BT_15_day_upper_bound for j in datetimes_7dayw])] bt_15day_087 = BT_15_days[:, 0] #bt_15day_108 = BT_15_days[:, 1] bt_15day_120 = BT_15_days[:, 2] bt_15day_087_mean = np.nanmean(bt_15day_087, axis=0) #bt_15day_108_mean = np.nanmean(bt_15day_108, axis=0) bt_15day_120_mean = np.nanmean(bt_15day_120, axis=0) btdiff_2_15daymean = bt_15day_120_mean - bt_15day_087_mean orig_lons, orig_lats = np.meshgrid(orig_lons, orig_lats) bt087_regridded = pinkdust.regrid_data(orig_lons, orig_lats, cloud_lons, cloud_lats, bt087) bt12_regridded = pinkdust.regrid_data(orig_lons, orig_lats, cloud_lons, cloud_lats, bt12) btdiff_2_15daymean_regridded = pinkdust.regrid_data(orig_lons, orig_lats, cloud_lons, cloud_lats, btdiff_2_15daymean) btdiff_2 = bt12_regridded - bt087_regridded btdiff_2_anom = btdiff_2 - btdiff_2_15daymean_regridded btdiff_2_anom[clouds_now > 0] = np.nan if btdiff_2_anom_prev_3 != None: # Get the difference between this timestep and the one two # timesteps before btdiff_2_anom_diff = btdiff_2_anom - btdiff_2_anom_prev_3 orig_btdiff_2_anom_diff = deepcopy(btdiff_2_anom_diff) btdiff_2_anom_diff += \ orig_btdiff_2_anom_diff - btdiff_2_anom_prev_2 btdiff_2_anom_diff += \ orig_btdiff_2_anom_diff - btdiff_2_anom_prev else: btdiff_2_anom_diff = np.zeros((btdiff_2_anom.shape)) if date_i == 0: btdiff_2_anom_prev = btdiff_2_anom elif date_i == 1: btdiff_2_anom_prev_2 = deepcopy(btdiff_2_anom_prev) btdiff_2_anom_prev = btdiff_2_anom elif date_i == 2: btdiff_2_anom_prev_3 = deepcopy(btdiff_2_anom_prev_2) btdiff_2_anom_prev_2 = deepcopy(btdiff_2_anom_prev) btdiff_2_anom_prev = deepcopy(btdiff_2_anom) elif date_i > 2: btdiff_2_anom_prev_3 = deepcopy(btdiff_2_anom_prev_2) btdiff_2_anom_prev_2 = deepcopy(btdiff_2_anom_prev) btdiff_2_anom_prev = deepcopy(btdiff_2_anom) lat_grad, lon_grad = np.gradient(btdiff_2_anom) total_grad = np.sqrt(lat_grad ** 2 + lon_grad ** 2) convolution = scipy.signal.convolve2d(clouds_now, np.ones((5, 5)), mode='same') clouds_now = convolution > 0 total_grad[clouds_now == 1] = np.nan ### PASS I ### # In the FIRST PASS the LORD sayeth unto the image, 'Let all # whose BTD is below -10K be classified as CPOs, and remove the # tiny ones' # And those who fulfilled this condition were classified, # and it was good convolution = scipy.signal.convolve2d(clouds_now, np.ones((5, 5)), mode='same') clouds_now = convolution > 0 btdiff_2_anom_diff_um = deepcopy(btdiff_2_anom_diff) btdiff_2_anom_diff[clouds_now > 0] = np.nan cpo_mask_pass_1 = btdiff_2_anom_diff < -7 label_objects, nb_labels = ndi.label(cpo_mask_pass_1) sizes = np.bincount(label_objects.ravel()) # Set clusters smaller than size 20 to zero mask_sizes = sizes > 20 mask_sizes[0] = 0 cpo_mask_pass_1 = mask_sizes[label_objects] ### PASS II ### # In the SECOND PASS the LORD sayeth unto the image, 'Let all # those included in the first pass which contain pixels which # are below -15K be classified' # And those who fulfilled this condition were classified, # and it was better cpo_mask_pass_2 = deepcopy(cpo_mask_pass_1) # Label the image and get all connected elements cpo_mask_pass_1, num = measurements.label(cpo_mask_pass_1) # Then loop through each labeled blob and find if one of the # pixels has a -15K. If it does, the whole blob is a freaking CPO. blob_ids = np.unique(cpo_mask_pass_1) blob_ids = blob_ids[blob_ids != 0] for i in np.arange(0, len(blob_ids)): target_region = cpo_mask_pass_1 == blob_ids[i] # First check if this region is within the original # time-gradient identified region (i.e. not one introduced # with the new generous gradient checker) if np.any(btdiff_2_anom_diff[target_region == 1] < -15): # Next check if there is a generous gradient contained # within this region somewhere pass else: cpo_mask_pass_2[target_region == 1] = 0 # For identified CPO regions, undo the convolution on the cloud # mask cpo_mask_um = btdiff_2_anom_diff_um < -7 # Label the image and get all connected elements cpo_mask_um, num = measurements.label(cpo_mask_um) # Then loop through each labeled blob and find if one of the # pixels has a -15K. If it does, the whole blob is a freaking CPO. blob_ids = np.unique(cpo_mask_um) blob_ids = blob_ids[blob_ids != 0] for i in np.arange(0, len(blob_ids)): target_region = cpo_mask_um == blob_ids[i] if np.any(cpo_mask_pass_2[target_region == 1] == 1): cpo_mask_pass_2[target_region == 1] = 1 return cpo_mask_pass_2, btdiff_2_anom_prev, btdiff_2_anom_prev_2,\ btdiff_2_anom_prev_3