year_upper = 2012
month_lower = 6
month_upper = 6
day_lower = 1
day_upper = 30
hour_lower = 8
hour_upper = 23
minute_lower = 0
minute_upper = 45
time_params = np.array([
year_lower, year_upper, month_lower, month_upper, day_lower, day_upper,
hour_lower, hour_upper, minute_lower, minute_upper
])
datetimes = utilities.get_datetime_objects(time_params)
datestrings = [j.strftime("%Y%m%d%H%M") for j in datetimes]
lonlats = Dataset(
'/ouce-home/data/satellite/meteosat/seviri/15-min/native/'
'lonlats.NA_MiddleEast.nc')
# These need to be regridded to regular for consistency with cloud mask
lons = lonlats.variables['longitude'][:]
lats = lonlats.variables['latitude'][:]
lonmask = lons > 360
latmask = lats > 90
lons = np.ma.array(lons, mask=lonmask)
lats = np.ma.array(lats, mask=latmask)
sdf_previous = None
clouds_previous = None
month_lower = 6
month_upper = 8
day_lower = 1
day_upper = 31
hour_lower = 0
hour_upper = 23
minute_lower = 0
minute_upper = 45
# Generate the full set of datetimes for the entire period
time_params = np.array([year_lower, year_upper, month_lower,
month_upper, day_lower, day_upper,
hour_lower, hour_upper, minute_lower,
minute_upper])
SDF_datetimes = utilities.get_datetime_objects(time_params)
SDF_datestrings = [j.strftime("%Y%m%d%H%M") for j in SDF_datetimes]
# Get Ian's lats and lons
sdf_test = Dataset(
'/soge-home/data_not_backed_up/satellite/meteosat/seviri'
'/15-min/0.03x0.03/sdf/nc/JUNE2010/SDF_v2/SDF_v2.'
'201006031500.nc')
ianlons = sdf_test.variables['longitude'][:]
ianlats = sdf_test.variables['latitude'][:]
# Get my lats and lons
tch_lonlats = Dataset('/soge-home/projects/seviri_dust/raw_seviri_data'
'/bt_nc/native_bt_lonlats.nc')
tchlons = tch_lonlats.variables['longitude'][:]
def wrapper(yearmonth):
year_lower = yearmonth[0]
year_upper = yearmonth[0]
month_lower = yearmonth[-1][0]
month_upper = yearmonth[-1][-1]
day_lower = 1
day_upper = 31
hour_lower = 0
hour_upper = 23
minute_lower = 0
minute_upper = 45
time_params = np.array([
year_lower, year_upper, month_lower, month_upper, day_lower, day_upper,
hour_lower, hour_upper, minute_lower, minute_upper
])
# Get datetime objects between the above bounds
window_datetime_lower = datetime.datetime(year_lower, 6,
1, 0, 0) \
- datetime.timedelta(days=7)
window_datetime_upper = datetime.datetime(year_upper, 8,
31, 23, 45) \
+ datetime.timedelta(days=7)
datetimes = utilities.get_datetime_objects(time_params)
datestrings = [j.strftime("%Y%m%d%H%M") for j in datetimes]
# Get lats and lons
sdf_test = Dataset(
'/soge-home/data_not_backed_up/satellite/meteosat/seviri'
'/15-min/0.03x0.03/sdf/nc/JUNE2010/SDF_v2/SDF_v2.'
'201006031500.nc')
lons, lats = np.meshgrid(sdf_test.variables['longitude'][:],
sdf_test.variables['latitude'][:])
lonmask = lons > 360
latmask = lats > 90
lons = np.ma.array(lons, mask=lonmask)
lats = np.ma.array(lats, mask=latmask)
sdf_previous = None
raw_sdf_prev = []
clouds_previous = None
ids_previous = []
cloud_ids_previous = []
deep_conv_IDs_prev = None
LLJ_plumes_IDs_prev = []
k = 0
available_colours = np.arange(0, 41)
# To pick back up where you left off, simply add all the keys in plume
# archive to used IDs here - perhaps include an option to do that
used_ids = []
used_cloud_ids = []
used_colour_IDs = {}
plume_objects = []
flicker_ids = []
reintroduced_ids = []
last_10_ids = []
last_10_ids = np.asarray(last_10_ids)
prev_dust_assoc_clouds = None
# Restrict the lons and lats to the CWS alone
lonbool = np.asarray([j >= -20 and j <= 10 for j in lons[0]])
if run:
plume_objects = {}
plume_archive = shelve.open(
'/soge-home/projects/seviri_dust/plumetracker/'
'plume_archive_flicker_v4_prob_v4_'
'' + str(yearmonth[0]))
if pickup:
archived_ids = np.asarray([j for j in plume_archive])
for i in archived_ids:
used_ids.append(int(i))
with open('date_i_' + str(yearmonth[0]) + '.txt', 'r') as f:
pickup_date_i = f.read()
datestrings = datestrings[int(pickup_date_i):]
datetimes = datetimes[int(pickup_date_i):]
for date_i in np.arange(0, len(datestrings)):
runtime = datetimes[date_i] - datetimes[0]
print '\n' + datestrings[date_i] + '\n'
totaltest = datetime.datetime.now()
if tch_sdfs == False:
sdf_root = '/soge-home/data_not_backed_up/satellite/meteosat' \
'/seviri/15' \
'-min/0.03x0.03/sdf/nc/'+datetimes[date_i].strftime(
'%B').upper()+str(datetimes[date_i].year)+'/SDF_v2/'
else:
sdf_root = '/soge-home/projects/seviri_dust/sdf/'\
+datetimes[date_i].strftime('%B')\
+str(datetimes[date_i].year)+'/'
if os.path.isfile(sdf_root+'SDF_v2.' + \
datestrings[date_i] + '.nc'):
sdf = Dataset(
sdf_root + 'SDF_v2.' + \
datestrings[date_i] + '.nc')
found_file = True
#print sdf
else:
print 'No SDF file found for this date'
found_file = False
if daily_bt_files:
# Pull BTs from daily nc files
bt = Dataset(daily_bt_root +
datetimes[date_i].strftime("%B%Y") + '/BT_' +
datetimes[date_i].strftime("%Y%m%d") + '.nc')
found_file = True
else:
try:
bt = Dataset(
'/ouce-home/data/satellite/meteosat/seviri/15-min/'
'0.03x0.03/bt'
'/nc/' + datetimes[date_i].strftime("%B").upper() +
str(datetimes[date_i].year) + '/H-000-MSG2__'
'-MSG2________-'
'IR_BrightnessTemperatures___'
'-000005___-' + datestrings[date_i] + '-__.nc')
found_file = True
#print bt12.shape
#print clouds_now.shape
#cloud_lons, cloud_lats = np.meshgrid(cloud_lons, cloud_lats)
except:
if os.path.isfile(
'/ouce-home/data/satellite/meteosat/seviri/'
'15-min/'
'0.03x0.03/bt'
'/nc/' + datetimes[date_i].strftime("%B").upper() +
str(datetimes[date_i].year) + '/H-000-MSG1__'
'-MSG1________-'
'IR_BrightnessTemperatures___'
'-000005___-' + datestrings[date_i] + '-__.nc'):
bt = Dataset(
'/ouce-home/data/satellite/meteosat/seviri/15-min/'
'0.03x0.03/bt'
'/nc/' + datetimes[date_i].strftime("%B").upper() +
str(datetimes[date_i].year) + '/H-000-MSG1__'
'-MSG1________-'
'IR_BrightnessTemperatures___'
'-000005___-' + datestrings[date_i] + '-__.nc')
found_file = True
else:
found_file = False
if found_file:
# If the SDF file has its own longitudes and latitudes,
# use these rather than the preset
if 'longitude' in sdf.variables:
lons = sdf.variables['longitude'][:]
lonmask = lons > 360
lons = np.ma.array(lons, mask=lonmask)
if 'latitude' in sdf.variables:
lats = sdf.variables['latitude'][:]
latmask = lats > 90
lats = np.ma.array(lats, mask=latmask)
lons, lats = np.meshgrid(lons, lats)
# clouds_now = np.load('/ouce-home/students/hert4173/'
# 'cloud_mask_numpy_files/cloudmask'
# ''+datetimes[date_i]
# .strftime(
# "%Y%m%d%H%M%S")+'.npy')
# Some SDF files have a time dimension. For these index it out.
if tch_sdfs:
sdf_now = sdf.variables['SDF'][:]
elif 'time' in sdf.variables:
sdf_now = sdf.variables['bt108'][0]
else:
sdf_now = sdf.variables['bt108'][:]
if daily_bt_files:
# We should get data from the HDF files instead
time_params_7dayw = np.array([
window_datetime_lower.year, window_datetime_upper.year,
window_datetime_lower.month,
window_datetime_upper.month, window_datetime_lower.day,
window_datetime_upper.day, datetimes[date_i].hour,
datetimes[date_i].hour, datetimes[date_i].minute,
datetimes[date_i].minute
])
datetimes_7dayw = utilities.get_daily_datetime_objects(
time_params_7dayw)
indices = np.arange(0, len(datetimes_7dayw))
current_ind = datetimes_7dayw == datetimes[date_i]
current_ind = indices[current_ind][0]
f = tables.open_file(
'/soge-home/projects/seviri_dust/sdf/intermediary_files/bt_15d_'
+ datetimes[date_i].strftime("%Y_%H_%M") + '.hdf')
bt_15day = f.root.data[current_ind]
f.close()
# We need to extract the right timestep from the daily
# nc file
bt_108 = bt_15day[1]
# Instead get BTs from the intermediary hdf files (
# already regridded)
elif 'time' in bt.variables:
bt_108 = bt.variables['bt108'][:][0]
else:
bt_108 = bt.variables['bt108'][:]
clouds = bt_108 < 270
# Add the reintroduced plumes back into the running
for i in np.arange(0, len(reintroduced_ids)):
#print 'Reintroducing plume', reintroduced_ids[i]
sdf_previous[sdf_previous == flicker_ids[i]] = \
reintroduced_ids[i]
ids_previous = np.append(ids_previous, reintroduced_ids[i])
# Get plumes first by scanning for them
sdf_plumes, new_ids, plume_ids, merge_ids = plumes.\
scan_for_plumes(
sdf_now,
sdf_previous,
used_ids,
clouds)
for i in np.arange(0, len(reintroduced_ids)):
if reintroduced_ids[i] not in plume_ids:
#print 'But plume '+str(reintroduced_ids[i])+' sadly ' \
# 'died.'
pass
# Clear flicker IDs and reintroduced IDs
flicker_ids = []
reintroduced_ids = []
# We could here do infilling, then at least you'll have it for
# the next iteration. But if you've labelled them already you
# don't have a binary.
# Get clouds by scanning for them
# clouds, new_cloud_ids, cloud_ids, merge_cloud_ids = \
# plumes.scan_for_plumes(clouds_now,
# clouds_previous,
# used_cloud_ids)
for i in new_ids:
used_ids.append(i)
# for i in new_cloud_ids:
# used_cloud_ids.append(i)
old_bool = np.asarray([j in ids_previous for j in plume_ids])
if len(old_bool) > 0:
old_ids = plume_ids[old_bool]
else:
old_ids = []
# old_cloud_bool = np.asarray([j in clouds_previous for
# j in cloud_ids])
# old_cloud_ids = cloud_ids[old_cloud_bool]
# Then, for each new ID, we initialise plume objects
for i in np.arange(0, len(new_ids)):
if debug:
print 'Creating new plume', new_ids[i]
plume = plumes.Plume(new_ids[i], datetimes[date_i])
plume.update_position(lats, lons, sdf_plumes, new_ids[i])
plume.update_duration(datetimes[date_i])
#if plume.plume_id == 18:
# print plume.dates_observed
# print plume.dates_observed
plume.update_bbox()
plume.update_majorminor_axes(lons, lats)
plume.update_area()
plume.update_max_extent()
plume.update_centroid_speed()
plume.update_centroid_direction()
plume.check_conv_distance(lats, lons, clouds)
plume.update_most_likely_source()
# plume.update_leading_edge_4(sdf_plumes, lons, lats)
plume_objects[str(new_ids[i])] = plume
if flicker:
missing_plume, missing_id, flickered = \
plume.chain_flickerchecker(
raw_sdf_prev)
else:
flickered = False
missing_plume = []
if flickered:
raise ValueError('Found an overlaping plume in the '
'previous timestep larger than size '
'250 - a new plume should not be '
'initiated here')
steps_back = 1
# As long as there is an overlapping previous plume,
# keep updating it back in time
while len(missing_plume) > 0:
if debug:
print 'Rolling back plume', new_ids[i]
# We can only step back to the first timestep and no
# earlier
if (date_i - steps_back) < 0:
missing_plume = []
break
else:
missing_date = datetimes[date_i - steps_back]
missing_sdf_plumes = np.zeros(missing_plume.shape)
missing_plume = missing_plume == 1
missing_sdf_plumes[missing_plume] = missing_id
# Run all the updates that would be used for a
# new plume
plume.update_position(lats, lons,
missing_sdf_plumes,
missing_id)
plume.update_duration(missing_date)
#if plume.plume_id == 18:
# print plume.dates_observed
plume.update_bbox()
plume.update_majorminor_axes(lons, lats)
plume.update_area()
plume.update_max_extent()
plume.update_centroid_speed()
plume.update_centroid_direction()
plume.check_conv_distance(lats, lons, clouds)
plume.update_most_likely_source()
plume.process_missing_plume()
#print 'Updated missing plume back '+str(
# steps_back)+' steps'
steps_back += 1
if (date_i - steps_back) < 0:
missing_plume = []
break
# Pull out data from the timestep before to
# continue the chain
try:
if tch_sdfs:
raw_sdf_prev_prev_data = Dataset(sdf_root+'SDF_v2.' + \
datestrings[date_i-steps_back] + '.nc')
else:
raw_sdf_prev_prev_data = Dataset(
'/soge-home/data_not_backed_up/satellite/'
'meteosat/'
'seviri/'
'15-min/0.03x0.03/sdf/nc/' +
datetimes[date_i-steps_back].strftime(
"%B").upper(
) + str(datetimes[date_i-steps_back].year) +
'/SDF_v2/SDF_v2.' + \
datestrings[date_i-steps_back] + '.nc')
except:
print 'Adding date to list of missing dates'
with open('missing_dates.txt', 'a') as my_file:
my_file.write('\n' +
datestrings[date_i -
steps_back])
break
if tch_sdfs:
raw_sdf_prev_prev = \
raw_sdf_prev_prev_data.variables['SDF'][:]
elif 'time' in raw_sdf_prev_prev_data.variables:
raw_sdf_prev_prev = \
raw_sdf_prev_prev_data.variables[
'bt108'][0]
else:
raw_sdf_prev_prev = \
raw_sdf_prev_prev_data.variables[
'bt108'][:]
missing_plume, missing_id, flickered = \
plume.chain_flickerchecker(
raw_sdf_prev_prev)
if flickered:
#print 'Found a flickered plume. Searching ' \
# 'for the corresponding archived plume.'
# We have a plume in a previous timestep
# which flickered
plume_archive_keys = last_10_ids.astype(int)
# Sort the keys in reverse order as the
# plume we want is most likely to have a
# high ID
plume_archive_keys[::-1].sort()
missing_plume = missing_plume == 1
missing_sdf_plumes = np.zeros(
missing_plume.shape)
missing_sdf_plumes[missing_plume] = missing_id
plume_bool = missing_sdf_plumes == missing_id
search_centroid_lon = \
np.nanmean(lons[plume_bool])
search_centroid_lat = \
np.nanmean(lats[plume_bool])
plume_lons = lons[plume_bool]
plume_lats = lats[plume_bool]
search_date = datetimes[date_i - steps_back]
#print search_date
found_plume = False
for key in plume_archive_keys:
#print 'Searching in plume archive'
if search_centroid_lon == \
plume_archive[\
str(key)].centroid_lon and \
search_centroid_lat ==\
plume_archive[str(
key)].centroid_lat and \
plume_archive[str(
key)].dates_observed[-1] == \
search_date:
#print 'Found it in plume archive. ' \
# 'ID is', \
# plume_archive[str(key)].plume_id
found_plume = True
correct_plume = plume_archive[str(key)]
plume_to_append = plume
# Append the flickered plume to the
# old one which was archived
correct_plume.append_missing_plume(
plume_to_append)
# Add it to plume objects and remove
# it from archives
plume_objects[str(key)] = correct_plume
del plume_archive[str(key)]
# Add it to old IDs, replacing the
# ID of the plume which was found to be
# flickered
flicker_ids.append(plume.plume_id)
reintroduced_ids.append(key)
missing_plume = []
# Reintroduced plumes also get removed
# from the record of the last 10 ids
index = np.argwhere(last_10_ids == key)
last_10_ids = np.delete(
last_10_ids, index)
break
# If we didn't find the plume in the plume
# archive, it must still be active
if found_plume == False:
plume_object_keys = np.asarray(
[int(i) for i in plume_objects])
# Sort the keys in reverse order as the
# plume we want is most likely to have a
# high ID
plume_object_keys[::-1].sort()
for key in plume_object_keys:
#print 'Searching in plume objects'
if search_centroid_lon == \
plume_objects[ \
str(
key)].centroid_lon and \
search_centroid_lat == \
plume_objects[str(
key)].centroid_lat and \
plume_objects[str(
key)].dates_observed[
-1] == \
search_date:
#print 'Found it in plume ' \
# 'objects. ID is', \
# plume_archive[
# str(key)].plume_id
found_plume = True
correct_plume = plume_archive[str(
key)]
plume_to_append = plume
# Append the flickered plume to the
# old one which was archived
correct_plume.append_missing_plume(
plume_to_append)
# Add it to plume objects and
# remove it from archives
plume_objects[str(
key)] = correct_plume
del plume_archive[str(key)]
# Add it to old IDs, replacing the
# ID of the plume which was found
# to be flickered
flicker_ids.append(plume.plume_id)
reintroduced_ids.append(key)
missing_plume = []
index = np.argwhere(
last_10_ids == key)
last_10_ids = np.delete(
last_10_ids, index)
break
break
# Remove any IDs which were actually flickers
for i in np.arange(0, len(flicker_ids)):
index = np.argwhere(new_ids == flicker_ids[i])
new_ids = np.delete(new_ids, index)
index = np.argwhere(ids_previous == flicker_ids[i])
ids_previous = np.delete(ids_previous, index)
index = np.argwhere(plume_ids == flicker_ids[i])
plume_ids = np.delete(plume_ids, index)
del plume_objects[str(flicker_ids[i])]
#For merged IDs, we move the tracks to pre-merge tracks
for i in np.arange(0, len(merge_ids)):
plume = plume_objects[str(merge_ids[i])]
plume.merge()
# For old IDs, we just run an update.
for i in np.arange(0, len(old_ids)):
if debug:
print 'Updating plume', old_ids[i]
plume = plume_objects[str(old_ids[i])]
#if plume.plume_id == 2:
# print plume.dates_observed
plume.update_position(lats, lons, sdf_plumes, old_ids[i])
plume.update_duration(datetimes[date_i])
#if plume.plume_id == 18:
# print plume.dates_observed
plume.update_bbox()
plume.update_majorminor_axes(lons, lats)
plume.update_area()
plume.update_centroid_speed()
plume.update_centroid_direction()
plume.update_max_extent()
#plume_assoc_clouds, plume_check_bool \
# = plume.flag_dust_associated_convection(
# clouds, prev_dust_assoc_clouds)
#dust_assoc_clouds += plume_assoc_clouds
#check_bool += plume_check_bool.astype(int)
# plume.update_leading_edge_4(sdf_plumes, lons, lats)
plume_objects[str(old_ids[i])] = plume
# Plumes which no longer exist are removed and archived
if len(ids_previous) == 0:
removed_ids = []
else:
removed_bool = np.asarray(
[j not in plume_ids for j in ids_previous])
removed_ids = ids_previous[removed_bool]
for i in np.arange(0, len(removed_ids)):
if debug:
print 'Archiving plume', removed_ids[i]
plume = plume_objects[str(removed_ids[i])]
plume.update_GPE_speed()
# plume.update_mechanism_likelihood()
plume.update_mean_axis_offset()
plume.update_llj_probability(trace)
plume_archive[str(removed_ids[i])] = plume
del plume_objects[str(removed_ids[i])]
last_10_ids = np.append(last_10_ids, removed_ids[i])
if len(last_10_ids) > 10:
last_10_ids = last_10_ids[0:10]
if len(np.unique(sdf_plumes)) < 2:
sdf_previous = None
ids_previous = []
raw_sdf_prev = []
else:
sdf_previous = sdf_plumes
ids_previous = plume_ids
raw_sdf_prev = sdf_now
else:
print 'Adding date to list of missing dates'
with open('missing_dates.txt', 'a') as my_file:
my_file.write('\n' + datestrings[date_i])
# Print date_i so we know where we got up to in case we have to
# restart
with open('date_i_' + str(yearmonth[0]) + '.txt', 'w') as f:
f.write('%d' % date_i)
# After the script has finished, add remaining plumes to the plume archive
for i in plume_objects:
plume_archive[i] = plume_objects[i]
plume_archive.close()
def wrapper(yearmonth):
year_lower = yearmonth[0]
year_upper = yearmonth[0]
month_lower = yearmonth[-1][0]
#month_lower = 7
month_upper = yearmonth[-1][-1]
day_lower = 1
day_upper = 31
hour_lower = 0
hour_upper = 23
minute_lower = 0
minute_upper = 45
time_params = np.array([
year_lower, year_upper, month_lower, month_upper, day_lower, day_upper,
hour_lower, hour_upper, minute_lower, minute_upper
])
datetimes = utilities.get_datetime_objects(time_params)
datestrings = [j.strftime("%Y%m%d%H%M") for j in datetimes]
# Get lats and lons
sdf_test = Dataset(
'/soge-home/data_not_backed_up/satellite/meteosat/seviri'
'/15-min/0.03x0.03/sdf/nc/JUNE2010/SDF_v2/SDF_v2.'
'201006031500.nc')
lons, lats = np.meshgrid(sdf_test.variables['longitude'][:],
sdf_test.variables['latitude'][:])
lonmask = lons > 360
latmask = lats > 90
lons = np.ma.array(lons, mask=lonmask)
lats = np.ma.array(lats, mask=latmask)
cpo_previous = None
raw_cpo_prev = []
clouds_previous = None
ids_previous = []
cloud_ids_previous = []
deep_conv_IDs_prev = None
LLJ_plumes_IDs_prev = []
k = 0
available_colours = np.arange(0, 41)
# To pick back up where you left off, simply add all the keys in plume
# archive to used IDs here - perhaps include an option to do that
used_ids = []
used_cloud_ids = []
used_colour_IDs = {}
flicker_ids = []
reintroduced_ids = []
last_10_ids = []
last_10_ids = np.asarray(last_10_ids)
prev_dust_assoc_clouds = None
found_file = True
# Restrict the lons and lats to the CWS alone
lonbool = np.asarray([j >= -20 and j <= 10 for j in lons[0]])
cpo_objects = {}
cpo_archive = shelve.open('/soge-home/projects/seviri_dust/plumetracker/'
'cpo_archive_v4'
'' + str(yearmonth[0]))
if pickup:
archived_ids = np.asarray([j for j in cpo_archive])
for i in archived_ids:
used_ids.append(int(i))
with open('date_i_' + str(yearmonth[0]) + '.txt', 'r') as f:
pickup_date_i = f.read()
print 'Picking up from timestep', pickup_date_i
datestrings = datestrings[int(pickup_date_i):]
datetimes = datetimes[int(pickup_date_i):]
cpo_archive.close()
year_lower = datetimes[0].year
year_upper = datetimes[-1].year
month_lower = datetimes[0].month
month_upper = datetimes[-1].month
day_lower = datetimes[0].day
day_upper = datetimes[-1].day
hour_lower = 0
hour_upper = 23
minute_lower = 0
minute_upper = 45
time_params_oneday = np.array([
year_lower, year_lower, month_lower, month_lower, day_lower, day_lower,
hour_lower, hour_upper, minute_lower, minute_upper
])
oneday_datetimes = utilities.get_datetime_objects(time_params_oneday)
if moving_window:
# Here run the moving window for our selected time period
print 'Generating cloud masks for all timesteps'
# Loop through each time in a single day
pool = multiprocessing.Pool()
for i in np.arange(0, len(oneday_datetimes)):
pool.apply_async(cpo_detection.cloud_mask_mw,
args=(i, datetimes, oneday_datetimes, ianlons,
ianlats))
# Pull a netCDF dataset object out so the projection coordinates can
# be obtained
pool.close()
pool.join()
btdiff_2_anom_prev = None
btdiff_2_anom_prev_2 = None
btdiff_2_anom_prev_3 = None
for date_i in np.arange(0, len(datestrings)):
cpo_archive = shelve.open(
'/soge-home/projects/seviri_dust/plumetracker/'
'cpo_archive_v4_'
'' + str(yearmonth[0]))
runtime = datetimes[date_i] - datetimes[0]
print '\n' + datestrings[date_i] + '\n'
totaltest = datetime.datetime.now()
if found_file:
#if 'time' in bt.variables:
# bt_108 = bt.variables['bt108'][:][0]
#else:
# bt_108 = bt.variables['bt108'][:]
#clouds = bt_108 < 270
# Here run CPO detection
# IF A RAW DETECTED CPO FILE EXISTS, USE THAT (assuming the
# algorithm hasn't changed)
if calculate_cpof:
cpo_now, btdiff_2_anom_prev, btdiff_2_anom_prev_2, \
btdiff_2_anom_prev_3 = cpo_detection.detect_cpo(
btdiff_2_anom_prev, btdiff_2_anom_prev_2,
btdiff_2_anom_prev_3, datetimes, datestrings, date_i,
lons, lats,
cloud_lons, cloud_lats,
daily_cloudmask, double_digits, mesh, daily_bt)
cpo_now[cpo_now > 0] = 1
else:
cpo_data = Dataset('/ouce-home/projects/seviri_dust/cpof/' +
datetimes[date_i].strftime("%B%Y") +
'/CPOF_' +
datetimes[date_i].strftime("%Y%m%d%H%M") +
'.nc')
cpo_now = cpo_data.variables['CPOF'][:]
# Save the raw CPO detection to file
# These should be stored in the project directory, not home
#np.save('raw_detected_cpo_'+datestrings[date_i], cpo_now)
# Add the reintroduced plumes back into the running
for i in np.arange(0, len(reintroduced_ids)):
#print 'Reintroducing CPO', reintroduced_ids[i]
cpo_previous[cpo_previous == flicker_ids[i]] = \
reintroduced_ids[i]
ids_previous = np.append(ids_previous, reintroduced_ids[i])
# Get plumes first by scanning for them
cpo_blobs, new_ids, blob_ids, merge_ids = cpo.\
scan_for_blobs(
cpo_now,
cpo_previous,
used_ids)
#print 'New IDs', new_ids
for i in np.arange(0, len(reintroduced_ids)):
if reintroduced_ids[i] not in blob_ids:
#print 'But plume '+str(reintroduced_ids[i])+' sadly ' \
# 'died.'
pass
# Clear flicker IDs and reintroduced IDs
flicker_ids = []
reintroduced_ids = []
for i in new_ids:
used_ids.append(i)
old_bool = np.asarray([j in ids_previous for j in blob_ids])
if len(old_bool) > 0:
old_ids = blob_ids[old_bool]
else:
old_ids = []
# Then, for each new ID, we initialise plume objects
for i in np.arange(0, len(new_ids)):
if debug:
print 'Creating new CPO', new_ids[i]
cold_pool = cpo.CPO(new_ids[i], datetimes[date_i])
cold_pool.update_position(lats, lons, cpo_blobs, new_ids[i])
cold_pool.update_duration(datetimes[date_i])
#if plume.plume_id == 18:
# print plume.dates_observed
# print plume.dates_observed
cold_pool.update_bbox()
cold_pool.update_majorminor_axes(lons, lats)
cold_pool.update_area()
cold_pool.update_max_extent()
cold_pool.update_centroid_speed()
cold_pool.update_centroid_direction()
#cold_pool.check_conv_distance(lats, lons, clouds)
cold_pool.update_most_likely_source()
# plume.update_leading_edge_4(sdf_plumes, lons, lats)
cpo_objects[str(new_ids[i])] = cold_pool
missing_cold_pool, missing_id, flickered = \
cold_pool.chain_flickerchecker(
raw_cpo_prev)
if flickered:
raise ValueError('Found an overlapping CPO in the '
'previous timestep larger than size '
'50 - a new CPO should not be '
'initiated here')
steps_back = 1
# As long as there is an overlapping previous plume,
# keep updating it back in time
while len(missing_cold_pool) > 0:
if debug:
print 'Rolling back CPO', new_ids[i]
# We can only step back to the first timestep and no
# earlier
if (date_i - steps_back) < 0:
missing_cold_pool = []
break
else:
missing_date = datetimes[date_i - steps_back]
missing_cpo_blobs = np.zeros(missing_cold_pool.shape)
missing_cold_pool = missing_cold_pool == 1
missing_cpo_blobs[missing_cold_pool] = missing_id
# Run all the updates that would be used for a
# new plume
cold_pool.update_position(lats, lons,
missing_cpo_blobs,
missing_id)
cold_pool.update_duration(missing_date)
#if plume.plume_id == 18:
# print plume.dates_observed
cold_pool.update_bbox()
cold_pool.update_majorminor_axes(lons, lats)
cold_pool.update_area()
cold_pool.update_max_extent()
cold_pool.update_centroid_speed()
cold_pool.update_centroid_direction()
#cold_pool.check_conv_distance(lats, lons, clouds)
cold_pool.update_most_likely_source()
cold_pool.process_missing_plume()
#print 'Updated missing plume back '+str(
# steps_back)+' steps'
steps_back += 1
if (date_i - steps_back) < 0:
missing_cpo = []
break
# Pull out data from the timestep before to
# continue the chain
try:
raw_cpo_prev_prev = \
np.load('raw_detected_cpo_'+datestrings[
date_i]+'.npy')
except:
#print 'Adding date to list of missing dates'
with open('missing_dates.txt', 'a') as my_file:
my_file.write('\n' +
datestrings[date_i - steps_back])
break
missing_cold_pool, missing_id, flickered = \
cold_pool.chain_flickerchecker(
raw_cpo_prev_prev)
if flickered:
#print 'Found a flickered plume. Searching ' \
# 'for the corresponding archived plume.'
# We have a plume in a previous timestep
# which flickered
cold_pool_archive_keys = last_10_ids.astype(int)
# Sort the keys in reverse order as the
# plume we want is most likely to have a
# high ID
cold_pool_archive_keys[::-1].sort()
missing_cold_pool = missing_cold_pool == 1
missing_cpo_blobs = np.zeros(
missing_cold_pool.shape)
missing_cpo_blobs[missing_cold_pool] = missing_id
cold_pool_bool = missing_cpo_blobs == missing_id
search_centroid_lon = \
np.nanmean(lons[cold_pool_bool])
search_centroid_lat = \
np.nanmean(lats[cold_pool_bool])
cold_pool_lons = lons[cold_pool_bool]
cold_pool_lats = lats[cold_pool_bool]
search_date = datetimes[date_i - steps_back]
#print search_date
found_cold_pool = False
for key in cold_pool_archive_keys:
#print 'Searching in plume archive'
if search_centroid_lon == \
cpo_archive[\
str(key)].centroid_lon and \
search_centroid_lat ==\
cpo_archive[str(
key)].centroid_lat and \
cpo_archive[str(
key)].dates_observed[-1] == \
search_date:
#print 'Found it in plume archive. ' \
# 'ID is', \
# plume_archive[str(key)].plume_id
found_cold_pool = True
correct_cold_pool = cpo_archive[str(key)]
cold_pool_to_append = cold_pool
# Append the flickered plume to the
# old one which was archived
correct_cold_pool.append_missing_cold_pool(
cold_pool_to_append)
# Add it to plume objects and remove
# it from archives
cpo_objects[str(key)] = correct_cold_pool
del cpo_archive[str(key)]
# Add it to old IDs, replacing the
# ID of the plume which was found to be
# flickered
flicker_ids.append(cold_pool.plume_id)
reintroduced_ids.append(key)
missing_cold_pool = []
# Reintroduced plumes also get removed
# from the record of the last 10 ids
index = np.argwhere(last_10_ids == key)
last_10_ids = np.delete(last_10_ids, index)
break
# If we didn't find the plume in the plume
# archive, it must still be active
if found_cold_pool == False:
cold_pool_object_keys = np.asarray(
[int(i) for i in cpo_objects])
# Sort the keys in reverse order as the
# plume we want is most likely to have a
# high ID
cold_pool_object_keys[::-1].sort()
for key in cold_pool_object_keys:
#print 'Searching in plume objects'
if search_centroid_lon == \
cpo_objects[ \
str(
key)].centroid_lon and \
search_centroid_lat == \
cpo_objects[str(
key)].centroid_lat and \
cpo_objects[str(
key)].dates_observed[
-1] == \
search_date:
found_cold_pool = True
correct_cold_pool = cpo_archive[str(
key)]
cold_pool_to_append = cold_pool
# Append the flickered plume to the
# old one which was archived
correct_cold_pool.append_missing_cold_pool(
cold_pool_to_append)
# Add it to plume objects and
# remove it from archives
cpo_objects[str(
key)] = correct_cold_pool
del cpo_archive[str(key)]
# Add it to old IDs, replacing the
# ID of the plume which was found
# to be flickered
flicker_ids.append(cold_pool.plume_id)
reintroduced_ids.append(key)
missing_cold_pool = []
index = np.argwhere(last_10_ids == key)
last_10_ids = np.delete(
last_10_ids, index)
break
break
# Remove any IDs which were actually flickers
for i in np.arange(0, len(flicker_ids)):
index = np.argwhere(new_ids == flicker_ids[i])
new_ids = np.delete(new_ids, index)
index = np.argwhere(ids_previous == flicker_ids[i])
ids_previous = np.delete(ids_previous, index)
index = np.argwhere(blob_ids == flicker_ids[i])
blob_ids = np.delete(blob_ids, index)
del cpo_objects[str(flicker_ids[i])]
#For merged IDs, we move the tracks to pre-merge tracks
for i in np.arange(0, len(merge_ids)):
cold_pool = cpo_objects[str(merge_ids[i])]
cold_pool.merge()
# For old IDs, we just run an update.
for i in np.arange(0, len(old_ids)):
if debug:
print 'Updating CPO', old_ids[i]
cold_pool = cpo_objects[str(old_ids[i])]
#if plume.plume_id == 2:
# print plume.dates_observed
cold_pool.update_position(lats, lons, cpo_blobs, old_ids[i])
cold_pool.update_duration(datetimes[date_i])
#if plume.plume_id == 18:
# print plume.dates_observed
cold_pool.update_bbox()
cold_pool.update_majorminor_axes(lons, lats)
cold_pool.update_area()
cold_pool.update_centroid_speed()
cold_pool.update_centroid_direction()
cold_pool.update_max_extent()
#plume_assoc_clouds, plume_check_bool \
# = plume.flag_dust_associated_convection(
# clouds, prev_dust_assoc_clouds)
#dust_assoc_clouds += plume_assoc_clouds
#check_bool += plume_check_bool.astype(int)
# plume.update_leading_edge_4(sdf_plumes, lons, lats)
cpo_objects[str(old_ids[i])] = cold_pool
# Plumes which no longer exist are removed and archived
if len(ids_previous) == 0:
removed_ids = []
else:
removed_bool = np.asarray(
[j not in blob_ids for j in ids_previous])
removed_ids = ids_previous[removed_bool]
for i in np.arange(0, len(removed_ids)):
if debug:
print 'Archiving CPO', removed_ids[i]
cold_pool = cpo_objects[str(removed_ids[i])]
cold_pool.update_GPE_speed()
# plume.update_mechanism_likelihood()
cold_pool.update_mean_axis_offset()
if cold_pool.duration.total_seconds() >= 3600:
cpo_archive[str(removed_ids[i])] = cold_pool
del cpo_objects[str(removed_ids[i])]
print '== CPO detected =='
print 'CPO emission time:', cold_pool.emission_time
print 'CPO emission lat:', cold_pool.track_centroid_lat[0]
print 'CPO emission lon:', cold_pool.track_centroid_lon[0]
last_10_ids = np.append(last_10_ids, removed_ids[i])
else:
#print 'Too short duration - not archived'
del cpo_objects[str(removed_ids[i])]
#print 'Plume', plume.plume_id,'removed. Final lons and \
# lats:'
#print plume.centroid_lon
#print plume.centroid_lat
if len(last_10_ids) > 10:
last_10_ids = last_10_ids[0:10]
if len(np.unique(cpo_blobs)) < 2:
cpo_previous = None
ids_previous = []
raw_cpo_prev = []
else:
cpo_previous = cpo_blobs
ids_previous = blob_ids
raw_cpo_prev = cpo_now
else:
print 'Adding date to list of missing dates'
with open('missing_dates.txt', 'a') as my_file:
my_file.write('\n' + datestrings[date_i])
with open('date_i_' + str(yearmonth[0]) + '.txt', 'w') as f:
f.write('%d' % date_i)
cpo_archive.close()
def wrapper(y):
Year_lower = years_list[y]
Year_upper = years_list[y]
Month_lower = 5
Month_upper = 9
Day_lower = 24
Day_upper = 8
Hour_lower = 0
Hour_upper = 23
Minute_lower = 0
Minute_upper = 45
# Generate datetime objects corresponding to these time bounds
time_params = np.array([
Year_lower, Year_upper, Month_lower, Month_upper, Day_lower, Day_upper,
Hour_lower, Hour_upper, Minute_lower, Minute_upper
])
datetimes = utilities.get_datetime_objects(time_params)
years = np.unique(np.asarray(([j.year for j in datetimes])))
months = np.unique(np.asarray(([j.month for j in datetimes])))
days = np.unique(np.asarray(([j.day for j in datetimes])))
for m in np.arange(0, len(years)):
for k in np.arange(0, len(months)):
for i in np.arange(0, len(days)):
day_datetimes_bool = np.asarray([
j.day == days[i] and j.month == months[k]
and j.year == years[m] for j in datetimes
])
day_datetimes = datetimes[day_datetimes_bool]
if len(day_datetimes) == 0:
continue
ncfile = pinkdust.create_time_cloudmask_nc_file(
'/soge-home/projects/seviri_dust/raw_seviri_data'
'/cloudmask_nc/' + day_datetimes[0].strftime('%B%Y') +
'/cloudmask_' + day_datetimes[0].strftime('%Y%m%d') +
'.nc', day_datetimes, lats, lons)
#sub_wrapper(day_datetimes, 0)
pool = multiprocessing.Pool()
for j in np.arange(0, len(day_datetimes)):
pool.apply_async(sub_wrapper, args=(
day_datetimes,
j,
))
pool.close()
pool.join()
for j in np.arange(0, len(day_datetimes)):
f = tables.open_file(
'/soge-home/projects/seviri_dust/raw_seviri_data'
'/intermediary_files/cloudmask_' +
day_datetimes[j].strftime('%Y%m%d%H%M%S.hdf'))
arrobj = f.get_node('/data')
cloudmask = arrobj.read()
f.close()
ncfile = pinkdust.save_cloudmask_to_existing_nc(
ncfile, cloudmask, day_datetimes[j])
print 'Day ' + str(days[i]) + ' done'
ncfile.close()
print 'Month ' + str(months[k]) + ' done'
def wrapper(i):
year_lower = i
year_upper = i
month_lower = 6
month_upper = 8
day_lower = 1
day_upper = 31
hour_lower = 0
hour_upper = 23
minute_lower = 0
minute_upper = 45
time_params = np.array([year_lower, year_upper, month_lower,
month_upper, day_lower, day_upper,
hour_lower, hour_upper, minute_lower,
minute_upper])
datetimes = utilities.get_datetime_objects(time_params)
for date_i in np.arange(0, len(datetimes)):
date = datetimes[date_i]
print date
year_bool = np.asarray(years == date.year)
root = np.asarray(year_cm_roots)[year_bool][0]
if root == root2:
# We're using TCH's daily cloudmask files. These are values from
# 0-2 (I think), and need selecting and regridding.
try:
clouddata = Dataset(root+date.strftime("%B%Y")+'/cloudmask_' + datetimes[
date_i].strftime("%Y%m%d") + '.nc')
cloudmask_times = num2date(clouddata.variables['time'][:],
clouddata.variables['time'].units)
cloudmask_times = np.asarray([dt.datetime(j.year, j.month,
j.day, j.hour,
j.minute) for j
in cloudmask_times])
cloudmask_bool = cloudmask_times == date
clouds_data = clouddata.variables['cloud_mask'][cloudmask_bool]
clouds_now = clouds_data#[cloudmask_bool]
clouds_now_regridded = pinkdust.regrid_data(root2_lons, root2_lats,
target_lons,
target_lats,
clouds_now, mesh=True)
data_array[clouds_now_regridded > 1] += 1
except:
print 'No cloud data for', date
else:
# We're using Ian's original cloud mask files. These are just a
# binary one or zero and won't be regridded.
try:
clouddata = Dataset(root+
datetimes[date_i].strftime("%B").upper(
) + str(datetimes[date_i].year) + '_CLOUDS/eumetsat.cloud.'
+ datetimes[
date_i].strftime("%Y%m%d%H%M") + '.nc')
clouds_now = clouddata.variables['cmask'][:][0]
data_array[clouds_now == 1] += 1
except:
print 'No cloud data for', date
if date_i == 100:
print data_array
np.save('cloudcover_array', data_array)
extent = (
np.min(target_lons), np.max(target_lons), np.min(target_lats),
np.max(target_lats))
m = Basemap(projection='cyl', llcrnrlon=extent[0],
urcrnrlon=extent[1],
llcrnrlat=extent[2], urcrnrlat=extent[3],
resolution='i')
m.drawcoastlines(linewidth=0.5)
m.drawcountries(linewidth=0.5)
parallels = np.arange(10., 40, 2.)
# labels = [left,right,top,bottom]
m.drawparallels(parallels, labels=[False, True, True, False],
linewidth=0.5)
meridians = np.arange(-20., 17., 4.)
m.drawmeridians(meridians, labels=[True, False, False, True],
linewidth=0.5)
min = 0
max = 70000
levels = MaxNLocator(nbins=15).tick_values(min, max)
# discrete_cmap = utilities.cmap_discretize(cm.RdYlBu_r, 10)
cmap = cm.get_cmap('RdYlBu_r')
norm = BoundaryNorm(levels, ncolors=cmap.N, clip=True)
data_array = np.ma.masked_where(np.isnan(data_array),
data_array)
m.pcolormesh(target_lons, target_lats, data_array,
cmap=cmap, norm=norm, vmin=min, vmax=max)
cbar = plt.colorbar(orientation='horizontal', fraction=0.056,
pad=0.06)
cbar.ax.set_xlabel('Counts of cloud occurrence')
plt.tight_layout()
plt.savefig('CPO_multiyear_cloud_frequency_2004_2012.png',
bbox_inches='tight')
plt.close()
