def read_sat(t0, dtRange, satdir, pre=False):
""" Generator reading the satellite data.
The loop is infinite; sat data are called when required until the end of
the parcel loop. """
# get dt from satmap
dt = dtRange
# initial time
current_time = t0
while True:
fname = os.path.join(satdir, current_time.strftime('%Y%m%d%H_TB230'))
dat = readidx107(fname, quiet=True)
""" Generate the sequence of time ranges.
This procedure works with empty time slots """
# get the list of discontinuities, note that it is turned to a list
id = list(np.where(dat['ir_start'][1:] - dat['ir_start'][:-1])[0])
#test print(dat['ir_start'][id+1],dat['ir_start'][0])
# append and prepend last and pre-first positions
id.append(len(dat['ir_start']) - 1)
id[:0] = [-1]
#test print(id)
dat['dtRange'] = dt
dat['nt'] = int(cloudtop_step / dtRange)
#test print('nt ',dat['nt'])
tf = current_time + cloudtop_step / 2
# Generate list of time intervals
dat['time'] = [[tf - dt, tf]]
if pre:
off = timedelta()
while tf >= current_time - cloudtop_step / 2:
dat['time'].append([tf, tf + dt])
if debug: print('times', [tf, tf + dt])
tf -= dt
else:
off = -dt
while tf > current_time - cloudtop_step / 2:
tf -= dt
dat['time'].append([tf - dt, tf])
dat['time'].reverse()
#test print(len(dat['time']))
dat['numRange'] = np.zeros(dat['nt'], dtype='int')
dat['indexRange'] = np.empty(shape=(dat['nt'], 2), dtype='int')
dat['indexRange'].fill(-999)
# index in the list of time segments
nc = dat['nt'] - 1
# process the list of crossing from the last one, backward in time
while len(id) > 1:
idc = id.pop()
# find the corresponding segment, skipping empty ones
while off+current_time+timedelta(seconds=int(dat['ir_start'][idc])) \
!= dat['time'][nc][0]:
nc -= 1
dat['indexRange'][nc, :] = [id[-1] + 1, idc + 1]
dat['numRange'][nc] = idc - id[-1]
nc -= 1
# check that all parcels are sorted
print('check sorting ', np.sum(dat['numRange']), dat['numpart'])
# iterate time
current_time -= cloudtop_step
yield dat
def main():
global IDX_ORGN
global lowpcut, highpcut
parser = argparse.ArgumentParser()
parser.add_argument("-y", "--year", type=int, help="year")
parser.add_argument("-m",
"--month",
type=int,
choices=1 + np.arange(12),
help="month")
parser.add_argument("-a",
"--advect",
type=str,
choices=["EID", "EIZ"],
help="source of advecting winds")
parser.add_argument("-s", "--suffix", type=str, help="suffix")
#parser.add_argument("-l","--level",type=int,help="P level")
parser.add_argument("-q",
"--quiet",
type=str,
choices=["y", "n"],
help="quiet (y) or not (n)")
parser.add_argument("-ct",
"--cloud_type",
type=str,
choices=["meanhigh", "veryhigh"],
help="cloud type filter")
parser.add_argument("-t",
"--step",
type=int,
help="step in hours between two part files")
parser.add_argument("-d",
"--duration",
type=int,
help="duration of integration in hours")
parser.add_argument("-gs",
"--granule_size",
type=int,
help="size of the granule")
parser.add_argument("-gn",
"--granule_step",
type=int,
help="number of granules in a step")
parser.add_argument("-ab", "--age_bound", type=int, help="age_bound")
parser.add_argument("-binx",
"--binx",
type=int,
help="number of grid points in longitude direction")
parser.add_argument("-biny",
"--biny",
type=int,
help="number of grid points in latitude direction")
parser.add_argument("-hmax",
"--hmax",
type=int,
help='maximum number of hours in traczilla simulation')
parser.add_argument("-username", "--username", type=str, help='username')
parser.add_argument("-userout", "--userout", type=str, help='userout')
""" Parsed parameters"""
# Parsed parameters
# Interval between two part files (in hours)
step = 6
# Largest time to be processed (in hours)
hmax = 1464
# Age limit in days
age_bound = 30
# start date of the backward run, corresponding to itime=0
year = 2017
# 8 +1 means we start on September 1 at 0h and cover the whole month of August
month = 6 + 1
advect = 'EIZ'
suffix = '_150_150hPa_500'
quiet = False
cloud_type = 'veryhigh'
# Bound on the age of the parcel (in days)
age_bound = 30
# Number of parcels launched per time slot (grid size)
binx = 320
biny = 224
# Number of granules in a step betwwen two part files
granule_step = 6
""" Non parsed parameters"""
# Time width of the parcel slice
slice_width = timedelta(minutes=5)
# dtRange (now defined in satmap definition)
#dtRange={'MSG1':timedelta(minutes=15),'Hima':timedelta(minutes=20)}
# day=1 should not be changed
day = 1
# low p cut applied in exiter
lowpcut = 3000
# high p cut applied in exiter
highpcut = 50000
args = parser.parse_args()
if args.year is not None: year = args.year
if args.month is not None: month = args.month + 1
if args.advect is not None: advect = args.advect
if args.suffix is not None: suffix = args.suffix
#if args.level is not None: level=args.level
if args.quiet is not None:
if args.quiet == 'y': quiet = True
else: quiet = False
if args.cloud_type is not None: cloud_type = args.cloud_type
if args.step is not None: step = args.step
if args.hmax is not None: hmax = args.hmax
if args.binx is not None: binx = args.binx
if args.biny is not None: biny = args.biny
granule_size = binx * biny
if args.granule_size is not None: granule_size = args.granule_size
if args.duration is not None: hmax = args.duration
if args.age_bound is not None: age_bound = args.age_bound
if args.granule_step is not None: granule_step = args.granule_step
if args.username is not None: username = args.username
if args.userout is not None: userout = args.userout
# Define main directories
main_sat_dir = '/bdd/STRATOCLIM/flexpart_in'
if 'ciclad' in socket.gethostname():
traj_dir = os.path.join('/data/', username, 'flexout', 'COCHIN',
'BACK')
out_dir = os.path.join('/data', userout, 'STC')
elif ('climserv' in socket.gethostname()) | ('polytechnique'
in socket.gethostname()):
traj_dir = os.path.join('/homedata/', username, 'flexout', 'COCHIN',
'BACK')
out_dir = os.path.join('/homedata', userout, 'STC')
else:
print('CANNOT RECOGNIZE HOST - DO NOT RUN ON NON DEFINED HOSTS')
exit()
# Output diretories
# Update the out_dir with the cloud type
out_dir = os.path.join(out_dir, 'STC-BACK-OUT-Cochin-SAF-' + cloud_type)
sdate = datetime(year, month, day)
# fdate defined to make output under the name of the month where parcels are released
fdate = sdate - timedelta(days=1)
# Number of slices between two outputs
dstep = timedelta(hours=step)
nb_slices = int(dstep / slice_width)
# size of granules launched during a step
granule_quanta = granule_size * granule_step
# Manage the file that receives the print output
if quiet:
# Output file
print_file = os.path.join(
out_dir, 'out',
'BACK-' + advect + fdate.strftime('-%b-%Y') + suffix + '.out')
fsock = open(print_file, 'w')
sys.stdout = fsock
print('year', year, 'month', month, 'day', day)
print('advect', advect)
print('suffix', suffix)
# Directory of the backward trajectories
ftraj = os.path.join(traj_dir,
'BACK-' + advect + fdate.strftime('-%b-%Y') + suffix)
# Output file
out_file = os.path.join(
out_dir,
'BACK-' + advect + fdate.strftime('-%b-%Y') + suffix + '.hdf5z')
#out_file1 = os.path.join(out_dir,'BACK-'+advect+fdate.strftime('-%b-%Y-')+str(level)+'K'+suffix+'.hdf5b')
#out_file2 = os.path.join(out_dir,'BACK-'+advect+fdate.strftime('-%b-%Y-')+str(level)+'K'+suffix+'.pkl')
# Directories for the satellite cloud top files
satdir ={'MSG1':os.path.join(main_sat_dir,'msg1','S_NWC'),\
'Hima':os.path.join(main_sat_dir,'himawari','S_NWC')}
""" Initialization of the calculation """
# Initialize the grid
gg = geosat.GeoGrid('FullAMA_SAFBox')
# Initialize the dictionary of the parcel dictionaries
partStep = {}
satmap = pixmap(gg)
# Build the satellite field generator
get_sat = {'MSG1': read_sat(sdate,'MSG1',satmap.zone['MSG1']['dtRange'],satdir['MSG1'],pre=True),\
'Hima': read_sat(sdate,'Hima',satmap.zone['Hima']['dtRange'],satdir['Hima'],pre=True)}
# Open the part_000 file that contains the initial positions
part0 = readidx107(os.path.join(ftraj, 'part_000'), quiet=True)
print('numpart', part0['numpart'])
numpart = part0['numpart']
numpart_s = granule_size
# stamp_date not set in these runs
# current_date actually shifted by one day / sdate
current_date = sdate
# check flag is clean
print('check flag is clean ',((part0['flag']&I_HIT)!=0).sum(),((part0['flag']&I_DEAD)!=0).sum(),\
((part0['flag']&I_CROSSED)!=0).sum())
# check idx_orgn
if part0['idx_orgn'] != 0:
print('MINCHIA, IDX_ORGN NOT 0 AS ASSUMED, CORRECTED WITH READ VALUE')
print('VALUE ', part0['idx_orgn'])
IDX_ORGN = part0['idx_orgn']
idx1 = IDX_ORGN
# Build a dictionary to host the results
prod0 = defaultdict(dict)
prod0['src']['x'] = np.full(part0['numpart'],
fill_value=np.nan,
dtype='float')
prod0['src']['y'] = np.full(part0['numpart'],
fill_value=np.nan,
dtype='float')
prod0['src']['p'] = np.full(part0['numpart'],
fill_value=np.nan,
dtype='float')
prod0['src']['t'] = np.full(part0['numpart'],
fill_value=np.nan,
dtype='float')
prod0['src']['age'] = np.full(part0['numpart'],
fill_value=np.nan,
dtype='int')
prod0['flag_source'] = part0['flag']
prod0['rvs'] = np.full(part0['numpart'], 0.01, dtype='float')
# truncate eventually to 32 bits at the output stage
# read the part_000 file for the first granule
partStep[0] = {}
partStep[0]['x'] = part0['x'][:granule_size]
partStep[0]['y'] = part0['y'][:granule_size]
partStep[0]['t'] = part0['t'][:granule_size]
partStep[0]['p'] = part0['p'][:granule_size]
partStep[0]['t'] = part0['t'][:granule_size]
partStep[0]['idx_back'] = part0['idx_back'][:granule_size]
partStep[0]['ir_start'] = part0['ir_start'][:granule_size]
partStep[0]['itime'] = 0
# number of hists and exits
nhits = 0
nexits = 0
ndborne = 0
nnew = granule_size
nold = 0
# used to get non borne parcels
new = np.empty(part0['numpart'], dtype='bool')
new.fill(False)
print('Initialization completed')
""" Main loop on the output time steps """
for hour in range(step, hmax + 1, step):
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use: {:4.2f} gb'.format(memoryUse))
# Get rid of dictionary no longer used
if hour >= 2 * step: del partStep[hour - 2 * step]
# Read the new data
partStep[hour] = readpart107(hour, ftraj, quiet=True)
# Link the names as views
partante = partStep[hour - step]
partpost = partStep[hour]
if partpost['nact'] > 0:
print('hour ', hour, ' numact ', partpost['nact'], ' max p ',
partpost['p'].max())
else:
print('hour ', hour, ' numact ', partpost['nact'])
# New date valid for partpost
current_date -= dstep
# Processing of water mixing ratio
# Select non stopped parcels in partante
selec = (prod0['flag_source'][partante['idx_back'] - IDX_ORGN]
& I_STOP) == 0
idx = partante['idx_back'][selec]
prod0['rvs'][idx-IDX_ORGN] = np.minimum(prod0['rvs'][idx-IDX_ORGN],\
satratio(partante['p'][selec],partante['t'][selec]))
""" Select the parcels that are common to the two steps
ketp_a is a logical field with same length as partante
kept_p is a logical field with same length as partpost
After the launch of the earliest parcel along the flight track, there
should not be any member in new.
"""
kept_a = np.in1d(partante['idx_back'],
partpost['idx_back'],
assume_unique=True)
kept_p = np.in1d(partpost['idx_back'],
partante['idx_back'],
assume_unique=True)
#new_p = ~np.in1d(partpost['idx_back'],partpost['idx_back'],assume_unique=True)
print('kept a, p ', len(kept_a), len(kept_p), kept_a.sum(),
kept_p.sum(), ' new ',
len(partpost['x']) - kept_p.sum())
nnew += len(partpost['x']) - kept_p.sum()
""" PROCESSING OF DEADBORNE PARCELS
Manage the parcels launched during the last 6-hour which have already
exited and do not appear in posold or posact (borne dead parcels).
These parcels are stored in the last part of posact, at most
the last granule_quanta parcels. """
if numpart_s < numpart:
print("manage deadborne", flush=True)
# First index of the current quanta """
numpart_s += granule_quanta
print("idx1", idx1, " numpart_s", numpart_s)
# Extract the last granule_size indexes from posacti, this is where the new parcels should be
if hour == step:
idx_act = partpost['idx_back']
else:
idx_act = partpost['idx_back'][-granule_quanta:]
# Generate the list of indexes that should be found in this range
# ACHTUNG ACHTUNG : this works because IDX_ORGN=1, FIX THAT
idx_theor = np.arange(idx1, numpart_s + IDX_ORGN)
# Find the missing indexes in idx_act (make a single line after validation)
kept_borne = np.in1d(idx_theor, idx_act, assume_unique=True)
idx_deadborne = idx_theor[~kept_borne]
# Process these parcels by assigning exit at initial location
prod0['flag_source'][
idx_deadborne -
IDX_ORGN] = prod0['flag_source'][idx_deadborne -
IDX_ORGN] | I_DEAD + I_DBORNE
prod0['src']['x'][idx_deadborne -
IDX_ORGN] = part0['x'][idx_deadborne - IDX_ORGN]
prod0['src']['y'][idx_deadborne -
IDX_ORGN] = part0['y'][idx_deadborne - IDX_ORGN]
prod0['src']['p'][idx_deadborne -
IDX_ORGN] = part0['p'][idx_deadborne - IDX_ORGN]
prod0['src']['t'][idx_deadborne -
IDX_ORGN] = part0['t'][idx_deadborne - IDX_ORGN]
prod0['src']['age'][idx_deadborne - IDX_ORGN] = 0.
print("number of deadborne ", len(idx_deadborne))
ndborne += len(idx_deadborne)
idx1 = numpart_s + IDX_ORGN
""" PROCESSING OF CROSSED PARCELS """
if len(kept_a) > 0:
exits = exiter(int((partante['itime']+partpost['itime'])/2), \
partante['x'][~kept_a],partante['y'][~kept_a],partante['p'][~kept_a],\
partante['t'][~kept_a],partante['idx_back'][~kept_a],\
prod0['flag_source'],prod0['src']['x'],prod0['src']['y'],\
prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],\
part0['ir_start'], satmap.range)
nexits += exits
print('exit ', nexits, exits, np.sum(~kept_a),
len(kept_a) - len(kept_p))
""" PROCESSING OF PARCELS WHICH ARE COMMON TO THE TWO OUTPUTS """
# Select the kept parcels which have not been hit yet
# !!! Never use and between two lists, the result is wrong
if kept_p.sum() == 0:
live_a = live_p = kept_p
else:
live_a = np.logical_and(
kept_a, (prod0['flag_source'][partante['idx_back'] - IDX_ORGN]
& I_DEAD) == 0)
live_p = np.logical_and(
kept_p, (prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_DEAD) == 0)
print('live a, b ', live_a.sum(), live_p.sum())
del kept_a
del kept_p
""" Correction step that moves partante['x'] to avoid big jumps at the periodicity frontier on the x-axis """
diffx = partpost['x'][live_p] - partante['x'][live_a]
bb = np.zeros(len(diffx))
bb[diffx > 180] = 360
bb[diffx < -180] = -360
partante['x'][live_a] += bb
del bb
del diffx
# DOES not work in the following WAY
#partante['x'][live_a][diffx>180] += 360
#partante['x'][live_a][diffx<-180] -= 360
# Build generator for parcel locations of the 5' slices
gsp = get_slice_part(partante, partpost, live_a, live_p, current_date,
dstep, slice_width)
if verbose: print('built parcel generator for ', current_date)
""" MAIN LOOP ON THE PARCEL TIME SLICES """
for i in range(nb_slices):
# get the slice for the particles
datpart = next(gsp)
if verbose: print('part slice ', i, datpart['time'])
# Check whether the present satellite image is valid
# The while should ensure that the run synchronizes
# when it starts.
while satmap.check('MSG1', datpart['time']) is False:
# if not get next satellite image
datsat1 = next(get_sat['MSG1'])
# Check that the image is available
if datsat1 is not None:
pm1 = geosat.SatGrid(datsat1, gg)
pm1._sat_togrid('CTTH_PRESS')
#print('pm1 diag',len(datsat1.var['CTTH_PRESS'][:].compressed()),
# len(pm1.var['CTTH_PRESS'][:].compressed()))
pm1._sat_togrid('CT')
pm1.attr = datsat1.attr.copy()
satmap.fill('MSG1', pm1, cloud_type)
del pm1
del datsat1
else:
# if the image is missing, extend the lease
try:
satmap.extend('MSG1')
except:
# This handle the unlikely case where the first image is missing
continue
while satmap.check('Hima', datpart['time']) is False:
# if not get next satellite image
datsath = next(get_sat['Hima'])
# Check that the image is available
if datsath is not None:
pmh = geosat.SatGrid(datsath, gg)
pmh._sat_togrid('CTTH_PRESS')
#print('pmh diag',len(datsath.var['CTTH_PRESS'][:].compressed()),
# len(pmh.var['CTTH_PRESS'][:].compressed()))
pmh._sat_togrid('CT')
pmh.attr = datsath.attr.copy()
satmap.fill('Hima', pmh, cloud_type)
del datsath
del pmh
else:
# if the image is missing, extend the lease
try:
satmap.extend('Hima')
except:
# This handle the unlikely case where the first image is missing
continue
""" Select the parcels located within the domain """
# TODO TODO the values used here should be derived from parameters defined above
indomain = np.all((datpart['x'] > -10, datpart['x'] < 160,
datpart['y'] > 0, datpart['y'] < 50),
axis=0)
""" PROCESS THE COMPARISON OF PARCEL PRESSURES TO CLOUDS """
if indomain.sum() > 0:
nhits += convbirth(datpart['itime'],
datpart['x'][indomain],datpart['y'][indomain],datpart['p'][indomain],\
datpart['t'][indomain],datpart['idx_back'][indomain],\
prod0['flag_source'],prod0['src']['x'],prod0['src']['y'],\
prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],\
satmap.ptop, part0['ir_start'],\
satmap.range[0,0],satmap.range[1,0],satmap.stepx,satmap.stepy,satmap.binx,satmap.biny)
sys.stdout.flush()
""" End of of loop on slices """
# Check the age limit (easier to do it here)
print("Manage age limit", flush=True)
age_sec = part0['ir_start'][partante['idx_back'] -
IDX_ORGN] - partante['itime']
IIold_o = age_sec > (age_bound - 0.25) * 86400
IIold_o = IIold_o & (
(prod0['flag_source'][partante['idx_back'] - IDX_ORGN] & I_STOP)
== 0)
idx_IIold = partante['idx_back'][IIold_o]
j_IIold_o = np.where(IIold_o)
prod0['flag_source'][idx_IIold - IDX_ORGN] = prod0['flag_source'][
idx_IIold - IDX_ORGN] | I_DEAD + I_OLD
prod0['src']['x'][idx_IIold - IDX_ORGN] = partante['x'][j_IIold_o]
prod0['src']['y'][idx_IIold - IDX_ORGN] = partante['y'][j_IIold_o]
prod0['src']['p'][idx_IIold - IDX_ORGN] = partante['p'][j_IIold_o]
prod0['src']['t'][idx_IIold - IDX_ORGN] = partante['t'][j_IIold_o]
prod0['src']['age'][idx_IIold - IDX_ORGN] = (
(part0['ir_start'][idx_IIold - IDX_ORGN] - partante['itime']) /
86400)
print("number of IIold ", len(idx_IIold))
nold += len(idx_IIold)
# find parcels still alive if kept_p.sum()==0:
try:
nlive = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_DEAD) == 0).sum()
n_nohit = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_HIT) == 0).sum()
except:
nlive = 0
n_nohit = 0
print('end hour ', hour, ' numact', partpost['nact'], ' nexits',
nexits, ' nhits', nhits, ' nlive', nlive, ' nohit', n_nohit)
# check that nlive + nhits + nexits = numpart, should be true after the first day
if part0['numpart'] != nexits + nhits + nlive + ndborne:
print('@@@ ACHTUNG numpart not equal to sum ', part0['numpart'],
nexits + nhits + nlive + ndborne)
""" End of the procedure and storage of the result """
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use before clean: {:4.2f} gb'.format(memoryUse))
del partante
del partpost
del live_a
del live_p
del datpart
prod0['rvs'] = prod0['rvs'].astype(np.float32)
for var in ['age', 'p', 't', 'x', 'y']:
prod0['src'][var] = prod0['src'][var].astype(np.float32)
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use after clean: {:4.2f} gb'.format(memoryUse))
#output file
#pickle.dump(prod0,gzip.open(out_file,'wb'))
#try:
# dd.io.save(out_file1,prod0,compression='blosc')
#except:
# print('error with dd blosc')
try:
dd.io.save(out_file, prod0, compression='zlib')
except:
print('error with dd zlib')
""" End of the procedure and storage of the result """
#output file
#dd.io.save(out_file2,prod0,compression='zlib')
#pickle.dump(prod0,gzip.open(out_file,'wb'))
# close the print file
if quiet: fsock.close()
def main():
global IDX_ORGN
parser = argparse.ArgumentParser()
parser.add_argument("-y", "--year", type=int, help="year")
parser.add_argument("-m",
"--month",
type=int,
choices=1 + np.arange(12),
help="month")
parser.add_argument("-d1",
"--day1",
type=int,
choices=1 + np.arange(31),
help="day1")
parser.add_argument("-d2",
"--day2",
type=int,
choices=1 + np.arange(31),
help="day2")
parser.add_argument(
"-a",
"--advect",
type=str,
choices=["OPZ", "EAD", "EAZ", "EID", "EIZ", "EID-FULL", "EIZ-FULL"],
help="source of advecting winds")
parser.add_argument("-s",
"--suffix",
type=str,
help="suffix for special cases")
parser.add_argument("-q",
"--quiet",
type=str,
choices=["y", "n"],
help="quiet (y) or not (n)")
#parser.add_argument("-c","--clean0",type=bool,help="clean part_000")
parser.add_argument("-t",
"--step",
type=int,
help="step in hour between two part files")
parser.add_argument("-ct",
"--cloud_type",
type=str,
choices=["meanhigh", "veryhigh", "silviahigh"],
help="cloud type filter")
parser.add_argument("-k",
"--diffus",
type=str,
choices=['01', '1', '001'],
help='diffusivity parameter')
parser.add_argument("-v",
"--vshift",
type=int,
choices=[0, 1, 2],
help='vertical shift')
parser.add_argument("-hm",
"--hmax",
type=int,
help='maximum considered integration time')
# to be updated
# Define main directories
if 'ciclad' in socket.gethostname():
main_sat_dir = '/data/legras/flexpart_in/SAFNWC'
#SVC_Dir = '/bdd/CFMIP/SEL2'
traj_dir = '/data/akottayil/flexout/STC/BACK'
out_dir = '/data/legras/STC'
elif ('climserv' in socket.gethostname()) | ('polytechnique'
in socket.gethostname()):
main_sat_dir = '/data/legras/flexpart_in/SAFNWC'
#SVC_Dir = '/bdd/CFMIP/SEL2'
traj_dir = '/data/akottayil/flexout/STC/BACK'
out_dir = '/homedata/legras/STC'
else:
print('CANNOT RECOGNIZE HOST - DO NOT RUN ON NON DEFINED HOSTS')
exit()
""" Parameters """
step = 3
hmax = 732
dstep = timedelta(hours=step)
# time width of the parcel slice
slice_width = timedelta(minutes=5)
# number of slices between two outputs
nb_slices = int(dstep / slice_width)
# default values of parameters
# date of the flight
year = 2017
month = 7 + 1
day1 = 1
day2 = 10
advect = 'EAD'
suffix = ''
quiet = False
clean0 = True
cloud_type = 'silviahigh'
diffus = '01'
vshift = 0
super = ''
args = parser.parse_args()
if args.year is not None: year = args.year
if args.month is not None: month = args.month + 1
if args.advect is not None: advect = args.advect
if args.day1 is not None: day1 = args.day1
if args.day2 is not None: day2 = args.day2
if args.suffix is not None: suffix = '-' + args.suffix
if args.hmax is not None: hmax = args.hmax
if args.quiet is not None:
if args.quiet == 'y': quiet = True
else: quiet = False
#if args.clean0 is not None: clean0 = args.clean0
if args.cloud_type is not None: cloud_type = args.cloud_type
if args.step is not None: step = args.step
if args.diffus is not None: diffus = args.diffus
diffus = '-D' + diffus
if args.vshift is not None:
vshift = args.vshift
if vshift > 0:
super = 'super' + str(vshift)
# Update the out_dir with the cloud type and the super paramater
out_dir = os.path.join(out_dir, 'SVC-BACK-OUT-SAF-' + super + cloud_type)
try:
os.mkdir(out_dir)
os.mkdir(out_dir + '/out')
except:
print('out_dir directory already created')
# Dates beginning and end
date_beg = datetime(year=year, month=month, day=day1, hour=0)
date_end = datetime(year=year, month=month, day=day2, hour=0)
# Manage the file that receives the print output
if quiet:
# Output file
print_file = os.path.join(
out_dir, 'out',
'BACK-SVC-EID-FULL-' + date_beg.strftime('%b-%Y-day%d-') +
date_end.strftime('%d-D01') + '.out')
fsock = open(print_file, 'w')
sys.stdout = fsock
# initial time to read the sat files
# should be after the end of the flight
sdate = date_end + timedelta(days=1)
print('year', year, 'month', month, 'day', day2)
print('advect', advect)
print('suffix', suffix)
# patch to fix the problem of the data hole on the evening of 2 August 2017
if sdate == datetime(2017, 8, 2): sdate = datetime(2017, 8, 3, 6)
# Directories of the backward trajectories and name of the output file
ftraj = os.path.join(
traj_dir, 'BACK-SVC-EID-FULL-' + date_beg.strftime('%b-%Y-day%d-') +
date_end.strftime('%d-D01'))
out_file2 = os.path.join(
out_dir, 'BACK-SVC-EID-FULL-' + date_beg.strftime('%b-%Y-day%d-') +
date_end.strftime('%d-D01') + '.hdf5')
# Directories for the satellite cloud top files
satdir ={'MSG1':os.path.join(main_sat_dir,'msg1','S_NWC'),\
'Hima':os.path.join(main_sat_dir,'himawari','S_NWC')}
""" Initialization of the calculation """
# Initialize the grid
gg = geosat.GeoGrid('FullAMA_SAFBox')
# Initialize the dictionary of the parcel dictionaries
partStep = {}
satmap = pixmap(gg)
# Build the satellite field generator
get_sat = {'MSG1': read_sat(sdate,'MSG1',satmap.zone['MSG1']['dtRange'],satdir['MSG1'],pre=True,vshift=vshift),\
'Hima': read_sat(sdate,'Hima',satmap.zone['Hima']['dtRange'],satdir['Hima'],pre=True,vshift=vshift)}
# Read the index file that contains the initial positions
part0 = readidx107(os.path.join(ftraj, 'part_000'), quiet=True)
print('numpart', part0['numpart'])
# stamp_date not set in these runs
# current_date actually shifted by one day / sdate
# We assume here that part time is defined from this day at 0h
# sdate defined above should be equal or posterior to current_date
current_date = sdate
# check flag is clean
print('check flag is clean ',((part0['flag']&I_HIT)!=0).sum(),((part0['flag']&I_DEAD)!=0).sum(),\
((part0['flag']&I_CROSSED)!=0).sum())
# check idx_orgn
if part0['idx_orgn'] != 0:
print('MINCHIA, IDX_ORGN NOT 0 AS ASSUMED, CORRECTED WITH READ VALUE')
print('VALUE ', part0['idx_orgn'])
IDX_ORGN = part0['idx_orgn']
# Build a dictionary to host the results
prod0 = defaultdict(dict)
prod0['src']['x'] = np.empty(part0['numpart'], dtype='float')
prod0['src']['y'] = np.empty(part0['numpart'], dtype='float')
prod0['src']['p'] = np.empty(part0['numpart'], dtype='float')
prod0['src']['t'] = np.empty(part0['numpart'], dtype='float')
prod0['src']['age'] = np.empty(part0['numpart'], dtype='int')
prod0['flag_source'] = part0['flag']
# truncate eventually to 32 bits at the output stage
# read the part_000 file
partStep[0] = readpart107(0, ftraj, quiet=True)
# cleaning is necessary for runs starting in the fake restart mode
# otherwise all parcels are thought to exit at the first step
if clean0:
partStep[0]['idx_back'] = []
# number of hists and exits
nhits = 0
nexits = 0
ndborne = 0
nnew = 0
# used to get non borne parcels
new = np.empty(part0['numpart'], dtype='bool')
new.fill(False)
print('Initialization completed')
""" Main loop on the output time steps """
for hour in range(step, hmax + 1, step):
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use: {:4.2f} gb'.format(memoryUse))
# Get rid of dictionary no longer used
if hour >= 2 * step: del partStep[hour - 2 * step]
# Read the new data
partStep[hour] = readpart107(hour, ftraj, quiet=True)
# Link the names
partante = partStep[hour - step]
partpost = partStep[hour]
if partpost['nact'] > 0:
print('hour ', hour, ' numact ', partpost['nact'], ' max p ',
partpost['p'].max())
else:
print('hour ', hour, ' numact ', partpost['nact'])
# New date valid for partpost
current_date -= dstep
""" Select the parcels that are common to the two steps
ketp_a is a logical field with same length as partante
kept_p is a logical field with same length as partpost
After the launch of the earliest parcel along the flight track, there
should not be any member in new
The parcels
"""
kept_a = np.in1d(partante['idx_back'],
partpost['idx_back'],
assume_unique=True)
kept_p = np.in1d(partpost['idx_back'],
partante['idx_back'],
assume_unique=True)
#new_p = ~np.in1d(partpost['idx_back'],partpost['idx_back'],assume_unique=True)
print('kept a, p ', len(kept_a), len(kept_p), kept_a.sum(),
kept_p.sum(), ' new ',
len(partpost['x']) - kept_p.sum())
""" IDENTIFY AND TAKE CARE OF DEADBORNE AS NON BORNE PARCELS """
if (hour < ((day2 - day1 + 5) * 24)) & (partpost['nact'] > 0):
new[partpost['idx_back'][~kept_p] - IDX_ORGN] = True
nnew += len(partpost['x']) - kept_p.sum()
if hour == ((day2 - day1 + 5) * 24):
ndborne = np.sum(~new)
prod0['flag_source'][~new] |= I_DBORNE + I_DEAD
prod0['src']['x'][~new] = part0['x'][~new]
prod0['src']['y'][~new] = part0['y'][~new]
prod0['src']['p'][~new] = part0['p'][~new]
prod0['src']['t'][~new] = part0['t'][~new]
prod0['src']['age'][~new] = 0
print('number of dead borne', ndborne, part0['numpart'] - nnew)
del new
""" INSERT HERE CODE FOR NEW PARCELS """
# nothing to be done for new parcels, just wait and see
""" PROCESSING OF CROSSED PARCELS """
if len(kept_a) > 0:
exits = exiter(int((partante['itime']+partpost['itime'])/2), \
partante['x'][~kept_a],partante['y'][~kept_a],partante['p'][~kept_a],\
partante['t'][~kept_a],partante['idx_back'][~kept_a],\
prod0['flag_source'],prod0['src']['x'],prod0['src']['y'],\
prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],\
part0['ir_start'], satmap.range)
nexits += exits
print('exit ', nexits, exits, np.sum(~kept_a),
len(kept_a) - len(kept_p))
""" PROCESSING OF PARCELS WHICH ARE COMMON TO THE TWO OUTPUTS """
# Select the kept parcels which have not been hit yet
# !!! Never use and between two lists, the result is wrong
if kept_p.sum() == 0:
live_a = live_p = kept_p
else:
live_a = np.logical_and(
kept_a, (prod0['flag_source'][partante['idx_back'] - IDX_ORGN]
& I_DEAD) == 0)
live_p = np.logical_and(
kept_p, (prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_DEAD) == 0)
print('live a, b ', live_a.sum(), live_p.sum())
# Build generator for parcel locations of the 5' slices
gsp = get_slice_part(partante, partpost, live_a, live_p, current_date,
dstep, slice_width)
if verbose: print('built parcel generator for ', current_date)
""" MAIN LOOP ON THE PARCEL TIME SLICES """
for i in range(nb_slices):
# get the slice for the particles
datpart = next(gsp)
if verbose: print('part slice ', i, datpart['time'])
# Check whether the present satellite image is valid
# The while should ensure that the run synchronizes
# when it starts.
while satmap.check('MSG1', datpart['time']) is False:
# if not get next satellite image
datsat1 = next(get_sat['MSG1'])
# Check that the image is available
if datsat1 is not None:
pm1 = geosat.SatGrid(datsat1, gg)
pm1._sat_togrid('CTTH_PRESS')
#print('pm1 diag',len(datsat1.var['CTTH_PRESS'][:].compressed()),
# len(pm1.var['CTTH_PRESS'][:].compressed()))
pm1._sat_togrid('CT')
if vshift > 0: pm1._sat_togrid('CTTH_TEMPER')
pm1.attr = datsat1.attr.copy()
satmap.fill('MSG1', pm1, cloud_type, vshift=vshift)
del pm1
del datsat1
else:
# if the image is missing, extend the lease
try:
satmap.extend('MSG1')
except:
# This handle the unlikely case where the first image is missing
continue
while satmap.check('Hima', datpart['time']) is False:
# if not get next satellite image
datsath = next(get_sat['Hima'])
# Check that the image is available
if datsath is not None:
pmh = geosat.SatGrid(datsath, gg)
pmh._sat_togrid('CTTH_PRESS')
#print('pmh diag',len(datsath.var['CTTH_PRESS'][:].compressed()),
# len(pmh.var['CTTH_PRESS'][:].compressed()))
pmh._sat_togrid('CT')
if vshift > 0: pmh._sat_togrid('CTTH_TEMPER')
pmh.attr = datsath.attr.copy()
satmap.fill('Hima', pmh, cloud_type, vshift=vshift)
del datsath
del pmh
else:
# if the image is missing, extend the lease
try:
satmap.extend('Hima')
except:
# This handle the unlikely case where the first image is missing
continue
""" PROCESS THE COMPARISON OF PARCEL PRESSURES TO CLOUDS """
if len(datpart['x']) > 0:
nhits += convbirth(datpart['itime'],
datpart['x'],datpart['y'],datpart['p'],datpart['t'],datpart['idx_back'],\
prod0['flag_source'],prod0['src']['x'],prod0['src']['y'],\
prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],\
satmap.ptop, part0['ir_start'],\
satmap.range[0,0],satmap.range[1,0],satmap.stepx,satmap.stepy,satmap.binx,satmap.biny)
sys.stdout.flush()
""" End of of loop on slices """
# find parcels still alive if kept_p.sum()==0:
try:
nlive = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_DEAD) == 0).sum()
n_nohit = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_HIT) == 0).sum()
except:
nlive = 0
n_nohit = 0
print('end hour ', hour, ' numact', partpost['nact'], ' nexits',
nexits, ' nhits', nhits, ' nlive', nlive, ' nohit', n_nohit)
# check that nlive + nhits + nexits = numpart, should be true after the first day
if part0['numpart'] != nexits + nhits + nlive + ndborne:
print('@@@ ACHTUNG numpart not equal to sum ', part0['numpart'],
nexits + nhits + nlive + ndborne)
""" End of the procedure and storage of the result """
#output file
dd.io.save(out_file2, prod0, compression='zlib')
#pickle.dump(prod0,gzip.open(out_file,'wb'))
# close the print file
if quiet: fsock.close()
def main():
global IDX_ORGN
parser = argparse.ArgumentParser()
parser.add_argument("-y","--year",type=int,help="year")
parser.add_argument("-m","--month",type=int,choices=1+np.arange(12),help="month")
parser.add_argument("-a","--advect",type=str,choices=["EID-FULL","EIZ-FULL"],help="source of advecting winds")
parser.add_argument("-l","--level",type=int,help="PT level")
parser.add_argument("-s","--suffix",type=str,help="suffix for special cases")
parser.add_argument("-q","--quiet",type=str,choices=["y","n"],help="quiet (y) or not (n)")
# to be updated
if socket.gethostname() == 'graphium':
pass
elif 'ciclad' in socket.gethostname():
traj_dir = '/data/legras/flexout/STC/BACK'
out_dir = '/data/legras/STC'
elif socket.gethostname() == 'grapelli':
pass
elif socket.gethostname() == 'coltrane':
pass
else:
print ('CANNOT RECOGNIZE HOST - DO NOT RUN ON NON DEFINED HOSTS')
exit()
""" Parameters """
# to do (perhaps) : some parameters might be parsed from command line
# step and max output time
step = 6
hmax = 1824
dstep = timedelta (hours=step)
# age limit in days
age_bound = 44.
# time width of the parcel slice
slice_width = timedelta(hours=1)
# number of slices between two outputs
nb_slices = int(dstep/slice_width)
# defines here the offset for the detrainment (100h)
detr_offset = 1/(100*3600.)
# defines the domain
domain = np.array([[-10.,160.],[0.,50.]])
# default values of parameters
# start date of the backward run, corresponding to itime=0
year=2017
# 8 +1 means we start on September 1 at 0h and cover the whole month of August
month=8+1
# Should not be changed
day=1
advect = 'EAD'
suffix =''
quiet = False
level = 380
args = parser.parse_args()
if args.year is not None:
year=args.year
if args.month is not None:
month=args.month+1
if args.advect is not None:
advect=args.advect
if args.level is not None:
level=args.level
if args.suffix is not None:
suffix='-'+args.suffix
if args.quiet is not None:
if args.quiet=='y':
quiet=True
else:
quiet=False
# Update the out_dir with the platform
out_dir = os.path.join(out_dir,'STC-BACK-DETR-OUT')
sdate = datetime(year,month,day)
# fdate defined to make output under the name of the month where parcels are released
fdate = sdate - timedelta(days=1)
""" Define granule_size and granule_quanta
granule_size: Number of parcels launched per time slot (1 per degree on a 170x50 grid)
granule_step: number of granules in 6 hours
granula_quanta: size of granules launched during 6 hours """
if 'FULL' in advect:
granule_size = 28800
granule_step = 6
granule_quanta = granule_size * granule_step
else:
granule_size = 8500
granule_step = 6*4
granule_quanta = granule_size * granule_step
# Manage the file that receives the print output
if quiet:
# Output file
print_file = os.path.join(out_dir,'out','BACK-'+advect+fdate.strftime('-%b-%Y-')+str(level)+'K'+suffix+'.out')
fsock = open(print_file,'w')
sys.stdout=fsock
# initial time to read the sat files
# should be after the end of the flight
# and a 12h or 0h boundary
print('year',year,'month',month,'day',day)
print('advect',advect)
print('suffix',suffix)
# Directory of the backward trajectories
ftraj = os.path.join(traj_dir,'BACK-'+advect+fdate.strftime('-%b-%Y-')+str(level)+'K'+suffix)
# Output file
out_file = os.path.join(out_dir,'BACK-'+advect+fdate.strftime('-%b-%Y-')+str(level)+'K'+suffix+'.hdf5b')
out_file1 = os.path.join(out_dir,'BACK-'+advect+fdate.strftime('-%b-%Y-')+str(level)+'K'+suffix+'.hdf5z')
out_file2 = os.path.join(out_dir,'BACK-'+advect+fdate.strftime('-%b-%Y-')+str(level)+'K'+suffix+'.pkl')
""" Initialization of the calculation """
# Initialize the dictionary of the parcel dictionaries
partStep={}
# Open the part_000 file that contains the initial positions
part0 = readidx107(os.path.join(ftraj,'part_000'),quiet=False)
print('numpart',part0['numpart'])
numpart = part0['numpart']
numpart_s = granule_size
# stamp_date not set in these runs
# current_date actually shifted by one day / sdate
current_date = sdate
# check flag is clean
print('check flag is clean ',((part0['flag']&I_HIT)!=0).sum(),((part0['flag']&I_DEAD)!=0).sum(),\
((part0['flag']&I_CROSSED)!=0).sum())
# check idx_orgn
if part0['idx_orgn'] != 0:
print('MINCHIA, IDX_ORGN NOT 0 AS ASSUMED, CORRECTED WITH READ VALUE')
print('VALUE ',part0['idx_orgn'])
IDX_ORGN = part0['idx_orgn']
idx1 = IDX_ORGN
# Build a dictionary to host the results
prod0 = defaultdict(dict)
# Locations of the crossing and detrainement
nsrc = 6
prod0['src']['x'] = np.full(shape=(nsrc,part0['numpart']),fill_value=np.nan,dtype='float')
prod0['src']['y'] = np.full(shape=(nsrc,part0['numpart']),fill_value=np.nan,dtype='float')
prod0['src']['p'] = np.full(shape=(nsrc,part0['numpart']),fill_value=np.nan,dtype='float')
prod0['src']['t'] = np.full(shape=(nsrc,part0['numpart']),fill_value=np.nan,dtype='float')
prod0['src']['age'] = np.full(shape=(nsrc,part0['numpart']),fill_value=np.nan,dtype='float')
# Flag is copied from index
prod0['flag_source'] = part0['flag']
# Make a source array to accumulate the chi
# Dimension is that of the ERA5 field (201,681)
prod0['source'] = np.zeros(shape=(201,681),dtype='float')
# truncate eventually to 32 bits at the output stage
# Inintialize the erosion
prod0['chi'] = np.full(part0['numpart'],1.,dtype='float')
prod0['passed'] = np.full(part0['numpart'],10,dtype='int')
# Build the interpolator to the hybrid level
fhyb, void = tohyb()
#vfhyb = np.vectorize(fhyb)
# Read the part_000 file for the first granule
partStep[0] = {}
partStep[0]['x']=part0['x'][:granule_size]
partStep[0]['y']=part0['y'][:granule_size]
partStep[0]['t']=part0['t'][:granule_size]
partStep[0]['p']=part0['p'][:granule_size]
partStep[0]['t']=part0['t'][:granule_size]
partStep[0]['idx_back']=part0['idx_back'][:granule_size]
partStep[0]['ir_start']=part0['ir_start'][:granule_size]
partStep[0]['itime'] = 0
# number of hists and exits
nhits = np.array([0,0,0,0,0,0])
nexits = 0
ndborne = 0
nnew = granule_size
nold = 0
nradada = 0
# used to get non borne parcels
new = np.empty(part0['numpart'],dtype='bool')
new.fill(False)
print('Initialization completed')
""" Main loop on the output time steps """
for hour in range(step,hmax+1,step):
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0]/2**30
print('memory use: {:4.2f} gb'.format(memoryUse))
# Get rid of dictionary no longer used
if hour >= 2*step: del partStep[hour-2*step]
# Read the new data
partStep[hour] = readpart107(hour,ftraj,quiet=True)
# Link the names
partante = partStep[hour-step]
partpost = partStep[hour]
if partpost['nact']>0:
print('hour ',hour,' numact ', partpost['nact'], ' max p ',partpost['p'].max())
else:
print('hour ',hour,' numact ', partpost['nact'])
# New date valid for partpost
current_date -= dstep
""" Select the parcels that are common to the two steps
ketp_a is a logical field with same length as partante
kept_p is a logical field with same length as partpost
After the launch of the earliest parcel along the flight track, there
should not be any member in new.
"""
kept_a = np.in1d(partante['idx_back'],partpost['idx_back'],assume_unique=True)
kept_p = np.in1d(partpost['idx_back'],partante['idx_back'],assume_unique=True)
#new_p = ~np.in1d(partpost['idx_back'],partpost['idx_back'],assume_unique=True)
print('kept a, p ',len(kept_a),len(kept_p),kept_a.sum(),kept_p.sum(),' new ',len(partpost['x'])-kept_p.sum())
nnew += len(partpost['x'])-kept_p.sum()
""" PROCESSING OF DEADBORNE PARCELS
Manage the parcels launched during the last 6-hour which have already
exited and do not appear in posold or posact (borne dead parcels).
These parcels are stored in the last part of posact, at most
the last granule_quanta parcels.
PB: this does not process the first parcels launched at time 0 since initially
numpart_s = granule_size"""
if numpart_s < numpart :
print("manage deadborne",flush=True)
# First index of the current quanta """
numpart_s += granule_quanta
print("numpart_s ",numpart_s)
# Extract the last granule_size indexes from posact
if hour==step:
idx_act = partpost['idx_back']
else:
idx_act = partpost['idx_back'][-granule_quanta:]
# Generate the list of indexes that should be found in this range
idx_theor = np.arange(idx1,numpart_s+IDX_ORGN)
# Find the missing indexes in idx_act (make a single line after validation)
kept_borne = np.in1d(idx_theor,idx_act,assume_unique=True)
idx_deadborne = idx_theor[~kept_borne]
# Process these parcels by assigning exit at initial location
prod0['flag_source'][idx_deadborne-IDX_ORGN] = prod0['flag_source'][idx_deadborne-IDX_ORGN] | I_DEAD+I_DBORNE
prod0['src']['x'][0,idx_deadborne-IDX_ORGN] = part0['x'][idx_deadborne-IDX_ORGN]
prod0['src']['y'][0,idx_deadborne-IDX_ORGN] = part0['y'][idx_deadborne-IDX_ORGN]
prod0['src']['p'][0,idx_deadborne-IDX_ORGN] = part0['p'][idx_deadborne-IDX_ORGN]
prod0['src']['t'][0,idx_deadborne-IDX_ORGN] = part0['t'][idx_deadborne-IDX_ORGN]
prod0['src']['age'][0,idx_deadborne-IDX_ORGN] = 0.
print("number of deadborne ",len(idx_deadborne))
ndborne += len(idx_deadborne)
idx1 = numpart_s + IDX_ORGN
""" PROCESSING OF CROSSED PARCELS """
# last known location before crossing stored in the index 0 of src fields
if len(kept_a)>0:
exits = exiter(int((partante['itime']+partpost['itime'])/2), \
partante['x'][~kept_a],partante['y'][~kept_a],partante['p'][~kept_a],\
partante['t'][~kept_a],partante['idx_back'][~kept_a],\
prod0['flag_source'],prod0['src']['x'],prod0['src']['y'],\
prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],\
part0['ir_start'], domain)
nexits += exits
#nhits[0] += exits
print('exit ',nexits, exits, np.sum(~kept_a), len(kept_a) - len(kept_p))
""" PROCESSING OF PARCELS WHICH ARE COMMON TO THE TWO OUTPUTS """
# Select the kept parcels which have not been hit yet
# !!! Never use and between two lists, the result is wrong
if kept_p.sum()==0:
live_a = live_p = kept_p
else:
live_a = np.logical_and(kept_a,(prod0['flag_source'][partante['idx_back']-IDX_ORGN] & I_DEAD) == 0)
live_p = np.logical_and(kept_p,(prod0['flag_source'][partpost['idx_back']-IDX_ORGN] & I_DEAD) == 0)
print('live a, p ',live_a.sum(),live_p.sum())
del kept_a
del kept_p
""" Correction step that moves partante['x'] to avoid big jumps at the periodicity frontier on the x-axis """
diffx = partpost['x'][live_p] - partante['x'][live_a]
bb = np.zeros(len(diffx))
bb[diffx>180] = 360
bb[diffx<-180] = -360
partante['x'][live_a] += bb
del bb
del diffx
# DOES not work in the following WAY
#partante['x'][live_a][diffx>180] += 360
#partante['x'][live_a][diffx<-180] -= 360
# Build generator for live parcel locations of the 1h slices
gsp = get_slice_part(partante,partpost,live_a,live_p,current_date,dstep,slice_width)
if verbose: print('built parcel generator for ',current_date)
""" MAIN LOOP ON THE 1H PARCEL TIME SLICES """
for i in range(nb_slices):
# get the 1h slice for the particles
datpart = next(gsp)
# skip if no particles
if datpart['ti'] == None:
continue
print('current_date ',datpart['ti'])
#@@ test
# print('ti in main ',datpart['ti'])
# print('pi in main ',np.min(datpart['pi']),np.max(datpart['pi']))
# print('idx_back ',np.min(datpart['idx_back']),np.max(datpart['idx_back']))
# #@@ end test
# as the ECMWF files are also available every hour
datrean = read_ECMWF(datpart['ti'])
"""
Calculate the -log surface pressure at parcel location at time ti
create a 2D linear interpolar from the surface pressure field
This interpolator is meant to generate the surface pressure which is in turn used to determine
sigma and the hybrid level.
The interpolator is only defined in the area where ECMWF ERA5 data are available."""
lsp = RegularGridInterpolator((datrean.attr['lats'],datrean.attr['lons']),\
-np.log(datrean.var['SP']),method='linear')
""" Select the pairs (i,f) entirely located within the domain """
indomain = np.all((datpart['xi']>-10,datpart['xi']<160,datpart['yi']>0,datpart['yi']<50,
datpart['xf']>-10,datpart['xf']<160,datpart['yf']>0,datpart['yf']<50),axis=0)
# perform the interpolation for the location of live parcels at time ti
# ACHTUNG: this interpolation is only meaningful for parcels laying within
# the domain of the ERA5 data
lspi = lsp(np.transpose([datpart['yi'][indomain],datpart['xi'][indomain]]))
#@@ test
# print('surface pressure ',np.exp(-np.min(datpart['lspi'])),np.exp(-np.max(datpart['lspi'])))
# print('particle pressure ',np.min(datpart['pi']),np.max(datpart['pi']))
#@@ end test
# get the closest hybrid level at time ti
# define first -log sigma = -log(p) - -log(ps)
lsig = - np.log(datpart['pi'][indomain]) - lspi
#@@ test
# print('sigma ',np.exp(-np.max(lsig)),np.exp(-np.min(lsig)))
#@@ end test
# get the hybrid level, the rank of the first retained level is substracted to have hyb starting from 0
hyb = np.floor(fhyb(np.transpose([lsig,lspi]))+0.5).astype(np.int64)-datrean.attr['levs'][0]
#@@ test the extreme values of sigma end ps
if np.min(lsig) < - np.log(0.95):
print('large sigma detected ',np.exp(-np.min(lsig)))
if np.max(lspi) > -np.log(45000):
print('small ps detected ',np.exp(-np.max(lspi)))
""" PROCESS THE PARCELS WHICH ARE TOO CLOSE TO GROUND
These parcels are flagged as crossed and dead, their last location is stored in the
index 0 of src fields.
This test handles also the cases outside the interpolation domain as NaN produced by fhyb
generates very large value of hyb.
The trajectories which are stopped here have exited the domain where winds are available to flexpart
and therefore are wrong from this point. For this reason we label them from their last valid position.
This last sentence is only valid in simulations using ERA5 as ERA-Interim contains data down to the ground.
"""
if np.max(hyb)> 100 :
selec = hyb>100
nr = radada(datpart['itime'],
datpart['xf'][indomain][selec],datpart['yf'][indomain][selec],datpart['pf'][indomain][selec],
datpart['tempf'][indomain][selec],datpart['idx_back'][indomain][selec],
prod0['flag_source'],prod0['src']['x'],prod0['src']['y'],
prod0['src']['p'],prod0['src']['t'],prod0['src']['age'] ,prod0['source'],
part0['ir_start'])
nradada += nr
nhits[0] += nr
""" PROCESS THE (ADJOINT) DETRAINMENT """
n1 = detrainer(datpart['itime'],
datpart['xi'][indomain],datpart['yi'][indomain],datpart['pi'][indomain],datpart['tempi'][indomain],hyb,
datpart['xf'][indomain],datpart['yf'][indomain], datrean.var['UDR'], datpart['idx_back'][indomain],\
prod0['flag_source'],part0['ir_start'], prod0['chi'],prod0['passed'],\
prod0['src']['x'],prod0['src']['y'],prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],prod0['source'],\
datrean.attr['Lo1'],datrean.attr['La1'],datrean.attr['dlo'],datrean.attr['dla'],detr_offset)
nhits += n1
#@@ test
# print('return from detrainer',nhits)
#@@ end test
del datpart
sys.stdout.flush()
""" End of of loop on slices """
# Check the age limit (easier to do it here)
print("Manage age limit",flush=True)
age_sec = part0['ir_start'][partante['idx_back']-IDX_ORGN]-partante['itime']
IIold_o = age_sec > (age_bound-0.25) * 86400
IIold_o = IIold_o & ((prod0['flag_source'][partante['idx_back']-IDX_ORGN] & I_STOP)==0)
idx_IIold = partante['idx_back'][IIold_o]
j_IIold_o = np.where(IIold_o)
prod0['flag_source'][idx_IIold-IDX_ORGN] = prod0['flag_source'][idx_IIold-IDX_ORGN] | I_DEAD+I_OLD
prod0['src']['x'][0,idx_IIold-IDX_ORGN] = partante['x'][j_IIold_o]
prod0['src']['y'][0,idx_IIold-IDX_ORGN] = partante['y'][j_IIold_o]
prod0['src']['p'][0,idx_IIold-IDX_ORGN] = partante['p'][j_IIold_o]
prod0['src']['t'][0,idx_IIold-IDX_ORGN] = partante['t'][j_IIold_o]
prod0['src']['age'][0,idx_IIold-IDX_ORGN] = ((part0['ir_start'][idx_IIold-IDX_ORGN]- partante['itime'])/86400)
print("number of IIold ",len(idx_IIold))
nold += len(idx_IIold)
# find parcels still alive if kept_p.sum()==0:
try:
# number of parcels still alive
nlive = ((prod0['flag_source'][partpost['idx_back']-IDX_ORGN] & I_DEAD) == 0).sum()
# number of parcels still alive and not hit
nprist = ((prod0['flag_source'][partpost['idx_back']-IDX_ORGN] & (I_DEAD+I_HIT)) == 0).sum()
# number of parcels which have hit and crossed
nouthit = ((prod0['flag_source'][partpost['idx_back']-IDX_ORGN] & I_HIT+I_CROSSED) == I_HIT+I_CROSSED).sum()
# number of parcels which heve crossed without hit
noutprist = ((prod0['flag_source'][partpost['idx_back']-IDX_ORGN] & I_HIT+I_CROSSED) == I_CROSSED).sum()
# number of parcels which have hit without crossing
nhitpure = ((prod0['flag_source'][partpost['idx_back']-IDX_ORGN] & I_HIT+I_CROSSED) == I_HIT).sum()
except:
nlive = 0
nprist =0
nouthit = 0
noutprist = 0
nhitpure = 0
nprist = part0['numpart']
print('end hour ',hour,' numact', partpost['nact'], ' nnew',nnew,' nexits',nexits,' nold',nold,' ndborne',ndborne)
print('nhits',nhits)
print('nlive', nlive,' nprist',nprist,' nouthit',nouthit,' noutprist',noutprist,' nhitpure',nhitpure)
# check that nprist + nhits + nexits + nold = nnew
#if partpost['nact'] != nprist + nouthit + noutprist + nhitpure + ndborne:
# print('@@@ ACHTUNG numact not equal to sum ',partpost['nact'],nprist + nouthit + noutprist + nhitpure + ndborne)
""" End of the procedure and storage of the result """
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0]/2**30
print('memory use before clean: {:4.2f} gb'.format(memoryUse))
del partante
del partpost
del live_a
del live_p
# reduction of the size of prod0 by converting float64 into float32
prod0['chi'] = prod0['chi'].astype(np.float32)
prod0['passed'] = prod0['passed'].astype(np.int32)
prod0['source'] = prod0['source'].astype(np.float32)
for var in ['age','p','t','x','y']:
prod0['src'][var] = prod0['src'][var].astype(np.float32)
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0]/2**30
print('memory use after clean: {:4.2f} gb'.format(memoryUse))
#output file
try:
dd.io.save(out_file1,prod0,compression='blosc')
except:
print('error with dd blosc')
try:
dd.io.save(out_file,prod0,compression='zlib')
except:
print('error with dd zlib')
try:
pickle.dump(prod0,open(out_file2,'wb'))
except:
print('error with pickle')
# close the print file
if quiet: fsock.close()
def main():
global IDX_ORGN
parser = argparse.ArgumentParser()
parser.add_argument("-y", "--year", type=int, help="year")
parser.add_argument("-m",
"--month",
type=int,
choices=1 + np.arange(12),
help="month")
parser.add_argument("-d",
"--day",
type=int,
choices=1 + np.arange(31),
help="day")
parser.add_argument("-a",
"--advect",
type=str,
choices=["OPZ", "EAD", "EAZ", "EID", "EIZ"],
help="source of advecting winds")
parser.add_argument("-p",
"--platform",
type=str,
choices=["M55", "GLO", "BAL"],
help="measurement platform")
parser.add_argument("-n",
"--launch_number",
type=int,
help="balloon launch number within a day")
parser.add_argument("-s",
"--suffix",
type=str,
help="suffix for special cases")
parser.add_argument("-q",
"--quiet",
type=str,
choices=["y", "n"],
help="quiet (y) or not (n)")
parser.add_argument("-c", "--clean0", type=bool, help="clean part_000")
parser.add_argument("-r",
"--reanalysis",
type=str,
choices=["ERA5", "ERAI"],
help="reanalysis for detrainement")
# to be updated
if 'ciclad' in socket.gethostname():
#root_dir = '/home/legras/STC/STC-M55'
traj_dir = '/data/legras/flexout/STC/M55'
out_dir = '/data/legras/STC'
mask_dir = '/home/legras/STC/mkSTCmask'
elif ('climserv' in socket.gethostname()) | ('polytechnique'
in socket.gethostname()):
#root_dir = '/home/legras/STC/STC-M55'
traj_dir = '/bdd/STRATOCLIM/flexout/M55'
out_dir = '/homedata/legras/STC'
mask_dir = '/home/legras/STC/mkSTCmask'
else:
print('CANNOT RECOGNIZE HOST - DO NOT RUN ON NON DEFINED HOSTS')
exit()
""" Parameters """
# fixed parameters
# to do (perhaps) : some parameters might be parsed from command line
# step and max output time
step = 6
hmax = 732
dstep = timedelta(hours=step)
# time width of the parcel slice
# slice_width cannot be chosen independently of the step
# see below the main loop
# ACHTUNG ACHTUNG!!!! If you change this value, change also the chi erosion in detrainer which is hard coded
# to occur by 1-hour steps
slice_width = timedelta(hours=1)
# number of slices between two outputs
nb_slices = int(dstep / slice_width)
# defines here the offset for the detrainment (100h)
detr_offset = 1 / (100 * 3600.)
# defines the domain where M55 parcels are living (no FULL trajectories for this setup)
domain = np.array([[-10., 160.], [0., 50.]])
# Size of each packet of parcels
segl = 1000
# default values of changeable parameters
# date of the flight
year = 2017
month = 7
day = 29
platform = 'M55'
advect = 'EAZ'
# choice of the reanalysis from which detrainement rates are extracted
rea = 'ERA5'
suffix = ''
launch_number = ''
quiet = False
clean0 = False
args = parser.parse_args()
if args.year is not None: year = args.year
if args.month is not None: month = args.month
if args.day is not None: day = args.day
if args.advect is not None: advect = args.advect
if args.platform is not None: platform = args.platform
if args.launch_number is not None:
launch_number = '-' + str(args.launch_number)
if args.suffix is not None: suffix = '-' + args.suffix
if args.quiet is not None:
if args.quiet == 'y':
quiet = True
else:
quiet = False
if args.clean0 is not None: clean0 = args.clean0
if args.reanalysis is not None: rea = args.reanalysis
# Update the out_dir with the platform
out_dir = os.path.join(out_dir, 'STC-' + platform + '-DETR-OUT')
fdate = datetime(year, month, day)
# Manage the file that receives the print output
if quiet:
# Output file
print_file = os.path.join(
out_dir, 'out',
platform + fdate.strftime('-%Y%m%d') + launch_number + '-' +
advect + '-D01' + suffix + '-' + rea + '.out')
fsock = open(print_file, 'w')
sys.stdout = fsock
# initial time to read the detrainment files
# should be after the end of the flight
# and a 12h or 0h boundary
print('year', year, 'month', month, 'day', day)
print('advect', advect)
print('platform', platform)
print('launch_number', launch_number)
print('suffix', suffix)
# Read the region mask
# ACHTUNG: the mask should fit the domain and dimensions of the prodO['source']
# defined below
if rea == 'ERA5':
mm = pickle.load(
gzip.open(os.path.join(mask_dir, 'MaskCartopy2-ERA5-STC.pkl')))
elif rea == 'ERAI':
mm = pickle.load(
gzip.open(os.path.join(mask_dir, 'MaskCartopy2-ERA-I.pkl')))
mask = mm['mask']
# Directory of the backward trajectories
ftraj = os.path.join(
traj_dir, platform + fdate.strftime('-%Y%m%d') + launch_number + '-' +
advect + '-D01' + suffix)
# Output file
#out_file = os.path.join(out_dir,platform+fdate.strftime('-%Y%m%d')+launch_number+'-'+advect+'-D01'+suffix+'.pkl')
out_file2 = os.path.join(
out_dir, platform + fdate.strftime('-%Y%m%d') + launch_number + '-' +
advect + '-D01' + suffix + '-' + rea + '.hdf5')
""" Initialization of the calculation """
# Initialize the dictionary of the parcel dictionaries
partStep = {}
# Read the index file that contains the initial positions
part0 = readidx107(os.path.join(ftraj, 'index_old'), quiet=False)
print('numpart', part0['numpart'])
# stamp_date not set in these runs
# current_date actually shifted by one day / sdate
current_date = fdate + timedelta(days=1)
# check flag is clean
print('check flag is clean ',((part0['flag']&I_HIT)!=0).sum(),((part0['flag']&I_DEAD)!=0).sum(),\
((part0['flag']&I_CROSSED)!=0).sum())
# check idx_orgn
if part0['idx_orgn'] != 0:
print('MINCHIA, IDX_ORGN NOT 0 AS ASSUMED, CORRECTED WITH READ VALUE')
print('VALUE ', part0['idx_orgn'])
IDX_ORGN = part0['idx_orgn']
# Build a dictionary to host the results
prod0 = defaultdict(dict)
# Locations of the crossing and detrainement
nsrc = 6
prod0['src']['x'] = np.full(shape=(nsrc, part0['numpart']),
fill_value=np.nan,
dtype='float')
prod0['src']['y'] = np.full(shape=(nsrc, part0['numpart']),
fill_value=np.nan,
dtype='float')
prod0['src']['p'] = np.full(shape=(nsrc, part0['numpart']),
fill_value=np.nan,
dtype='float')
prod0['src']['t'] = np.full(shape=(nsrc, part0['numpart']),
fill_value=np.nan,
dtype='float')
prod0['src']['age'] = np.full(shape=(nsrc, part0['numpart']),
fill_value=np.nan,
dtype='float')
# Flag is copied from index
prod0['flag_source'] = part0['flag']
# Make a source array to accumulate the chi
# For ERA5: Dimension is that of the STC ERA5 fields (201,681) at 0.25° resolution
# For ERAI: The domain is the reduced domain (10-50N,10W,160E) at 1° resolution, that is (51,171) size
# Both latitudes and longitudes are growing
if rea == 'ERA5':
prod0['source'] = np.zeros(shape=(201, 681), dtype='float')
xshift = 0
yshift = 0
elif rea == 'ERAI':
prod0['source'] = np.zeros(shape=(51, 171), dtype='float')
# shift of the source grid in the original ERAI grid with origins at (90S, 179W)
xshift = -169
yshift = -90
# truncate eventually to 32 bits at the output stage
# Source array that cumulates within regions as a function of time
prod0['pl'] = np.zeros(shape=(len(mm['regcode']) + 1,
int(part0['numpart'] / segl)),
dtype='float')
# Initialize the erosion
prod0['chi'] = np.full(part0['numpart'], 1., dtype='float')
prod0['passed'] = np.full(part0['numpart'], 10, dtype='int')
# Build the interpolator to the hybrid level
fhyb, void = tohyb(rea)
#vfhyb = np.vectorize(fhyb)
# Read the part_000 file
partStep[0] = readpart107(0, ftraj, quiet=True)
# cleaning is necessary for runs starting in the fake restart mode
# otherwise all parcels are thought to exit at the first step
if clean0:
partStep[0]['idx_back'] = []
# number of hists and exits
nhits = np.array([0, 0, 0, 0, 0, 0])
nexits = 0
ndborne = 0
nnew = 0
nradada = 0
# used to get non borne parcels
new = np.empty(part0['numpart'], dtype='bool')
new.fill(False)
print('Initialization completed')
""" Main loop on the output time steps """
for hour in range(step, hmax + 1, step):
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use: {:4.2f} gb'.format(memoryUse))
# Get rid of dictionary no longer used
if hour >= 2 * step: del partStep[hour - 2 * step]
# Read the new data
partStep[hour] = readpart107(hour, ftraj, quiet=True)
# Link the names
partante = partStep[hour - step]
partpost = partStep[hour]
if partpost['nact'] > 0:
print('hour ', hour, ' numact ', partpost['nact'], ' max p ',
partpost['p'].max())
else:
print('hour ', hour, ' numact ', partpost['nact'])
# New date valid for partpost
current_date -= dstep
""" Select the parcels that are common to the two steps
ketp_a is a logical field with same length as partante
kept_p is a logical field with same length as partpost
After the launch of the earliest parcel along the flight track, there
should not be any member in new.
"""
kept_a = np.in1d(partante['idx_back'],
partpost['idx_back'],
assume_unique=True)
kept_p = np.in1d(partpost['idx_back'],
partante['idx_back'],
assume_unique=True)
#new_p = ~np.in1d(partpost['idx_back'],partpost['idx_back'],assume_unique=True)
print('kept a, p ', len(kept_a), len(kept_p), kept_a.sum(),
kept_p.sum(), ' new ',
len(partpost['x']) - kept_p.sum())
""" IDENTIFY AND TAKE CARE OF DEADBORNE AS NON BORNE PARCELS """
# Find and count the parcels in partpost which where not in partante, flag them as new
if (hour <= 30) & (partpost['nact'] > 0):
new[partpost['idx_back'][~kept_p] - IDX_ORGN] = True
nnew += len(partpost['x']) - kept_p.sum()
# When the release of parcels has ended, flag the ones that did not show up as dead
# source their last location as the launching location
if hour == 30:
ndborne = np.sum(~new)
prod0['flag_source'][~new] |= I_DBORNE + I_DEAD
prod0['src']['x'][0, ~new] = part0['x'][~new]
prod0['src']['y'][0, ~new] = part0['y'][~new]
prod0['src']['p'][0, ~new] = part0['p'][~new]
prod0['src']['t'][0, ~new] = part0['t'][~new]
prod0['src']['age'][0, ~new] = 0
print('number of dead borne', ndborne, part0['numpart'] - nnew)
# get rid of new
del new
""" PROCESSING OF CROSSED PARCELS """
# last known location before crossing stored in the index 0 of src fields
if len(kept_a) > 0:
exits = exiter(int((partante['itime']+partpost['itime'])/2), \
partante['x'][~kept_a],partante['y'][~kept_a],partante['p'][~kept_a],\
partante['t'][~kept_a],partante['idx_back'][~kept_a],\
prod0['flag_source'],prod0['src']['x'],prod0['src']['y'],\
prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],\
part0['ir_start'], domain)
nexits += exits
nhits[0] += exits
print('exit ', nexits, exits, np.sum(~kept_a),
len(kept_a) - len(kept_p))
""" PROCESSING OF PARCELS WHICH ARE COMMON TO THE TWO OUTPUTS """
# Select the kept parcels which have not been hit yet
# !!! Never use and between two lists, the result is wrong
if kept_p.sum() == 0:
live_a = live_p = kept_p
else:
live_a = np.logical_and(
kept_a, (prod0['flag_source'][partante['idx_back'] - IDX_ORGN]
& I_DEAD) == 0)
live_p = np.logical_and(
kept_p, (prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_DEAD) == 0)
print('live a, p ', live_a.sum(), live_p.sum())
del kept_a
del kept_p
# Build generator for live parcel locations of the 1h slices
gsp = get_slice_part(partante, partpost, live_a, live_p, current_date,
dstep, slice_width)
if verbose: print('built parcel generator for ', current_date)
""" MAIN LOOP ON THE PARCEL TIME SLICES """
for i in range(nb_slices):
# get the next slice for the particles
datpart = next(gsp)
# skip if no particles
if datpart['ti'] == None:
continue
print('current_date ', datpart['ti'])
#@@ test
# print('ti in main ',datpart['ti'])
# print('pi in main ',np.min(datpart['pi']),np.max(datpart['pi']))
# print('idx_back ',np.min(datpart['idx_back']),np.max(datpart['idx_back']))
# #@@ end test
# What follows is not independent of the choices of step and slice_width
# TO DO : a more general version that does not have this dependency
# This should not be difficult using a generator with validity times
# We just need to make sure the entrainment data and SP data are valid
# during the interval [ti tf]
# Read ERA5 data as the files are also available every hour
# this might not work if output step is changed without changing slice width
# the detrainement is actually defined as an average over the next hour that is between ti and tf
if rea == 'ERA5':
datrean = read_ECMWF(datpart['ti'], rea)
# calculate the -log surface pressure at parcel location at time ti
# create a 2D linear interpolar from the surface pressure field
lsp = RegularGridInterpolator((datrean.attr['lats'],datrean.attr['lons']),\
-np.log(datrean.var['SP']))
# In the case of ERA-Interim, we read the detrainment which is averaged over a 3h-interval
# if ti corresponds to the first or the fourth interval of a 6-hour step
# This might not work if the output step is cahnged without changing slice width
elif rea == 'ERAI' and ((datpart['ti'].hour % 3) == 2):
datrean = read_ECMWF(datpart['ti'] - timedelta(hours=2), rea)
lsp = RegularGridInterpolator((datrean.attr['lats'],np.append(datrean.attr['lons'],181)),\
-np.log(np.append(datrean.var['SP'].T,[datrean.var['SP'][:,0]],axis=0).T))
# perform the interpolation for the location of live parcels at time 0.5*(ti+tf)
datpart['lsp'] = lsp(
np.transpose([
0.5 * (datpart['yi'] + datpart['yf']),
0.5 * (datpart['xi'] + datpart['xf'])
]))
#@@ test
# print('surface pressure ',np.exp(-np.min(datpart['lsp'])),np.exp(-np.max(datpart['lsp'])))
# print('particle pressure ',np.min(datpart['pi']),np.max(datpart['pi']))
#@@ end test
# get the closest hybrid level at time ti
# define first -log sigma = -log(p) - -log(ps) = - log(p/ps)
lsig = -np.log(0.5 *
(datpart['pi'] + datpart['pf'])) - datpart['lsp']
#@@ test
# print('sigma ',np.exp(-np.max(lsig)),np.exp(-np.min(lsig)))
#@@ end test
# get the hybrid level at time ti, the rank of the first retained level is substracted to have hyb starting from 0
# +1 because the levels are counted from 1, not 0 and +0.5 because we get the closest neighbour
hyb = np.floor(fhyb(np.transpose([lsig, datpart['lsp']])) +
1.5).astype(np.int64) - datrean.attr['levs'][0]
#@@ test the extreme values of sigma end ps
if np.min(lsig) < -np.log(0.95):
print('large sigma detected ', np.exp(-np.min(lsig)))
if np.max(datpart['lsp']) > -np.log(45000):
print('small ps detected ', np.exp(-np.max(datpart['lsp'])))
""" PROCESS THE PARCELS WHICH ARE TOO CLOSE TO GROUND
These parcels are flagged as crossed and dead, their last location is stored in the
index 0 of src fields.
This test handles also the cases outside the interpolation domain as NaN produced by fhyb
generates very large value of hyb.
The trajectories which are stopped here have exited the domain where winds are available to flexpart
and therefore are wrong from this point. For this reason we label them from their last valid position.
The threshold 100 is valid for the particular STC ERA5 archive only.
With ERA-I, this section should not operate."""
if np.max(hyb) > 100:
selec = hyb > 100
nr = radada(datpart['itime'], datpart['xf'][selec],
datpart['yf'][selec], datpart['pf'][selec],
datpart['tempf'][selec],
datpart['idx_back'][selec], prod0['flag_source'],
prod0['src']['x'], prod0['src']['y'],
prod0['src']['p'], prod0['src']['t'],
prod0['src']['age'], part0['ir_start'])
nradada += nr
nhits[0] += nr
""" PROCESS THE (ADJOINT) DETRAINMENT """
n1 = detrainer(datpart['itime'],
datpart['xi'],datpart['yi'],datpart['pi'],datpart['tempi'],hyb,
datpart['xf'],datpart['yf'], datrean.var['UDR'], datpart['idx_back'],\
prod0['flag_source'],part0['ir_start'], prod0['chi'],prod0['passed'],\
prod0['src']['x'],prod0['src']['y'],prod0['src']['p'],prod0['src']['t'],\
prod0['src']['age'],prod0['source'],prod0['pl'],\
datrean.attr['Lo1'],datrean.attr['La1'],datrean.attr['dlo'],datrean.attr['dla'],\
xshift,yshift,detr_offset,segl,mask)
nhits += n1
#@@ test
print('return from detrainer', nhits)
#@@ end test
sys.stdout.flush()
""" End of of loop on slices """
# find parcels still alive if kept_p.sum()==0:
try:
# number of parcels still alive
nlive = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_DEAD) == 0).sum()
# number of parcels still alive and not hit
nprist = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN] &
(I_DEAD + I_HIT)) == 0).sum()
# number of parcels which have hit and crossed
nouthit = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_HIT + I_CROSSED) == I_HIT + I_CROSSED).sum()
# number of parcels which heve crossed without hit
noutprist = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_HIT + I_CROSSED) == I_CROSSED).sum()
# number of parcels which have hit without crossing
nhitpure = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_HIT + I_CROSSED) == I_HIT).sum()
except:
nlive = 0
nprist = 0
nouthit = 0
noutprist = 0
nhitpure = 0
nprist = part0['numpart']
print('end hour ', hour, ' numact', partpost['nact'], ' nexits',
nexits, ' nhits', nhits)
print('nlive', nlive, ' nprist', nprist, ' nouthit', nouthit,
' noutprist', noutprist, ' nhitpure', nhitpure)
# check that nlive + nhits + nexits = numpart, should be true after the first day
if partpost[
'nact'] != nprist + nouthit + noutprist + nhitpure + ndborne:
print('@@@ ACHTUNG numact not equal to sum ', partpost['nact'],
nprist + nouthit + noutprist + nhitpure + ndborne)
""" End of the procedure and storage of the result """
# clear other fields before the output as this step requires memory
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use before clean: {:4.2f} gb'.format(memoryUse))
del partante
del partpost
del partStep
del datpart
del datrean
del hyb
del live_a
del live_p
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use after clean: {:4.2f} gb'.format(memoryUse))
sys.stdout.flush()
# reduction of the size of prod0 by converting float64 into float32
prod0['chi'] = prod0['chi'].astype(np.float32)
prod0['passed'] = prod0['passed'].astype(np.int32)
for var in ['age', 'p', 't', 'x', 'y']:
prod0['src'][var] = prod0['src'][var].astype(np.float32)
#output file
dd.io.save(out_file2, prod0, compression='zlib')
#pickle.dump(prod0,gzip.open(out_file,'wb'))
# close the print file
print('completed run')
if quiet: fsock.close()
return
def main():
global IDX_ORGN
parser = argparse.ArgumentParser()
parser.add_argument("-y", "--year", type=int, help="year")
parser.add_argument("-m",
"--month",
type=int,
choices=1 + np.arange(12),
help="month")
parser.add_argument(
"-a",
"--advect",
type=str,
choices=["OPZ", "EAD", "EAZ", "EID", "EIZ", "EID-FULL", "EIZ-FULL"],
help="source of advecting winds")
parser.add_argument("-l", "--level", type=int, help="PT level")
parser.add_argument("-s",
"--suffix",
type=str,
help="suffix for special cases")
parser.add_argument("-q",
"--quiet",
type=str,
choices=["y", "n"],
help="quiet (y) or not (n)")
# to be updated
if socket.gethostname() == 'graphium':
pass
elif 'ciclad' in socket.gethostname():
#root_dir = '/home/legras/STC/STC-M55'
main_sat_dir = '/bdd/STRATOCLIM/flexpart_in'
traj_dir = '/data/legras/flexout/STC/BACK'
out_dir = '/data/legras/STC'
elif socket.gethostname() == 'grapelli':
pass
elif socket.gethostname() == 'coltrane':
pass
else:
print('CANNOT RECOGNIZE HOST - DO NOT RUN ON NON DEFINED HOSTS')
exit()
""" Parameters """
# to do (perhaps) : some parameters might be parsed from command line
# step and max output time
step = 6
hmax = 1824
dstep = timedelta(hours=step)
# age limit in days
age_bound = 44.
# time width of the parcel slice
slice_width = timedelta(minutes=5)
# dtRange
dtRange = {'MSG1': timedelta(minutes=30), 'Hima': timedelta(minutes=20)}
# number of slices between two outputs
nb_slices = int(dstep / slice_width)
# default values of parameters
# start date of the backward run, corresponding to itime=0
year = 2017
# 8 +1 means we start on September 1 at 0h and cover the whole month of August
month = 8 + 1
# day=1 should not be changed
day = 1
advect = 'EAD'
suffix = ''
quiet = False
level = 380
args = parser.parse_args()
if args.year is not None:
year = args.year
if args.month is not None:
month = args.month + 1
if args.advect is not None:
advect = args.advect
if args.level is not None:
level = args.level
if args.suffix is not None:
suffix = '-' + args.suffix
if args.quiet is not None:
if args.quiet == 'y':
quiet = True
else:
quiet = False
# Update the out_dir with the platform
out_dir = os.path.join(out_dir, 'STC-BACK-OUT')
sdate = datetime(year, month, day)
# fdate defined to make output under the name of the month where parcels are released
fdate = sdate - timedelta(days=1)
# Number of parcels launched per time slot (1 per degree on a 170x50 grid)
granule_size = 8500
# number of granules in 6 hours
granule_step = 4 * 6
# size of granules launched during 6 hours
granule_quanta = granule_size * granule_step
# Modification of granule_size and granule_quanta
if 'FULL' in advect:
granule_size = 28800
granule_step = 6
granule_quanta = granule_size * granule_step
# Manage the file that receives the print output
if quiet:
# Output file
print_file = os.path.join(
out_dir, 'out', 'BACK-' + advect + fdate.strftime('-%b-%Y-') +
str(level) + 'K' + suffix + '.out')
fsock = open(print_file, 'w')
sys.stdout = fsock
print('year', year, 'month', month, 'day', day)
print('advect', advect)
print('suffix', suffix)
# Directory of the backward trajectories
ftraj = os.path.join(
traj_dir, 'BACK-' + advect + fdate.strftime('-%b-%Y-') + str(level) +
'K' + suffix)
# Output file
out_file = os.path.join(
out_dir, 'BACK-' + advect + fdate.strftime('-%b-%Y-') + str(level) +
'K' + suffix + '.hdf5z')
out_file1 = os.path.join(
out_dir, 'BACK-' + advect + fdate.strftime('-%b-%Y-') + str(level) +
'K' + suffix + '.hdf5b')
out_file2 = os.path.join(
out_dir, 'BACK-' + advect + fdate.strftime('-%b-%Y-') + str(level) +
'K' + suffix + '.pkl')
# Directories for the satellite cloud top files
satdir ={'MSG1':os.path.join(main_sat_dir,'StratoClim+1kmD_msg1-c'),\
'Hima':os.path.join(main_sat_dir,'StratoClim+1kmD_himawari-d')}
""" Initialization of the calculation """
# Initialize the slice map to be used as a buffer for the cloudtops
satmap = pixmap()
satfill = {}
datsat = {}
# Initialize the dictionary of the parcel dictionaries
partStep = {}
# Build the satellite field generator
get_sat = {'MSG1': read_sat(sdate,dtRange['MSG1'],satdir['MSG1']),\
'Hima': read_sat(sdate,dtRange['Hima'],satdir['Hima'])}
# Open the part_000 file that contains the initial positions
part0 = readidx107(os.path.join(ftraj, 'part_000'), quiet=True)
print('numpart', part0['numpart'])
numpart = part0['numpart']
numpart_s = granule_size
# stamp_date not set in these runs
# current_date actually shifted by one day / sdate
current_date = sdate
# check flag is clean
print('check flag is clean ',((part0['flag']&I_HIT)!=0).sum(),((part0['flag']&I_DEAD)!=0).sum(),\
((part0['flag']&I_CROSSED)!=0).sum())
# check idx_orgn
if part0['idx_orgn'] != 0:
print('MINCHIA, IDX_ORGN NOT 0 AS ASSUMED, CORRECTED WITH READ VALUE')
print('VALUE ', part0['idx_orgn'])
IDX_ORGN = part0['idx_orgn']
idx1 = IDX_ORGN
# Build a dictionary to host the results
prod0 = defaultdict(dict)
prod0['src']['x'] = np.full(part0['numpart'],
fill_value=np.nan,
dtype='float')
prod0['src']['y'] = np.full(part0['numpart'],
fill_value=np.nan,
dtype='float')
prod0['src']['p'] = np.full(part0['numpart'],
fill_value=np.nan,
dtype='float')
prod0['src']['t'] = np.full(part0['numpart'],
fill_value=np.nan,
dtype='float')
prod0['src']['age'] = np.full(part0['numpart'],
fill_value=np.nan,
dtype='int')
prod0['flag_source'] = part0['flag']
prod0['rvs'] = np.full(part0['numpart'], 0.01, dtype='float')
# truncate eventually to 32 bits at the output stage
# read the part_000 file for the first granule
partStep[0] = {}
partStep[0]['x'] = part0['x'][:granule_size]
partStep[0]['y'] = part0['y'][:granule_size]
partStep[0]['t'] = part0['t'][:granule_size]
partStep[0]['p'] = part0['p'][:granule_size]
partStep[0]['t'] = part0['t'][:granule_size]
partStep[0]['idx_back'] = part0['idx_back'][:granule_size]
partStep[0]['ir_start'] = part0['ir_start'][:granule_size]
partStep[0]['itime'] = 0
# number of hists and exits
nhits = 0
nexits = 0
ndborne = 0
nnew = granule_size
nold = 0
# used to get non borne parcels
new = np.empty(part0['numpart'], dtype='bool')
new.fill(False)
print('Initialization completed')
""" Main loop on the output time steps """
for hour in range(step, hmax + 1, step):
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use: {:4.2f} gb'.format(memoryUse))
# Get rid of dictionary no longer used
if hour >= 2 * step: del partStep[hour - 2 * step]
# Read the new data
partStep[hour] = readpart107(hour, ftraj, quiet=True)
# Link the names
partante = partStep[hour - step]
partpost = partStep[hour]
if partpost['nact'] > 0:
print('hour ', hour, ' numact ', partpost['nact'], ' max p ',
partpost['p'].max())
else:
print('hour ', hour, ' numact ', partpost['nact'])
# New date valid for partpost
current_date -= dstep
# Processing of water mixing ratio
# Select non stopped parcels in partante
selec = (prod0['flag_source'][partante['idx_back'] - IDX_ORGN]
& I_STOP) == 0
idx = partante['idx_back'][selec]
prod0['rvs'][idx-IDX_ORGN] = np.minimum(prod0['rvs'][idx-IDX_ORGN],\
satratio(partante['p'][selec],partante['t'][selec]))
""" Select the parcels that are common to the two steps
ketp_a is a logical field with same length as partante
kept_p is a logical field with same length as partpost
"""
kept_a = np.in1d(partante['idx_back'],
partpost['idx_back'],
assume_unique=True)
kept_p = np.in1d(partpost['idx_back'],
partante['idx_back'],
assume_unique=True)
#new_p = ~np.in1d(partpost['idx_back'],partpost['idx_back'],assume_unique=True)
print('kept a, p ', len(kept_a), len(kept_p), kept_a.sum(),
kept_p.sum(), ' new ',
len(partpost['x']) - kept_p.sum())
nnew += len(partpost['x']) - kept_p.sum()
""" PROCESSING OF DEADBORNE PARCELS
Manage the parcels launched during the last 6-hour which have already
exited and do not appear in posold or posact (borne dead parcels).
These parcels are stored in the last part of posact, at most
the last granule_quanta parcels. """
if numpart_s < numpart:
print("manage deadborne", flush=True)
# First index of the current quanta """
numpart_s += granule_quanta
print("idx1", idx1, " numpart_s", numpart_s)
# Extract the last granule_size indexes from posacti, this is where the new parcels should be
if hour == step:
idx_act = partpost['idx_back']
else:
idx_act = partpost['idx_back'][-granule_quanta:]
# Generate the list of indexes that should be found in this range
# ACHTUNG ACHTUNG : this works because IDX_ORGN=1, FIX THAT
idx_theor = np.arange(idx1, numpart_s + IDX_ORGN)
# Find the missing indexes in idx_act (make a single line after validation)
kept_borne = np.in1d(idx_theor, idx_act, assume_unique=True)
idx_deadborne = idx_theor[~kept_borne]
# Process these parcels by assigning exit at initial location
prod0['flag_source'][
idx_deadborne -
IDX_ORGN] = prod0['flag_source'][idx_deadborne -
IDX_ORGN] | I_DEAD + I_DBORNE
prod0['src']['x'][idx_deadborne -
IDX_ORGN] = part0['x'][idx_deadborne - IDX_ORGN]
prod0['src']['y'][idx_deadborne -
IDX_ORGN] = part0['y'][idx_deadborne - IDX_ORGN]
prod0['src']['p'][idx_deadborne -
IDX_ORGN] = part0['p'][idx_deadborne - IDX_ORGN]
prod0['src']['t'][idx_deadborne -
IDX_ORGN] = part0['t'][idx_deadborne - IDX_ORGN]
prod0['src']['age'][idx_deadborne - IDX_ORGN] = 0.
print("number of deadborne ", len(idx_deadborne))
ndborne += len(idx_deadborne)
idx1 = numpart_s + IDX_ORGN
""" PROCESSING OF CROSSED PARCELS """
if len(kept_a) > 0:
exits = exiter(int((partante['itime']+partpost['itime'])/2), \
partante['x'][~kept_a],partante['y'][~kept_a],partante['p'][~kept_a],\
partante['t'][~kept_a],partante['idx_back'][~kept_a],\
prod0['flag_source'],prod0['src']['x'],prod0['src']['y'],\
prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],\
part0['ir_start'], satmap.range)
nexits += exits
print('exit ', nexits, exits, np.sum(~kept_a),
len(kept_a) - len(kept_p))
""" PROCESSING OF PARCELS WHICH ARE COMMON TO THE TWO OUTPUTS """
# Select the kept parcels which have not been hit yet
# !!! Never use and between two lists, the result is wrong
if kept_p.sum() == 0:
live_a = live_p = kept_p
else:
live_a = np.logical_and(
kept_a, (prod0['flag_source'][partante['idx_back'] - IDX_ORGN]
& I_DEAD) == 0)
live_p = np.logical_and(
kept_p, (prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_DEAD) == 0)
print('live a, p ', live_a.sum(), live_p.sum())
del kept_a
del kept_p
# Build generator for parcel locations of the 5' slices
gsp = get_slice_part(partante, partpost, live_a, live_p, current_date,
dstep, slice_width)
if verbose: print('built parcel generator for ', current_date)
""" MAIN LOOP ON THE PARCEL TIME SLICES """
for i in range(nb_slices):
# get the slice for the particles
datpart = next(gsp)
if verbose: print('part slice ', i, datpart['time'])
# Make sure the present satellite slice is OK
# The while should ensure that the run synchronizes
# when it starts.
while satmap.check('MSG1', datpart['time']) is False:
# if not get next satellite slice
try:
void = next(satfill['MSG1'])
# read new satellite file if the slice generator is over
# make a new slice generator and get first slice
except:
datsat['MSG1'] = next(get_sat['MSG1'])
satfill['MSG1'] = satmap.fill('MSG1', datsat)
void = next(satfill['MSG1'])
finally:
if verbose:
print('check MSG1 ',
satmap.check('MSG1', datpart['time']), '##',
datpart['time'], '##', satmap.zone['MSG1']['ti'],
'##', satmap.zone['MSG1']['tf'])
while satmap.check('Hima', datpart['time']) is False:
try:
void = next(satfill['Hima'])
except:
datsat['Hima'] = next(get_sat['Hima'])
satfill['Hima'] = satmap.fill('Hima', datsat)
void = next(satfill['Hima'])
finally:
if verbose:
print('check Hima ',
satmap.check('Hima', datpart['time']), '##',
datpart['time'], '##', satmap.zone['Hima']['ti'],
'##', satmap.zone['Hima']['tf'])
""" PROCESS THE COMPARISON OF PARCEL PRESSURES TO CLOUDS """
if len(datpart['x']) > 0:
nhits += convbirth(datpart['itime'],
datpart['x'],datpart['y'],datpart['p'],datpart['t'],datpart['idx_back'],\
prod0['flag_source'],prod0['src']['x'],prod0['src']['y'],\
prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],\
satmap.ptop, part0['ir_start'],\
satmap.range[0,0],satmap.range[1,0],satmap.stepx,satmap.stepy,satmap.binx,satmap.biny)
sys.stdout.flush()
""" End of of loop on slices """
# Check the age limit (easier to do it here)
print("Manage age limit", flush=True)
age_sec = part0['ir_start'][partante['idx_back'] -
IDX_ORGN] - partante['itime']
IIold_o = age_sec > (age_bound - 0.25) * 86400
IIold_o = IIold_o & (
(prod0['flag_source'][partante['idx_back'] - IDX_ORGN] & I_STOP)
== 0)
idx_IIold = partante['idx_back'][IIold_o]
j_IIold_o = np.where(IIold_o)
prod0['flag_source'][idx_IIold - IDX_ORGN] = prod0['flag_source'][
idx_IIold - IDX_ORGN] | I_DEAD + I_OLD
prod0['src']['x'][idx_IIold - IDX_ORGN] = partante['x'][j_IIold_o]
prod0['src']['y'][idx_IIold - IDX_ORGN] = partante['y'][j_IIold_o]
prod0['src']['p'][idx_IIold - IDX_ORGN] = partante['p'][j_IIold_o]
prod0['src']['t'][idx_IIold - IDX_ORGN] = partante['t'][j_IIold_o]
prod0['src']['age'][idx_IIold - IDX_ORGN] = (
(part0['ir_start'][idx_IIold - IDX_ORGN] - partante['itime']) /
86400)
print("number of IIold ", len(idx_IIold))
nold += len(idx_IIold)
# find parcels still alive if kept_p.sum()==0:
try:
nlive = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_DEAD) == 0).sum()
#n_nohit = ((prod0['flag_source'][partpost['idx_back']-IDX_ORGN] & I_HIT) == 0).sum()
except:
nlive = 0
#n_nohit =0
print('end hour ',hour,' numact', partpost['nact'],' nnew',nnew, ' nexits',nexits,' nhits',nhits, ' nlive',nlive,\
' nold',nold,' ndborne',ndborne)
# check that nold + nlive + nhits + nexits = nnew
if nnew != nexits + nhits + nlive + nold:
print('@@@ ACHTUNG nnew not equal to sum ', nnew,
nexits + nhits + nlive + nold)
""" End of the procedure and storage of the result """
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use before clean: {:4.2f} gb'.format(memoryUse))
del partante
del partpost
del live_a
del live_p
del datsat
del datpart
del satfill
prod0['rvs'] = prod0['rvs'].astype(np.float32)
for var in ['age', 'p', 't', 'x', 'y']:
prod0['src'][var] = prod0['src'][var].astype(np.float32)
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use after clean: {:4.2f} gb'.format(memoryUse))
#output file
#pickle.dump(prod0,gzip.open(out_file,'wb'))
try:
dd.io.save(out_file1, prod0, compression='blosc')
except:
print('error with dd blosc')
try:
dd.io.save(out_file, prod0, compression='zlib')
except:
print('error with dd zlib')
try:
pickle.dump(prod0, open(out_file2, 'wb'))
except:
print('error with pickle')
# close the print file
if quiet: fsock.close()
def main():
global IDX_ORGN
parser = argparse.ArgumentParser()
parser.add_argument("-y", "--year", type=int, help="year")
parser.add_argument("-m1",
"--month1",
type=int,
choices=1 + np.arange(12),
help="month1")
parser.add_argument("-m2",
"--month2",
type=int,
choices=1 + np.arange(12),
help="month2")
parser.add_argument("-d1",
"--day1",
type=int,
choices=1 + np.arange(31),
help="day1")
parser.add_argument("-d2",
"--day2",
type=int,
choices=1 + np.arange(31),
help="day2")
parser.add_argument("-a",
"--advect",
type=str,
choices=["OPZ", "EAD", "EAZ", "EID", "EIZ"],
help="source of advecting winds")
parser.add_argument("-s",
"--suffix",
type=str,
help="suffix for special cases")
parser.add_argument("-q",
"--quiet",
type=str,
choices=["y", "n"],
help="quiet (y) or not (n)")
parser.add_argument("-l", "--level", type=int, help="PT level")
#parser.add_argument("-c","--clean0",type=bool,help="clean part_000")
parser.add_argument("-t",
"--step",
type=int,
help="step in hour between two part files")
#parser.add_argument("-ct","--cloud_type",type=str,choices=["meanhigh","veryhigh","silviahigh"],help="cloud type filter")
#parser.add_argument("-k","--diffus",type=str,choices=['01','1','001'],help='diffusivity parameter')
parser.add_argument("-v",
"--vshift",
type=int,
choices=[0, 10],
help='vertical shift')
parser.add_argument("-hm",
"--hmax",
type=int,
help='maximum considered integration time (hour)')
parser.add_argument("-b",
"--bak",
type=str,
choices=["y", "n"],
help="backup (y) or not (n)")
parser.add_argument("-c",
"--cont",
type=str,
choices=["y", "n"],
help="continuation (y) or not (n)")
parser.add_argument("-bs",
"--bakstep",
type=int,
help='backup step (hour)')
# to be updated
# Define main directories
if 'ciclad' in socket.gethostname():
#main_sat_dir = '/data/legras/flexpart_in/SAFNWC'
#SVC_Dir = '/bdd/CFMIP/SEL2'
traj_dir = '/data/legras/flexout/STC/Sivan'
out_dir = '/data/legras/STC'
#out_dir = '/data/legras/STC'
elif ('climserv' in socket.gethostname()) | ('polytechnique'
in socket.gethostname()):
traj_dir = '/data/legras/flexout/STC/Sivan'
out_dir = '/homedata/legras/STC'
else:
print('CANNOT RECOGNIZE HOST - DO NOT RUN ON NON DEFINED HOSTS')
exit()
""" Parameters """
# Output step of the trajectories in hour
step = 6
hmax = 3360
#hmax = 18
dstep = timedelta(hours=step)
# time width of the parcel slice
# might be better passed as a parameter
age_bound = 44.5
# time width of the parcel slice
slice_width = timedelta(minutes=5)
# number of slices between two outputs
nb_slices = int(dstep / slice_width)
# Number of parcels launched per time slot (1 per degree on a 170x50 grid)
granule_size = 8500
# number of granules in step hours
granule_step = 6
# size of granules launched during step hours
granule_quanta = granule_size * granule_step
# default values of parameters
# date of the flight
year = 2017
month1 = 7
month2 = 9
day1 = 1
day2 = 30
advect = 'EAD'
suffix = ''
quiet = False
clean0 = True
backup = False
backup_step = 40 * step
restart = False
#cloud_type = 'silviahigh'
#diffus = '01'
vshift = 0
level = 380
super = ''
# Step of GridSat
dtRange = timedelta(hours=3)
args = parser.parse_args()
if args.year is not None: year = args.year
if args.month1 is not None: month1 = args.month1
if args.month2 is not None: month2 = args.month2
if args.advect is not None: advect = args.advect
if args.day1 is not None: day1 = args.day1
if args.day2 is not None: day2 = args.day2
if args.level is not None: level = args.level
if args.hmax is not None: hmax = args.hmax
if args.suffix is not None: suffix = '-' + args.suffix
if args.quiet is not None:
if args.quiet == 'y': quiet = True
else: quiet = False
#if args.clean0 is not None: clean0 = args.clean0
#if args.cloud_type is not None: cloud_type = args.cloud_type
if args.step is not None: step = args.step
#if args.diffus is not None: diffus = args.diffus
#diffus = '-D' + diffus
if args.vshift is not None:
vshift = args.vshift
if vshift > 0:
super = '-super' + str(vshift)
if args.bak is not None:
if args.bak == 'y': backup = True
else: backup = False
if args.bakstep is not None: backup_step = args.bakstep
if args.cont is not None:
if args.cont == 'y': restart = True
else: restart = False
# Update the out_dir with the cloud type and the super paramater
out_dir = os.path.join(out_dir, 'STC-Sivan-OUT-GridSat-WMO' + super)
try:
os.mkdir(out_dir)
os.mkdir(out_dir + '/out')
except:
print('out_dir directory already created')
# Dates beginning and end
date_beg = datetime(year=year, month=month1, day=day1, hour=0)
date_end = datetime(year=year, month=month2, day=day2, hour=0)
# Manage the file that receives the print output
if quiet:
# Output file
#print_file = os.path.join(out_dir,'out','BACK-SVC-EAD-'+date_beg.strftime('%b-%Y-day%d-')+date_end.strftime('%d-D01')+'.out')
print_file = os.path.join(
out_dir, 'out', 'Sivan-T2-' + advect + '-JAS-' +
date_beg.strftime('%Y-') + str(level) + 'K.out')
fsock = open(print_file, 'w')
sys.stdout = fsock
# initial time to read the sat files
# should be after the latest launch date
sdate = date_end + timedelta(days=1)
print('year', year, 'month1', month1, 'day1', day1, 'month2', month2,
'day2', day2)
print('advect', advect)
print('suffix', suffix)
# patch to fix the problem of the data hole on the evening of 2 August 2017
# should not happen
#if sdate == datetime(2017,8,2): sdate = datetime(2017,8,3,6)
# Directory of the backward trajectories (input) and name of the output file
ftraj = os.path.join(
traj_dir, 'Sivan-' + advect + '-JAS-' + date_beg.strftime('%Y-') +
str(level) + 'K')
out_file2 = os.path.join(
out_dir, 'Sivan-T2-' + advect + '-JAS-' + date_beg.strftime('%Y-') +
str(level) + 'K.hdf5')
# backup file
if backup:
bak_file_prod0 = os.path.join(
out_dir, 'Sivan-T2-' + advect + '-JAS-' +
date_beg.strftime('%Y-') + str(level) + 'K-backup-prod0.hdf5')
bak_file_params = os.path.join(
out_dir, 'Sivan-T2-' + advect + '-JAS-' +
date_beg.strftime('%Y-') + str(level) + 'K-backup-params.hdf5')
""" Initialization of the calculation """
# Initialize the dictionary of the parcel dictionaries
partStep = {}
# initialize a grid that will be used to before actually doing any read
gg = geosat.GeoGrid('GridSat')
# Build the satellite field generator
get_sat = read_sat(sdate, dtRange, pre=True, vshift=vshift)
# Build the ECMWF field generator (both available at 3 hour interval)
get_ERA5 = read_ERA5(sdate, dtRange, pre=True)
# Read the index file that contains the initial positions
part0 = readidx107(os.path.join(ftraj, 'part_000'), quiet=False)
print('numpart', part0['numpart'])
numpart = part0['numpart']
numpart_s = granule_size # because of parcels launched at time 0
# stamp_date not set in these runs
# current_date actually shifted by one day / sdate
# We assume here that part time is defined from this day at 0h
# sdate defined above should be equal or posterior to current_date
current_date = sdate
# check flag is clean
print('check flag is clean ',((part0['flag']&I_HIT)!=0).sum(),((part0['flag']&I_DEAD)!=0).sum(),\
((part0['flag']&I_CROSSED)!=0).sum())
# check idx_orgn
if part0['idx_orgn'] != 0:
print(
'MINCHIA, IDX_ORGN NOT 0 ASSTC-M55-OUT-SAF-super1silviahigh ASSUMED, CORRECTED WITH READ VALUE'
)
print('VALUE ', part0['idx_orgn'])
IDX_ORGN = part0['idx_orgn']
idx1 = IDX_ORGN
# Build a dictionary to host the results
prod0 = defaultdict(dict)
prod0['src']['x'] = np.empty(part0['numpart'], dtype='float')
prod0['src']['y'] = np.empty(part0['numpart'], dtype='float')
prod0['src']['p'] = np.empty(part0['numpart'], dtype='float')
prod0['src']['t'] = np.empty(part0['numpart'], dtype='float')
prod0['src']['age'] = np.empty(part0['numpart'], dtype='int')
prod0['flag_source'] = part0['flag']
# Set rvs at arbitrary large value
prod0['rvs'] = np.full(part0['numpart'], 0.01, dtype='float')
# truncate eventually to 32 bits at the output stage
# read the part_000 file
if not restart:
partStep[0] = readpart107(0, ftraj, quiet=False)
# cleaning is necessary for runs starting in the fake restart mode
# otherwise all parcels are thought to exit at the first step
if clean0:
partStep[0]['idx_back'] = []
# parcels with longitude east of zero degree are set to negative values
partStep[0]['x'][partStep[0]['x'] > 180] -= 360
# number of hists and exits
nhits = 0
nexits = 0
ndborne = 0
nnew = granule_size # because of parcels launched at time 0
nold = 0
offset = 0
# initialize datsat to None to force first read
datsat = None
# used to get non borne parcels (presently unused)
#new = np.empty(part0['numpart'],dtype='bool')
#new.fill(False)
print('Initialization completed')
if restart:
print('restart run')
try:
params = fl.load(bak_file_params)
prod0 = fl.load(bak_file_prod0)
except:
print('cannot load backup')
return -1
[
offset, nhits, nexits, nold, ndborne, nnew, idx1, numpart_s,
current_date
] = params['params']
partStep[offset] = readpart107(offset, ftraj, quiet=True)
partStep[offset]['x'][partStep[offset]['x'] > 180] -= 360
# Initialize sat and ERA5 yield one step ahead as a precaution
get_sat = read_sat(current_date + dstep,
dtRange,
pre=True,
vshift=vshift)
get_ERA5 = read_ERA5(current_date + dstep, dtRange, pre=True)
""" Main loop on the output time steps """
for hour in range(step + offset, hmax + 1, step):
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use: {:4.2f} gb'.format(memoryUse))
# Get rid of dictionary no longer used
if hour >= 2 * step:
try:
del partStep[hour - 2 * step]
except:
print('nothing to delete')
# Read the new data
partStep[hour] = readpart107(hour, ftraj, quiet=True)
# Link the names as views
partante = partStep[hour - step]
partpost = partStep[hour]
# parcels with longitude east of zero degree are set to negative values
# done with try to prevent errors when the file is empty
try:
partpost['x'][partpost['x'] > 180] -= 360
#print("partpost x ",partpost['x'].min(),partpost['x'].max())
except:
pass
if partpost['nact'] > 0:
print('hour ', hour, ' numact ', partpost['nact'], ' max p ',
partpost['p'].max())
else:
print('hour ', hour, ' numact ', partpost['nact'])
# New date valid for partpost
current_date -= dstep
# Processing of water mixing ratio
# Select non stopped parcels in partante
if len(partante['idx_back']) > 0:
selec = (prod0['flag_source'][partante['idx_back'] - IDX_ORGN]
& I_STOP) == 0
idx = partante['idx_back'][selec]
prod0['rvs'][idx-IDX_ORGN] = np.minimum(prod0['rvs'][idx-IDX_ORGN],\
satratio(partante['p'][selec],partante['t'][selec]))
""" Select the parcels that are common to the two steps
ketp_a is a logical field with same length as partante
kept_p is a logical field with same length as partpost
After the launch of the earliest parcel along the flight track, there
should not be any member in new
The parcels
"""
kept_a = np.in1d(partante['idx_back'],
partpost['idx_back'],
assume_unique=True)
kept_p = np.in1d(partpost['idx_back'],
partante['idx_back'],
assume_unique=True)
#new_p = ~np.in1d(partpost['idx_back'],partpost['idx_back'],assume_unique=True)
print('kept a, p ', len(kept_a), len(kept_p), kept_a.sum(),
kept_p.sum(), ' new ',
len(partpost['x']) - kept_p.sum())
nnew += len(partpost['x']) - kept_p.sum()
""" PROCESSING OF DEADBORNE PARCELS
Manage the parcels launched during the last 6-hour which have already
exited and do not appear in posold or posact (borne dead parcels).
These parcels are stored in the last part of posact, at most
the last granule_quanta parcels. """
if numpart_s < numpart:
# First index of the current quanta """
numpart_s += granule_quanta
print("manage deadborne idx1", idx1, " numpart_s", numpart_s)
# Extract the last granule_size indexes from posacti, this is where the new parcels should be
if hour == step:
idx_act = partpost['idx_back']
else:
idx_act = partpost['idx_back'][-granule_quanta:]
# Generate the list of indexes that should be found in this range
# This should works for both values of IDX_orgn
idx_theor = np.arange(idx1, numpart_s + IDX_ORGN)
# Find the missing indexes in idx_act (make a single line after validation)
kept_borne = np.in1d(idx_theor, idx_act, assume_unique=True)
idx_deadborne = idx_theor[~kept_borne]
# Process these parcels by assigning exit at initial location
prod0['flag_source'][
idx_deadborne -
IDX_ORGN] = prod0['flag_source'][idx_deadborne -
IDX_ORGN] | I_DEAD + I_DBORNE
prod0['src']['x'][idx_deadborne -
IDX_ORGN] = part0['x'][idx_deadborne - IDX_ORGN]
prod0['src']['y'][idx_deadborne -
IDX_ORGN] = part0['y'][idx_deadborne - IDX_ORGN]
prod0['src']['p'][idx_deadborne -
IDX_ORGN] = part0['p'][idx_deadborne - IDX_ORGN]
prod0['src']['t'][idx_deadborne -
IDX_ORGN] = part0['t'][idx_deadborne - IDX_ORGN]
prod0['src']['age'][idx_deadborne - IDX_ORGN] = 0.
print("number of deadborne ", len(idx_deadborne))
ndborne += len(idx_deadborne)
idx1 = numpart_s + IDX_ORGN
""" PROCESSING OF CROSSED PARCELS """
if len(kept_a) > 0:
exits = exiter(int((partante['itime']+partpost['itime'])/2), \
partante['x'][~kept_a],partante['y'][~kept_a],partante['p'][~kept_a],\
partante['t'][~kept_a],partante['idx_back'][~kept_a],\
prod0['flag_source'],prod0['src']['x'],prod0['src']['y'],\
prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],\
part0['ir_start'], gg.box_range)
nexits += exits
print('exit ', nexits, exits, np.sum(~kept_a),
len(kept_a) - len(kept_p))
""" PROCESSING OF PARCELS WHICH ARE COMMON TO THE TWO OUTPUTS """
# Select the kept parcels which have not been hit yet
# !!! Never use and between two lists, the result is wrong
if kept_p.sum() == 0:
live_a = live_p = kept_p
else:
live_a = np.logical_and(
kept_a, (prod0['flag_source'][partante['idx_back'] - IDX_ORGN]
& I_DEAD) == 0)
live_p = np.logical_and(
kept_p, (prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_DEAD) == 0)
print('live a, b ', live_a.sum(), live_p.sum())
""" Correction step that moves partante['x'] to avoid big jumps at the periodicity frontier on the x-axis
To be activated in the FULL version"""
# if kept_p.sum()>0:
# diffx = partpost['x'][live_p] - partante['x'][live_a]
# bb = np.zeros(len(diffx))
# bb[diffx>180] = 360
# bb[diffx<-180] = -360
# partante['x'][live_a] += bb
# del bb
# del diffx
# del kept_p
# del kept_a
# DOES not work in the following WAY
#partante['x'][live_a][diffx>180] += 360
#partante['x'][live_a][diffx<-180] -= 360
# Build generator for parcel locations of the 5' slices
gsp = get_slice_part(partante, partpost, live_a, live_p, current_date,
dstep, slice_width)
if verbose: print('built parcel generator for ', current_date)
""" MAIN LOOP ON THE PARCEL TIME SLICES """
for i in range(nb_slices):
# get the slice for the particles
datpart = next(gsp)
# Check x within (-180,180), necessary when GridSat is used
#if datpart['x'] != []:
if len(datpart['x']) > 0:
datpart['x'] = (datpart['x'] + 180) % 360 - 180
if verbose: print('part slice ', i, datpart['time'])
# Check whether the present satellite image is valid
# The while should ensure that the run synchronizes
# when it starts.
while check(datsat, datpart['time']) is False:
# if not get next satellite image
datsat = next(get_sat)
datera = next(get_ERA5)
""" PROCESS THE COMPARISON OF PARCEL PRESSURES TO CLOUDS """
if len(datpart['x']) > 0:
# tropopause pressure at the location of the parcels
#print('ptropo x ',datpart['x'].min(),datpart['x'].max())
ptrop = datera.fP(np.transpose([datpart['y'], datpart['x']]))
nhits += convbirth(datpart['itime'],
datpart['x'],datpart['y'],datpart['p'],datpart['t'],datpart['idx_back'],\
prod0['flag_source'],ptrop,prod0['src']['x'],prod0['src']['y'],\
prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],\
datsat.var['IR0'], part0['ir_start'],\
datsat.geogrid.box_range[0,0],datsat.geogrid.box_range[1,0],datsat.geogrid.stepx,\
datsat.geogrid.stepy,datsat.geogrid.box_binx,datsat.geogrid.box_biny)
""" End of of loop on slices """
""" INSERT HERE CODE FOR PARCELS ENDING BY AGE
Important: if the age limit is set in the run, the age_bound needs to
be decremented by one output step. Otherwise, the parcel disappeared
and is wrongly processed as crossed."""
# Check the age limit (easier to do it here)
if len(partante['idx_back']) > 0:
age_sec = part0['ir_start'][partante['idx_back'] -
IDX_ORGN] - partante['itime']
IIold_o = age_sec > (age_bound - (step / 24)) * 86400
IIold_o = IIold_o & (
(prod0['flag_source'][partante['idx_back'] - IDX_ORGN]
& I_STOP) == 0)
idx_IIold = partante['idx_back'][IIold_o]
j_IIold_o = np.where(IIold_o)
prod0['flag_source'][idx_IIold - IDX_ORGN] = prod0['flag_source'][
idx_IIold - IDX_ORGN] | I_DEAD + I_OLD
prod0['src']['x'][idx_IIold - IDX_ORGN] = partante['x'][j_IIold_o]
prod0['src']['y'][idx_IIold - IDX_ORGN] = partante['y'][j_IIold_o]
prod0['src']['p'][idx_IIold - IDX_ORGN] = partante['p'][j_IIold_o]
prod0['src']['t'][idx_IIold - IDX_ORGN] = partante['t'][j_IIold_o]
prod0['src']['age'][idx_IIold - IDX_ORGN] = (
(part0['ir_start'][idx_IIold - IDX_ORGN] - partante['itime']) /
86400)
print("manage age limit: number of IIold ", len(idx_IIold))
nold += len(idx_IIold)
# find parcels still alive after processing this step if kept_p.sum()==0:
try:
nlive = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_DEAD) == 0).sum()
n_nohit = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_HIT) == 0).sum()
except:
nlive = 0
n_nohit = 0
# check that nlive + nhits + nexits = numpart, should be true after the first day
if numpart_s != nexits + nhits + nlive + ndborne + nold:
print('@@@ ACHTUNG numpart_s not equal to sum ', numpart_s,
nexits + nhits + nlive + ndborne + nold)
print('end hour ',
hour,
' numact',
partpost['nact'],
' nexits',
nexits,
' nhits',
nhits,
' nlive',
nlive,
' nohit',
n_nohit,
' nold',
nold,
flush=True)
sys.stdout.flush()
if backup & (hour % backup_step == 0):
fl.save(bak_file_prod0, prod0)
fl.save(
bak_file_params, {
'params': [
hour, nhits, nexits, nold, ndborne, nnew, idx1,
numpart_s, current_date
]
})
""" End of the procedure and storage of the result """
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use before clean: {:4.2f} gb'.format(memoryUse))
del partante
del partpost
del live_a
del live_p
del datpart
prod0['rvs'] = prod0['rvs'].astype(np.float32)
for var in ['age', 'p', 't', 'x', 'y']:
prod0['src'][var] = prod0['src'][var].astype(np.float32)
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use after clean: {:4.2f} gb'.format(memoryUse))
#output file
fl.save(out_file2, prod0, compression='zlib')
#pickle.dump(prod0,gzip.open(out_file,'wb'))
# close the print file
if quiet: fsock.close()
def main():
global IDX_ORGN
parser = argparse.ArgumentParser()
parser.add_argument("-y", "--year", type=int, help="year")
parser.add_argument("-m",
"--month",
type=int,
choices=1 + np.arange(12),
help="month")
parser.add_argument("-d",
"--day",
type=int,
choices=1 + np.arange(31),
help="day")
parser.add_argument("-a",
"--advect",
type=str,
choices=["OPZ", "EAD", "EAZ", "EID", "EIZ"],
help="source of advecting winds")
parser.add_argument("-p",
"--platform",
type=str,
choices=["M55", "GLO", "BAL"],
help="measurement platform")
parser.add_argument("-n",
"--launch_number",
type=int,
help="balloon launch number within a day")
parser.add_argument("-s",
"--suffix",
type=str,
help="suffix for special cases")
parser.add_argument("-q",
"--quiet",
type=str,
choices=["y", "n"],
help="quiet (y) or not (n)")
parser.add_argument("-t",
"--tau",
type=int,
help="cloud cover time scale (h)")
# to be updated
if socket.gethostname() == 'graphium':
pass
elif 'ciclad' in socket.gethostname():
#root_dir = '/home/legras/STC/STC-M55'
traj_dir = '/data/legras/flexout/STC/M55'
out_dir = '/data/legras/STC'
elif ('climserv' in socket.gethostname()) | ('polytechnique'
in socket.gethostname()):
#root_dir = '/home/legras/STC/STC-M55'
traj_dir = '/bdd/STRATOCLIM/flexout/M55'
out_dir = '/homedata/legras/STC'
elif socket.gethostname() == 'grapelli':
pass
elif socket.gethostname() == 'gort':
pass
else:
print('CANNOT RECOGNIZE HOST - DO NOT RUN ON NON DEFINED HOSTS')
exit()
""" Parameters """
# to do (perhaps) : some parameters might be parsed from command line
# step and max output time
step = 6
hmax = 732
dstep = timedelta(hours=step)
# time width of the parcel slice
slice_width = timedelta(hours=1)
# number of slices between two outputs
nb_slices = int(dstep / slice_width)
# defines here the offset for the cloud-cover
cc_offset = 0.001
# defines the domain
domain = np.array([[-10., 160.], [0., 50.]])
# default values of parameters
# date of the flight
year = 2017
month = 7
day = 29
platform = 'GLO'
advect = 'EAZ'
suffix = ''
launch_number = ''
quiet = False
tau = 3
args = parser.parse_args()
if args.year is not None:
year = args.year
if args.month is not None:
month = args.month
if args.day is not None:
day = args.day
if args.advect is not None:
advect = args.advect
if args.platform is not None:
platform = args.platform
if args.launch_number is not None:
launch_number = '-' + str(args.launch_number)
if args.suffix is not None:
suffix = '-' + args.suffix
if args.quiet is not None:
if args.quiet == 'y':
quiet = True
else:
quiet = False
if args.tau is not None:
tau = args.tau
# Change tau into its inverse in s**-1
tauf = '-{:d}H'.format(tau)
tau = 1 / (3600 * tau)
# Update the out_dir with the platform
out_dir = os.path.join(out_dir, 'STC-' + platform + '-CC-OUT')
fdate = datetime(year, month, day)
# Manage the file that receives the print output
if quiet:
# Output file
print_file = os.path.join(
out_dir, 'out', platform + fdate.strftime('-%Y%m%d') +
launch_number + '-' + advect + '-D01' + suffix + tauf + '.out')
fsock = open(print_file, 'w')
sys.stdout = fsock
# initial time to read the sat files
# should be after the end of the flight
# and a 12h or 0h boundary
print('year', year, 'month', month, 'day', day)
print('advect', advect)
print('platform', platform)
print('launch_number', launch_number)
print('suffix', suffix)
print('tau', tau)
# Directory of the backward trajectories
ftraj = os.path.join(
traj_dir, platform + fdate.strftime('-%Y%m%d') + launch_number + '-' +
advect + '-D01' + suffix)
# Output file
out_file = os.path.join(
out_dir, platform + fdate.strftime('-%Y%m%d') + launch_number + '-' +
advect + '-D01' + suffix + tauf + '.hdf5')
""" Initialization of the calculation """
# Initialize the dictionary of the parcel dictionaries
partStep = {}
# Read the index file that contains the initial positions
part0 = readidx107(os.path.join(ftraj, 'index_old'), quiet=False)
print('numpart', part0['numpart'])
# stamp_date not set in these runs
# current_date actually shifted by one day / sdate
current_date = fdate + timedelta(days=1)
# check flag is clean
print('check flag is clean ',((part0['flag']&I_HIT)!=0).sum(),((part0['flag']&I_DEAD)!=0).sum(),\
((part0['flag']&I_CROSSED)!=0).sum())
# check idx_orgn
if part0['idx_orgn'] != 0:
print('MINCHIA, IDX_ORGN NOT 0 AS ASSUMED, CORRECTED WITH READ VALUE')
print('VALUE ', part0['idx_orgn'])
IDX_ORGN = part0['idx_orgn']
# Build a dictionary to host the results
prod0 = defaultdict(dict)
# Locations of the crossing and detrainement
nsrc = 6
prod0['src']['x'] = np.empty(shape=(nsrc, part0['numpart']), dtype='float')
prod0['src']['y'] = np.empty(shape=(nsrc, part0['numpart']), dtype='float')
prod0['src']['p'] = np.empty(shape=(nsrc, part0['numpart']), dtype='float')
prod0['src']['t'] = np.empty(shape=(nsrc, part0['numpart']), dtype='float')
prod0['src']['age'] = np.empty(shape=(nsrc, part0['numpart']),
dtype='float')
prod0['src']['x'].fill(np.nan)
prod0['src']['y'].fill(np.nan)
prod0['src']['p'].fill(np.nan)
prod0['src']['t'].fill(np.nan)
prod0['src']['age'].fill(np.nan)
# Flag is copied from index
prod0['flag_source'] = part0['flag']
# Ininitalize the erosion
prod0['chi'] = np.empty(part0['numpart'], dtype='float')
prod0['passed'] = np.empty(part0['numpart'], dtype='int')
prod0['chi'].fill(1.)
prod0['passed'].fill(10)
# Build the interpolator to the hybrid level
fhyb, void = tohyb()
#vfhyb = np.vectorize(fhyb)
# Read the part_000 file
partStep[0] = readpart107(0, ftraj, quiet=True)
# number of hists and exits
nhits = np.array([0, 0, 0, 0, 0, 0])
nexits = 0
ndborne = 0
nnew = 0
nradada = 0
# used to get non borne parcels
new = np.empty(part0['numpart'], dtype='bool')
new.fill(False)
print('Initialization completed')
""" Main loop on the output time steps """
for hour in range(step, hmax + 1, step):
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use: {:4.2f} gb'.format(memoryUse))
# Get rid of dictionary no longer used
if hour >= 2 * step: del partStep[hour - 2 * step]
# Read the new data
partStep[hour] = readpart107(hour, ftraj, quiet=True)
# Link the names
partante = partStep[hour - step]
partpost = partStep[hour]
if partpost['nact'] > 0:
print('hour ', hour, ' numact ', partpost['nact'], ' max p ',
partpost['p'].max())
else:
print('hour ', hour, ' numact ', partpost['nact'])
# New date valid for partpost
current_date -= dstep
""" Select the parcels that are common to the two steps
ketp_a is a logical field with same length as partante
kept_p is a logical field with same length as partpost
After the launch of the earliest parcel along the flight track, there
should not be any member in new.
"""
kept_a = np.in1d(partante['idx_back'],
partpost['idx_back'],
assume_unique=True)
kept_p = np.in1d(partpost['idx_back'],
partante['idx_back'],
assume_unique=True)
#new_p = ~np.in1d(partpost['idx_back'],partpost['idx_back'],assume_unique=True)
print('kept a, p ', len(kept_a), len(kept_p), kept_a.sum(),
kept_p.sum(), ' new ',
len(partpost['x']) - kept_p.sum())
""" IDENTIFY AND TAKE CARE OF DEADBORNE AS NON BORNE PARCELS """
# Find and count the parcels in partpost which where not in partante, flag them as new
if (hour <= 30) & (partpost['nact'] > 0):
new[partpost['idx_back'][~kept_p] - IDX_ORGN] = True
nnew += len(partpost['x']) - kept_p.sum()
# When the release of parcels has ended, flag the ones that did not show up as dead
# source their last location as the launching location
if hour == 30:
ndborne = np.sum(~new)
prod0['flag_source'][~new] |= I_DBORNE + I_DEAD
prod0['src']['x'][0, ~new] = part0['x'][~new]
prod0['src']['y'][0, ~new] = part0['y'][~new]
prod0['src']['p'][0, ~new] = part0['p'][~new]
prod0['src']['t'][0, ~new] = part0['t'][~new]
prod0['src']['age'][0, ~new] = 0
print('number of dead borne', ndborne, part0['numpart'] - nnew)
# get rid of new
del new
""" INSERT HERE CODE FOR NEW PARCELS """
# nothing to be done for new parcels, just wait and see
""" PROCESSING OF CROSSED PARCELS """
# last known location before crossing stored in the index 0 of src fields
if len(kept_a) > 0:
exits = exiter(int((partante['itime']+partpost['itime'])/2), \
partante['x'][~kept_a],partante['y'][~kept_a],partante['p'][~kept_a],\
partante['t'][~kept_a],partante['idx_back'][~kept_a],\
prod0['flag_source'],prod0['src']['x'],prod0['src']['y'],\
prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],\
part0['ir_start'], domain)
nexits += exits
nhits[0] += exits
print('exit ', nexits, exits, np.sum(~kept_a),
len(kept_a) - len(kept_p))
""" PROCESSING OF PARCELS WHICH ARE COMMON TO THE TWO OUTPUTS """
# Select the kept parcels which have not been hit yet
# !!! Never use and between two lists, the result is wrong
if kept_p.sum() == 0:
live_a = live_p = kept_p
else:
live_a = np.logical_and(
kept_a, (prod0['flag_source'][partante['idx_back'] - IDX_ORGN]
& I_DEAD) == 0)
live_p = np.logical_and(
kept_p, (prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_DEAD) == 0)
print('live a, b ', live_a.sum(), live_p.sum())
del kept_a
del kept_p
# Build generator for live parcel locations of the 1h slices
gsp = get_slice_part(partante, partpost, live_a, live_p, current_date,
dstep, slice_width)
if verbose: print('built parcel generator for ', current_date)
""" MAIN LOOP ON THE PARCEL TIME SLICES """
for i in range(nb_slices):
# get the 1h slice for the particles
datpart = next(gsp)
# skip if no particles
if datpart['ti'] == None:
continue
print('current_date ', datpart['ti'])
#@@ test
# print('ti in main ',datpart['ti'])
# print('pi in main ',np.min(datpart['pi']),np.max(datpart['pi']))
# print('idx_back ',np.min(datpart['idx_back']),np.max(datpart['idx_back']))
# #@@ end test
# as the ECMWF files are alaos available every hour
datrean = read_ECMWF(datpart['ti'])
# calculate the -log surface pressure at parcel location at time ti
# create a 2D linear interpolar from the surface pressure field
lsp = RegularGridInterpolator((datrean.attr['lats'],datrean.attr['lons']),\
-np.log(datrean.var['SP']))
# perform the interpolation for the location of live parcels at time ti
datpart['lspi'] = lsp(np.transpose([datpart['yi'], datpart['xi']]))
#@@ test
# print('surface pressure ',np.exp(-np.min(datpart['lspi'])),np.exp(-np.max(datpart['lspi'])))
# print('particle pressure ',np.min(datpart['pi']),np.max(datpart['pi']))
#@@ end test
# get the closest hybrid level at time ti
# define first -log sigma = -log(p) - -log(ps)
lsig = -np.log(datpart['pi']) - datpart['lspi']
#@@ test
# print('sigma ',np.exp(-np.max(lsig)),np.exp(-np.min(lsig)))
#@@ end test
# get the hybrid level, the rank of the first retained level is substracted to have hyb starting from 0
hyb = np.floor(fhyb(np.transpose([lsig, datpart['lspi']])) +
0.5).astype(np.int64) - datrean.attr['levs'][0]
#@@ test the extreme values of sigma end ps
if np.min(lsig) < -np.log(0.95):
print('large sigma detected ', np.exp(-np.min(lsig)))
if np.max(datpart['lspi']) > -np.log(45000):
print('small ps detected ', np.exp(-np.max(datpart['lspi'])))
del lsig
""" PROCESS THE PARCELS WHICH ARE TOO CLOSE TO GROUND
These parcels are flagged as crossed and dead, their last location is stored in the
index 0 of src fields.
This test handles also the cases outside the interpolation domain as NaN produced by fhyb
generates very large value of hyb.
The trajectories which are stopped here have exited the domain where winds are avilable to flexpart
and therefore are wrong from this point. For this reason we label them from their last valid position."""
if np.max(hyb) > 100:
selec = hyb > 100
nr = radada(datpart['itime'], datpart['xf'][selec],
datpart['yf'][selec], datpart['pf'][selec],
datpart['tempf'][selec],
datpart['idx_back'][selec], prod0['flag_source'],
prod0['src']['x'], prod0['src']['y'],
prod0['src']['p'], prod0['src']['t'],
prod0['src']['age'], part0['ir_start'])
nradada += nr
nhits[0] += nr
""" PROCESS THE (ADJOINT) DETRAINMENT """
n1 = detrainer(datpart['itime'],
datpart['xi'],datpart['yi'],datpart['pi'],datpart['tempi'],hyb,
datpart['xf'],datpart['yf'], datrean.var['CC'], datpart['idx_back'],\
prod0['flag_source'],part0['ir_start'], prod0['chi'],prod0['passed'],\
prod0['src']['x'],prod0['src']['y'],prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],\
datrean.attr['Lo1'],datrean.attr['La1'],datrean.attr['dlo'],datrean.attr['dla'],cc_offset,tau)
nhits += n1
#@@ test
print('return from detrainer', nhits)
#@@ end test
sys.stdout.flush()
""" End of of loop on slices """
# find parcels still alive if kept_p.sum()==0:
try:
# number of parcels still alive
nlive = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_DEAD) == 0).sum()
# number of parcels still alive and not hit
nprist = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN] &
(I_DEAD + I_HIT)) == 0).sum()
# number of parcels which have hit and crossed
nouthit = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_HIT + I_CROSSED) == I_HIT + I_CROSSED).sum()
# number of parcels which heve crossed without hit
noutprist = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_HIT + I_CROSSED) == I_CROSSED).sum()
# number of parcels which have hit without crossing
nhitpure = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_HIT + I_CROSSED) == I_HIT).sum()
except:
nlive = 0
nprist = 0
nouthit = 0
noutprist = 0
nhitpure = 0
nprist = part0['numpart']
print('end hour ', hour, ' numact', partpost['nact'], ' nexits',
nexits, ' nhits', nhits)
print('nlive', nlive, ' nprist', nprist, ' nouthit', nouthit,
' noutprist', noutprist, ' nhitpure', nhitpure)
# check that nlive + nhits + nexits = numpart, should be true after the first day
if partpost[
'nact'] != nprist + nouthit + noutprist + nhitpure + ndborne:
print('@@@ ACHTUNG numact not equal to sum ', partpost['nact'],
nprist + nouthit + noutprist + nhitpure + ndborne)
""" End of the procedure and storage of the result """
# clear memory as pickle dump is memory consuming
del partante
del partpost
del partStep
del datrean
del datpart
del hyb
del live_p
del live_a
prod0['chi'] = prod0['chi'].astype(np.float32)
prod0['passed'] = prod0['passed'].astype(np.int32)
for var in ['age', 'p', 't', 'x', 'y']:
prod0['src'][var] = prod0['src'][var].astype(np.float32)
#output file
#pickle.dump(prod0,gzip.open(out_file,'wb'))
dd.io.save(out_file, prod0, compression='zlib')
# close the print file
if quiet: fsock.close()
def main():
global IDX_ORGN
parser = argparse.ArgumentParser()
parser.add_argument("-y","--year",type=int,help="year")
parser.add_argument("-m","--month",type=int,choices=1+np.arange(12),help="month")
parser.add_argument("-d","--day",type=int,choices=1+np.arange(31),help="day")
parser.add_argument("-a","--advect",type=str,choices=["OPZ","EAD","EAZ","EID","EIZ"],help="source of advecting winds")
parser.add_argument("-p","--platform",type=str,choices=["M55","GLO","BAL"],help="measurement platform")
parser.add_argument("-n","--launch_number",type=int,help="balloon launch number within a day")
parser.add_argument("-s","--suffix",type=str,help="suffix for special cases")
parser.add_argument("-q","--quiet",type=str,choices=["y","n"],help="quiet (y) or not (n)")
parser.add_argument("-c","--clean0",type=bool,help="clean part_000")
# to be updated
if socket.gethostname() == 'graphium':
pass
elif 'ciclad' in socket.gethostname():
#root_dir = '/home/legras/STC/STC-M55'
main_sat_dir = '/bdd/STRATOCLIM/flexpart_in'
traj_dir = '/data/legras/flexout/STC/M55'
out_dir = '/data/legras/STC'
elif ('climserv' in socket.gethostname()) | ('polytechnique' in socket.gethostname()):
#root_dir = '/home/legras/STC/STC-M55'
main_sat_dir = '/bdd/STRATOCLIM/flexpart_in'
traj_dir = '/bdd/STRATOCLIM/flexout/M55'
out_dir = '/homedata/legras/STC'
elif socket.gethostname() == 'grapelli':
pass
elif socket.gethostname() == 'gort':
pass
else:
print ('CANNOT RECOGNIZE HOST - DO NOT RUN ON NON DEFINED HOSTS')
exit()
""" Parameters """
# to do (perhaps) : some parameters might be parsed from command line
# step and max output time
step = 1
hmax = 732
dstep = timedelta (hours=step)
# time width of the parcel slice
slice_width = timedelta(minutes=5)
# dtRange
dtRange={'MSG1':timedelta(minutes=30),'Hima':timedelta(minutes=20)}
# number of slices between two outputs
nb_slices = int(dstep/slice_width)
# default values of parameters
# date of the flight
year=2017
month=7
day=27
platform = 'M55'
advect = 'OPZ'
suffix =''
launch_number=''
quiet = False
clean0 = False
args = parser.parse_args()
if args.year is not None:
year=args.year
if args.month is not None:
month=args.month
if args.day is not None:
day=args.day
if args.advect is not None:
advect=args.advect
if args.platform is not None:
platform=args.platform
if args.launch_number is not None:
launch_number='-'+str(args.launch_number)
if args.suffix is not None:
suffix='-'+args.suffix
if args.quiet is not None:
if args.quiet=='y':
quiet=True
else:
quiet=False
if args.clean0 is not None:
clean0 = args.clean0
# Update the out_dir with the platform
out_dir = os.path.join(out_dir,'STC-'+platform+'-OUT')
fdate = datetime(year,month,day)
# Manage the file that receives the print output
if quiet:
# Output file
print_file = os.path.join(out_dir,'out',platform+fdate.strftime('-%Y%m%d')+launch_number+'-'+advect+'-D01'+suffix+'.out')
saveout = sys.stdout
fsock = open(print_file,'w')
sys.stdout=fsock
# initial time to read the sat files
# should be after the end of the flight
# and a 12h or 0h boundary
sdate = fdate + timedelta(days=1)
print('year',year,'month',month,'day',day)
print('advect',advect)
print('platform',platform)
print('launch_number',launch_number)
print('suffix',suffix)
# Directory of the backward trajectories
ftraj = os.path.join(traj_dir,platform+fdate.strftime('-%Y%m%d')+launch_number+'-'+advect+'-D01'+suffix)
# Output file
out_file = os.path.join(out_dir,platform+fdate.strftime('-%Y%m%d')+launch_number+'-'+advect+'-D01'+suffix+'.pkl')
out_file2 = os.path.join(out_dir,platform+fdate.strftime('-%Y%m%d')+launch_number+'-'+advect+'-D01'+suffix+'.hdf5')
# Directories for the satellite cloud top files
satdir ={'MSG1':os.path.join(main_sat_dir,'StratoClim+1kmD_msg1-c'),\
'Hima':os.path.join(main_sat_dir,'StratoClim+1kmD_himawari-d')}
""" Initialization of the calculation """
# Initialize the slice map to be used as a buffer for the cloudtops
satmap = pixmap()
satfill = {}
datsat = {}
# Initialize the dictionary of the parcel dictionaries
partStep={}
# Build the satellite field generator
get_sat = {'MSG1': read_sat(sdate,dtRange['MSG1'],satdir['MSG1']),\
'Hima': read_sat(sdate,dtRange['Hima'],satdir['Hima'])}
# Read the index file that contains the initial positions
part0 = readidx107(os.path.join(ftraj,'index_old'),quiet=True)
print('numpart',part0['numpart'])
# stamp_date not set in these runs
# current_date actually shifted by one day / sdate
current_date = fdate + timedelta(days=1)
# check flag is clean
print('check flag is clean ',((part0['flag']&I_HIT)!=0).sum(),((part0['flag']&I_DEAD)!=0).sum(),\
((part0['flag']&I_CROSSED)!=0).sum())
# check idx_orgn
if part0['idx_orgn'] != 0:
print('MINCHIA, IDX_ORGN NOT 0 AS ASSUMED, CORRECTED WITH READ VALUE')
print('VALUE ',part0['idx_orgn'])
IDX_ORGN = part0['idx_orgn']
# Build a dictionary to host the results
prod0 = defaultdict(dict)
prod0['src']['x'] = np.empty(part0['numpart'],dtype='float')
prod0['src']['y'] = np.empty(part0['numpart'],dtype='float')
prod0['src']['p'] = np.empty(part0['numpart'],dtype='float')
prod0['src']['t'] = np.empty(part0['numpart'],dtype='float')
prod0['src']['age'] = np.empty(part0['numpart'],dtype='int')
prod0['flag_source'] = part0['flag']
# truncate eventually to 32 bits at the output stage
# read the part_000 file
partStep[0] = readpart107(0,ftraj,quiet=True)
# cleaning is necessary for runs starting in the fake restart mode
# otherwise all parcels are thought to exit at the first step
if clean0:
partStep[0]['idx_back']=[]
# number of hists and exits
nhits = 0
nexits = 0
ndborne = 0
nnew = 0
# used to get non borne parcels
new = np.empty(part0['numpart'],dtype='bool')
new.fill(False)
print('Initialization completed')
""" Main loop on the output time steps """
for hour in range(step,hmax+1,step):
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0]/2**30
print('memory use: {:4.2f} gb'.format(memoryUse))
# Get rid of dictionary no longer used
if hour >= 2*step: del partStep[hour-2*step]
# Read the new data
partStep[hour] = readpart107(hour,ftraj,quiet=True)
# Link the names
partante = partStep[hour-step]
partpost = partStep[hour]
if partpost['nact']>0:
print('hour ',hour,' numact ', partpost['nact'], ' max p ',partpost['p'].max())
else:
print('hour ',hour,' numact ', partpost['nact'])
# New date valid for partpost
current_date -= dstep
""" Select the parcels that are common to the two steps
ketp_a is a logical field with same length as partante
kept_p is a logical field with same length as partpost
After the launch of the earliest parcel along the flight track, there
should not be any member in new
The parcels
"""
kept_a = np.in1d(partante['idx_back'],partpost['idx_back'],assume_unique=True)
kept_p = np.in1d(partpost['idx_back'],partante['idx_back'],assume_unique=True)
#new_p = ~np.in1d(partpost['idx_back'],partpost['idx_back'],assume_unique=True)
print('kept a, p ',len(kept_a),len(kept_p),kept_a.sum(),kept_p.sum(),' new ',len(partpost['x'])-kept_p.sum())
""" IDENTIFY AND TAKE CARE OF DEADBORNE AS NON BORNE PARCELS """
if (hour <= 30) & (partpost['nact']>0):
new[partpost['idx_back'][~kept_p]-IDX_ORGN] = True
nnew += len(partpost['x'])-kept_p.sum()
if hour == 30:
ndborne = np.sum(~new)
prod0['flag_source'][~new] |= I_DBORNE + I_DEAD
prod0['src']['x'][~new] = part0['x'][~new]
prod0['src']['y'][~new] = part0['y'][~new]
prod0['src']['p'][~new] = part0['p'][~new]
prod0['src']['t'][~new] = part0['t'][~new]
prod0['src']['age'][~new] = 0
print('number of dead borne',ndborne,part0['numpart']-nnew)
del new
""" INSERT HERE CODE FOR NEW PARCELS """
# nothing to be done for new parcels, just wait and see
""" PROCESSING OF CROSSED PARCELS """
if len(kept_a)>0:
exits = exiter(int((partante['itime']+partpost['itime'])/2), \
partante['x'][~kept_a],partante['y'][~kept_a],partante['p'][~kept_a],\
partante['t'][~kept_a],partante['idx_back'][~kept_a],\
prod0['flag_source'],prod0['src']['x'],prod0['src']['y'],\
prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],\
part0['ir_start'], satmap.range)
nexits += exits
print('exit ',nexits, exits, np.sum(~kept_a), len(kept_a) - len(kept_p))
""" PROCESSING OF PARCELS WHICH ARE COMMON TO THE TWO OUTPUTS """
# Select the kept parcels which have not been hit yet
# !!! Never use and between two lists, the result is wrong
if kept_p.sum()==0:
live_a = live_p = kept_p
else:
live_a = np.logical_and(kept_a,(prod0['flag_source'][partante['idx_back']-IDX_ORGN] & I_DEAD) == 0)
live_p = np.logical_and(kept_p,(prod0['flag_source'][partpost['idx_back']-IDX_ORGN] & I_DEAD) == 0)
print('live a, b ',live_a.sum(),live_p.sum())
# Build generator for parcel locations of the 5' slices
gsp = get_slice_part(partante,partpost,live_a,live_p,current_date,dstep,slice_width)
if verbose: print('built parcel generator for ',current_date)
""" MAIN LOOP ON THE PARCEL TIME SLICES """
for i in range(nb_slices):
# get the slice for the particles
datpart = next(gsp)
if verbose: print('part slice ',i, datpart['time'])
# Make sure the present satellite slice is OK
# The while should ensure that the run synchronizes
# when it starts.
while satmap.check('MSG1',datpart['time']) is False:
# if not get next satellite slice
try:
void = next(satfill['MSG1'])
# read new satellite file if the slice generator is over
# make a new slice generator and get first slice
except:
datsat['MSG1'] = next(get_sat['MSG1'])
satfill['MSG1'] = satmap.fill('MSG1',datsat)
void = next(satfill['MSG1'])
finally:
if verbose: print('check MSG1 ',satmap.check('MSG1',datpart['time']),'##',datpart['time'],
'##',satmap.zone['MSG1']['ti'],'##',satmap.zone['MSG1']['tf'])
while satmap.check('Hima',datpart['time']) is False:
try:
void = next(satfill['Hima'])
except:
datsat['Hima'] = next(get_sat['Hima'])
satfill['Hima'] = satmap.fill('Hima',datsat)
void = next(satfill['Hima'])
finally:
if verbose: print('check Hima ',satmap.check('Hima',datpart['time']),'##',datpart['time'],
'##',satmap.zone['Hima']['ti'],'##',satmap.zone['Hima']['tf'])
""" PROCESS THE COMPARISON OF PARCEL PRESSURES TO CLOUDS """
if len(datpart['x'])>0:
nhits += convbirth(datpart['itime'],
datpart['x'],datpart['y'],datpart['p'],datpart['t'],datpart['idx_back'],\
prod0['flag_source'],prod0['src']['x'],prod0['src']['y'],\
prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],\
satmap.ptop, part0['ir_start'],\
satmap.range[0,0],satmap.range[1,0],satmap.stepx,satmap.stepy,satmap.binx,satmap.biny)
sys.stdout.flush()
""" End of of loop on slices """
# find parcels still alive if kept_p.sum()==0:
try:
nlive = ((prod0['flag_source'][partpost['idx_back']-IDX_ORGN] & I_DEAD) == 0).sum()
n_nohit = ((prod0['flag_source'][partpost['idx_back']-IDX_ORGN] & I_HIT) == 0).sum()
except:
nlive = 0
n_nohit =0
print('end hour ',hour,' numact', partpost['nact'], ' nexits',nexits,' nhits',nhits, ' nlive',nlive,' nohit',n_nohit)
# check that nlive + nhits + nexits = numpart, should be true after the first day
if part0['numpart'] != nexits + nhits + nlive + ndborne:
print('@@@ ACHTUNG numpart not equal to sum ',part0['numpart'],nexits+nhits+nlive+ndborne)
""" End of the procedure and storage of the result """
#output file
dd.io.save(out_file2,prod0,compression='zlib')
#pickle.dump(prod0,gzip.open(out_file,'wb'))
# close the print file
if quiet: fsock.close()