def main():
global IDX_ORGN
global lowpcut, highpcut
parser = argparse.ArgumentParser()
parser.add_argument("-y", "--year", type=int, help="year")
parser.add_argument("-m",
"--month",
type=int,
choices=1 + np.arange(12),
help="month")
parser.add_argument("-a",
"--advect",
type=str,
choices=["EID", "EIZ"],
help="source of advecting winds")
parser.add_argument("-s", "--suffix", type=str, help="suffix")
#parser.add_argument("-l","--level",type=int,help="P level")
parser.add_argument("-q",
"--quiet",
type=str,
choices=["y", "n"],
help="quiet (y) or not (n)")
parser.add_argument("-ct",
"--cloud_type",
type=str,
choices=["meanhigh", "veryhigh"],
help="cloud type filter")
parser.add_argument("-t",
"--step",
type=int,
help="step in hours between two part files")
parser.add_argument("-d",
"--duration",
type=int,
help="duration of integration in hours")
parser.add_argument("-gs",
"--granule_size",
type=int,
help="size of the granule")
parser.add_argument("-gn",
"--granule_step",
type=int,
help="number of granules in a step")
parser.add_argument("-ab", "--age_bound", type=int, help="age_bound")
parser.add_argument("-binx",
"--binx",
type=int,
help="number of grid points in longitude direction")
parser.add_argument("-biny",
"--biny",
type=int,
help="number of grid points in latitude direction")
parser.add_argument("-hmax",
"--hmax",
type=int,
help='maximum number of hours in traczilla simulation')
parser.add_argument("-username", "--username", type=str, help='username')
parser.add_argument("-userout", "--userout", type=str, help='userout')
""" Parsed parameters"""
# Parsed parameters
# Interval between two part files (in hours)
step = 6
# Largest time to be processed (in hours)
hmax = 1464
# Age limit in days
age_bound = 30
# start date of the backward run, corresponding to itime=0
year = 2017
# 8 +1 means we start on September 1 at 0h and cover the whole month of August
month = 6 + 1
advect = 'EIZ'
suffix = '_150_150hPa_500'
quiet = False
cloud_type = 'veryhigh'
# Bound on the age of the parcel (in days)
age_bound = 30
# Number of parcels launched per time slot (grid size)
binx = 320
biny = 224
# Number of granules in a step betwwen two part files
granule_step = 6
""" Non parsed parameters"""
# Time width of the parcel slice
slice_width = timedelta(minutes=5)
# dtRange (now defined in satmap definition)
#dtRange={'MSG1':timedelta(minutes=15),'Hima':timedelta(minutes=20)}
# day=1 should not be changed
day = 1
# low p cut applied in exiter
lowpcut = 3000
# high p cut applied in exiter
highpcut = 50000
args = parser.parse_args()
if args.year is not None: year = args.year
if args.month is not None: month = args.month + 1
if args.advect is not None: advect = args.advect
if args.suffix is not None: suffix = args.suffix
#if args.level is not None: level=args.level
if args.quiet is not None:
if args.quiet == 'y': quiet = True
else: quiet = False
if args.cloud_type is not None: cloud_type = args.cloud_type
if args.step is not None: step = args.step
if args.hmax is not None: hmax = args.hmax
if args.binx is not None: binx = args.binx
if args.biny is not None: biny = args.biny
granule_size = binx * biny
if args.granule_size is not None: granule_size = args.granule_size
if args.duration is not None: hmax = args.duration
if args.age_bound is not None: age_bound = args.age_bound
if args.granule_step is not None: granule_step = args.granule_step
if args.username is not None: username = args.username
if args.userout is not None: userout = args.userout
# Define main directories
main_sat_dir = '/bdd/STRATOCLIM/flexpart_in'
if 'ciclad' in socket.gethostname():
traj_dir = os.path.join('/data/', username, 'flexout', 'COCHIN',
'BACK')
out_dir = os.path.join('/data', userout, 'STC')
elif ('climserv' in socket.gethostname()) | ('polytechnique'
in socket.gethostname()):
traj_dir = os.path.join('/homedata/', username, 'flexout', 'COCHIN',
'BACK')
out_dir = os.path.join('/homedata', userout, 'STC')
else:
print('CANNOT RECOGNIZE HOST - DO NOT RUN ON NON DEFINED HOSTS')
exit()
# Output diretories
# Update the out_dir with the cloud type
out_dir = os.path.join(out_dir, 'STC-BACK-OUT-Cochin-SAF-' + cloud_type)
sdate = datetime(year, month, day)
# fdate defined to make output under the name of the month where parcels are released
fdate = sdate - timedelta(days=1)
# Number of slices between two outputs
dstep = timedelta(hours=step)
nb_slices = int(dstep / slice_width)
# size of granules launched during a step
granule_quanta = granule_size * granule_step
# Manage the file that receives the print output
if quiet:
# Output file
print_file = os.path.join(
out_dir, 'out',
'BACK-' + advect + fdate.strftime('-%b-%Y') + suffix + '.out')
fsock = open(print_file, 'w')
sys.stdout = fsock
print('year', year, 'month', month, 'day', day)
print('advect', advect)
print('suffix', suffix)
# Directory of the backward trajectories
ftraj = os.path.join(traj_dir,
'BACK-' + advect + fdate.strftime('-%b-%Y') + suffix)
# Output file
out_file = os.path.join(
out_dir,
'BACK-' + advect + fdate.strftime('-%b-%Y') + suffix + '.hdf5z')
#out_file1 = os.path.join(out_dir,'BACK-'+advect+fdate.strftime('-%b-%Y-')+str(level)+'K'+suffix+'.hdf5b')
#out_file2 = os.path.join(out_dir,'BACK-'+advect+fdate.strftime('-%b-%Y-')+str(level)+'K'+suffix+'.pkl')
# Directories for the satellite cloud top files
satdir ={'MSG1':os.path.join(main_sat_dir,'msg1','S_NWC'),\
'Hima':os.path.join(main_sat_dir,'himawari','S_NWC')}
""" Initialization of the calculation """
# Initialize the grid
gg = geosat.GeoGrid('FullAMA_SAFBox')
# Initialize the dictionary of the parcel dictionaries
partStep = {}
satmap = pixmap(gg)
# Build the satellite field generator
get_sat = {'MSG1': read_sat(sdate,'MSG1',satmap.zone['MSG1']['dtRange'],satdir['MSG1'],pre=True),\
'Hima': read_sat(sdate,'Hima',satmap.zone['Hima']['dtRange'],satdir['Hima'],pre=True)}
# Open the part_000 file that contains the initial positions
part0 = readidx107(os.path.join(ftraj, 'part_000'), quiet=True)
print('numpart', part0['numpart'])
numpart = part0['numpart']
numpart_s = granule_size
# stamp_date not set in these runs
# current_date actually shifted by one day / sdate
current_date = sdate
# check flag is clean
print('check flag is clean ',((part0['flag']&I_HIT)!=0).sum(),((part0['flag']&I_DEAD)!=0).sum(),\
((part0['flag']&I_CROSSED)!=0).sum())
# check idx_orgn
if part0['idx_orgn'] != 0:
print('MINCHIA, IDX_ORGN NOT 0 AS ASSUMED, CORRECTED WITH READ VALUE')
print('VALUE ', part0['idx_orgn'])
IDX_ORGN = part0['idx_orgn']
idx1 = IDX_ORGN
# Build a dictionary to host the results
prod0 = defaultdict(dict)
prod0['src']['x'] = np.full(part0['numpart'],
fill_value=np.nan,
dtype='float')
prod0['src']['y'] = np.full(part0['numpart'],
fill_value=np.nan,
dtype='float')
prod0['src']['p'] = np.full(part0['numpart'],
fill_value=np.nan,
dtype='float')
prod0['src']['t'] = np.full(part0['numpart'],
fill_value=np.nan,
dtype='float')
prod0['src']['age'] = np.full(part0['numpart'],
fill_value=np.nan,
dtype='int')
prod0['flag_source'] = part0['flag']
prod0['rvs'] = np.full(part0['numpart'], 0.01, dtype='float')
# truncate eventually to 32 bits at the output stage
# read the part_000 file for the first granule
partStep[0] = {}
partStep[0]['x'] = part0['x'][:granule_size]
partStep[0]['y'] = part0['y'][:granule_size]
partStep[0]['t'] = part0['t'][:granule_size]
partStep[0]['p'] = part0['p'][:granule_size]
partStep[0]['t'] = part0['t'][:granule_size]
partStep[0]['idx_back'] = part0['idx_back'][:granule_size]
partStep[0]['ir_start'] = part0['ir_start'][:granule_size]
partStep[0]['itime'] = 0
# number of hists and exits
nhits = 0
nexits = 0
ndborne = 0
nnew = granule_size
nold = 0
# used to get non borne parcels
new = np.empty(part0['numpart'], dtype='bool')
new.fill(False)
print('Initialization completed')
""" Main loop on the output time steps """
for hour in range(step, hmax + 1, step):
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use: {:4.2f} gb'.format(memoryUse))
# Get rid of dictionary no longer used
if hour >= 2 * step: del partStep[hour - 2 * step]
# Read the new data
partStep[hour] = readpart107(hour, ftraj, quiet=True)
# Link the names as views
partante = partStep[hour - step]
partpost = partStep[hour]
if partpost['nact'] > 0:
print('hour ', hour, ' numact ', partpost['nact'], ' max p ',
partpost['p'].max())
else:
print('hour ', hour, ' numact ', partpost['nact'])
# New date valid for partpost
current_date -= dstep
# Processing of water mixing ratio
# Select non stopped parcels in partante
selec = (prod0['flag_source'][partante['idx_back'] - IDX_ORGN]
& I_STOP) == 0
idx = partante['idx_back'][selec]
prod0['rvs'][idx-IDX_ORGN] = np.minimum(prod0['rvs'][idx-IDX_ORGN],\
satratio(partante['p'][selec],partante['t'][selec]))
""" Select the parcels that are common to the two steps
ketp_a is a logical field with same length as partante
kept_p is a logical field with same length as partpost
After the launch of the earliest parcel along the flight track, there
should not be any member in new.
"""
kept_a = np.in1d(partante['idx_back'],
partpost['idx_back'],
assume_unique=True)
kept_p = np.in1d(partpost['idx_back'],
partante['idx_back'],
assume_unique=True)
#new_p = ~np.in1d(partpost['idx_back'],partpost['idx_back'],assume_unique=True)
print('kept a, p ', len(kept_a), len(kept_p), kept_a.sum(),
kept_p.sum(), ' new ',
len(partpost['x']) - kept_p.sum())
nnew += len(partpost['x']) - kept_p.sum()
""" PROCESSING OF DEADBORNE PARCELS
Manage the parcels launched during the last 6-hour which have already
exited and do not appear in posold or posact (borne dead parcels).
These parcels are stored in the last part of posact, at most
the last granule_quanta parcels. """
if numpart_s < numpart:
print("manage deadborne", flush=True)
# First index of the current quanta """
numpart_s += granule_quanta
print("idx1", idx1, " numpart_s", numpart_s)
# Extract the last granule_size indexes from posacti, this is where the new parcels should be
if hour == step:
idx_act = partpost['idx_back']
else:
idx_act = partpost['idx_back'][-granule_quanta:]
# Generate the list of indexes that should be found in this range
# ACHTUNG ACHTUNG : this works because IDX_ORGN=1, FIX THAT
idx_theor = np.arange(idx1, numpart_s + IDX_ORGN)
# Find the missing indexes in idx_act (make a single line after validation)
kept_borne = np.in1d(idx_theor, idx_act, assume_unique=True)
idx_deadborne = idx_theor[~kept_borne]
# Process these parcels by assigning exit at initial location
prod0['flag_source'][
idx_deadborne -
IDX_ORGN] = prod0['flag_source'][idx_deadborne -
IDX_ORGN] | I_DEAD + I_DBORNE
prod0['src']['x'][idx_deadborne -
IDX_ORGN] = part0['x'][idx_deadborne - IDX_ORGN]
prod0['src']['y'][idx_deadborne -
IDX_ORGN] = part0['y'][idx_deadborne - IDX_ORGN]
prod0['src']['p'][idx_deadborne -
IDX_ORGN] = part0['p'][idx_deadborne - IDX_ORGN]
prod0['src']['t'][idx_deadborne -
IDX_ORGN] = part0['t'][idx_deadborne - IDX_ORGN]
prod0['src']['age'][idx_deadborne - IDX_ORGN] = 0.
print("number of deadborne ", len(idx_deadborne))
ndborne += len(idx_deadborne)
idx1 = numpart_s + IDX_ORGN
""" PROCESSING OF CROSSED PARCELS """
if len(kept_a) > 0:
exits = exiter(int((partante['itime']+partpost['itime'])/2), \
partante['x'][~kept_a],partante['y'][~kept_a],partante['p'][~kept_a],\
partante['t'][~kept_a],partante['idx_back'][~kept_a],\
prod0['flag_source'],prod0['src']['x'],prod0['src']['y'],\
prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],\
part0['ir_start'], satmap.range)
nexits += exits
print('exit ', nexits, exits, np.sum(~kept_a),
len(kept_a) - len(kept_p))
""" PROCESSING OF PARCELS WHICH ARE COMMON TO THE TWO OUTPUTS """
# Select the kept parcels which have not been hit yet
# !!! Never use and between two lists, the result is wrong
if kept_p.sum() == 0:
live_a = live_p = kept_p
else:
live_a = np.logical_and(
kept_a, (prod0['flag_source'][partante['idx_back'] - IDX_ORGN]
& I_DEAD) == 0)
live_p = np.logical_and(
kept_p, (prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_DEAD) == 0)
print('live a, b ', live_a.sum(), live_p.sum())
del kept_a
del kept_p
""" Correction step that moves partante['x'] to avoid big jumps at the periodicity frontier on the x-axis """
diffx = partpost['x'][live_p] - partante['x'][live_a]
bb = np.zeros(len(diffx))
bb[diffx > 180] = 360
bb[diffx < -180] = -360
partante['x'][live_a] += bb
del bb
del diffx
# DOES not work in the following WAY
#partante['x'][live_a][diffx>180] += 360
#partante['x'][live_a][diffx<-180] -= 360
# Build generator for parcel locations of the 5' slices
gsp = get_slice_part(partante, partpost, live_a, live_p, current_date,
dstep, slice_width)
if verbose: print('built parcel generator for ', current_date)
""" MAIN LOOP ON THE PARCEL TIME SLICES """
for i in range(nb_slices):
# get the slice for the particles
datpart = next(gsp)
if verbose: print('part slice ', i, datpart['time'])
# Check whether the present satellite image is valid
# The while should ensure that the run synchronizes
# when it starts.
while satmap.check('MSG1', datpart['time']) is False:
# if not get next satellite image
datsat1 = next(get_sat['MSG1'])
# Check that the image is available
if datsat1 is not None:
pm1 = geosat.SatGrid(datsat1, gg)
pm1._sat_togrid('CTTH_PRESS')
#print('pm1 diag',len(datsat1.var['CTTH_PRESS'][:].compressed()),
# len(pm1.var['CTTH_PRESS'][:].compressed()))
pm1._sat_togrid('CT')
pm1.attr = datsat1.attr.copy()
satmap.fill('MSG1', pm1, cloud_type)
del pm1
del datsat1
else:
# if the image is missing, extend the lease
try:
satmap.extend('MSG1')
except:
# This handle the unlikely case where the first image is missing
continue
while satmap.check('Hima', datpart['time']) is False:
# if not get next satellite image
datsath = next(get_sat['Hima'])
# Check that the image is available
if datsath is not None:
pmh = geosat.SatGrid(datsath, gg)
pmh._sat_togrid('CTTH_PRESS')
#print('pmh diag',len(datsath.var['CTTH_PRESS'][:].compressed()),
# len(pmh.var['CTTH_PRESS'][:].compressed()))
pmh._sat_togrid('CT')
pmh.attr = datsath.attr.copy()
satmap.fill('Hima', pmh, cloud_type)
del datsath
del pmh
else:
# if the image is missing, extend the lease
try:
satmap.extend('Hima')
except:
# This handle the unlikely case where the first image is missing
continue
""" Select the parcels located within the domain """
# TODO TODO the values used here should be derived from parameters defined above
indomain = np.all((datpart['x'] > -10, datpart['x'] < 160,
datpart['y'] > 0, datpart['y'] < 50),
axis=0)
""" PROCESS THE COMPARISON OF PARCEL PRESSURES TO CLOUDS """
if indomain.sum() > 0:
nhits += convbirth(datpart['itime'],
datpart['x'][indomain],datpart['y'][indomain],datpart['p'][indomain],\
datpart['t'][indomain],datpart['idx_back'][indomain],\
prod0['flag_source'],prod0['src']['x'],prod0['src']['y'],\
prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],\
satmap.ptop, part0['ir_start'],\
satmap.range[0,0],satmap.range[1,0],satmap.stepx,satmap.stepy,satmap.binx,satmap.biny)
sys.stdout.flush()
""" End of of loop on slices """
# Check the age limit (easier to do it here)
print("Manage age limit", flush=True)
age_sec = part0['ir_start'][partante['idx_back'] -
IDX_ORGN] - partante['itime']
IIold_o = age_sec > (age_bound - 0.25) * 86400
IIold_o = IIold_o & (
(prod0['flag_source'][partante['idx_back'] - IDX_ORGN] & I_STOP)
== 0)
idx_IIold = partante['idx_back'][IIold_o]
j_IIold_o = np.where(IIold_o)
prod0['flag_source'][idx_IIold - IDX_ORGN] = prod0['flag_source'][
idx_IIold - IDX_ORGN] | I_DEAD + I_OLD
prod0['src']['x'][idx_IIold - IDX_ORGN] = partante['x'][j_IIold_o]
prod0['src']['y'][idx_IIold - IDX_ORGN] = partante['y'][j_IIold_o]
prod0['src']['p'][idx_IIold - IDX_ORGN] = partante['p'][j_IIold_o]
prod0['src']['t'][idx_IIold - IDX_ORGN] = partante['t'][j_IIold_o]
prod0['src']['age'][idx_IIold - IDX_ORGN] = (
(part0['ir_start'][idx_IIold - IDX_ORGN] - partante['itime']) /
86400)
print("number of IIold ", len(idx_IIold))
nold += len(idx_IIold)
# find parcels still alive if kept_p.sum()==0:
try:
nlive = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_DEAD) == 0).sum()
n_nohit = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_HIT) == 0).sum()
except:
nlive = 0
n_nohit = 0
print('end hour ', hour, ' numact', partpost['nact'], ' nexits',
nexits, ' nhits', nhits, ' nlive', nlive, ' nohit', n_nohit)
# check that nlive + nhits + nexits = numpart, should be true after the first day
if part0['numpart'] != nexits + nhits + nlive + ndborne:
print('@@@ ACHTUNG numpart not equal to sum ', part0['numpart'],
nexits + nhits + nlive + ndborne)
""" End of the procedure and storage of the result """
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use before clean: {:4.2f} gb'.format(memoryUse))
del partante
del partpost
del live_a
del live_p
del datpart
prod0['rvs'] = prod0['rvs'].astype(np.float32)
for var in ['age', 'p', 't', 'x', 'y']:
prod0['src'][var] = prod0['src'][var].astype(np.float32)
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use after clean: {:4.2f} gb'.format(memoryUse))
#output file
#pickle.dump(prod0,gzip.open(out_file,'wb'))
#try:
# dd.io.save(out_file1,prod0,compression='blosc')
#except:
# print('error with dd blosc')
try:
dd.io.save(out_file, prod0, compression='zlib')
except:
print('error with dd zlib')
""" End of the procedure and storage of the result """
#output file
#dd.io.save(out_file2,prod0,compression='zlib')
#pickle.dump(prod0,gzip.open(out_file,'wb'))
# close the print file
if quiet: fsock.close()
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
This is testing the SAFBox
Created on Fri Jun 8 13:33:15 2018
@author: Bernard Legras
"""
import geosat
from SAFNWCnc import SAFNWC_CTTH
from datetime import datetime
date = datetime(2017, 7, 22, 3, 45)
dat = SAFNWC_CTTH(date, 'msg1', BBname='SAFBox')
gg = geosat.GeoGrid('FullAMA_SAFBox')
dat._CTTH_PRESS()
p1 = geosat.SatGrid(dat, gg)
p1._sat_togrid('CTTH_PRESS')
p1.chart('CTTH_PRESS')
# Himawari read
hima = geosat.Himawari(date)
# read IR0
hima._get_IR0()
hima._get_var('WV_062')
hima._get_var('WV_073')
hima.close()
# MSG1 read
msg1 = geosat.MSG1(date)
msg1._get_IR0()
msg1._get_var('WV_062')
msg1._get_var('WV_073')
msg1.close()
# grid projection and match
for grid in ['KTM', 'MesoInd', 'FullAMA']:
gg = geosat.GeoGrid(grid)
# associate grid and sat
phima = geosat.SatGrid(hima, gg)
pmsg1 = geosat.SatGrid(msg1, gg)
# project by closest neigbour
# and do angular correction
for var in ['IR0', 'WV_062', 'WV_073']:
phima._sat_togrid(var)
pmsg1._sat_togrid(var)
# angular correction for 'IR0'
phima._sza_correc()
pmsg1._sza_correc()
# patching
ppt = pmsg1.patch(phima, 90.75, ['IR0', 'WV_062', 'WV_073'])
ppt.chart('IR0', txt='brightness temperature infrared 10.8')
def main():
global IDX_ORGN
parser = argparse.ArgumentParser()
parser.add_argument("-y", "--year", type=int, help="year")
parser.add_argument("-m",
"--month",
type=int,
choices=1 + np.arange(12),
help="month")
parser.add_argument("-d1",
"--day1",
type=int,
choices=1 + np.arange(31),
help="day1")
parser.add_argument("-d2",
"--day2",
type=int,
choices=1 + np.arange(31),
help="day2")
parser.add_argument(
"-a",
"--advect",
type=str,
choices=["OPZ", "EAD", "EAZ", "EID", "EIZ", "EID-FULL", "EIZ-FULL"],
help="source of advecting winds")
parser.add_argument("-s",
"--suffix",
type=str,
help="suffix for special cases")
parser.add_argument("-q",
"--quiet",
type=str,
choices=["y", "n"],
help="quiet (y) or not (n)")
#parser.add_argument("-c","--clean0",type=bool,help="clean part_000")
parser.add_argument("-t",
"--step",
type=int,
help="step in hour between two part files")
parser.add_argument("-ct",
"--cloud_type",
type=str,
choices=["meanhigh", "veryhigh", "silviahigh"],
help="cloud type filter")
parser.add_argument("-k",
"--diffus",
type=str,
choices=['01', '1', '001'],
help='diffusivity parameter')
parser.add_argument("-v",
"--vshift",
type=int,
choices=[0, 1, 2],
help='vertical shift')
parser.add_argument("-hm",
"--hmax",
type=int,
help='maximum considered integration time')
# to be updated
# Define main directories
if 'ciclad' in socket.gethostname():
main_sat_dir = '/data/legras/flexpart_in/SAFNWC'
#SVC_Dir = '/bdd/CFMIP/SEL2'
traj_dir = '/data/akottayil/flexout/STC/BACK'
out_dir = '/data/legras/STC'
elif ('climserv' in socket.gethostname()) | ('polytechnique'
in socket.gethostname()):
main_sat_dir = '/data/legras/flexpart_in/SAFNWC'
#SVC_Dir = '/bdd/CFMIP/SEL2'
traj_dir = '/data/akottayil/flexout/STC/BACK'
out_dir = '/homedata/legras/STC'
else:
print('CANNOT RECOGNIZE HOST - DO NOT RUN ON NON DEFINED HOSTS')
exit()
""" Parameters """
step = 3
hmax = 732
dstep = timedelta(hours=step)
# time width of the parcel slice
slice_width = timedelta(minutes=5)
# number of slices between two outputs
nb_slices = int(dstep / slice_width)
# default values of parameters
# date of the flight
year = 2017
month = 7 + 1
day1 = 1
day2 = 10
advect = 'EAD'
suffix = ''
quiet = False
clean0 = True
cloud_type = 'silviahigh'
diffus = '01'
vshift = 0
super = ''
args = parser.parse_args()
if args.year is not None: year = args.year
if args.month is not None: month = args.month + 1
if args.advect is not None: advect = args.advect
if args.day1 is not None: day1 = args.day1
if args.day2 is not None: day2 = args.day2
if args.suffix is not None: suffix = '-' + args.suffix
if args.hmax is not None: hmax = args.hmax
if args.quiet is not None:
if args.quiet == 'y': quiet = True
else: quiet = False
#if args.clean0 is not None: clean0 = args.clean0
if args.cloud_type is not None: cloud_type = args.cloud_type
if args.step is not None: step = args.step
if args.diffus is not None: diffus = args.diffus
diffus = '-D' + diffus
if args.vshift is not None:
vshift = args.vshift
if vshift > 0:
super = 'super' + str(vshift)
# Update the out_dir with the cloud type and the super paramater
out_dir = os.path.join(out_dir, 'SVC-BACK-OUT-SAF-' + super + cloud_type)
try:
os.mkdir(out_dir)
os.mkdir(out_dir + '/out')
except:
print('out_dir directory already created')
# Dates beginning and end
date_beg = datetime(year=year, month=month, day=day1, hour=0)
date_end = datetime(year=year, month=month, day=day2, hour=0)
# Manage the file that receives the print output
if quiet:
# Output file
print_file = os.path.join(
out_dir, 'out',
'BACK-SVC-EID-FULL-' + date_beg.strftime('%b-%Y-day%d-') +
date_end.strftime('%d-D01') + '.out')
fsock = open(print_file, 'w')
sys.stdout = fsock
# initial time to read the sat files
# should be after the end of the flight
sdate = date_end + timedelta(days=1)
print('year', year, 'month', month, 'day', day2)
print('advect', advect)
print('suffix', suffix)
# patch to fix the problem of the data hole on the evening of 2 August 2017
if sdate == datetime(2017, 8, 2): sdate = datetime(2017, 8, 3, 6)
# Directories of the backward trajectories and name of the output file
ftraj = os.path.join(
traj_dir, 'BACK-SVC-EID-FULL-' + date_beg.strftime('%b-%Y-day%d-') +
date_end.strftime('%d-D01'))
out_file2 = os.path.join(
out_dir, 'BACK-SVC-EID-FULL-' + date_beg.strftime('%b-%Y-day%d-') +
date_end.strftime('%d-D01') + '.hdf5')
# Directories for the satellite cloud top files
satdir ={'MSG1':os.path.join(main_sat_dir,'msg1','S_NWC'),\
'Hima':os.path.join(main_sat_dir,'himawari','S_NWC')}
""" Initialization of the calculation """
# Initialize the grid
gg = geosat.GeoGrid('FullAMA_SAFBox')
# Initialize the dictionary of the parcel dictionaries
partStep = {}
satmap = pixmap(gg)
# Build the satellite field generator
get_sat = {'MSG1': read_sat(sdate,'MSG1',satmap.zone['MSG1']['dtRange'],satdir['MSG1'],pre=True,vshift=vshift),\
'Hima': read_sat(sdate,'Hima',satmap.zone['Hima']['dtRange'],satdir['Hima'],pre=True,vshift=vshift)}
# Read the index file that contains the initial positions
part0 = readidx107(os.path.join(ftraj, 'part_000'), quiet=True)
print('numpart', part0['numpart'])
# stamp_date not set in these runs
# current_date actually shifted by one day / sdate
# We assume here that part time is defined from this day at 0h
# sdate defined above should be equal or posterior to current_date
current_date = sdate
# check flag is clean
print('check flag is clean ',((part0['flag']&I_HIT)!=0).sum(),((part0['flag']&I_DEAD)!=0).sum(),\
((part0['flag']&I_CROSSED)!=0).sum())
# check idx_orgn
if part0['idx_orgn'] != 0:
print('MINCHIA, IDX_ORGN NOT 0 AS ASSUMED, CORRECTED WITH READ VALUE')
print('VALUE ', part0['idx_orgn'])
IDX_ORGN = part0['idx_orgn']
# Build a dictionary to host the results
prod0 = defaultdict(dict)
prod0['src']['x'] = np.empty(part0['numpart'], dtype='float')
prod0['src']['y'] = np.empty(part0['numpart'], dtype='float')
prod0['src']['p'] = np.empty(part0['numpart'], dtype='float')
prod0['src']['t'] = np.empty(part0['numpart'], dtype='float')
prod0['src']['age'] = np.empty(part0['numpart'], dtype='int')
prod0['flag_source'] = part0['flag']
# truncate eventually to 32 bits at the output stage
# read the part_000 file
partStep[0] = readpart107(0, ftraj, quiet=True)
# cleaning is necessary for runs starting in the fake restart mode
# otherwise all parcels are thought to exit at the first step
if clean0:
partStep[0]['idx_back'] = []
# number of hists and exits
nhits = 0
nexits = 0
ndborne = 0
nnew = 0
# used to get non borne parcels
new = np.empty(part0['numpart'], dtype='bool')
new.fill(False)
print('Initialization completed')
""" Main loop on the output time steps """
for hour in range(step, hmax + 1, step):
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use: {:4.2f} gb'.format(memoryUse))
# Get rid of dictionary no longer used
if hour >= 2 * step: del partStep[hour - 2 * step]
# Read the new data
partStep[hour] = readpart107(hour, ftraj, quiet=True)
# Link the names
partante = partStep[hour - step]
partpost = partStep[hour]
if partpost['nact'] > 0:
print('hour ', hour, ' numact ', partpost['nact'], ' max p ',
partpost['p'].max())
else:
print('hour ', hour, ' numact ', partpost['nact'])
# New date valid for partpost
current_date -= dstep
""" Select the parcels that are common to the two steps
ketp_a is a logical field with same length as partante
kept_p is a logical field with same length as partpost
After the launch of the earliest parcel along the flight track, there
should not be any member in new
The parcels
"""
kept_a = np.in1d(partante['idx_back'],
partpost['idx_back'],
assume_unique=True)
kept_p = np.in1d(partpost['idx_back'],
partante['idx_back'],
assume_unique=True)
#new_p = ~np.in1d(partpost['idx_back'],partpost['idx_back'],assume_unique=True)
print('kept a, p ', len(kept_a), len(kept_p), kept_a.sum(),
kept_p.sum(), ' new ',
len(partpost['x']) - kept_p.sum())
""" IDENTIFY AND TAKE CARE OF DEADBORNE AS NON BORNE PARCELS """
if (hour < ((day2 - day1 + 5) * 24)) & (partpost['nact'] > 0):
new[partpost['idx_back'][~kept_p] - IDX_ORGN] = True
nnew += len(partpost['x']) - kept_p.sum()
if hour == ((day2 - day1 + 5) * 24):
ndborne = np.sum(~new)
prod0['flag_source'][~new] |= I_DBORNE + I_DEAD
prod0['src']['x'][~new] = part0['x'][~new]
prod0['src']['y'][~new] = part0['y'][~new]
prod0['src']['p'][~new] = part0['p'][~new]
prod0['src']['t'][~new] = part0['t'][~new]
prod0['src']['age'][~new] = 0
print('number of dead borne', ndborne, part0['numpart'] - nnew)
del new
""" INSERT HERE CODE FOR NEW PARCELS """
# nothing to be done for new parcels, just wait and see
""" PROCESSING OF CROSSED PARCELS """
if len(kept_a) > 0:
exits = exiter(int((partante['itime']+partpost['itime'])/2), \
partante['x'][~kept_a],partante['y'][~kept_a],partante['p'][~kept_a],\
partante['t'][~kept_a],partante['idx_back'][~kept_a],\
prod0['flag_source'],prod0['src']['x'],prod0['src']['y'],\
prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],\
part0['ir_start'], satmap.range)
nexits += exits
print('exit ', nexits, exits, np.sum(~kept_a),
len(kept_a) - len(kept_p))
""" PROCESSING OF PARCELS WHICH ARE COMMON TO THE TWO OUTPUTS """
# Select the kept parcels which have not been hit yet
# !!! Never use and between two lists, the result is wrong
if kept_p.sum() == 0:
live_a = live_p = kept_p
else:
live_a = np.logical_and(
kept_a, (prod0['flag_source'][partante['idx_back'] - IDX_ORGN]
& I_DEAD) == 0)
live_p = np.logical_and(
kept_p, (prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_DEAD) == 0)
print('live a, b ', live_a.sum(), live_p.sum())
# Build generator for parcel locations of the 5' slices
gsp = get_slice_part(partante, partpost, live_a, live_p, current_date,
dstep, slice_width)
if verbose: print('built parcel generator for ', current_date)
""" MAIN LOOP ON THE PARCEL TIME SLICES """
for i in range(nb_slices):
# get the slice for the particles
datpart = next(gsp)
if verbose: print('part slice ', i, datpart['time'])
# Check whether the present satellite image is valid
# The while should ensure that the run synchronizes
# when it starts.
while satmap.check('MSG1', datpart['time']) is False:
# if not get next satellite image
datsat1 = next(get_sat['MSG1'])
# Check that the image is available
if datsat1 is not None:
pm1 = geosat.SatGrid(datsat1, gg)
pm1._sat_togrid('CTTH_PRESS')
#print('pm1 diag',len(datsat1.var['CTTH_PRESS'][:].compressed()),
# len(pm1.var['CTTH_PRESS'][:].compressed()))
pm1._sat_togrid('CT')
if vshift > 0: pm1._sat_togrid('CTTH_TEMPER')
pm1.attr = datsat1.attr.copy()
satmap.fill('MSG1', pm1, cloud_type, vshift=vshift)
del pm1
del datsat1
else:
# if the image is missing, extend the lease
try:
satmap.extend('MSG1')
except:
# This handle the unlikely case where the first image is missing
continue
while satmap.check('Hima', datpart['time']) is False:
# if not get next satellite image
datsath = next(get_sat['Hima'])
# Check that the image is available
if datsath is not None:
pmh = geosat.SatGrid(datsath, gg)
pmh._sat_togrid('CTTH_PRESS')
#print('pmh diag',len(datsath.var['CTTH_PRESS'][:].compressed()),
# len(pmh.var['CTTH_PRESS'][:].compressed()))
pmh._sat_togrid('CT')
if vshift > 0: pmh._sat_togrid('CTTH_TEMPER')
pmh.attr = datsath.attr.copy()
satmap.fill('Hima', pmh, cloud_type, vshift=vshift)
del datsath
del pmh
else:
# if the image is missing, extend the lease
try:
satmap.extend('Hima')
except:
# This handle the unlikely case where the first image is missing
continue
""" PROCESS THE COMPARISON OF PARCEL PRESSURES TO CLOUDS """
if len(datpart['x']) > 0:
nhits += convbirth(datpart['itime'],
datpart['x'],datpart['y'],datpart['p'],datpart['t'],datpart['idx_back'],\
prod0['flag_source'],prod0['src']['x'],prod0['src']['y'],\
prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],\
satmap.ptop, part0['ir_start'],\
satmap.range[0,0],satmap.range[1,0],satmap.stepx,satmap.stepy,satmap.binx,satmap.biny)
sys.stdout.flush()
""" End of of loop on slices """
# find parcels still alive if kept_p.sum()==0:
try:
nlive = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_DEAD) == 0).sum()
n_nohit = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_HIT) == 0).sum()
except:
nlive = 0
n_nohit = 0
print('end hour ', hour, ' numact', partpost['nact'], ' nexits',
nexits, ' nhits', nhits, ' nlive', nlive, ' nohit', n_nohit)
# check that nlive + nhits + nexits = numpart, should be true after the first day
if part0['numpart'] != nexits + nhits + nlive + ndborne:
print('@@@ ACHTUNG numpart not equal to sum ', part0['numpart'],
nexits + nhits + nlive + ndborne)
""" End of the procedure and storage of the result """
#output file
dd.io.save(out_file2, prod0, compression='zlib')
#pickle.dump(prod0,gzip.open(out_file,'wb'))
# close the print file
if quiet: fsock.close()
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
This script tests the behavior of geosat with a MSG1 image with missing lines
Created on Sat Jun 10 16:16:30 2017
@author: Bernard Legras
"""
import geosat
from datetime import datetime
date = datetime(2017, 6, 8, 7)
ah = geosat.MSG1(date)
gg = geosat.GeoGrid('MesoInd')
ah._get_IR0()
ph = geosat.SatGrid(ah, gg)
ph._sat_togrid('IR0')
ah.show('IR0')
ph.chart('IR0', txt='BT MSG1')
def main():
# to be updated
if socket.gethostname() == 'graphium':
pass
elif 'ciclad' in socket.gethostname():
#main_sat_dirSAF = '/data/legras/flexpart_in/SAFNWC'
main_sat_dir1 = '/bdd/STRATOCLIM/flexpart_in'
output_dir = '/data/legras/STC/pixmaps'
elif ('climserv' in socket.gethostname()) | ('polytechnique' in socket.gethostname()):
#main_sat_dirSAF = '/data/legras/flexpart_in/SAFNWC'
main_sat_dir1 = '/bdd/STRATOCLIM/flexpart_in'
else:
print ('CANNOT RECOGNIZE HOST - DO NOT RUN ON NON DEFINED HOSTS')
exit()
parser = argparse.ArgumentParser()
#parser.add_argument("-y","--year",type=int,help="year")
parser.add_argument("-m1","--month1",type=int,choices=1+np.arange(12),help="month")
parser.add_argument("-d1","--day1",type=int,choices=1+np.arange(31),help="day")
parser.add_argument("-m2","--month2",type=int,choices=1+np.arange(12),help="month")
parser.add_argument("-d2","--day2",type=int,choices=1+np.arange(31),help="day")
#parser.add_argument("-N","--NewSAF",type=bool,help='NewSAF')
#parser.add_argument("-H","--doHima",type=bool,help='doHima')
""" Parameters """
# Starting time
# This date must fall on a cloudtop boundary, that it 0 or 12h
month1 = 8
day1 = 16
#sdate = datetime(2017,month1,day1,0)
#sdate = datetime(2017,7,21,0)
# Ending time (backward)
month2 = 7
day2 = 13
#edate = datetime(2017,7,13,0)
# time between two comparisons
dstep = timedelta (hours=2)
# dtRange
#dtRangeSAF = {'MSG1':timedelta(minutes=15),'Hima':timedelta(minutes=20)}
#dtRange1 = {'MSG1':timedelta(minutes=30),'Hima':timedelta(minutes=20)}
# SAF filters
#good = True
#opaq = True
args = parser.parse_args()
if args.month1 is not None: month1 = args.month1
if args.month2 is not None: month2 = args.month2
if args.day1 is not None: day1 = args.day1
if args.day2 is not None: day2 = args.day2
#if args.NewSAF is not None: NewSAF = args.NewSAF
#if args.doHima is not None: doHima = args.doHima
sdate = datetime(2017,month1,day1,0)
edate = datetime(2017,month2,day2,0)
# Directories for the satellite cloud top files
#satdirSAF ={'MSG1':os.path.join(main_sat_dirSAF,'msg1','S_NWC'),\
# 'Hima':os.path.join(main_sat_dirSAF,'himawari','S_NWC')}
satdir1 = {'MSG1':os.path.join(main_sat_dir1,'StratoClim+1kmD_msg1-c'),\
'Hima':os.path.join(main_sat_dir1,'StratoClim+1kmD_himawari-d')}
""" Initialization of the calculation """
# Initialize the grid
if NewSAF:
gg = geosat.GeoGrid('FullAMA_SAFBox')
else:
gg = geosat.GeoGrid('FullAMA')
output_dir += '-StdSAF'
satmapSAF = pixmap(gg,'SAF')
satmap1 = pixmap(gg,'1')
satfill = {}
# Build the satellite field generator
get_sat1 = {'MSG1': read_sat1(sdate,satmap1.zone['MSG1']['dtRange1'],satdir1['MSG1'],pre=True),\
'Hima': read_sat1(sdate,satmap1.zone['Hima']['dtRange1'],satdir1['Hima'],pre=True)}
#get_satSAF = {'MSG1': read_satSAF(sdate,'MSG1',satmapSAF.zone['MSG1']['dtRangeSAF'],satdirSAF['MSG1'],pre=True),\
# 'Hima': read_satSAF(sdate,'Hima',satmapSAF.zone['Hima']['dtRangeSAF'],satdirSAF['Hima'],pre=True)}
get_satSAF = {'MSG1': read_satSAF(sdate,'MSG1',satmapSAF.zone['MSG1']['dtRangeSAF'],pre=True),\
'Hima': read_satSAF(sdate,'Hima',satmapSAF.zone['Hima']['dtRangeSAF'],pre=True)}
current_date = sdate
""" Main loop on the output time steps """
while current_date > edate:
print('searching ',current_date)
# Fill the pixmap for cloudtops
while satmap1.check('MSG1',current_date) is False:
# if not get next satellite slice
try:
next(satfill['MSG1'])
# read new satellite file if the slice generator is over
except:
datsatm = next(get_sat1['MSG1'])
satfill['MSG1'] = satmap1.fill1('MSG1',datsatm)
next(satfill['MSG1'])
if verbose: print('check 1 MSG1 ',satmap1.check('MSG1',current_date),'##',current_date,
'##',satmap1.zone['MSG1']['ti'],'##',satmap1.zone['MSG1']['tf'])
if doHima:
while satmap1.check('Hima',current_date) is False:
try:
next(satfill['Hima'])
except:
datsath = next(get_sat1['Hima'])
satfill['Hima'] = satmap1.fill1('Hima',datsath)
next(satfill['Hima'])
if verbose: print('check 1 Hima ',satmap1.check('Hima',current_date),'##',current_date,
'##',satmap1.zone['Hima']['ti'],'##',satmap1.zone['Hima']['tf'])
# Fill the pixmap for SAF
# Check whether the present satellite image is valid
# The while should ensure that the run synchronizes when it starts.
while satmapSAF.check('MSG1',current_date) is False:
# if not get next satellite image
datsatm = next(get_satSAF['MSG1'])
# Check that the image is available
if datsatm is not None:
# all the data need to be read because of possible gaps
# and extensions to be done in such cases
pmm = geosat.SatGrid(datsatm,gg)
pmm._sat_togrid('CTTH_PRESS')
#print('pm1 diag',len(datsat1.var['CTTH_PRESS'][:].compressed()),
# len(pm1.var['CTTH_PRESS'][:].compressed()))
pmm._sat_togrid('ctth_alti')
pmm._sat_togrid('ctth_tempe')
pmm._sat_togrid('ctth_quality')
pmm._sat_togrid('ctth_conditions')
pmm._sat_togrid('ctth_status_flag')
pmm._sat_togrid('ctth_method')
pmm._sat_togrid('CT')
pmm._sat_togrid('ct_cumuliform')
pmm._sat_togrid('ct_multilayer')
pmm._sat_togrid('ct_status_flag')
pmm._sat_togrid('CMa')
pmm._sat_togrid('cma_status_flag')
pmm._sat_togrid('cma_quality')
pmm.attr = datsatm.attr.copy()
satmapSAF.fillSAF('MSG1',pmm)
del pmm
#del datsat1
else:
# if the image is missing, extend the lease of previous image
try:
satmapSAF.extend('MSG1')
if debug2: print ('MSG1 extension ',satmapSAF.zone['MSG1']['ti'])
except:
# This handle the unlikely case where the first image is missing
continue
#print('match MSG1 ',datsatm.attr['date'],datsatm.attr['lease_time'])
if verbose: print('check SAF MSG1 ',satmapSAF.check('MSG1',current_date),'##',current_date,
'##',satmapSAF.zone['MSG1']['ti'],'##',satmapSAF.zone['MSG1']['tf'])
if doHima:
while satmapSAF.check('Hima',current_date) is False:
# if not get next satellite image
datsath = next(get_satSAF['Hima'])
# Check that the image is available
if datsath is not None:
# not elegant as it duplicates the while test but saves
# some processing
#if (current_date <= datsath.attr['lease_time']) | \
# (current_date > datsath.attr['date']): continue
# not a good odea when some image is missing
pmh = geosat.SatGrid(datsath,gg)
pmh._sat_togrid('CTTH_PRESS')
#print('pmh diag',len(datsath.var['CTTH_PRESS'][:].compressed()),
# len(pmh.var['CTTH_PRESS'][:].compressed()))
pmh._sat_togrid('ctth_tempe')
pmh._sat_togrid('ctth_alti')
pmh._sat_togrid('ctth_quality')
pmh._sat_togrid('ctth_conditions')
pmh._sat_togrid('ctth_status_flag')
pmh._sat_togrid('ctth_method')
pmh._sat_togrid('CT')
pmh._sat_togrid('ct_cumuliform')
pmh._sat_togrid('ct_multilayer')
pmh._sat_togrid('ct_status_flag')
pmh._sat_togrid('CMa')
pmh._sat_togrid('cma_status_flag')
pmh._sat_togrid('cma_quality')
pmh.attr = datsath.attr
satmapSAF.fillSAF('Hima',pmh)
#del datsath
del pmh
else:
# if the image is missing, extend the leaseof previous image
try:
satmapSAF.extend('Hima')
if debug2: print ('Hima extension ',satmapSAF.zone['Hima']['ti'])
except:
# This handle the unlikely case where the first image is missing
continue
#print('match Hima ',datsath.attr['date'],datsath.attr['lease_time'])
if verbose: print('check SAF Hima ',satmapSAF.check('Hima',current_date),'##',current_date,
'##',satmapSAF.zone['Hima']['ti'],'##',satmapSAF.zone['Hima']['tf'])
outfile = os.path.join(output_dir,current_date.strftime('%Y-%m-%dT%H%M'))
with gzip.open(outfile,'wb') as f:
pickle.dump([satmapSAF,satmap1],f)
print('OUTPUT for time ',current_date,'\n')
current_date -= dstep
sys.stdout.flush()
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-s",
"--segment",
type=str,
choices=[
"May.11", "May.21", "Jun.01", "Jun.11", "Jun.21",
"Jul.01", "Jul.11", "Jul.21", "Aug.01", "Aug.11",
"Aug.21", "Sep.01", "Sep.11", "Sep.21", "OctBeg"
],
help="segment to be processed")
seg_dates = {
'Jul.01': [datetime(2017, 7, 1, 0),
datetime(2017, 7, 11, 0)],
'Jul.11': [datetime(2017, 7, 11, 0),
datetime(2017, 7, 21, 0)],
'Jul.21': [datetime(2017, 7, 21, 0),
datetime(2017, 8, 1, 0)],
'Aug.01': [datetime(2017, 8, 1, 0),
datetime(2017, 8, 11, 0)],
'Aug.11': [datetime(2017, 8, 11, 0),
datetime(2017, 8, 21, 0)],
'Aug.21': [datetime(2017, 8, 21, 0),
datetime(2017, 9, 1, 0)],
'Jun.01': [datetime(2017, 6, 1, 0),
datetime(2017, 6, 11, 0)],
'Jun.11': [datetime(2017, 6, 11, 0),
datetime(2017, 6, 21, 0)],
'Jun.21': [datetime(2017, 6, 21, 0),
datetime(2017, 7, 1, 0)],
'Sep.01': [datetime(2017, 9, 1, 0),
datetime(2017, 9, 11, 0)],
'Sep.11': [datetime(2017, 9, 11, 0),
datetime(2017, 9, 21, 0)],
'Sep.21': [datetime(2017, 9, 21, 0),
datetime(2017, 10, 1, 0)],
'May.21': [datetime(2017, 5, 21, 0),
datetime(2017, 6, 1, 0)],
'May.11': [datetime(2017, 5, 11, 0),
datetime(2017, 5, 21, 0)],
'OctBeg': [datetime(2017, 10, 1, 0),
datetime(2017, 10, 3, 0)]
}
seg = 'Jul.11'
args = parser.parse_args()
if args.segment is not None:
seg = args.segment
date1 = seg_dates[seg][0]
date2 = seg_dates[seg][1]
gg = geosat.GeoGrid('FullAMA_SAFBox')
date = date1
output_dir = '/data/legras/STC/STC-SAFNWC-OUT'
gM = get_msg1(date1, date2, gg)
gH = get_himawari(date1, date2, gg)
# date1 should correspond to a date where both images are available
# in order to start properly
phM = next(gM)
phH = next(gH)
make_new_comp = True
while date < date2:
# Check new msg1 to be loaded
if phM.expiration == date:
pp = next(gM)
# If available image, get it
# if not, extend validity of previous image
if pp is not None:
phM = pp
make_new_comp = True
else:
phM.expiration += timedelta(minutes=delta_msg1)
if phH.expiration == date:
pp = next(gH)
# If available image, get it
# if not, extend validity of previous image
if pp is not None:
phH = pp
make_new_comp = True
else:
phH.expiration += timedelta(minutes=delta_himawari)
if make_new_comp:
comb = phM.patch(phH, 90.75, [
'CTTH_PRESS', 'CT', 'ctth_status_flag', 'ctth_quality',
'ct_multilayer'
])
comb.var['CTTH_PRESS'][comb.var['CTTH_PRESS'] < 0] = np.ma.masked
comb.var['CTTH_PRESS'][
comb.var['CTTH_PRESS'] > 65000] = np.ma.masked
# compact the CT field and the flags into a single 32 bit variable
comb.var['cloud_flag'] = (comb.var['CT'].astype(np.uint32) << 24) \
+ (comb.var['ctth_quality'].astype(np.uint32) << 16) \
+ (comb.var['ct_multilayer'].astype(np.uint32) << 8) \
+ comb.var['ctth_status_flag'].astype(np.uint32)
comb.var['cloud_flag'].__setmask__(comb.var['CTTH_PRESS'].mask)
filename = date.strftime('SAFNWC-PTOP-%Y-%m-%d-%H:%M.pkl')
day_dir = os.path.join(output_dir, date.strftime('%Y/%m/%Y-%m-%d'))
try:
os.mkdir(day_dir)
except:
pass
# output containing the cloud top pressure and the good/opaq flag
fullname = os.path.join(day_dir, filename)
with gzip.open(fullname, 'wb') as f:
pickle.dump([comb.var['CTTH_PRESS'], comb.var['cloud_flag']],
f,
protocol=3)
print('new file', date, np.ma.count_masked(comb.var['CTTH_PRESS']))
del comb
date += timedelta(minutes=delta_time)
make_new_comp = False
qual_data = v2018_data.ncid.variables['ctth_quality'][:].data
# for i in range(len(flag_values)):
if fname == 'All':
fmask = np.zeros(flag_data.shape).astype(bool)
else:
fi = np.where(np.asarray(flag_meaning) == fname)[0][0]
fval = flag_values[fi]
fmask = ~(flag_data == fval)
# qmaks = (qual_data == qual_values[2+3]) | (qual_data == qual_values[2+2]) | (qual_data == qual_values[2+4])
# mask = qmaks | fmask
mask = fmask
v2018_data._CTTH_PRESS()
v2018_data.var['CTTH_PRESS'].data[mask] = v2018_maskprod.fill_value
#: Change to new grid
v2018_gg = geosat.GeoGrid('FullAMA_SAFBox')
v2018_p1 = geosat.SatGrid(v2018_data, v2018_gg)
v2018_p1._sat_togrid(grid_type)
#: Decide which one to use
if fm:
#: hPa
use_data = np.where(v2018_prod != v2018_maskprod.fill_value,
v2018_prod / 100, v2018_prod)
else:
use_data = v2018_p1.var[grid_type].data
#: Create mask width valid data
v2018_valid_mask = createClimMask(use_data, clim[0], clim[1],
v2018_maskprod.fill_value)
#: Add to results
Created on Thu Apr 13 17:52:48 2017
This script tests and demonstrates the main features of geosat.py.
@author: Bernard Legras
"""
import geosat
#import pickle,gzip
#import numpy as np
from datetime import datetime
#import matplotlib.pyplot as plt
#%%
date = datetime(year=2017, month=3, day=15, hour=19)
# Himawari read and plot
ah = geosat.Himawari(date)
gg = geosat.GeoGrid('FullAMA')
ah._get_IR0()
ph = geosat.SatGrid(ah, gg)
ph._sat_togrid('IR0')
ah.show('IR0')
ph.chart('IR0', txt='BT himawari')
# MSG1 read and plot
a1 = geosat.MSG1(date)
a1._get_IR0()
p1 = geosat.SatGrid(a1, gg)
p1._sat_togrid('IR0')
a1.show('IR0')
p1.chart('IR0', txt='BT msg1')
# MSG3 read and plot
a3 = geosat.MSG3(date)
a3._get_IR0()
def main():
global IDX_ORGN
parser = argparse.ArgumentParser()
parser.add_argument("-y", "--year", type=int, help="year")
parser.add_argument("-m1",
"--month1",
type=int,
choices=1 + np.arange(12),
help="month1")
parser.add_argument("-m2",
"--month2",
type=int,
choices=1 + np.arange(12),
help="month2")
parser.add_argument("-d1",
"--day1",
type=int,
choices=1 + np.arange(31),
help="day1")
parser.add_argument("-d2",
"--day2",
type=int,
choices=1 + np.arange(31),
help="day2")
parser.add_argument("-a",
"--advect",
type=str,
choices=["OPZ", "EAD", "EAZ", "EID", "EIZ"],
help="source of advecting winds")
parser.add_argument("-s",
"--suffix",
type=str,
help="suffix for special cases")
parser.add_argument("-q",
"--quiet",
type=str,
choices=["y", "n"],
help="quiet (y) or not (n)")
parser.add_argument("-l", "--level", type=int, help="PT level")
#parser.add_argument("-c","--clean0",type=bool,help="clean part_000")
parser.add_argument("-t",
"--step",
type=int,
help="step in hour between two part files")
#parser.add_argument("-ct","--cloud_type",type=str,choices=["meanhigh","veryhigh","silviahigh"],help="cloud type filter")
#parser.add_argument("-k","--diffus",type=str,choices=['01','1','001'],help='diffusivity parameter')
parser.add_argument("-v",
"--vshift",
type=int,
choices=[0, 10],
help='vertical shift')
parser.add_argument("-hm",
"--hmax",
type=int,
help='maximum considered integration time (hour)')
parser.add_argument("-b",
"--bak",
type=str,
choices=["y", "n"],
help="backup (y) or not (n)")
parser.add_argument("-c",
"--cont",
type=str,
choices=["y", "n"],
help="continuation (y) or not (n)")
parser.add_argument("-bs",
"--bakstep",
type=int,
help='backup step (hour)')
# to be updated
# Define main directories
if 'ciclad' in socket.gethostname():
#main_sat_dir = '/data/legras/flexpart_in/SAFNWC'
#SVC_Dir = '/bdd/CFMIP/SEL2'
traj_dir = '/data/legras/flexout/STC/Sivan'
out_dir = '/data/legras/STC'
#out_dir = '/data/legras/STC'
elif ('climserv' in socket.gethostname()) | ('polytechnique'
in socket.gethostname()):
traj_dir = '/data/legras/flexout/STC/Sivan'
out_dir = '/homedata/legras/STC'
else:
print('CANNOT RECOGNIZE HOST - DO NOT RUN ON NON DEFINED HOSTS')
exit()
""" Parameters """
# Output step of the trajectories in hour
step = 6
hmax = 3360
#hmax = 18
dstep = timedelta(hours=step)
# time width of the parcel slice
# might be better passed as a parameter
age_bound = 44.5
# time width of the parcel slice
slice_width = timedelta(minutes=5)
# number of slices between two outputs
nb_slices = int(dstep / slice_width)
# Number of parcels launched per time slot (1 per degree on a 170x50 grid)
granule_size = 8500
# number of granules in step hours
granule_step = 6
# size of granules launched during step hours
granule_quanta = granule_size * granule_step
# default values of parameters
# date of the flight
year = 2017
month1 = 7
month2 = 9
day1 = 1
day2 = 30
advect = 'EAD'
suffix = ''
quiet = False
clean0 = True
backup = False
backup_step = 40 * step
restart = False
#cloud_type = 'silviahigh'
#diffus = '01'
vshift = 0
level = 380
super = ''
# Step of GridSat
dtRange = timedelta(hours=3)
args = parser.parse_args()
if args.year is not None: year = args.year
if args.month1 is not None: month1 = args.month1
if args.month2 is not None: month2 = args.month2
if args.advect is not None: advect = args.advect
if args.day1 is not None: day1 = args.day1
if args.day2 is not None: day2 = args.day2
if args.level is not None: level = args.level
if args.hmax is not None: hmax = args.hmax
if args.suffix is not None: suffix = '-' + args.suffix
if args.quiet is not None:
if args.quiet == 'y': quiet = True
else: quiet = False
#if args.clean0 is not None: clean0 = args.clean0
#if args.cloud_type is not None: cloud_type = args.cloud_type
if args.step is not None: step = args.step
#if args.diffus is not None: diffus = args.diffus
#diffus = '-D' + diffus
if args.vshift is not None:
vshift = args.vshift
if vshift > 0:
super = '-super' + str(vshift)
if args.bak is not None:
if args.bak == 'y': backup = True
else: backup = False
if args.bakstep is not None: backup_step = args.bakstep
if args.cont is not None:
if args.cont == 'y': restart = True
else: restart = False
# Update the out_dir with the cloud type and the super paramater
out_dir = os.path.join(out_dir, 'STC-Sivan-OUT-GridSat-WMO' + super)
try:
os.mkdir(out_dir)
os.mkdir(out_dir + '/out')
except:
print('out_dir directory already created')
# Dates beginning and end
date_beg = datetime(year=year, month=month1, day=day1, hour=0)
date_end = datetime(year=year, month=month2, day=day2, hour=0)
# Manage the file that receives the print output
if quiet:
# Output file
#print_file = os.path.join(out_dir,'out','BACK-SVC-EAD-'+date_beg.strftime('%b-%Y-day%d-')+date_end.strftime('%d-D01')+'.out')
print_file = os.path.join(
out_dir, 'out', 'Sivan-T2-' + advect + '-JAS-' +
date_beg.strftime('%Y-') + str(level) + 'K.out')
fsock = open(print_file, 'w')
sys.stdout = fsock
# initial time to read the sat files
# should be after the latest launch date
sdate = date_end + timedelta(days=1)
print('year', year, 'month1', month1, 'day1', day1, 'month2', month2,
'day2', day2)
print('advect', advect)
print('suffix', suffix)
# patch to fix the problem of the data hole on the evening of 2 August 2017
# should not happen
#if sdate == datetime(2017,8,2): sdate = datetime(2017,8,3,6)
# Directory of the backward trajectories (input) and name of the output file
ftraj = os.path.join(
traj_dir, 'Sivan-' + advect + '-JAS-' + date_beg.strftime('%Y-') +
str(level) + 'K')
out_file2 = os.path.join(
out_dir, 'Sivan-T2-' + advect + '-JAS-' + date_beg.strftime('%Y-') +
str(level) + 'K.hdf5')
# backup file
if backup:
bak_file_prod0 = os.path.join(
out_dir, 'Sivan-T2-' + advect + '-JAS-' +
date_beg.strftime('%Y-') + str(level) + 'K-backup-prod0.hdf5')
bak_file_params = os.path.join(
out_dir, 'Sivan-T2-' + advect + '-JAS-' +
date_beg.strftime('%Y-') + str(level) + 'K-backup-params.hdf5')
""" Initialization of the calculation """
# Initialize the dictionary of the parcel dictionaries
partStep = {}
# initialize a grid that will be used to before actually doing any read
gg = geosat.GeoGrid('GridSat')
# Build the satellite field generator
get_sat = read_sat(sdate, dtRange, pre=True, vshift=vshift)
# Build the ECMWF field generator (both available at 3 hour interval)
get_ERA5 = read_ERA5(sdate, dtRange, pre=True)
# Read the index file that contains the initial positions
part0 = readidx107(os.path.join(ftraj, 'part_000'), quiet=False)
print('numpart', part0['numpart'])
numpart = part0['numpart']
numpart_s = granule_size # because of parcels launched at time 0
# stamp_date not set in these runs
# current_date actually shifted by one day / sdate
# We assume here that part time is defined from this day at 0h
# sdate defined above should be equal or posterior to current_date
current_date = sdate
# check flag is clean
print('check flag is clean ',((part0['flag']&I_HIT)!=0).sum(),((part0['flag']&I_DEAD)!=0).sum(),\
((part0['flag']&I_CROSSED)!=0).sum())
# check idx_orgn
if part0['idx_orgn'] != 0:
print(
'MINCHIA, IDX_ORGN NOT 0 ASSTC-M55-OUT-SAF-super1silviahigh ASSUMED, CORRECTED WITH READ VALUE'
)
print('VALUE ', part0['idx_orgn'])
IDX_ORGN = part0['idx_orgn']
idx1 = IDX_ORGN
# Build a dictionary to host the results
prod0 = defaultdict(dict)
prod0['src']['x'] = np.empty(part0['numpart'], dtype='float')
prod0['src']['y'] = np.empty(part0['numpart'], dtype='float')
prod0['src']['p'] = np.empty(part0['numpart'], dtype='float')
prod0['src']['t'] = np.empty(part0['numpart'], dtype='float')
prod0['src']['age'] = np.empty(part0['numpart'], dtype='int')
prod0['flag_source'] = part0['flag']
# Set rvs at arbitrary large value
prod0['rvs'] = np.full(part0['numpart'], 0.01, dtype='float')
# truncate eventually to 32 bits at the output stage
# read the part_000 file
if not restart:
partStep[0] = readpart107(0, ftraj, quiet=False)
# cleaning is necessary for runs starting in the fake restart mode
# otherwise all parcels are thought to exit at the first step
if clean0:
partStep[0]['idx_back'] = []
# parcels with longitude east of zero degree are set to negative values
partStep[0]['x'][partStep[0]['x'] > 180] -= 360
# number of hists and exits
nhits = 0
nexits = 0
ndborne = 0
nnew = granule_size # because of parcels launched at time 0
nold = 0
offset = 0
# initialize datsat to None to force first read
datsat = None
# used to get non borne parcels (presently unused)
#new = np.empty(part0['numpart'],dtype='bool')
#new.fill(False)
print('Initialization completed')
if restart:
print('restart run')
try:
params = fl.load(bak_file_params)
prod0 = fl.load(bak_file_prod0)
except:
print('cannot load backup')
return -1
[
offset, nhits, nexits, nold, ndborne, nnew, idx1, numpart_s,
current_date
] = params['params']
partStep[offset] = readpart107(offset, ftraj, quiet=True)
partStep[offset]['x'][partStep[offset]['x'] > 180] -= 360
# Initialize sat and ERA5 yield one step ahead as a precaution
get_sat = read_sat(current_date + dstep,
dtRange,
pre=True,
vshift=vshift)
get_ERA5 = read_ERA5(current_date + dstep, dtRange, pre=True)
""" Main loop on the output time steps """
for hour in range(step + offset, hmax + 1, step):
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use: {:4.2f} gb'.format(memoryUse))
# Get rid of dictionary no longer used
if hour >= 2 * step:
try:
del partStep[hour - 2 * step]
except:
print('nothing to delete')
# Read the new data
partStep[hour] = readpart107(hour, ftraj, quiet=True)
# Link the names as views
partante = partStep[hour - step]
partpost = partStep[hour]
# parcels with longitude east of zero degree are set to negative values
# done with try to prevent errors when the file is empty
try:
partpost['x'][partpost['x'] > 180] -= 360
#print("partpost x ",partpost['x'].min(),partpost['x'].max())
except:
pass
if partpost['nact'] > 0:
print('hour ', hour, ' numact ', partpost['nact'], ' max p ',
partpost['p'].max())
else:
print('hour ', hour, ' numact ', partpost['nact'])
# New date valid for partpost
current_date -= dstep
# Processing of water mixing ratio
# Select non stopped parcels in partante
if len(partante['idx_back']) > 0:
selec = (prod0['flag_source'][partante['idx_back'] - IDX_ORGN]
& I_STOP) == 0
idx = partante['idx_back'][selec]
prod0['rvs'][idx-IDX_ORGN] = np.minimum(prod0['rvs'][idx-IDX_ORGN],\
satratio(partante['p'][selec],partante['t'][selec]))
""" Select the parcels that are common to the two steps
ketp_a is a logical field with same length as partante
kept_p is a logical field with same length as partpost
After the launch of the earliest parcel along the flight track, there
should not be any member in new
The parcels
"""
kept_a = np.in1d(partante['idx_back'],
partpost['idx_back'],
assume_unique=True)
kept_p = np.in1d(partpost['idx_back'],
partante['idx_back'],
assume_unique=True)
#new_p = ~np.in1d(partpost['idx_back'],partpost['idx_back'],assume_unique=True)
print('kept a, p ', len(kept_a), len(kept_p), kept_a.sum(),
kept_p.sum(), ' new ',
len(partpost['x']) - kept_p.sum())
nnew += len(partpost['x']) - kept_p.sum()
""" PROCESSING OF DEADBORNE PARCELS
Manage the parcels launched during the last 6-hour which have already
exited and do not appear in posold or posact (borne dead parcels).
These parcels are stored in the last part of posact, at most
the last granule_quanta parcels. """
if numpart_s < numpart:
# First index of the current quanta """
numpart_s += granule_quanta
print("manage deadborne idx1", idx1, " numpart_s", numpart_s)
# Extract the last granule_size indexes from posacti, this is where the new parcels should be
if hour == step:
idx_act = partpost['idx_back']
else:
idx_act = partpost['idx_back'][-granule_quanta:]
# Generate the list of indexes that should be found in this range
# This should works for both values of IDX_orgn
idx_theor = np.arange(idx1, numpart_s + IDX_ORGN)
# Find the missing indexes in idx_act (make a single line after validation)
kept_borne = np.in1d(idx_theor, idx_act, assume_unique=True)
idx_deadborne = idx_theor[~kept_borne]
# Process these parcels by assigning exit at initial location
prod0['flag_source'][
idx_deadborne -
IDX_ORGN] = prod0['flag_source'][idx_deadborne -
IDX_ORGN] | I_DEAD + I_DBORNE
prod0['src']['x'][idx_deadborne -
IDX_ORGN] = part0['x'][idx_deadborne - IDX_ORGN]
prod0['src']['y'][idx_deadborne -
IDX_ORGN] = part0['y'][idx_deadborne - IDX_ORGN]
prod0['src']['p'][idx_deadborne -
IDX_ORGN] = part0['p'][idx_deadborne - IDX_ORGN]
prod0['src']['t'][idx_deadborne -
IDX_ORGN] = part0['t'][idx_deadborne - IDX_ORGN]
prod0['src']['age'][idx_deadborne - IDX_ORGN] = 0.
print("number of deadborne ", len(idx_deadborne))
ndborne += len(idx_deadborne)
idx1 = numpart_s + IDX_ORGN
""" PROCESSING OF CROSSED PARCELS """
if len(kept_a) > 0:
exits = exiter(int((partante['itime']+partpost['itime'])/2), \
partante['x'][~kept_a],partante['y'][~kept_a],partante['p'][~kept_a],\
partante['t'][~kept_a],partante['idx_back'][~kept_a],\
prod0['flag_source'],prod0['src']['x'],prod0['src']['y'],\
prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],\
part0['ir_start'], gg.box_range)
nexits += exits
print('exit ', nexits, exits, np.sum(~kept_a),
len(kept_a) - len(kept_p))
""" PROCESSING OF PARCELS WHICH ARE COMMON TO THE TWO OUTPUTS """
# Select the kept parcels which have not been hit yet
# !!! Never use and between two lists, the result is wrong
if kept_p.sum() == 0:
live_a = live_p = kept_p
else:
live_a = np.logical_and(
kept_a, (prod0['flag_source'][partante['idx_back'] - IDX_ORGN]
& I_DEAD) == 0)
live_p = np.logical_and(
kept_p, (prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_DEAD) == 0)
print('live a, b ', live_a.sum(), live_p.sum())
""" Correction step that moves partante['x'] to avoid big jumps at the periodicity frontier on the x-axis
To be activated in the FULL version"""
# if kept_p.sum()>0:
# diffx = partpost['x'][live_p] - partante['x'][live_a]
# bb = np.zeros(len(diffx))
# bb[diffx>180] = 360
# bb[diffx<-180] = -360
# partante['x'][live_a] += bb
# del bb
# del diffx
# del kept_p
# del kept_a
# DOES not work in the following WAY
#partante['x'][live_a][diffx>180] += 360
#partante['x'][live_a][diffx<-180] -= 360
# Build generator for parcel locations of the 5' slices
gsp = get_slice_part(partante, partpost, live_a, live_p, current_date,
dstep, slice_width)
if verbose: print('built parcel generator for ', current_date)
""" MAIN LOOP ON THE PARCEL TIME SLICES """
for i in range(nb_slices):
# get the slice for the particles
datpart = next(gsp)
# Check x within (-180,180), necessary when GridSat is used
#if datpart['x'] != []:
if len(datpart['x']) > 0:
datpart['x'] = (datpart['x'] + 180) % 360 - 180
if verbose: print('part slice ', i, datpart['time'])
# Check whether the present satellite image is valid
# The while should ensure that the run synchronizes
# when it starts.
while check(datsat, datpart['time']) is False:
# if not get next satellite image
datsat = next(get_sat)
datera = next(get_ERA5)
""" PROCESS THE COMPARISON OF PARCEL PRESSURES TO CLOUDS """
if len(datpart['x']) > 0:
# tropopause pressure at the location of the parcels
#print('ptropo x ',datpart['x'].min(),datpart['x'].max())
ptrop = datera.fP(np.transpose([datpart['y'], datpart['x']]))
nhits += convbirth(datpart['itime'],
datpart['x'],datpart['y'],datpart['p'],datpart['t'],datpart['idx_back'],\
prod0['flag_source'],ptrop,prod0['src']['x'],prod0['src']['y'],\
prod0['src']['p'],prod0['src']['t'],prod0['src']['age'],\
datsat.var['IR0'], part0['ir_start'],\
datsat.geogrid.box_range[0,0],datsat.geogrid.box_range[1,0],datsat.geogrid.stepx,\
datsat.geogrid.stepy,datsat.geogrid.box_binx,datsat.geogrid.box_biny)
""" End of of loop on slices """
""" INSERT HERE CODE FOR PARCELS ENDING BY AGE
Important: if the age limit is set in the run, the age_bound needs to
be decremented by one output step. Otherwise, the parcel disappeared
and is wrongly processed as crossed."""
# Check the age limit (easier to do it here)
if len(partante['idx_back']) > 0:
age_sec = part0['ir_start'][partante['idx_back'] -
IDX_ORGN] - partante['itime']
IIold_o = age_sec > (age_bound - (step / 24)) * 86400
IIold_o = IIold_o & (
(prod0['flag_source'][partante['idx_back'] - IDX_ORGN]
& I_STOP) == 0)
idx_IIold = partante['idx_back'][IIold_o]
j_IIold_o = np.where(IIold_o)
prod0['flag_source'][idx_IIold - IDX_ORGN] = prod0['flag_source'][
idx_IIold - IDX_ORGN] | I_DEAD + I_OLD
prod0['src']['x'][idx_IIold - IDX_ORGN] = partante['x'][j_IIold_o]
prod0['src']['y'][idx_IIold - IDX_ORGN] = partante['y'][j_IIold_o]
prod0['src']['p'][idx_IIold - IDX_ORGN] = partante['p'][j_IIold_o]
prod0['src']['t'][idx_IIold - IDX_ORGN] = partante['t'][j_IIold_o]
prod0['src']['age'][idx_IIold - IDX_ORGN] = (
(part0['ir_start'][idx_IIold - IDX_ORGN] - partante['itime']) /
86400)
print("manage age limit: number of IIold ", len(idx_IIold))
nold += len(idx_IIold)
# find parcels still alive after processing this step if kept_p.sum()==0:
try:
nlive = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_DEAD) == 0).sum()
n_nohit = ((prod0['flag_source'][partpost['idx_back'] - IDX_ORGN]
& I_HIT) == 0).sum()
except:
nlive = 0
n_nohit = 0
# check that nlive + nhits + nexits = numpart, should be true after the first day
if numpart_s != nexits + nhits + nlive + ndborne + nold:
print('@@@ ACHTUNG numpart_s not equal to sum ', numpart_s,
nexits + nhits + nlive + ndborne + nold)
print('end hour ',
hour,
' numact',
partpost['nact'],
' nexits',
nexits,
' nhits',
nhits,
' nlive',
nlive,
' nohit',
n_nohit,
' nold',
nold,
flush=True)
sys.stdout.flush()
if backup & (hour % backup_step == 0):
fl.save(bak_file_prod0, prod0)
fl.save(
bak_file_params, {
'params': [
hour, nhits, nexits, nold, ndborne, nnew, idx1,
numpart_s, current_date
]
})
""" End of the procedure and storage of the result """
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use before clean: {:4.2f} gb'.format(memoryUse))
del partante
del partpost
del live_a
del live_p
del datpart
prod0['rvs'] = prod0['rvs'].astype(np.float32)
for var in ['age', 'p', 't', 'x', 'y']:
prod0['src'][var] = prod0['src'][var].astype(np.float32)
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2**30
print('memory use after clean: {:4.2f} gb'.format(memoryUse))
#output file
fl.save(out_file2, prod0, compression='zlib')
#pickle.dump(prod0,gzip.open(out_file,'wb'))
# close the print file
if quiet: fsock.close()