def makeBulkLCS(itnum, nrt):
if itnum < 1993001:
print('Error: LCS is only available after 1993.')
return
df_bw_adt = matToDF(nrt=nrt, forward=False, sla=False, itnum=itnum)
df_fw_adt = matToDF(nrt=nrt, forward=True, sla=False, itnum=itnum)
df_bw_sla = matToDF(nrt=nrt, forward=False, sla=True, itnum=itnum)
df_fw_sla = matToDF(nrt=nrt, forward=True, sla=True, itnum=itnum)
prefix = 'LCS_'
df = pd.DataFrame()
df['lat'] = df_bw_adt['lat']
df['lon'] = df_bw_adt['lon']
df['time'] = df_bw_adt['time']
df['ftle_bw_adt'] = df_bw_adt['ftle']
df['disp_bw_adt'] = df_bw_adt['disp']
df['ftle_fw_adt'] = df_fw_adt['ftle']
df['disp_fw_adt'] = df_fw_adt['disp']
df['ftle_bw_sla'] = df_bw_sla['ftle']
df['disp_bw_sla'] = df_bw_sla['disp']
df['ftle_fw_sla'] = df_fw_sla['ftle']
df['disp_fw_sla'] = df_fw_sla['disp']
df['ID'] = None
exportBase = cfgv.opedia_proj + 'db/dbInsert/export/'
export_path = '%s%s%d.csv' % (exportBase, prefix, itnum)
df.to_csv(export_path, index=False)
ip.mapTo180180(export_path, 'lon') # only use if necessary
ip.sortByLatLon(df, export_path, 'lon', 'lat')
return export_path
def makeBulkCHL(itnum, nrt):
if itnum < 1998001:
print('Error: CHL data is only availabe after 1998.')
return
if nrt:
path = cfgv.nrt_chl_raw + cfgv.nrt_chl_prefix + '%7.7d.nc' % itnum
else:
path = cfgv.rep_chl_raw + cfgv.rep_chl_prefix + '%7.7d.nc' % itnum
prefix = 'chl_oi_'
df = nc.ncToDF(path)
df = ip.removeColumn(['CHL_error'], df)
#df = ip.removeMissings(['CHL'], df) # remove land
df['ID'] = None
exportBase = cfgv.opedia_proj + 'db/dbInsert/export/'
export_path = '%s%s%d.csv' % (exportBase, prefix, itnum)
df.to_csv(export_path)
## seems like the code below is only needed for the NRT product
## longitude range in NRT products: 0 - 360
if nrt:
ip.mapTo180180(export_path, 'lon') # only use if necessary
###df = downsampleCHL(export_path)
ip.sortByLatLon(df, export_path, 'lon', 'lat')
return export_path
def makeBulkMLD(itnum, nrt):
if nrt:
path = cfgv.nrt_mercator_mld_raw + cfgv.nrt_mercator_mld_prefix + '%7.7d.nc' % itnum
else:
path = 'unknown'
#path = cfgv.rep_mercator_pisces_raw + cfgv.rep_mercator_pisces_prefix + '%7.7d.nc' % itnum
prefix = 'mercator_mld_'
df = nc.ncToDF(path)
## arrange the columns: making sure that the columns are arranged in the correct (consistent with the undelying table) order
#df = ip.arrangeColumns(['mlotst'], df)
df['ID'] = None
exportBase = cfgv.opedia_proj + 'db/dbInsert/export/'
export_path = '%s%s%d.csv' % (exportBase, prefix, itnum)
df.to_csv(export_path)
#ip.mapTo180180(export_path, 'longitude') # only use if necessary
ip.sortByLatLon(df, export_path, 'longitude', 'latitude')
return export_path
def makeBulkALT(itnum, nrt):
if itnum < 1993001:
print('Error: Altimetry data is only available after 1993.')
return
if nrt:
path = cfgv.nrt_alt_raw + cfgv.nrt_alt_prefix + '%7.7d.nc' % itnum
else:
path = cfgv.rep_alt_raw + cfgv.rep_alt_prefix + '%7.7d.nc' % itnum
prefix = 'alt_'
df = nc.ncToDF(path)
df = ip.removeColumn(['err'], df)
#df = ip.removeMissings(['sla'], df) # remove land
df['ID'] = None
exportBase = cfgv.opedia_proj + 'db/dbInsert/export/'
export_path = '%s%s%d.csv' % (exportBase, prefix, itnum)
df.to_csv(export_path)
ip.mapTo180180(export_path, 'longitude') # only use if necessary
ip.sortByLatLon(df, export_path, 'longitude', 'latitude')
return export_path
def makeBulkVort(itnum, nrt):
if itnum < 1993001:
print('Error: Relative Vorticity is only available after 1993.')
return
if nrt:
path = cfgv.nrt_vort_sla_raw + cfgv.nrt_vort_sla_prefix + '%10.10d.mat' % itnum
else:
path = cfgv.rep_vort_sla_raw + cfgv.rep_vort_sla_prefix + '%10.10d.mat' % itnum
prefix = 'vort_sla_'
df = matToDF(path, itnum)
#df = ip.removeColumn(['err'], df)
#df = ip.removeMissings(['vort'], df) # remove land
df['ID'] = None
exportBase = cfgv.opedia_proj + 'db/dbInsert/export/'
export_path = '%s%s%d.csv' % (exportBase, prefix, itnum)
df.to_csv(export_path, index=False)
ip.mapTo180180(export_path, 'lon') # only use if necessary
ip.sortByLatLon(df, export_path, 'lon', 'lat')
return export_path
def makeBulkSST(itnum, nrt):
if itnum < 1981244:
print('Error: SST data is only availabe after 1981224.')
return
if nrt:
path = cfgv.nrt_sst_raw + cfgv.nrt_sst_prefix + '%7.7d.nc' % itnum
else:
path = 'unknown'
#path = cfgv.rep_sst_raw + cfgv.rep_sst_prefix + '%7.7d.nc' % itnum
prefix = 'sst_'
df = nc.ncToDF(path)
df = ip.removeColumn(
['analysis_error', 'mask', 'sea_ice_fraction', 'lat_bnds', 'lon_bnds'],
df)
if 'time_bnds' in df.columns:
df = ip.removeColumn(['time_bnds'], df)
#df = ip.removeMissings(['analysed_sst'], df) # remove land
df['ID'] = None
exportBase = cfgv.opedia_proj + 'db/dbInsert/export/'
export_path = '%s%s%d.csv' % (exportBase, prefix, itnum)
df.to_csv(export_path)
#ip.mapTo180180(export_path, 'lon') # only use if necessary
ip.sortByLatLon(df, export_path, 'lon', 'lat')
## 1. only keep records with nv=1 (remove nv=0 records)
## 2. drop nv column
## 3. unit conversion; kelvin to centigrade
## 4. fix time stamp: one or some of the sst files have incorrecty timestamp.
df = pd.read_csv(export_path)
df = df[df['nv'] == 1]
df = ip.removeColumn(['nv'], df)
df['analysed_sst'] = df['analysed_sst'] - 273.15
if itnum in [
2016097
]: # this is to account for a little bug in the time variable in the SST netcdf file
df['time'] = pd.DatetimeIndex(df['time']) + pd.DateOffset(1)
df.to_csv(export_path, index=False)
####################################################
return export_path
def makeBulkWind(itnum, nrt, hour):
if itnum < 2012320:
print('Error: Wind data is only availabe after 2012320.')
return
if nrt:
path = cfgv.nrt_wind_raw + cfgv.nrt_wind_prefix + '%7.7d_%2.2dh.nc' % (
itnum, hour)
else:
path = 'unknown'
#path = cfgv.rep_wind_raw + cfgv.rep_wind_prefix + '%7.7d_%2.2dh.nc' % (itnum, hour)
prefix = 'wind_'
df = nc.ncToDF(path)
df = ip.removeColumn(['land_ice_mask', 'sampling_length'], df)
#df = ip.removeMissings(['wind_stress'], df) # remove land
## arrange the columns: making sure that the columns are arranged in the correct (consistent with the undelying table) order
df = ip.arrangeColumns([
'wind_speed_rms', 'eastward_wind_rms', 'wind_stress', 'eastward_wind',
'surface_downward_eastward_stress', 'wind_speed',
'surface_downward_northward_stress', 'northward_wind',
'northward_wind_rms'
], df)
df['hour'] = hour
df['ID'] = None
exportBase = cfgv.opedia_proj + 'db/dbInsert/export/'
export_path = '%s%s%d_%2.2dh.csv' % (exportBase, prefix, itnum, hour)
df.to_csv(export_path)
#ip.mapTo180180(export_path, 'longitude') # only use if necessary
ip.sortByLatLon(df, export_path, 'longitude', 'latitude')
########### drop depth column ###############
df = pd.read_csv(export_path)
df = ip.removeColumn(['depth'], df)
df.to_csv(export_path, index=False)
##############################################
return export_path
def makeBulkALT(itnum, nrt):
if itnum < 1993001:
print('Error: Altimetry data is only available after 1993.')
return
if nrt:
path = cfgv.nrt_alt_raw + cfgv.nrt_alt_prefix + '%7.7d.nc' % itnum
else:
path = cfgv.rep_alt_raw + cfgv.rep_alt_prefix + '%7.7d.nc' % itnum
prefix = 'alt_'
df = nc.ncToDF(path)
df = ip.removeColumn(['err'], df)
#df = ip.removeMissings(['sla'], df) # remove land
## arrange the columns: making sure that the columns are arranged in the correct (consistent with the undelying table) order
df = ip.arrangeColumns(['vgosa', 'vgos', 'sla', 'adt', 'ugosa', 'ugos'], df)
# df['ID'] = None
exportBase = cfgv.opedia_proj + 'db/dbInsert/export/'
export_path = '%s%s%d.csv' % (exportBase, prefix, itnum)
df.to_csv(export_path)
ip.mapTo180180(export_path, 'longitude') # only use if necessary
ip.sortByLatLon(df, export_path, 'longitude', 'latitude')
return export_path