def makeBulkLCS(itnum, nrt): if itnum < 1993001: print('Error: LCS is only available after 1993.') return df_bw_adt = matToDF(nrt=nrt, forward=False, sla=False, itnum=itnum) df_fw_adt = matToDF(nrt=nrt, forward=True, sla=False, itnum=itnum) df_bw_sla = matToDF(nrt=nrt, forward=False, sla=True, itnum=itnum) df_fw_sla = matToDF(nrt=nrt, forward=True, sla=True, itnum=itnum) prefix = 'LCS_' df = pd.DataFrame() df['lat'] = df_bw_adt['lat'] df['lon'] = df_bw_adt['lon'] df['time'] = df_bw_adt['time'] df['ftle_bw_adt'] = df_bw_adt['ftle'] df['disp_bw_adt'] = df_bw_adt['disp'] df['ftle_fw_adt'] = df_fw_adt['ftle'] df['disp_fw_adt'] = df_fw_adt['disp'] df['ftle_bw_sla'] = df_bw_sla['ftle'] df['disp_bw_sla'] = df_bw_sla['disp'] df['ftle_fw_sla'] = df_fw_sla['ftle'] df['disp_fw_sla'] = df_fw_sla['disp'] df['ID'] = None exportBase = cfgv.opedia_proj + 'db/dbInsert/export/' export_path = '%s%s%d.csv' % (exportBase, prefix, itnum) df.to_csv(export_path, index=False) ip.mapTo180180(export_path, 'lon') # only use if necessary ip.sortByLatLon(df, export_path, 'lon', 'lat') return export_path
def makeBulkCHL(itnum, nrt): if itnum < 1998001: print('Error: CHL data is only availabe after 1998.') return if nrt: path = cfgv.nrt_chl_raw + cfgv.nrt_chl_prefix + '%7.7d.nc' % itnum else: path = cfgv.rep_chl_raw + cfgv.rep_chl_prefix + '%7.7d.nc' % itnum prefix = 'chl_oi_' df = nc.ncToDF(path) df = ip.removeColumn(['CHL_error'], df) #df = ip.removeMissings(['CHL'], df) # remove land df['ID'] = None exportBase = cfgv.opedia_proj + 'db/dbInsert/export/' export_path = '%s%s%d.csv' % (exportBase, prefix, itnum) df.to_csv(export_path) ## seems like the code below is only needed for the NRT product ## longitude range in NRT products: 0 - 360 if nrt: ip.mapTo180180(export_path, 'lon') # only use if necessary ###df = downsampleCHL(export_path) ip.sortByLatLon(df, export_path, 'lon', 'lat') return export_path
def makeBulkMLD(itnum, nrt): if nrt: path = cfgv.nrt_mercator_mld_raw + cfgv.nrt_mercator_mld_prefix + '%7.7d.nc' % itnum else: path = 'unknown' #path = cfgv.rep_mercator_pisces_raw + cfgv.rep_mercator_pisces_prefix + '%7.7d.nc' % itnum prefix = 'mercator_mld_' df = nc.ncToDF(path) ## arrange the columns: making sure that the columns are arranged in the correct (consistent with the undelying table) order #df = ip.arrangeColumns(['mlotst'], df) df['ID'] = None exportBase = cfgv.opedia_proj + 'db/dbInsert/export/' export_path = '%s%s%d.csv' % (exportBase, prefix, itnum) df.to_csv(export_path) #ip.mapTo180180(export_path, 'longitude') # only use if necessary ip.sortByLatLon(df, export_path, 'longitude', 'latitude') return export_path
def makeBulkALT(itnum, nrt): if itnum < 1993001: print('Error: Altimetry data is only available after 1993.') return if nrt: path = cfgv.nrt_alt_raw + cfgv.nrt_alt_prefix + '%7.7d.nc' % itnum else: path = cfgv.rep_alt_raw + cfgv.rep_alt_prefix + '%7.7d.nc' % itnum prefix = 'alt_' df = nc.ncToDF(path) df = ip.removeColumn(['err'], df) #df = ip.removeMissings(['sla'], df) # remove land df['ID'] = None exportBase = cfgv.opedia_proj + 'db/dbInsert/export/' export_path = '%s%s%d.csv' % (exportBase, prefix, itnum) df.to_csv(export_path) ip.mapTo180180(export_path, 'longitude') # only use if necessary ip.sortByLatLon(df, export_path, 'longitude', 'latitude') return export_path
def makeBulkVort(itnum, nrt): if itnum < 1993001: print('Error: Relative Vorticity is only available after 1993.') return if nrt: path = cfgv.nrt_vort_sla_raw + cfgv.nrt_vort_sla_prefix + '%10.10d.mat' % itnum else: path = cfgv.rep_vort_sla_raw + cfgv.rep_vort_sla_prefix + '%10.10d.mat' % itnum prefix = 'vort_sla_' df = matToDF(path, itnum) #df = ip.removeColumn(['err'], df) #df = ip.removeMissings(['vort'], df) # remove land df['ID'] = None exportBase = cfgv.opedia_proj + 'db/dbInsert/export/' export_path = '%s%s%d.csv' % (exportBase, prefix, itnum) df.to_csv(export_path, index=False) ip.mapTo180180(export_path, 'lon') # only use if necessary ip.sortByLatLon(df, export_path, 'lon', 'lat') return export_path
def makeBulkSST(itnum, nrt): if itnum < 1981244: print('Error: SST data is only availabe after 1981224.') return if nrt: path = cfgv.nrt_sst_raw + cfgv.nrt_sst_prefix + '%7.7d.nc' % itnum else: path = 'unknown' #path = cfgv.rep_sst_raw + cfgv.rep_sst_prefix + '%7.7d.nc' % itnum prefix = 'sst_' df = nc.ncToDF(path) df = ip.removeColumn( ['analysis_error', 'mask', 'sea_ice_fraction', 'lat_bnds', 'lon_bnds'], df) if 'time_bnds' in df.columns: df = ip.removeColumn(['time_bnds'], df) #df = ip.removeMissings(['analysed_sst'], df) # remove land df['ID'] = None exportBase = cfgv.opedia_proj + 'db/dbInsert/export/' export_path = '%s%s%d.csv' % (exportBase, prefix, itnum) df.to_csv(export_path) #ip.mapTo180180(export_path, 'lon') # only use if necessary ip.sortByLatLon(df, export_path, 'lon', 'lat') ## 1. only keep records with nv=1 (remove nv=0 records) ## 2. drop nv column ## 3. unit conversion; kelvin to centigrade ## 4. fix time stamp: one or some of the sst files have incorrecty timestamp. df = pd.read_csv(export_path) df = df[df['nv'] == 1] df = ip.removeColumn(['nv'], df) df['analysed_sst'] = df['analysed_sst'] - 273.15 if itnum in [ 2016097 ]: # this is to account for a little bug in the time variable in the SST netcdf file df['time'] = pd.DatetimeIndex(df['time']) + pd.DateOffset(1) df.to_csv(export_path, index=False) #################################################### return export_path
def makeBulkWind(itnum, nrt, hour): if itnum < 2012320: print('Error: Wind data is only availabe after 2012320.') return if nrt: path = cfgv.nrt_wind_raw + cfgv.nrt_wind_prefix + '%7.7d_%2.2dh.nc' % ( itnum, hour) else: path = 'unknown' #path = cfgv.rep_wind_raw + cfgv.rep_wind_prefix + '%7.7d_%2.2dh.nc' % (itnum, hour) prefix = 'wind_' df = nc.ncToDF(path) df = ip.removeColumn(['land_ice_mask', 'sampling_length'], df) #df = ip.removeMissings(['wind_stress'], df) # remove land ## arrange the columns: making sure that the columns are arranged in the correct (consistent with the undelying table) order df = ip.arrangeColumns([ 'wind_speed_rms', 'eastward_wind_rms', 'wind_stress', 'eastward_wind', 'surface_downward_eastward_stress', 'wind_speed', 'surface_downward_northward_stress', 'northward_wind', 'northward_wind_rms' ], df) df['hour'] = hour df['ID'] = None exportBase = cfgv.opedia_proj + 'db/dbInsert/export/' export_path = '%s%s%d_%2.2dh.csv' % (exportBase, prefix, itnum, hour) df.to_csv(export_path) #ip.mapTo180180(export_path, 'longitude') # only use if necessary ip.sortByLatLon(df, export_path, 'longitude', 'latitude') ########### drop depth column ############### df = pd.read_csv(export_path) df = ip.removeColumn(['depth'], df) df.to_csv(export_path, index=False) ############################################## return export_path
def makeBulkALT(itnum, nrt): if itnum < 1993001: print('Error: Altimetry data is only available after 1993.') return if nrt: path = cfgv.nrt_alt_raw + cfgv.nrt_alt_prefix + '%7.7d.nc' % itnum else: path = cfgv.rep_alt_raw + cfgv.rep_alt_prefix + '%7.7d.nc' % itnum prefix = 'alt_' df = nc.ncToDF(path) df = ip.removeColumn(['err'], df) #df = ip.removeMissings(['sla'], df) # remove land ## arrange the columns: making sure that the columns are arranged in the correct (consistent with the undelying table) order df = ip.arrangeColumns(['vgosa', 'vgos', 'sla', 'adt', 'ugosa', 'ugos'], df) # df['ID'] = None exportBase = cfgv.opedia_proj + 'db/dbInsert/export/' export_path = '%s%s%d.csv' % (exportBase, prefix, itnum) df.to_csv(export_path) ip.mapTo180180(export_path, 'longitude') # only use if necessary ip.sortByLatLon(df, export_path, 'longitude', 'latitude') return export_path