def process_cumulative_plot(url_dest_list, start_ts_utc, end_ts_utc, output_dir, lat_min, lon_min, lat_max, lon_max): from_to = '%s-%s' % (start_ts_utc.strftime('%Y-%m-%d_%H:%M'), end_ts_utc.strftime('%Y-%m-%d_%H:%M')) cum_filename = os.path.join(output_dir, 'jaxa_sat_cum_rf_' + from_to + '.png') if not utils.file_exists_nonempty(cum_filename): total = None for url_dest in url_dest_list: if total is None: total = np.genfromtxt(url_dest[2] + '.archive', dtype=float) else: total += np.genfromtxt(url_dest[2] + '.archive', dtype=float) title = 'Cumulative rainfall ' + from_to # clevs = np.concatenate(([-1, 0], np.array([pow(2, i) for i in range(0, 9)]))) clevs_cum = 10 * np.array( [0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30, 50, 75, 100]) norm_cum = colors.BoundaryNorm(boundaries=clevs_cum, ncolors=256) cmap = plt.get_cmap('jet') ext_utils.create_contour_plot(total, cum_filename, lat_min, lon_min, lat_max, lon_max, title, clevs=clevs_cum, cmap=cmap, norm=norm_cum) else: logging.info('%s already exits' % cum_filename)
def _get_google_app_credentials(): g_app_cred_path = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] if utils.file_exists_nonempty(g_app_cred_path): with open(g_app_cred_path, 'r') as f: return f.read() else: return None
def clean_up_wrf_run(init_task_id, **context): config_str = docker_rf_utils.get_base64_decoded_str(context['task_instance'].xcom_pull(task_ids=init_task_id)) wrf_config = json.loads(config_str) metgrid_path = os.path.join(wrf_config.get('nfs_dir'), 'metgrid', wrf_config.get('run_id') + '_metgrid.zip') if utils.file_exists_nonempty(metgrid_path): logging.info('Cleaning up metgrid ' + metgrid_path) os.remove(metgrid_path) else: logging.info('No metgrid file')
def create_asc_file(data, lats, lons, out_file_path, cell_size=0.1, no_data_val=-99, overwrite=False): if not utils.file_exists_nonempty(out_file_path) or overwrite: with open(out_file_path, 'wb') as out_file: out_file.write(('NCOLS %d\n' % len(lons)).encode()) out_file.write(('NROWS %d\n' % len(lats)).encode()) out_file.write(('XLLCORNER %f\n' % lons[0]).encode()) out_file.write(('YLLCORNER %f\n' % lats[0]).encode()) out_file.write(('CELLSIZE %f\n' % cell_size).encode()) out_file.write(('NODATA_VALUE %d\n' % no_data_val).encode()) np.savetxt(out_file, data, fmt='%.4f') else: logging.info('%s already exits' % out_file_path)
def create_contour_plot(data, out_file_path, lat_min, lon_min, lat_max, lon_max, plot_title, basemap=None, clevs=None, cmap=plt.get_cmap('Reds'), overwrite=False, norm=None, additional_changes=None, **kwargs): """ create a contour plot using basemap :param additional_changes: :param norm: :param cmap: color map :param clevs: color levels :param basemap: creating basemap takes time, hence you can create it outside and pass it over :param plot_title: :param data: 2D grid data :param out_file_path: :param lat_min: :param lon_min: :param lat_max: :param lon_max: :param overwrite: :return: """ if not utils.file_exists_nonempty(out_file_path) or overwrite: fig = plt.figure(figsize=(8.27, 11.69)) # ax = fig.add_axes([0.1, 0.1, 0.8, 0.8]) if basemap is None: basemap = Basemap(projection='merc', llcrnrlon=lon_min, llcrnrlat=lat_min, urcrnrlon=lon_max, urcrnrlat=lat_max, resolution='h') basemap.drawcoastlines() parallels = np.arange( math.floor(lat_min) - 1, math.ceil(lat_max) + 1, 1) basemap.drawparallels(parallels, labels=[1, 0, 0, 0], fontsize=10) meridians = np.arange( math.floor(lon_min) - 1, math.ceil(lon_max) + 1, 1) basemap.drawmeridians(meridians, labels=[0, 0, 0, 1], fontsize=10) ny = data.shape[0] nx = data.shape[1] lons, lats = basemap.makegrid(nx, ny) if clevs is None: clevs = np.arange(-1, np.max(data) + 1, 1) # cs = basemap.contourf(lons, lats, data, clevs, cmap=cm.s3pcpn_l, latlon=True) cs = basemap.contourf(lons, lats, data, clevs, cmap=cmap, latlon=True, norm=norm) cbar = basemap.colorbar(cs, location='bottom', pad="5%") cbar.set_label('mm') if isinstance(plot_title, str): plt.title(plot_title) elif isinstance(plot_title, dict): plt.title(plot_title.pop('label'), **plot_title) # make any additional changes to the plot if additional_changes is not None: additional_changes(plt, data, **kwargs) # draw_center_of_mass(data) # com = ndimage.measurements.center_of_mass(data) # plt.plot(com[1], com[0], 'ro') plt.draw() plt.savefig(out_file_path) # fig.savefig(out_file_path) plt.close() else: logging.info('%s already exists' % out_file_path)
def extract_jaxa_satellite_data(start_ts_utc, end_ts_utc, output_dir, cleanup=True, cum=False, tmp_dir=None, lat_min=5.722969, lon_min=79.52146, lat_max=10.06425, lon_max=82.18992, output_prefix='jaxa_sat', db_adapter_config=None): start = utils.datetime_floor(start_ts_utc, 3600) end = utils.datetime_floor(end_ts_utc, 3600) login = '******' url0 = 'ftp://' + login + '@hokusai.eorc.jaxa.jp/realtime/txt/05_AsiaSS/YYYY/MM/DD/gsmap_nrt.YYYYMMDD.HH00.05_AsiaSS.csv.zip' url1 = 'ftp://' + login + '@hokusai.eorc.jaxa.jp/now/txt/05_AsiaSS/gsmap_now.YYYYMMDD.HH00_HH59.05_AsiaSS.csv.zip' def get_jaxa_url(ts): url_switch = (dt.datetime.utcnow() - ts) > dt.timedelta(hours=5) _url = url0 if url_switch else url1 ph = { 'YYYY': ts.strftime('%Y'), 'MM': ts.strftime('%m'), 'DD': ts.strftime('%d'), 'HH': ts.strftime('%H') } for k, v in list(ph.items()): _url = _url.replace(k, v) return _url # tmp_dir = os.path.join(output_dir, 'tmp_jaxa/') # if not os.path.exists(tmp_dir): # os.mkdir(tmp_dir) if tmp_dir is None: tmp_dir = tempfile.mkdtemp(prefix='tmp_jaxa') url_dest_list = [] for timestamp in np.arange(start, end, dt.timedelta(hours=1)).astype(dt.datetime): url = get_jaxa_url(timestamp) url_dest_list.append( (url, os.path.join(tmp_dir, os.path.basename(url)), os.path.join( output_dir, output_prefix + '_' + timestamp.strftime('%Y-%m-%d_%H:%M') + '.asc'))) procs = multiprocessing.cpu_count() logging.info('Downloading inventory in parallel') utils.download_parallel(url_dest_list, procs) logging.info('Downloading inventory complete') logging.info('Processing files in parallel') Parallel(n_jobs=procs)(delayed(process_jaxa_zip_file) (i[1], i[2], lat_min, lon_min, lat_max, lon_max, cum, output_prefix, db_adapter_config) for i in url_dest_list) logging.info('Processing files complete') logging.info('Creating sat rf gif for today') create_daily_gif(start, output_dir, output_prefix + '_today.gif', output_prefix) prev_day_gif = os.path.join(output_dir, output_prefix + '_yesterday.gif') if not utils.file_exists_nonempty(prev_day_gif) or start.strftime( '%H:%M') == '00:00': logging.info('Creating sat rf gif for yesterday') create_daily_gif( utils.datetime_floor(start, 3600 * 24) - dt.timedelta(days=1), output_dir, output_prefix + '_yesterday.gif', output_prefix) if cum: logging.info('Processing cumulative') process_cumulative_plot(url_dest_list, start_ts_utc, end_ts_utc, output_dir, lat_min, lon_min, lat_max, lon_max) logging.info('Processing cumulative complete') # clean up temp dir if cleanup: logging.info('Cleaning up') shutil.rmtree(tmp_dir) utils.delete_files_with_prefix(output_dir, '*.archive')
def process_jaxa_zip_file(zip_file_path, out_file_path, lat_min, lon_min, lat_max, lon_max, archive_data=False, output_prefix='jaxa_sat', db_adapter_config=None): sat_zip = zipfile.ZipFile(zip_file_path) sat = np.genfromtxt(sat_zip.open( os.path.basename(zip_file_path).replace('.zip', '')), delimiter=',', names=True) sat_filt = np.sort(sat[(sat['Lat'] <= lat_max) & (sat['Lat'] >= lat_min) & (sat['Lon'] <= lon_max) & (sat['Lon'] >= lon_min)], order=['Lat', 'Lon']) lats = np.sort(np.unique(sat_filt['Lat'])) lons = np.sort(np.unique(sat_filt['Lon'])) data = sat_filt['RainRate'].reshape(len(lats), len(lons)) ext_utils.create_asc_file(np.flip(data, 0), lats, lons, out_file_path) # clevs = np.concatenate(([-1, 0], np.array([pow(2, i) for i in range(0, 9)]))) # clevs = 10 * np.array([0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30]) # norm = colors.BoundaryNorm(boundaries=clevs, ncolors=256) # cmap = plt.get_cmap('jet') clevs = [ 0, 1, 2.5, 5, 7.5, 10, 15, 20, 30, 40, 50, 75, 100, 150, 200, 250, 300 ] # clevs = [0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30, 50, 75, 100] norm = None cmap = cm.s3pcpn ts = dt.datetime.strptime( os.path.basename(out_file_path).replace(output_prefix + '_', '').replace('.asc', ''), '%Y-%m-%d_%H:%M') lk_ts = utils.datetime_utc_to_lk(ts) title_opts = { 'label': output_prefix + ' ' + lk_ts.strftime('%Y-%m-%d %H:%M') + ' LK\n' + ts.strftime('%Y-%m-%d %H:%M') + ' UTC', 'fontsize': 30 } ext_utils.create_contour_plot(data, out_file_path + '.png', np.min(lats), np.min(lons), np.max(lats), np.max(lons), title_opts, clevs=clevs, cmap=cmap, norm=norm) if archive_data and not utils.file_exists_nonempty(out_file_path + '.archive'): np.savetxt(out_file_path + '.archive', data, fmt='%g') else: logging.info('%s already exits' % (out_file_path + '.archive')) if not db_adapter_config: logging.info('db_adapter not available. Unable to push data!') return db_adapter = ext_utils.get_curw_adapter(mysql_config=db_adapter_config) width = len(lons) height = len(lats) station_prefix = 'sat' run_name = 'Cloud-1' upsert = True def random_check_stations_exist(): for _ in range(10): _x = lons[int(random() * width)] _y = lats[int(random() * height)] _name = '%s_%.6f_%.6f' % (station_prefix, _x, _y) _query = {'name': _name} if db_adapter.get_station(_query) is None: logging.debug('Random stations check fail') return False logging.debug('Random stations check success') return True stations_exists = random_check_stations_exist() rf_ts = {} for y in range(height): for x in range(width): lat = lats[y] lon = lons[x] station_id = '%s_%.6f_%.6f' % (station_prefix, lon, lat) name = station_id if not stations_exists: logging.info('Creating station %s ...' % name) station = [ Station.Sat, station_id, name, str(lon), str(lat), str(0), "WRF point" ] db_adapter.create_station(station) # add rf series to the dict rf_ts[name] = [[lk_ts.strftime('%Y-%m-%d %H:%M:%S'), data[y, x]]] ext_utils.push_rainfall_to_db(db_adapter, rf_ts, source=station_prefix, name=run_name, types=['Observed'], upsert=upsert)
def extract_jaxa_satellite_data_every_half_hr(exe_ts_utc, output_dir, cleanup=True, cum=False, tmp_dir=None, lat_min=5.722969, lon_min=79.52146, lat_max=10.06425, lon_max=82.18992, output_prefix='jaxa_sat', db_adapter_config=None): exe_ts_utc = exe_ts_utc - dt.timedelta(minutes=2) print( '-------------extract_jaxa_satellite_data_half_hr---------------exe_ts_utc:', exe_ts_utc) exe_ts_utc = exe_ts_utc - dt.timedelta(minutes=30) run_minute = int(exe_ts_utc.strftime('%M')) print('run_minute : ', run_minute) year_str = exe_ts_utc.strftime('%Y') month_str = exe_ts_utc.strftime('%m') day_str = exe_ts_utc.strftime('%d') hour_str = exe_ts_utc.strftime('%H') hour_str1 = (exe_ts_utc + dt.timedelta(hours=1)).strftime('%H') # if run_minute == 0: # url = 'ftp://*****:*****@hokusai.eorc.jaxa.jp/now/txt/05_AsiaSS/gsmap_now.{}{}{}.{}00_{}59.05_AsiaSS.csv.zip' \ # .format(year_str, month_str, day_str, hour_str, hour_str) # else: # url = 'ftp://*****:*****@hokusai.eorc.jaxa.jp/now/txt/05_AsiaSS/gsmap_now.{}{}{}.{}30_{}29.05_AsiaSS.csv.zip' \ # .format(year_str, month_str, day_str, hour_str, hour_str1) if run_minute == 0: url = 'ftp://*****:*****@hokusai.eorc.jaxa.jp/now/txt/island/SriLanka/gsmap_now.{}{}{}.{}00_{}59.SriLanka.csv.zip' \ .format(year_str, month_str, day_str, hour_str, hour_str) else: url = 'ftp://*****:*****@hokusai.eorc.jaxa.jp/now/txt/island/SriLanka/gsmap_now.{}{}{}.{}30_{}29.SriLanka.csv.zip' \ .format(year_str, month_str, day_str, hour_str, hour_str1) print('Download url : ', url) start_time = exe_ts_utc end_time = start_time + dt.timedelta(hours=1) print('_get_start_end|[start_time, end_time] : ', [start_time, end_time]) if tmp_dir is None: tmp_dir = tempfile.mkdtemp(prefix='tmp_jaxa') print('Download url : ', url) url_dest_list = [(url, os.path.join(tmp_dir, os.path.basename(url)), os.path.join( output_dir, output_prefix + '_' + exe_ts_utc.strftime('%Y-%m-%d_%H:%M') + '.asc'))] procs = multiprocessing.cpu_count() logging.info('Downloading inventory in parallel') utils.download_parallel(url_dest_list, procs) logging.info('Downloading inventory complete') logging.info('Processing files in parallel') Parallel(n_jobs=procs)(delayed(process_jaxa_zip_file) (i[1], i[2], lat_min, lon_min, lat_max, lon_max, cum, output_prefix, db_adapter_config) for i in url_dest_list) logging.info('Processing files complete') logging.info('Creating sat rf gif for today') create_daily_gif(exe_ts_utc, output_dir, output_prefix + '_today.gif', output_prefix) prev_day_gif = os.path.join(output_dir, output_prefix + '_yesterday.gif') if not utils.file_exists_nonempty(prev_day_gif) or exe_ts_utc.strftime( '%H:%M') == '00:00': logging.info('Creating sat rf gif for yesterday') create_daily_gif( utils.datetime_floor(exe_ts_utc, 3600 * 24) - dt.timedelta(days=1), output_dir, output_prefix + '_yesterday.gif', output_prefix) if cum: logging.info('Processing cumulative') process_cumulative_plot(url_dest_list, start_time, end_time, output_dir, lat_min, lon_min, lat_max, lon_max) logging.info('Processing cumulative complete') # clean up temp dir if cleanup: logging.info('Cleaning up') shutil.rmtree(tmp_dir) utils.delete_files_with_prefix(output_dir, '*.archive')
def extract_jaxa_satellite_data_half_hr(exe_ts_utc, output_dir, cleanup=True, cum=False, tmp_dir=None, lat_min=5.722969, lon_min=79.52146, lat_max=10.06425, lon_max=82.18992, output_prefix='jaxa_sat', db_adapter_config=None): print( '-------------extract_jaxa_satellite_data_half_hr---------------exe_ts_utc:', exe_ts_utc) exe_ts_utc = exe_ts_utc - dt.timedelta(hours=1) login = '******' url_hour = 'ftp://' + login + '@hokusai.eorc.jaxa.jp/now/txt/05_AsiaSS/gsmap_now.YYYYMMDD.HH00_HH59.05_AsiaSS.csv.zip' url_half_hour = 'ftp://' + login + '@hokusai.eorc.jaxa.jp/now/txt/05_AsiaSS/gsmap_now.YYYYMMDD.HH30_KK29.05_AsiaSS.csv.zip' run_minute = int(exe_ts_utc.strftime('%M')) print('run_minute : ', run_minute) remainder = run_minute % 30 run_minute = run_minute - remainder start_time = exe_ts_utc - dt.timedelta(minutes=remainder) end_time = start_time + dt.timedelta(minutes=30) print('_get_start_end|[start_time, end_time] : ', [start_time, end_time]) if run_minute == 0: exe_ts_utc = exe_ts_utc.strftime('%Y-%m-%d %H:00:00') else: exe_ts_utc = exe_ts_utc.strftime('%Y-%m-%d %H:30:00') exe_ts_utc = dt.datetime.strptime(exe_ts_utc, '%Y-%m-%d %H:%M:%S') def _get_download_url(run_minute): remainder = run_minute % 30 run_minute = run_minute - remainder if run_minute == 0: return url_hour else: return url_half_hour def _format_url(url): ph = { 'YYYY': exe_ts_utc.strftime('%Y'), 'MM': exe_ts_utc.strftime('%m'), 'DD': exe_ts_utc.strftime('%d'), 'KK': (exe_ts_utc + dt.timedelta(hours=1)).strftime('%H'), 'HH': exe_ts_utc.strftime('%H') } for k, v in list(ph.items()): url = url.replace(k, v) print('url : ', url) return url if tmp_dir is None: tmp_dir = tempfile.mkdtemp(prefix='tmp_jaxa') url = _get_download_url(run_minute) formatted_url = _format_url(url) print('formatted_url : ', formatted_url) url_dest_list = [(formatted_url, os.path.join(tmp_dir, os.path.basename(formatted_url)), os.path.join( output_dir, output_prefix + '_' + exe_ts_utc.strftime('%Y-%m-%d_%H:%M') + '.asc'))] procs = multiprocessing.cpu_count() logging.info('Downloading inventory in parallel') utils.download_parallel(url_dest_list, procs) logging.info('Downloading inventory complete') logging.info('Processing files in parallel') Parallel(n_jobs=procs)(delayed(process_jaxa_zip_file) (i[1], i[2], lat_min, lon_min, lat_max, lon_max, cum, output_prefix, db_adapter_config) for i in url_dest_list) logging.info('Processing files complete') logging.info('Creating sat rf gif for today') create_daily_gif(exe_ts_utc, output_dir, output_prefix + '_today.gif', output_prefix) prev_day_gif = os.path.join(output_dir, output_prefix + '_yesterday.gif') if not utils.file_exists_nonempty(prev_day_gif) or exe_ts_utc.strftime( '%H:%M') == '00:00': logging.info('Creating sat rf gif for yesterday') create_daily_gif( utils.datetime_floor(exe_ts_utc, 3600 * 24) - dt.timedelta(days=1), output_dir, output_prefix + '_yesterday.gif', output_prefix) if cum: logging.info('Processing cumulative') process_cumulative_plot(url_dest_list, start_time, end_time, output_dir, lat_min, lon_min, lat_max, lon_max) logging.info('Processing cumulative complete') # clean up temp dir if cleanup: logging.info('Cleaning up') shutil.rmtree(tmp_dir) utils.delete_files_with_prefix(output_dir, '*.archive')