def download_global_analysis(args, global_analysis):
"""download global analysis/forecasts"""
global_analysis_on_s3 = args.global_analysis_on_s3_for_download
cur_analysis_datetime = args.start_analysis_date
while cur_analysis_datetime <= args.end_analysis_date:
cur_global_analysis_filename = \
'global_a{cur_analysis}_v{cur_valid}.nc'.format(
cur_analysis=cur_analysis_datetime.strftime('%Y%m%d%H'),
cur_valid=(cur_analysis_datetime +
timedelta(
seconds=3600*int(args.fcst_length))).strftime(
'%Y%m%d%H'))
cur_remote_fcst = os.path.join(global_analysis_on_s3,
cur_global_analysis_filename)
cur_local_fcst = os.path.join(global_analysis,
cur_global_analysis_filename)
if not os.path.exists(cur_local_fcst):
try:
resources.copy(cur_remote_fcst, cur_local_fcst)
except:
cur_analysis_datetime = cur_analysis_datetime + timedelta(
seconds=int(args.analysis_interval) * 3600)
pass
cur_analysis_datetime = cur_analysis_datetime + timedelta(
seconds=int(args.analysis_interval) * 3600)
def upload_processed_lam(args, blending):
"""upload blended data back to S3"""
cur_analysis_datetime = args.start_analysis_date
while cur_analysis_datetime <= args.end_analysis_date:
cur_blending = os.path.join(blending,
cur_analysis_datetime.strftime('%Y%m%d%H'))
if not os.path.exists(cur_blending):
os.makedirs(cur_blending)
cur_processed_lam_fcst = os.path.join(cur_blending, 'test.nc')
remote_processed_lam_dir = os.path.join(
args.lam_fcst_on_s3_for_upload, args.model_name,
cur_analysis_datetime.strftime('%y/%m/%d/%H'))
remote_processed_lam_filename = \
'wrf_hourly_{model_name}_d0{domain_id}_{model_time}'.format(
model_name=args.model_name, domain_id=args.domain_id,
model_time=(cur_analysis_datetime + timedelta(
seconds=3600*int(args.fcst_length))).strftime(
'%Y-%m-%d_%H:%M:%S'))
remote_processed_lam = os.path.join(remote_processed_lam_dir,
remote_processed_lam_filename)
try:
resources.copy(cur_processed_lam_fcst, remote_processed_lam)
print 'from {} to {}'.format(cur_processed_lam_fcst,
remote_processed_lam)
except:
cur_analysis_datetime = cur_analysis_datetime + timedelta(
seconds=int(args.analysis_interval) * 3600)
pass
cur_analysis_datetime = cur_analysis_datetime + timedelta(
seconds=int(args.analysis_interval) * 3600)
def _copy_from_s3_to_local(data_on_s3, data_on_local):
if not os.path.exists(data_on_local):
try:
print 'copy glb from {} to {}'.format(data_on_s3,
data_on_local)
resources.copy(data_on_s3, data_on_local)
except:
print ' --- failed coping {} to {}'.format(
data_on_s3, data_on_local)
pass
def download_global_bias(args, gmbias_dir):
"""download bias files for the global models"""
remote_gmbias = os.path.join(args.global_bias_on_s3,
'L{}'.format(args.fcst_length),
'gm_bias.GFS.15kmMIDUS.asc')
remote_warm_nl = os.path.join(args.global_bias_on_s3,
'L{}'.format(args.fcst_length), 'warm.nml')
local_gmbias = os.path.join(gmbias_dir, 'gm_bias.GFS.15kmMIDUS.asc')
local_warm_nl = os.path.join(gmbias_dir, 'warm.nml')
resources.copy(remote_gmbias, local_gmbias)
resources.copy(remote_warm_nl, local_warm_nl)
def download_fcst(args, fcst_dir):
"""download fcst files into $fcst_dir/$cur_analysis_time/wrf_hourly*
"""
#if not rinstance:
# raise Exception('download_fcst must be run in the EC2 box')
cur_analysis_time = args.start_analysis_time
while cur_analysis_time <= args.end_analysis_time:
cur_local_fcst_dir = \
os.path.join(fcst_dir, cur_analysis_time.strftime('%y%m%d%H'))
if args.download_fcst_source == 'archive':
archive_dir = 's3://metservice-research-us-west-2/research/archive-data/wrf_archive/wrfout'
cur_remote_fcst_dir = os.path.join(
archive_dir, args.model,
cur_analysis_time.strftime('%y/%m/%d/%H'))
elif args.download_fcst_source == 'internal':
internal_dir = 's3://metservice-research-us-west-2/research/internal-data/wrf/output'
cur_remote_fcst_dir = os.path.join(
internal_dir, cur_analysis_time.strftime('%Y%m%d%H'),
args.model, args.download_fcst_unique_id)
else:
cur_remote_fcst_dir = os.path.join(
args.download_fcst_source,
cur_analysis_time.strftime('%Y%m%d%H'), args.model,
args.download_fcst_unique_id)
for fcst_h in range(1, int(args.forecast_length) + 1):
cur_valid_t = cur_analysis_time + timedelta(seconds=fcst_h * 3600)
cur_remote_fcst_filename = \
'wrf_hourly_{}_d0{}_{}'.format(
args.model, args.domain_id,
cur_valid_t.strftime('%Y-%m-%d_%H:00:00')
)
cur_remote_fcst_path = os.path.join(cur_remote_fcst_dir,
cur_remote_fcst_filename)
cur_local_fcst_path = os.path.join(cur_local_fcst_dir,
cur_remote_fcst_filename)
resources.copy(cur_remote_fcst_path, cur_local_fcst_path)
cur_analysis_time = cur_analysis_time + \
timedelta(seconds = 3600*int(args.analysis_time_interval))
def download_lam_fcst(args, lam_fcst):
"""download LAM forecasts from S3"""
lam_fcst_on_s3 = args.lam_fcst_on_s3_for_download
cur_analysis_datetime = args.start_analysis_date
while cur_analysis_datetime <= args.end_analysis_date:
cur_lam_fcst_on_s3_dir = os.path.join(
lam_fcst_on_s3, args.model_name,
cur_analysis_datetime.strftime('%y/%m/%d/%H'))
cur_lam_fcst_dir = os.path.join(
lam_fcst, cur_analysis_datetime.strftime('%Y%m%d%H'))
if not os.path.join(cur_lam_fcst_dir):
os.makedirs(cur_lam_fcst_dir)
cur_fcst_filename = (
'wrf_hourly_{model_name}_'
'd0{domain_id}_{model_time}').format(
model_name=args.model_name,
domain_id=args.domain_id,
model_time=(cur_analysis_datetime +
timedelta(seconds=3600 * int(args.fcst_length))
).strftime('%Y-%m-%d_%H:%M:%S'))
cur_remote_fcst = os.path.join(cur_lam_fcst_on_s3_dir,
cur_fcst_filename)
cur_local_fcst = os.path.join(cur_lam_fcst_dir, cur_fcst_filename)
if not os.path.exists(cur_local_fcst):
try:
resources.copy(cur_remote_fcst, cur_local_fcst)
except:
print 'download failed: check the remote at {}'.format(
cur_remote_fcst)
cur_analysis_datetime = cur_analysis_datetime + timedelta(
seconds=int(args.analysis_interval) * 3600)
pass
cur_analysis_datetime = cur_analysis_datetime + timedelta(
seconds=int(args.analysis_interval) * 3600)