def single_downsample_dat(file_reso, file_list, counter, totals):
if file_reso == '10':
#print 'Downsampling '+ file_list[counter]+' into 2 km'
#print "\n"
#os.system('downsample '+ file_list[counter])
proc = subprocess.Popen(['downsample '+ file_list[counter]],\
stdout = subprocess.PIPE, shell=True)
out, err = proc.communicate()
#print "program output:", out
progress_bar(counter, totals, desc = file_list[counter]+' into 2 km',\
def_total = 10)
os.system('rm ' + file_list[counter])
elif file_reso == '05':
#print 'Downsampling '+ file_list[counter]+' into 1 km'
#print "\n"
#os.system('downsample '+ file_list[counter])
proc = subprocess.Popen(['downsample '+ file_list[counter]],\
stdout = subprocess.PIPE, shell=True)
out, err = proc.communicate()
#print "program output:", out
progress_bar(counter, totals, desc = file_list[counter]+' into 1 km',\
def_total = 10)
os.system('rm ' + file_list[counter])
else:
print "\nH8 DAT file beyond 2 km resolution : " + file_list[counter]
quit()
def single_hisd2hrit(file_h8, counter, totals):
#os.system('hisd2hrit '+ file_h8)
proc = subprocess.Popen(['hisd2hrit '+ file_h8],\
stdout = subprocess.PIPE, shell=True)
out, err = proc.communicate()
#print "program output:", out
progress_bar(counter, totals, desc=file_h8)
def single_preprocess_H8(number, totals, seg_no,\
segment_interest,\
hrit_segment_interest,\
llcrnrlon = x_ll,\
llcrnrlat = y_ll,\
urcrnrlon = x_ur,\
urcrnrlat = y_ur,\
dat_segment = dat_segment,\
hrit_list = hrit_list ,\
dat_listnum = dat_listnum, hrit_listb = hrit_listb,\
dat_listnuma = dat_listnuma, dat_listnumb = dat_listnumb,\
hrit_spa = hrit_spa, hrit_spb = hrit_spb):
#print "\nEXTRACTING DATA AT SEGMENT: ["+k+"]\n"
# read metadata
group_B_meta, group_B_gamma =\
get_dat_meta(segment_interest[number],\
dat_listnum = dat_listnum, hrit_listb = hrit_listb,\
dat_listnuma = dat_listnuma, dat_listnumb = dat_listnumb )
# get hrit data, lon, lat, and time
group_B_data, group_B_coors, group_B_time =\
get_hrit_data(hrit_segment_interest[number],\
llcrnrlon = llcrnrlon,\
llcrnrlat = llcrnrlat,\
urcrnrlon = urcrnrlon,\
urcrnrlat = urcrnrlat,\
hrit_listb = hrit_listb,\
hrit_list = hrit_list)
# convert to radiance
#rad = get_radiance( group_B_data, group_B_meta, group_B_gamma )
rad = get_radiance(group_B_data, group_B_meta, group_B_gamma,\
hrit_listb = hrit_listb, hrit_spa = hrit_spa,\
hrit_spb = hrit_spb)
h8_data = group_B_data
h8_radiance = rad
h8_coordinates = group_B_coors
h8_metadata = group_B_meta
#h8_gamma[k] = group_B_gamma
progress_bar(number, totals, desc=seg_no)
print "\n"
return h8_data, h8_radiance, h8_coordinates, h8_metadata
def single_download(combi, counter, totals,\
_dates_, _dates1_, path_out,\
dat_list = dat_list,\
dat_list_reso = dat_list_reso,\
user_name = "rednacky_gmail.com",\
user_password = "******",\
proxy_switch = "off" ):
# H8 filename to download
#print combi
#print _dates_, dat_list[ combi[0] ], dat_list_reso[ combi[0] ], combi[1]
file_dl = 'HS_H08_'+ _dates_ +'_'+ combi[0] +\
'_FLDK_'+ combi[1] +'_S'+ combi[2] +'10.DAT.bz2'
ftp_site = 'ftp://ftp.ptree.jaxa.jp/jma/hsd'
wget_c = "wget -o download.log -c "+\
"--directory-prefix="+ path_out +" "+\
"--tries=5 --read-timeout=20 "+\
"--passive-ftp --ftp-user="******" "+\
"--ftp-password="******" "+\
"-e use_proxy="+ proxy_switch +" "
#-o download.log
# command
#print wget_c + ftp_site + _dates1_ + file_dl
out = os.system(wget_c + ftp_site + _dates1_ + file_dl)
progress_bar(counter, totals, desc=file_dl)
# checks if data exist
if out == 0:
indicator = os.path.exists(os.path.join(path_out, file_dl))
if indicator == True:
return file_dl
else:
os.sys.exit("Data downloaded but missing: " + file_dl)
else:
# print"\n\n\tRetry download. Data download unsuccesfull: "+ file_dl
# internet = check_internet()
# if internet == True:
# print "\tConnected to the internet but data cannot be downloaded."
return N.nan
def single_bunzip_h8(datbz_fname, counter, totals):
#out = os.system("bunzip2 -q "+ datbz_fname)
proc = subprocess.Popen(["bunzip2 -q "+ datbz_fname],\
stdout = subprocess.PIPE, shell=True)
out, err = proc.communicate()
progress_bar(counter, totals, desc=datbz_fname)
def convert_BTD_BT(date_list, file_list, path_to_open, path_to_save, mode ="PH",\
tagx = tagx, tagy = tagy, tag_range = tag_range,\
h8_lvl1 = False, h8_data_type = "default",\
interpolate_lon_lat = True, replace = False):
"""
Write Himawari 8 data into BTD and BT data
Input:
path_to_open Path where to input data is located
path_to_save Path where to save ouput data
mode "PH" or "TAG"
Ouput:
"HS_H08_"+h08_time+"_"+mode+"_R20_S030405_BTD_BT.hdf5"
"""
# make lists of file
#file_list_H = open_h8_hdf(path_to_open, mode = "default")
# make lists of path
#file_list_H_open = [path_to_open+"/"+i for i in file_list_H]
# reads and process the data
totals = len(file_list)
print "\n"
print "=" * 75
print mode, os.path.basename(path_to_open)
print "\n"
for ctr, i in enumerate(file_list):
sdate_time = date_list[ctr]
sample_fout = os.path.join(path_to_save,\
"HS_H08_"+ sdate_time +"_"+mode+\
"_R20_S030405_BTD_BT.hdf5" )
if os.path.exists(sample_fout) == True:
if replace == True:
print ctr, sdate_time, "replacing."
else:
print ctr, sdate_time, "exists."
continue
else:
print "\n"
print ctr, sdate_time, os.path.basename(i)
print "\n"
if h8_data_type == "default":
print i
h08_data, h08_radiance, lon_geos, lat_geos, date_time = read_h8_hdf5_level1(
i)
elif h8_data_type == "netcdf":
try:
h08_data, date_time, lon_geos, lat_geos, units = read_h8_l1_netcdf(
i)
except IOError:
print "\n"
print ctr, sdate_time, os.path.basename(i), "is corrupted"
print "\n"
continue
if interpolate_lon_lat == True:
for j in hrit_list:
#print j
h08_data[j] = interpolate_data(h08_data[j], lon_geos,
lat_geos, h08_lon_ph,
h08_lat_ph)
lon_geos = h08_lon_ph
lat_geos = h08_lat_ph
# if os.path.exists(path_to_save+"/HS_H08_"+date_time+"_"+mode+"_R20_S030405_BTD_BT.hdf5") == True and mode == "PH":
# print "HS_H08_"+date_time+"_"+mode+"_R20_S030405_BTD_BT.hdf5 exists ==================="
# continue
# del h08_radiance
# Get H8 split window
BTD_BT = get_H8_IR_split_window(h08_data)
# save into separate file
if mode == "PH":
if h8_lvl1 == True:
for j in hrit_list:
BTD_BT[j] = h08_data[j]
save_H8_level2_hdf5(path_to_save, BTD_BT, sdate_time, lon_geos, lat_geos,\
mode = mode, h8_lvl1 = h8_lvl1)
# test if save georef point is the same as to other points
# <> edit
progress_bar(ctr, totals, desc = "PH"+" " +os.path.basename(path_to_open)+\
" "+ i, def_total=10)
elif mode == "TAG":
# get index address
index_x, index_y = subset_square(tagx, tagy, tag_range, lon_geos,
lat_geos)
# get lon and lat
map_subset_lon = get_subset_points(lon_geos, index_x, index_y)
map_subset_lat = get_subset_points(lat_geos, index_x, index_y)
#print N.shape(map_subset_lon)
# write subset data
map_subset = {}
for j in BTD_BT_listb:
map_subset[j] = get_subset_points(BTD_BT[j], index_x, index_y)
if h8_lvl1 == True:
for j in hrit_list:
map_subset[j] = get_subset_points(h08_data[j], index_x,
index_y)
save_H8_level2_hdf5(path_to_save, map_subset, sdate_time, map_subset_lon,\
map_subset_lat, mode = mode, h8_lvl1 = h8_lvl1)
progress_bar(ctr, totals, desc = "TAG"+" " +os.path.basename(path_to_open)+\
" "+ os.path.basename(i), def_total=10)
else:
os.sys.exit("Error: Creating H8 file\"" + mode + "\" not found")