def read_amsr_hdf4(filename):
from pyhdf.SD import SD, SDC
from pyhdf.HDF import HDF # HC
import pyhdf.VS
retv = AmsrObject()
h4file = SD(filename, SDC.READ)
# datasets = h4file.datasets()
# attributes = h4file.attributes()
# for idx, attr in enumerate(attributes.keys()):
# print idx, attr
for sds in ["Longitude", "Latitude", "High_res_cloud"]:
data = h4file.select(sds).get()
if sds in ["Longitude", "Latitude"]:
retv.all_arrays[sds.lower()] = data.ravel()
elif sds in ["High_res_cloud"]:
lwp_gain = h4file.select(sds).attributes()['Scale']
retv.all_arrays["lwp_mm"] = data.ravel() * lwp_gain
# print h4file.select(sds).info()
h4file = HDF(filename, SDC.READ)
vs = h4file.vstart()
data_info_list = vs.vdatainfo()
# print "1D data compound/Vdata"
for item in data_info_list:
# 1D data compound/Vdata
name = item[0]
# print name
if name in ["Time"]:
data_handle = vs.attach(name)
data = np.array(data_handle[:])
retv.all_arrays["sec1993"] = data
data_handle.detach()
else:
pass
# print name
# data = np.array(data_handle[:])
# attrinfo_dic = data_handle.attrinfo()
# factor = data_handle.findattr('factor')
# offset = data_handle.findattr('offset')
# print data_handle.factor
# data_handle.detach()
# print data_handle.attrinfo()
h4file.close()
# for key in retv.all_arrays.keys():
# print key, retv.all_arrays[key]
return retv
def read_amsr_hdf4(filename):
from pyhdf.SD import SD, SDC
from pyhdf.HDF import HDF, HC
import pyhdf.VS
retv = AmsrObject()
h4file = SD(filename, SDC.READ)
datasets = h4file.datasets()
attributes = h4file.attributes()
#for idx,attr in enumerate(attributes.keys()):
# print idx, attr
for sds in ["Longitude", "Latitude", "High_res_cloud"]:
data = h4file.select(sds).get()
if sds in ["Longitude", "Latitude"]:
retv.all_arrays[sds.lower()] = data.ravel()
elif sds in ["High_res_cloud"]:
lwp_gain = h4file.select(sds).attributes()['Scale']
retv.all_arrays["lwp_mm"] = data.ravel() * lwp_gain
#print h4file.select(sds).info()
h4file = HDF(filename, SDC.READ)
vs = h4file.vstart()
data_info_list = vs.vdatainfo()
#print "1D data compound/Vdata"
for item in data_info_list:
#1D data compound/Vdata
name = item[0]
#print name
if name in ["Time"]:
data_handle = vs.attach(name)
data = np.array(data_handle[:])
retv.all_arrays["sec1993"] = data
data_handle.detach()
else:
pass
#print name
#data = np.array(data_handle[:])
#attrinfo_dic = data_handle.attrinfo()
#factor = data_handle.findattr('factor')
#offset = data_handle.findattr('offset')
#print data_handle.factor
#data_handle.detach()
#print data_handle.attrinfo()
h4file.close()
#for key in retv.all_arrays.keys():
# print key, retv.all_arrays[key]
return retv
"C:\\\\Users\\\\Administrator\\\\Desktop\\\\MODIS\\\\HDF", "")
item[0] = item[0].replace("\\", "")
item[0] = item[0].replace(".hdf文件", "")
item = np.array(item)
aod_outcome_list_v2.append(item)
#避免输出结果字符串省略，四行设置都需要
pd.set_option('display.max_rows', None) #行
pd.set_option('display.max_columns', 1000) #列
pd.set_option('display.width', 1000)
pd.set_option('display.max_colwidth', 1000)
#AOD结果写入TXT
aod_outcome_list_v2 = pd.DataFrame(aod_outcome_list_v2)
aod_outcome_list_v2.columns = ['日期', '监测站', 'AOD值']
file = open('data.txt', 'w')
file.write(str(aod_outcome_list_v2))
file.close()
#计算耗时结果
endtime = datetime.datetime.now()
print(endtime - starttime)
#程序用时写入TXT
file = open('TIME.txt', 'w')
file.write(str(endtime - starttime))
file.close()
#程序运行完成后关机
def shutdown_computer(seconds):
print(str(seconds) + u' 秒后将会关机')
time.sleep(seconds)
print('关机')
os.system('shutdown -s -f -t 1')
def download(username, password, date_start, date_end, earthData_name,
fileList):
if not os.path.exists('./' + earthData_name):
os.mkdir('./' + earthData_name)
saveDir = './' + earthData_name # Set local directory to download to
pathNetrc = os.path.join(os.path.expanduser("~"), '.netrc')
if os.path.exists(pathNetrc):
os.remove(pathNetrc)
netrcFile = [
'machine urs.earthdata.nasa.gov', 'login ' + username,
'password ' + password
]
with open('.netrc', 'w') as f:
for item in netrcFile:
f.write("%s\n" % item)
shutil.copy('.netrc', os.path.expanduser("~"))
#fileList = DownloadList(date_start , date_end,earthData_name)
fileList = sorted(fileList)
#for i in fileList:
# print(i)
# -----------------------------------------DOWNLOAD FILE(S)-------------------------------------- #
# Loop through and download all files to the directory specified above, and keeping same filenames
count = 0
for f in fileList:
#print(' ',str((count+1)/len(fileList)*100)[:5] + ' % Completed')
date_of_file = f.split('/')[5].replace('.', '-')
path = os.path.join(saveDir, date_of_file)
if not os.path.exists(path):
os.mkdir(path)
saveName = os.path.join(path, f.split('/')[-1].strip())
if os.path.exists(saveName):
try:
if not earthData_name == 'IMERG':
f = SD(saveName, SDC.READ)
f.end()
else:
f = Dataset(saveName, 'r')
f.close()
count += 1
continue
except:
print('Damgeged file encountered, redownloading...')
# Create and submit request and download file
file_downloaded = 0
while file_downloaded == 0:
try:
with requests.get(f.strip(), stream=True) as response:
if response.status_code != 200:
print(
"Verify that your username and password are correct"
)
else:
response.raw.decode_content = True
content = response.raw
with open(saveName, 'wb') as d:
while True:
chunk = content.read(16 * 1024)
if not chunk:
break
d.write(chunk)
print('Downloaded file: {}'.format(saveName))
file_downloaded = 1
except:
print("Connection refused by the server..")
print("Let me sleep for 5 seconds")
print("ZZzzzz...")
time.sleep(10)
print("Was a nice sleep, now let me continue...")
continue
count += 1
def get_aod_multiprocessing(location_xy):
JCZ_file = pd.read_excel("F:\\毕业论文程序\\MODIS\\坐标\\多线程.xlsx", sheet_name=location_xy)
JCZ = []
# 批量导入监测站
for i in range(len(JCZ_file)):
exec('JCZ%s = [JCZ_file["经度"][i],JCZ_file["纬度"][i],JCZ_file["城市"][i]+"-"+JCZ_file["监测点名称"][i]]' % i)
exec("JCZ.append(JCZ%s)" % i) # exec可以执行字符串指令
for item in JCZ:
aod_outcome_list = [] # 每个监测站生成一个文件时
for hdf in file_name:
HDF_FILE_URL = file_path + hdf
file = SD(HDF_FILE_URL)
# data_sets_dic = file.datasets()
'''
#输出数据集名称
for idx, sds in enumerate(data_sets_dic.keys()):
print(idx, sds)
'''
sds_obj1 = file.select('Longitude') # 选择经度
sds_obj2 = file.select('Latitude') # 选择纬度
sds_obj3 = file.select('Optical_Depth_Land_And_Ocean') # 产品质量最高的AOD数据集
longitude = sds_obj1.get() # 读取数据
latitude = sds_obj2.get()
aod = sds_obj3.get()
longitude = np.array(longitude) # 格式转换
latitude = np.array(latitude)
aod = np.array(aod)
# 距离计算，提取监测站半径为r范围内的AOD值
aod_list = []
@jit(nogil=True)
# 此次,numpy切片的检索顺序是先"行"后"列"
def get_aod_list(longitude_df, latitude_df, aod_df):
for row in range(longitude_df.shape[0]): # 行 676
for column in range(longitude_df.shape[1]): # 列 451
d = geo_distance(longitude_df[row][column], latitude_df[row][column], item[0], item[1])
if (d > 0) and (d < r) and aod_df[row][column] > 0:
aod_list.append(aod_df[row][column])
get_aod_list(longitude, latitude, aod)
aod_outcome = "%s文件" % hdf, "%s" % item[2], np.average(aod_list)
# 进度
print("完成 %s文件" % hdf, "%s" % item[2])
aod_outcome_list.append(aod_outcome)
aod_outcome_list_v2 = []
for element in aod_outcome_list:
element = pd.Series(element) # 格式转换
# 截取文件名称,结果为获取数据的时间,格式为"年+第几天"
element[0] = str(element[0])[10:17] # 如2018123
# 修改日期格式为XX月XX日
element[0] = time.strptime(element[0], '%Y%j')
element[0] = time.strftime("%Y-%m-%d ", element[0])
element = np.array(element) # 格式转换
aod_outcome_list_v2.append(element)
# 避免输出结果字符串省略，四行设置都需要
pd.set_option('display.max_rows', None) # 行
pd.set_option('display.max_columns', 1000) # 列
pd.set_option('display.width', 1000)
pd.set_option('display.max_colwidth', 1000)
aod_outcome_list_v2 = pd.DataFrame(aod_outcome_list_v2) # 格式转换
# 重设列名
aod_outcome_list_v2.columns = ['日期', '监测站', "AOD值"]
# 同日期，多文件情况下的均值处理
aod_outcome_list_v2 = aod_outcome_list_v2.groupby(['日期', "监测站"]).mean()
# 美化group by均值计算后的数据框格式
aod_outcome_list_v2 = pd.Series(aod_outcome_list_v2["AOD值"]) # AOD值按分组计算的结果
aod_outcome_list_v2.to_excel(output_file_path+"%s.xlsx" % item[2]) # 完整结果存入excel
# 程序用时写入文件
file = open('程序耗时.txt', 'w')
end_time = datetime.datetime.now()
file.write(str(end_time - start_time))
file.close()
def read_cloudsat_hdf4(filename):
from pyhdf.SD import SD, SDC
from pyhdf.HDF import HDF, HC
import pyhdf.VS
def convert_data(data):
if len(data.shape) == 2:
if data.shape[1] == 1:
return data[:, 0]
elif data.shape[0] == 1:
return data[0, :]
return data
retv = CloudsatObject()
h4file = SD(filename, SDC.READ)
datasets = h4file.datasets()
attributes = h4file.attributes()
#for idx,attr in enumerate(attributes.keys()):
# print idx, attr
for idx,sds in enumerate(datasets.keys()):
#2D data, print idx, sds
data = h4file.select(sds).get()
#print h4file.select(sds).attributes().keys()
am_name = clsat_name_conversion(sds, retv)
if am_name in retv.all_arrays.keys():
retv.all_arrays[am_name] = convert_data(data)
#print h4file.select(sds).info()
h4file = HDF(filename, SDC.READ)
vs = h4file.vstart()
data_info_list = vs.vdatainfo()
for item in data_info_list:
#1D data compound/Vdata
name = item[0]
data_handle = vs.attach(name)
data = np.array(data_handle[:])
attrinfo_dic = data_handle.attrinfo()
factor = data_handle.findattr('factor')
offset = data_handle.findattr('offset')
#print data_handle.factor
am_name = clsat_name_conversion(name, retv)
if am_name in retv.all_arrays.keys():
#To save RAM and disk only read what we use!
if factor is None and offset is None:
retv.all_arrays[am_name] = convert_data(data)
elif np.float(factor.get()) == 1.0 and np.float(offset.get()) == 0.0:
retv.all_arrays[am_name] = convert_data(data)
else:
if factor is None:
factor = 1.0
if offset is None:
offset = 0.0
raise MatchupError("Not default offset and factor. Fix code")
#The code below is probably ok, but make sure:
#the_data_scaled = convert_data(data)*factor + offset
#retv.all_arrays[am_name] = the_data_scaled
data_handle.detach()
#print data_handle.attrinfo()
h4file.close()
# Convert from TAI time to UTC in seconds since 1970:
dsec = time.mktime((1993,1,1,0,0,0,0,0,0)) - time.timezone
retv.sec_1970 = retv.Profile_time.ravel() + retv.TAI_start + dsec
return retv