def mosaic_2018():
fdir = r'E:\before2000\unzip\\'
flist = os.listdir(fdir)
out_rasters = []
for f in flist:
out_rasters.append('_'.join(f.split('_')[:3]))
out_raster = list(set(out_rasters))
out_dir = r'E:\before2000\mosaic\\'
mk_dir(out_dir)
time_init = time.time()
flag = 0
for i in out_raster:
log.logger.info(i)
time_start = time.time()
in_rasters_list = []
for f in flist:
if i in f:
in_rasters_list.append(f)
mosaic(fdir, in_rasters_list, out_dir=out_dir, out_raster=i + '.tif')
time_end = time.time()
log_process.process_bar(flag, len(out_raster), time_init, time_start,
time_end, '\n')
flag += 1
pass
def gen_dependent_pdsi():
# 去除无效值
# key = 50136_200411
pdsi = np.load(this_root + 'PDSI\\PDSI_result.npz')
pdsi_out = this_root + 'PDSI\\PDSI_result_filter'
valid_year = []
for y in range(2000, 2018):
valid_year.append(y)
independent_pdsi = {}
time_init = time.time()
flag = 0
for sta in pdsi:
time_start = time.time()
for year in pdsi[sta].item():
if year in valid_year:
# print(type(i))
# print(year)
# print(sta,pdsi[sta].item()[year])
for mon, val in enumerate(pdsi[sta].item()[year]):
key = sta + '_' + str(year) + '%02d' % (mon + 1)
# print(key)
# print(val)
if -10 < val < 10:
independent_pdsi[key] = val
# time.sleep(0.3)
time_end = time.time()
log_process.process_bar(flag, len(pdsi), time_init, time_start,
time_end,
str(flag) + '/' + str(len(pdsi)))
flag += 1
print 'saving dic...'
np.save(pdsi_out, independent_pdsi)
def gen_shp_dic():
fdir = this_root + u'190509\\民权线路cad\\dwg_to_shp\\'
flist = os.listdir(fdir)
shp_dic = {}
time_init = time.time()
flag = 0
for folder in flist:
time_start = time.time()
print(folder)
shp_list = os.listdir(fdir + folder)
for shp in shp_list:
if shp.endswith('Annotation.shp'):
daShapefile = fdir + folder + '\\' + shp
driver = ogr.GetDriverByName("ESRI Shapefile")
dataSource = driver.Open(daShapefile, 0)
layer = dataSource.GetLayer()
for feature in layer:
geom = feature.GetGeometryRef()
x = geom.GetX()
y = geom.GetY()
name = feature.GetField("RefName")
name_gbk = name.decode('utf-8')
shp_dic[name_gbk] = [x, y]
# print x,y,name_gbk
# time.sleep(0.5)
time_end = time.time()
log_process.process_bar(flag, len(flist), time_init, time_start,
time_end)
flag += 1
print('saving shp dic...')
np.save(this_root + '190509\\民权线路cad\\shp_dic', shp_dic)
def run_dwg_to_shp(fdir,out_dir_):
# log = log_process.Logger('log.log')
# fdir = this_root+'190905\\сюЁг\\'
flist = os.listdir(fdir)
init_time = time.time()
flag = 0
for f in flist:
start = time.time()
out_dir = out_dir_+f.split('.')[0]
# print(f)
# print(out_dir)
# exit()
if os.path.isdir(out_dir):
end = time.time()
log_process.process_bar(flag, len(flist), init_time, start, end,
str(flag + 1) + '/' + str(len(flist)) + '\n')
flag += 1
print(out_dir+'is existed')
continue
mk_dir(out_dir)
# try:
print(out_dir.decode('gbk'))
# exit()
dwg_to_shp(fdir+f+'\\Annotation',out_dir)
dwg_to_shp(fdir+f+'\\Polyline',out_dir)
# log.logger.info('\\Annotation')
# log.logger.info('\\Polyline')
# except Exception as e:
# log.logger.error(e)
end = time.time()
log_process.process_bar(flag,len(flist),init_time,start,end,str(flag+1)+'/'+str(len(flist))+'\n')
flag += 1
def do_mosaic(year):
year = str(year)
fdir = 'E:\\30m_fvc_annual_unzip\\' + year + '\\'
flist = os.listdir(fdir)
out_rasters = []
for f in flist:
out_rasters.append('_'.join(f.split('_')[:3]))
out_raster = list(set(out_rasters))
out_dir = 'E:\\30m_fvc_annual_unzip\\' + year + '_mosaic\\'
mk_dir(out_dir)
time_init = time.time()
flag = 0
for i in out_raster:
log.logger.info(i)
time_start = time.time()
in_rasters_list = []
for f in flist:
if i in f:
in_rasters_list.append(f)
mosaic(fdir, in_rasters_list, out_dir=out_dir, out_raster=i + '.tif')
time_end = time.time()
log_process.process_bar(flag, len(out_raster), time_init, time_start,
time_end, '\n')
flag += 1
pass
def main():
print('initializing GEE')
ee.Initialize()
print('done')
product_dic = {
# 'LC08':"LANDSAT/LC08/C01/T1_32DAY_NDVI",
# 'LE07':'LANDSAT/LE07/C01/T1_32DAY_NDVI',
# 'LT05':"LANDSAT/LT05/C01/T1_32DAY_NDVI",
'006': "MODIS/006/MOD13Q1"
}
fdir = this_root + 'download_data\\'
re_download_dir = this_root + 're_download_data\\'
mk_dir(re_download_dir)
poligon_list = gen_polygons()
polygon_dic = {}
for i in range(len(poligon_list)):
polygon_dic['%02d' % (i + 1)] = poligon_list[i]
flist = os.listdir(fdir)
#
time_init = time.time()
flag = 0
invalid_list = []
for f in flist:
start = time.time()
if check(fdir + f):
invalid_list.append(f)
end = time.time()
logger.process_bar(flag, len(flist), time_init, start, end, 'checked')
flag += 1
time_init = time.time()
flag = 0
for f in invalid_list:
start = time.time()
# print(f)
fsplit = f.split('_')
product = fsplit[0]
dataset = product_dic[product]
date_start = str(fsplit[1])
date_end = str(fsplit[2])
num = str(fsplit[3].split('.')[0])
region = polygon_dic[num]
# print(dataset)
# print(date_start)
# print(date_end)
# print(num)
# print(region)
#
url = get_download_url(dataset, date_start, date_end, region)
file_name = '_'.join([product, date_start, date_end, num]) + '.zip'
# print(file_name)
download_data(url, re_download_dir + file_name)
end = time.time()
logger.process_bar(flag, len(invalid_list), time_init, start, end,
str(flag + 1) + '/' + str(len(invalid_list)) + '\n')
flag += 1
def gen_3_months_average_new():
# 用新数据
this_root = os.getcwd() + '\\'
# pre = np.load(this_root + 'data\\pre_anomaly_dic.npy').item()
temp = np.load(this_root + 'data\\tmp_anomaly_dic.npy').item()
# pre = dict(pre)
temp = dict(temp)
date_list = []
for year in range(2000, 2016):
for mon in range(1, 13):
date_list.append(str(year) + '%02d' % mon)
new_dic = {}
time_init = time.time()
flag = 0
for key in temp:
time_start = time.time()
key_split = key.split('_')
sta = key_split[0]
date_str = key_split[1]
year = int(date_str[:4])
mon = int(date_str[-2:])
date = datetime.datetime(year, mon, 15)
time_delta = datetime.timedelta(30)
date_1 = date - time_delta
date_1_str = '%s%02d' % (date_1.year, date_1.month)
date_2 = date - time_delta * 2
date_2_str = '%s%02d' % (date_2.year, date_2.month)
# print(date_1_str)
# print(date_2_str)
try:
val1 = temp[sta + '_' + date_str]
val2 = temp[sta + '_' + date_1_str]
val3 = temp[sta + '_' + date_2_str]
val_mean = np.mean([val1, val2, val3])
# print(val_mean)
new_dic[key] = val_mean
except Exception as e:
# print(e)
pass
time_end = time.time()
log_process.process_bar(flag, len(temp), time_init, time_start,
time_end)
flag += 1
print('saving dic...')
np.save(this_root + 'data\\temp_3_months_average_new', new_dic)
def gen_line_annotation():
'''
composite shp
将xian_dic_sort生成的shp合成为1个shp
作为annotation
:return:
'''
gdal.SetConfigOption("SHAPE_ENCODING", "GBK")
fdir = this_root + '\\190509\\民权线路cad\\dwg_to_shp\\'
flist = os.listdir(fdir)
time_init = time.time()
time_i = 0
for folder in flist:
time_start = time.time()
# print(folder.decode('gbk'))
shp_dir = fdir + folder + '\\annotation\\'
try:
shp_list = os.listdir(shp_dir)
inlist = []
for shp in shp_list:
# print(shp)
if shp.endswith('.shp'):
# print(shp.decode('gbk'))
daShapefile = shp_dir + shp
# print(daShapefile.decode('gbk'))
driver = ogr.GetDriverByName("ESRI Shapefile")
dataSource = driver.Open(daShapefile.decode('gbk'), 0)
layer = dataSource.GetLayer()
for feature in layer:
geom = feature.GetGeometryRef()
Points = geom.GetPoints()
val1 = feature.GetField("val1")
val2 = feature.GetField("val2")
val3 = feature.GetField("val3")
val4 = feature.GetField("val4")
inlist.append(
[Points[0], Points[1], val1, val2, val3, val4, ''])
# print(fdir.decode('gbk') + folder.decode('gbk') + '\\test.shp')
output_fn = fdir.decode('gbk').encode('utf-8') + folder.decode(
'gbk').encode('utf-8') + '\\line_annotation.shp'
# print output_fn
line_to_shp(inlist, output_fn)
time_end = time.time()
time_i += 1
log_process.process_bar(time_i, len(flist), time_init, time_start,
time_end)
except:
time_i += 1
pass
# exit()
pass
def gen_lon_lat_dic_non_normalize():
'''
生成栅格和经纬度对应的字典
数据格式:
dic[str(x.y)] = [lon,lat]
:return:
'''
# rasterized_tif = this_root + '/conf/my.tif'
this_root = r'e:\\MODIS\\'
DEM = this_root + 'DEM\\china_1km_dem_wgs1984_resample.tif'
_, im_width, im_height, im_geotrans, im_proj = tif_to_array(DEM)
array = np.load(this_root +
'DEM\\china_1km_dem_wgs1984_resample_non_normalize.npy')
# normalize(array)
# exit()
# clip_array = np.load(this_root+'\\conf\\china_T_F.npy')
# print(array)
# array = np.ma.masked_where(array<-9999,array)
# plt.imshow(array)
# plt.colorbar()
# plt.show()
print(im_geotrans)
lon_lat_dic = {}
time_init = time.time()
flag = 0
for y in range(len(array)):
time_start = time.time()
for x in range(len(array[y])):
if array[y][x] > -9000:
lon_start = im_geotrans[0]
lon_step = im_geotrans[1]
lon = lon_start + lon_step * x
lat_start = im_geotrans[3]
lat_step = im_geotrans[5]
lat = lat_start + lat_step * y
# print('*'*8)
# print(str(lon)+'_'+str(lat))
# print(array[y][x])
lon_lat_dic[str(lon) + '_' + str(lat)] = array[y][x]
# print(str(lon)+'_'+str(lat),array[y][x])
time_end = time.time()
log_process.process_bar(flag, len(array), time_init, time_start,
time_end)
flag += 1
np.save(this_root + '\\DEM\\DEM_data_transform', lon_lat_dic)
def extract_cci_from_sta():
this_root = 'e:\\MODIS\\'
npz = np.load(this_root + 'CCI\\time_interp.npz')
sta_pos_dic = np.load(this_root +
'ANN_input_para\\00_PDSI\\sta_pos_dic.npz')
date_list = []
for year in range(2003, 2017):
for mon in range(1, 13):
date_list.append(str(year) + '%02d' % mon)
CCI_sta_extract_dic = {}
import log_process
time_init = time.time()
flag = 0
for npy in npz:
time_start = time.time()
npy_split = npy.split('_')
lon = float(npy_split[0])
lat = float(npy_split[1])
vals = npz[npy]
if len(vals) < 10:
continue
for sta in sta_pos_dic:
pos = sta_pos_dic[sta]
lon_sta = pos[1]
lat_sta = pos[0]
lon_max = lon_sta + 0.25
lon_min = lon_sta - 0.25
lat_max = lat_sta + 0.25
lat_min = lat_sta - 0.25
if lon_min < lon < lon_max and lat_min < lat < lat_max:
for date in range(len(date_list)):
key = sta + '_' + date_list[date]
CCI_sta_extract_dic[key] = vals[date]
time_end = time.time()
log_process.process_bar(flag, len(npz), time_init, time_start,
time_end)
flag += 1
print('saving...')
np.save(this_root + 'CCI\\CCI_sta_extract_dic', CCI_sta_extract_dic)
def main():
movie_path = 'E:\\532_all_movie\\movie\\'
if not os.path.isdir(movie_path):
os.makedirs(movie_path)
f = open('movies_url_list.txt', 'r')
lines = f.readlines()
f.close()
import log_process
flag = 0
time_init = time.time()
for line in lines:
time_start = time.time()
url = line.split('\n')[0]
# print(url)
movie_name, urls = get_vedio_url(url)
# print(movie_name.decode('utf-8').encode('gbk'))
download_movies(url,movie_path)
time_end = time.time()
log_process.process_bar(flag,len(lines),time_init,time_start,time_end,'\n')
flag+=1
def gen_dependent_pdsi1():
# 去除无效值
# key = 2003
pdsi = np.load(this_root +
'PDSI\\project02_pdsi\\self_cal_all_sta_dic.npz')
pdsi_out = this_root + 'PDSI\\project02_pdsi\\PDSI_result_filter'
valid_year = []
# oneyear_filtered_pdsi = {}
for y in range(2000, 2018):
# oneyear_filtered_pdsi[y] = []
valid_year.append(y)
time_init = time.time()
flag = 0
filtered_pdsi = {}
for sta in pdsi:
oneyear_filtered_pdsi = {}
for y in range(2000, 2018):
oneyear_filtered_pdsi[y] = []
valid_year.append(y)
time_start = time.time()
onesta_pdsi_dic = pdsi[sta].item()
for year in onesta_pdsi_dic:
if year in valid_year:
vals = onesta_pdsi_dic[year]
for val in vals:
if -10 < val < 10:
oneyear_filtered_pdsi[year].append(val)
# print(oneyear_filtered_pdsi)
filtered_pdsi[sta] = oneyear_filtered_pdsi
time_end = time.time()
log_process.process_bar(flag, len(pdsi), time_init, time_start,
time_end,
str(flag) + '/' + str(len(pdsi)))
flag += 1
print('saving...')
np.savez(pdsi_out, **filtered_pdsi)
def trans_nc_to_lon_lat_list():
nc = r'E:\MODIS\CRU_precip\cru_ts4.02.1901.2017.pre.dat.nc'
ncin = Dataset(nc, 'r')
# print ncin.variables
lon = ncin['lon']
lat = ncin['lat']
lon_origin = lon[0]
lat_origin = lat[-1]
lon_size = lon[1]-lon[0]
lat_size = lat[1]-lat[0]
print(lon_origin)
print(lat_origin)
print(lon_size)
print(lat_size)
# exit()
npz = np.load(r'E:\MODIS\CRU_precip\2001.2018.npz')
time_init = time.time()
flag = 0
for date in npz:
# print(date)
time_start = time.time()
transform_dic = {}
arr = npz[date]
for i in range(len(arr)):
for j in range(len(arr[i])):
# print(i,j)
lon_i = lon_origin+lon_size*j
lat_i = lat_origin-lat_size*i
val = arr[i][j]
key = str(lon_i)+'_'+str(lat_i)
# print(key)
transform_dic[key] = val
np.save(this_root+r'CRU_precip\transform\\'+date,transform_dic)
time_end = time.time()
log_process.process_bar(flag,len(npz),time_init,time_start,time_end)
flag += 1
# coding=utf-8
def extract_value_from_stations():
# 1、获取站点经纬度，设置50km范围lon_min,lon_max,lat_min,lat_max，存入字典
# 2、建立空字典
sta_pos_dic = np.load(this_root + 'ANN_input_para\\00_PDSI\\sta_pos_dic.npz')
sta_pos_dic_range = {}
sta_pos_dic_void = {}
for sta in sta_pos_dic:
# print(sta)
sta_pos_dic_void[sta] = []
lat = sta_pos_dic[sta][0]
lon = sta_pos_dic[sta][1]
lon_min = lon-0.5
lon_max = lon+0.5
lat_min = lat-0.5
lat_max = lat+0.5
sta_pos_dic_range[sta] = [[lon_min,lon_max],
[lat_min,lat_max]]
#3、读取每个月的CRU数据
fdir = r'E:\MODIS\CRU_precip\transform\\'
flist = os.listdir(fdir)
CRU_vals_dic = {}
time_init = time.time()
flag = 0
for f in flist:
time_start = time.time()
arr = np.load(fdir + f).item()
dic = dict(arr)
# flag = 0
for key in dic:
key_split = key.split('_')
lon = float(key_split[0])
lat = float(key_split[1])
for sta in sta_pos_dic_range:
sta_pos_dic_range_dic = sta_pos_dic_range[sta]
lon_range = sta_pos_dic_range_dic[0]
lat_range = sta_pos_dic_range_dic[1]
lon_min = lon_range[0]
lon_max = lon_range[1]
lat_min = lat_range[0]
lat_max = lat_range[1]
if lon_min < lon < lon_max and lat_min < lat < lat_max:
val = dic[key]
sta_pos_dic_void[sta].append(val)
for sta in sta_pos_dic_void:
vals_list = sta_pos_dic_void[sta]
if len(vals_list) > 0:
key_name = sta+'_'+f[:-4]
val = np.mean(vals_list)
CRU_vals_dic[key_name] = val
time_end = time.time()
log_process.process_bar(flag,len(flist),time_init,time_start,time_end)
flag += 1
print('\nsaving dic...')
np.save(r'E:\MODIS\CRU_precip\CRU_precip_dic',CRU_vals_dic)
def gen_6_months_average():
# 用CRU数据
# this_root = os.getcwd()+'\\'
mode = 'tmp'
if mode == 'tmp':
data = np.load('E:\\MODIS\\CRU_tmp\\CRU_tmp_dic_filter_inf.npy').item()
data = dict(data)
path = 'CRU_tmp\\tmp'
elif mode == 'pre':
data = np.load('E:\\MODIS\\CRU_precip\\CRU_precip_dic_filter_inf.npy').item()
data = dict(data)
path = 'CRU_precip\\pre'
else:
data = None
path = None
# temp = dict(temp)
date_list = []
for year in range(2000, 2017):
for mon in range(1, 13):
date_list.append(str(year) + '%02d' % mon)
new_dic = {}
time_init = time.time()
flag = 0
for key in data:
time_start = time.time()
key_split = key.split('_')
sta = key_split[0]
date_str = key_split[1]
year = int(date_str[:4])
mon = int(date_str[-2:])
date = datetime.datetime(year, mon, 15)
time_delta = datetime.timedelta(30)
date_1 = date - time_delta * 1
date_1_str = '%s%02d' % (date_1.year, date_1.month)
date_2 = date - time_delta * 2
date_2_str = '%s%02d' % (date_2.year, date_2.month)
date_3 = date - time_delta * 3
date_3_str = '%s%02d' % (date_2.year, date_3.month)
date_4 = date - time_delta * 4
date_4_str = '%s%02d' % (date_2.year, date_4.month)
date_5 = date - time_delta * 5
date_5_str = '%s%02d' % (date_2.year, date_5.month)
# print(date_1_str)
# print(date_2_str)
try:
val1 = data[sta + '_' + date_str]
val2 = data[sta + '_' + date_1_str]
val3 = data[sta + '_' + date_2_str]
val4 = data[sta + '_' + date_3_str]
val5 = data[sta + '_' + date_4_str]
val6 = data[sta + '_' + date_5_str]
val_mean = np.mean([val1, val2, val3, val4, val5, val6])
# print(val_mean)
new_dic[key] = val_mean
except Exception as e:
# print(e)
pass
time_end = time.time()
log_process.process_bar(flag, len(data), time_init, time_start, time_end)
flag += 1
print('saving dic...')
np.save('E:\\MODIS\\'+ path +'_6_months_average', new_dic)
def line_annotation():
'''
生成离散的line shp
1、生成shp dic
根据所有的dwg to shp，生成电线杆的坐标，存入字典
2、打开excel，re筛选汉字，将汉字作为字典键值，生成电线杆字典，将设备名称，所属线路等属性加入字典
3、xian_dic 1-100排序 生成 xian_dic_sort
4、根据xian_dic_sort生成shp
5、下一步合成shp（功能不在此函数）
:return:
'''
npy = this_root + '190509\\民权线路cad\\shp_dic.npy'
shp_dic = np.load(npy).item()
shp_dic = dict(shp_dic)
f_excel = this_root + '190509\\民权线路cad\\民权台账 - 以此为准.xls'
bk = xlrd.open_workbook(f_excel)
sh = bk.sheet_by_index(0)
nrows = sh.nrows
# 筛选汉字
xx = u"([\u4e00-\u9fff]+)"
pattern = re.compile(xx)
# 建立字典
xian_list = []
for i in range(nrows):
shebei_name = sh.cell_value(i, 0)
results = pattern.findall(shebei_name)
xian_list.append(''.join(results))
xian_list = set(xian_list)
xian_dic = {}
xian_dic_sort = {}
for x in xian_list:
xian_dic[x] = []
xian_dic_sort[x] = []
# 读excel 信息
for i in range(nrows):
shebei_name = sh.cell_value(i, 0)
suoshu_xianlu = sh.cell_value(i, 2)
zhixian_mingcheng = sh.cell_value(i, 5)
daoxian_xinghao = sh.cell_value(i, 6)
results = pattern.findall(shebei_name)
hanzi_xian = ''.join(results)
xian_dic[hanzi_xian].append(
[shebei_name, suoshu_xianlu, zhixian_mingcheng, daoxian_xinghao])
# sort 1-100
error = 0
for key in xian_dic:
# line_list = []
for i in range(len(xian_dic[key])):
for j in range(len(xian_dic[key])):
if u'线'+str(i+1)+u'号杆' in xian_dic[key][j][0] or \
u'支' + str(i+1) + u'号杆' in xian_dic[key][j][0] or \
u'线' + str(i+1) + u'杆塔' in xian_dic[key][j][0]:
try:
a = shp_dic[xian_dic[key][j][0]]
xian_dic_sort[key].append(xian_dic[key][j])
except:
error += 1
pass
flag = 0
time_init = time.time()
for key in xian_dic_sort:
time_start = time.time()
inlist = []
for i in range(len(xian_dic_sort[key])):
if i == len(xian_dic_sort[key]) - 1:
continue
x1 = shp_dic[xian_dic_sort[key][i][0]][0]
y1 = shp_dic[xian_dic_sort[key][i][0]][1]
x2 = shp_dic[xian_dic_sort[key][i + 1][0]][0]
y2 = shp_dic[xian_dic_sort[key][i + 1][0]][1]
start = [x1, y1]
end = [x2, y2]
# name_gbk = name.decode('utf-8')
val1 = xian_dic_sort[key][i][0].encode(
'utf-8') + '-' + xian_dic_sort[key][i + 1][0].encode('utf-8')
val2 = xian_dic_sort[key][i][1].encode('utf-8')
val3 = xian_dic_sort[key][i][2].encode('utf-8')
val4 = xian_dic_sort[key][i][3].encode('utf-8')
# print(val4)
inlist.append([start, end, val1, val2, val3, val4, ''])
# print(key.encode('utf-8'))
# for i in inlist:
# print(i)
dwg_dir = this_root + u'190509\\民权线路cad\\dwg_to_shp\\' + val2.decode(
'utf-8') + '\\annotation\\'
mk_dir(dwg_dir)
dwg_dir = dwg_dir.encode('utf-8')
# print(dwg_dir)
# exit()
try:
line_to_shp(
inlist,
dwg_dir + '\\' + key.encode('utf-8') + 'line_annotation.shp')
except:
pass
time_end = time.time()
log_process.process_bar(flag, len(xian_dic_sort), time_init,
time_start, time_end)
flag += 1
pass
temp = []
for j in i:
if j < -999:
val = 255
elif -999 < j < 0:
val = 0
elif 0 <= j <=1:
val = int(j*100)
elif j > 1:
val = 100
else:
val = 255
temp.append(val)
# print val
new_array.append(temp)
# new_array = np.array(new_array,dtype=int)
new_array = np.array(new_array,dtype=int)
# print 'ploting'
# plt.imshow(new_array)
# plt.colorbar()
# print 'done'
# plt.show()
# exit()
# print 'saving'
newRasterfn = (save_dir+f).decode('gbk').encode('utf-8')
to_raster.array2raster(newRasterfn,originX,originY,pixelWidth,pixelHeight,new_array)
time_end = time.time()
log_process.process_bar(flag,len(flist),time_init,time_start,time_end,f)
flag+=1
def download_movies(url, movie_path, selected_episodes=range(1, int(1e5))):
'''
:param url: 'http://532movie.bnu.edu.cn/player/3379.html'
:return: None
download *.ts to RAM
write mp4 movie file to local disk from RAM
'''
movie_name, urls = get_vedio_url(url)
invalid_char = '/\:*"<>|?'
for ic in invalid_char:
if ic in movie_name:
movie_name = movie_name.replace(ic, '.')
try:
print(movie_name.decode('utf-8').encode('gbk', 'ignore'))
except:
print(movie_name.split()[0].decode('utf-8').encode('gbk', 'ignore'))
try:
movie_name_utf8 = movie_name.decode('utf-8').encode('gbk', 'ignore')
except:
movie_name_utf8 = movie_name.split()[0].decode('utf-8').encode(
'gbk', 'ignore')
episode = 0
flag = 0
time_init = time.time()
for i in urls:
time_start = time.time()
if len(urls) == 1:
if os.path.isfile(movie_path + movie_name_utf8 + '.mp4'):
print(movie_path + movie_name_utf8 + '.mp4 is already existed')
return None
ts = split_videos(i)
pool = ThreadPool(20)
bar_fmt = 'Downloading\t' + '|{bar}|{percentage:3.0f}%'
results = list(
tqdm(pool.imap(download_ts, ts),
total=len(ts),
ncols=50,
bar_format=bar_fmt))
pool.close()
pool.join()
# print('Writing to disk...')
movie = codecs.open(movie_path + movie_name_utf8 + '.mp4', 'wb')
bar_fmt1 = 'writing to disk\t' + '|{bar}|{percentage:3.0f}%'
for i in tqdm(range(len(results)), bar_format=bar_fmt1, ncols=50):
movie.write(results[i])
movie.close()
else:
episode += 1
if episode not in selected_episodes:
continue
TV_dir = movie_path + movie_name_utf8 + '\\'
if not os.path.isdir(TV_dir):
os.makedirs(TV_dir)
if os.path.isfile(TV_dir + 'Episode ' + '%02d' % episode + '.mp4'):
print(TV_dir + 'Episode ' + '%02d' % episode +
'.mp4 is already existed')
flag += 1
continue
pool = ThreadPool(20)
ts = split_videos(i)
bar_fmt = 'Episode %02d' % episode + '|{bar}|{percentage:3.0f}%'
results = list(
tqdm(pool.imap(download_ts, ts),
total=len(ts),
ncols=50,
bar_format=bar_fmt))
pool.close()
pool.join()
movie = codecs.open(
TV_dir + 'Episode ' + '%02d' % episode + '.mp4', 'wb')
for r in results:
movie.write(r)
time_end = time.time()
lenurl = len(urls)
len_selected = len(selected_episodes)
length = min([lenurl, len_selected])
try:
name = movie_name.decode('utf-8').encode('gbk', 'ignore')
except:
name = movie_name.split()[0].decode('utf-8').encode(
'gbk', 'ignore')
log_process.process_bar(flag, length, time_init, time_start,
time_end, name + '\n')
flag += 1
def extract_value_from_stations():
# 1、获取站点经纬度，设置50km范围lon_min,lon_max,lat_min,lat_max，存入字典
# 2、建立空字典
this_root = 'e:\\MODIS\\'
sta_pos_dic = np.load(this_root +
'ANN_input_para\\00_PDSI\\sta_pos_dic.npz')
sta_pos_dic_range = {}
sta_pos_dic_void = {}
for sta in sta_pos_dic:
# print(sta)
sta_pos_dic_void[sta] = []
lat = sta_pos_dic[sta][0]
lon = sta_pos_dic[sta][1]
lon_min = lon - 0.1
lon_max = lon + 0.1
lat_min = lat - 0.1
lat_max = lat + 0.1
sta_pos_dic_range[sta] = [[lon_min, lon_max], [lat_min, lat_max]]
date_list = []
for year in range(2003, 2017):
for month in range(1, 13):
date_list.append(str(year) + '%02d' % month)
DEM_vals_dic = {}
f = r'E:\MODIS\DEM\DEM_data_transform.npy'
arr = np.load(f).item()
dic = dict(arr)
flag = 0
time_init = time.time()
for key in dic:
time_start = time.time()
key_split = key.split('_')
lon = float(key_split[0])
lat = float(key_split[1])
for sta in sta_pos_dic_range:
sta_pos_dic_range_dic = sta_pos_dic_range[sta]
lon_range = sta_pos_dic_range_dic[0]
lat_range = sta_pos_dic_range_dic[1]
lon_min = lon_range[0]
lon_max = lon_range[1]
lat_min = lat_range[0]
lat_max = lat_range[1]
if lon_min < lon < lon_max and lat_min < lat < lat_max:
val = dic[key]
sta_pos_dic_void[sta].append(val)
time_end = time.time()
log_process.process_bar(flag, len(dic), time_init, time_start,
time_end)
flag += 1
for sta in sta_pos_dic_void:
vals_list = sta_pos_dic_void[sta]
if len(vals_list) > 0:
for date in date_list:
key_name = sta + '_' + date
val = np.mean(vals_list)
DEM_vals_dic[key_name] = val
print(key_name, val)
print('saving')
np.save(this_root + 'DEM\\DEM_extract_from_sta', DEM_vals_dic)
def merge_line_shp():
'''
composite shp
将xian_dic_sort生成的shp合成为1个shp
作为annotation
:return:
'''
gdal.SetConfigOption("SHAPE_ENCODING", "GBK")
fdir = this_root + '\\190714\\dwg_to_shp\\'
flist = os.listdir(fdir)
time_init = time.time()
time_i = 0
inlist = []
for folder in flist:
# print(folder.decode('gbk'))
# exit()
time_start = time.time()
# print(folder.decode('gbk'))
shp_dir = fdir + folder + '\\annotation\\'
try:
shp_list = os.listdir(shp_dir)
except:
continue
for shp in shp_list:
# print(shp.decode('gbk'))
# exit()
if shp.endswith('annotation.shp'):
# print(shp.decode('gbk'))
# exit()
daShapefile = shp_dir + shp
print(daShapefile.decode('gbk'))
# print(daShapefile.decode('gbk'))
driver = ogr.GetDriverByName("ESRI Shapefile")
dataSource = driver.Open(daShapefile.decode('gbk'), 0)
layer = dataSource.GetLayer()
# inlist_i = []
for feature in layer:
geom = feature.GetGeometryRef()
val1 = feature.GetField("val1")
val2 = feature.GetField("val2")
val3 = feature.GetField("val3")
val4 = feature.GetField("val4")
Points = geom.GetPoints()
if Points:
for i in range(len(Points)):
if i == len(Points) - 1:
break
inlist.append([
Points[i], Points[i + 1], val1, val2, val3,
val4, ''
])
# print(Points)
# print(inlist_i)
# exit()
# exit()
# inlist.append([Points[0],Points[1],val1,val2,val3,val4,''])
time_end = time.time()
time_i += 1
log_process.process_bar(time_i, len(flist), time_init, time_start,
time_end)
output_fn = this_root + '190714\\line_annotation_merge.shp'.decode(
'gbk').encode('utf-8')
print('exporting line shp...')
line_to_shp(inlist, output_fn)
pass
def gen_temp_anomaly_new_data():
this_root = os.getcwd() + '\\'
tmp_dic = dict(
np.load(this_root + 'data\\pre_monthly_composite_dic.npy').item())
sta_dic = np.load(this_root + 'data\\sta_num_list.npy').item()
sta_dic = list(sta_dic)
sta_dic.sort()
date_list = []
for year in range(1951, 2018):
for mon in range(1, 13):
date = str(year) + '%02d' % mon
date_list.append(date)
# flag = 0
mean_std_dic = {}
for sta in sta_dic:
# flag+=1
# print(flag,'/',len(sta_dic))
for mon in range(1, 13):
mon_str = '%02d' % mon
vals = []
for year in range(1951, 2018):
year_str = str(year)
key = sta + '_' + year_str + mon_str
try:
val = tmp_dic[key]
except Exception as e:
val = None
pass
if val != None:
vals.append(val)
# print(e)
if len(vals) > 10:
mean = np.mean(vals)
std = np.std(vals)
# print(sta+'_'+mon_str)
mean_std_dic[sta + '_' + mon_str] = [mean, std]
# else:
# print(sta,mon_str)
# # exit()
anomaly_dic = {}
flag = 0
time_init = time.time()
for key in tmp_dic:
# flag+=1
# print(flag,'/',len(tmp_dic))
time_start = time.time()
key_split = key.split('_')
sta = key_split[0]
mon = key_split[1][-2:]
val = tmp_dic[key]
try:
mean, std = mean_std_dic[sta + '_' + mon]
anomaly = (val - mean) / std
anomaly_dic[key] = anomaly
except Exception as e:
pass
time_end = time.time()
log_process.process_bar(flag, len(tmp_dic), time_init, time_start,
time_end)
flag += 1
print('saving ...')
np.save(this_root + 'data\\pre_anomaly_dic', anomaly_dic)
def gen_precip_anomaly():
valid_year = []
for y in range(2003, 2016):
valid_year.append(y)
date_list = []
for y in valid_year:
for m in range(1, 13):
date_list.append(str(y) + '%02d' % m)
tem_dic = np.load(this_root + 'PDSI\\pre_dic.npz')
print('processing temperature...')
sta_tem_dic = {}
time_init = time.time()
flag = 0
for sta in tem_dic:
if sta == '56247':
continue
time_start = time.time()
# print(sta)
# print(tem_dic[sta])
one_sta_tem_dic = tem_dic[sta].item()
one_sta = []
for key in date_list:
try:
one_sta.append(one_sta_tem_dic[key])
except:
pass
one_sta = np.array(one_sta)
try:
one_sta = one_sta.reshape(13, 12)
except:
continue
one_month_average = []
one_month_std = []
for i in one_sta.T:
mean_val = np.mean(i)
std_val = np.std(i)
one_month_average.append(mean_val)
one_month_std.append(std_val)
anomaly_val = []
for i in one_sta:
val = (i - one_month_average) / one_month_std
anomaly_val.append(val)
anomaly_val = np.array(anomaly_val)
# plt.figure()
# plt.plot(anomaly_val.flatten())
#
# plt.figure()
# plt.plot(one_sta.flatten())
# plt.show()
anomaly_val_result = anomaly_val.flatten()
# if len(one_sta) == 156:
sta_tem_dic[sta] = anomaly_val_result
time_end = time.time()
log_process.process_bar(flag, len(tem_dic), time_init, time_start,
time_end, sta)
flag += 1
print('\nsaving...')
np.save(this_root + 'PDSI\\pre_anomaly_2003-2015', sta_tem_dic)
