def mosaic_2018(): fdir = r'E:\before2000\unzip\\' flist = os.listdir(fdir) out_rasters = [] for f in flist: out_rasters.append('_'.join(f.split('_')[:3])) out_raster = list(set(out_rasters)) out_dir = r'E:\before2000\mosaic\\' mk_dir(out_dir) time_init = time.time() flag = 0 for i in out_raster: log.logger.info(i) time_start = time.time() in_rasters_list = [] for f in flist: if i in f: in_rasters_list.append(f) mosaic(fdir, in_rasters_list, out_dir=out_dir, out_raster=i + '.tif') time_end = time.time() log_process.process_bar(flag, len(out_raster), time_init, time_start, time_end, '\n') flag += 1 pass
def gen_dependent_pdsi(): # 去除无效值 # key = 50136_200411 pdsi = np.load(this_root + 'PDSI\\PDSI_result.npz') pdsi_out = this_root + 'PDSI\\PDSI_result_filter' valid_year = [] for y in range(2000, 2018): valid_year.append(y) independent_pdsi = {} time_init = time.time() flag = 0 for sta in pdsi: time_start = time.time() for year in pdsi[sta].item(): if year in valid_year: # print(type(i)) # print(year) # print(sta,pdsi[sta].item()[year]) for mon, val in enumerate(pdsi[sta].item()[year]): key = sta + '_' + str(year) + '%02d' % (mon + 1) # print(key) # print(val) if -10 < val < 10: independent_pdsi[key] = val # time.sleep(0.3) time_end = time.time() log_process.process_bar(flag, len(pdsi), time_init, time_start, time_end, str(flag) + '/' + str(len(pdsi))) flag += 1 print 'saving dic...' np.save(pdsi_out, independent_pdsi)
def gen_shp_dic(): fdir = this_root + u'190509\\民权线路cad\\dwg_to_shp\\' flist = os.listdir(fdir) shp_dic = {} time_init = time.time() flag = 0 for folder in flist: time_start = time.time() print(folder) shp_list = os.listdir(fdir + folder) for shp in shp_list: if shp.endswith('Annotation.shp'): daShapefile = fdir + folder + '\\' + shp driver = ogr.GetDriverByName("ESRI Shapefile") dataSource = driver.Open(daShapefile, 0) layer = dataSource.GetLayer() for feature in layer: geom = feature.GetGeometryRef() x = geom.GetX() y = geom.GetY() name = feature.GetField("RefName") name_gbk = name.decode('utf-8') shp_dic[name_gbk] = [x, y] # print x,y,name_gbk # time.sleep(0.5) time_end = time.time() log_process.process_bar(flag, len(flist), time_init, time_start, time_end) flag += 1 print('saving shp dic...') np.save(this_root + '190509\\民权线路cad\\shp_dic', shp_dic)
def run_dwg_to_shp(fdir,out_dir_): # log = log_process.Logger('log.log') # fdir = this_root+'190905\\сюЁг\\' flist = os.listdir(fdir) init_time = time.time() flag = 0 for f in flist: start = time.time() out_dir = out_dir_+f.split('.')[0] # print(f) # print(out_dir) # exit() if os.path.isdir(out_dir): end = time.time() log_process.process_bar(flag, len(flist), init_time, start, end, str(flag + 1) + '/' + str(len(flist)) + '\n') flag += 1 print(out_dir+'is existed') continue mk_dir(out_dir) # try: print(out_dir.decode('gbk')) # exit() dwg_to_shp(fdir+f+'\\Annotation',out_dir) dwg_to_shp(fdir+f+'\\Polyline',out_dir) # log.logger.info('\\Annotation') # log.logger.info('\\Polyline') # except Exception as e: # log.logger.error(e) end = time.time() log_process.process_bar(flag,len(flist),init_time,start,end,str(flag+1)+'/'+str(len(flist))+'\n') flag += 1
def do_mosaic(year): year = str(year) fdir = 'E:\\30m_fvc_annual_unzip\\' + year + '\\' flist = os.listdir(fdir) out_rasters = [] for f in flist: out_rasters.append('_'.join(f.split('_')[:3])) out_raster = list(set(out_rasters)) out_dir = 'E:\\30m_fvc_annual_unzip\\' + year + '_mosaic\\' mk_dir(out_dir) time_init = time.time() flag = 0 for i in out_raster: log.logger.info(i) time_start = time.time() in_rasters_list = [] for f in flist: if i in f: in_rasters_list.append(f) mosaic(fdir, in_rasters_list, out_dir=out_dir, out_raster=i + '.tif') time_end = time.time() log_process.process_bar(flag, len(out_raster), time_init, time_start, time_end, '\n') flag += 1 pass
def main(): print('initializing GEE') ee.Initialize() print('done') product_dic = { # 'LC08':"LANDSAT/LC08/C01/T1_32DAY_NDVI", # 'LE07':'LANDSAT/LE07/C01/T1_32DAY_NDVI', # 'LT05':"LANDSAT/LT05/C01/T1_32DAY_NDVI", '006': "MODIS/006/MOD13Q1" } fdir = this_root + 'download_data\\' re_download_dir = this_root + 're_download_data\\' mk_dir(re_download_dir) poligon_list = gen_polygons() polygon_dic = {} for i in range(len(poligon_list)): polygon_dic['%02d' % (i + 1)] = poligon_list[i] flist = os.listdir(fdir) # time_init = time.time() flag = 0 invalid_list = [] for f in flist: start = time.time() if check(fdir + f): invalid_list.append(f) end = time.time() logger.process_bar(flag, len(flist), time_init, start, end, 'checked') flag += 1 time_init = time.time() flag = 0 for f in invalid_list: start = time.time() # print(f) fsplit = f.split('_') product = fsplit[0] dataset = product_dic[product] date_start = str(fsplit[1]) date_end = str(fsplit[2]) num = str(fsplit[3].split('.')[0]) region = polygon_dic[num] # print(dataset) # print(date_start) # print(date_end) # print(num) # print(region) # url = get_download_url(dataset, date_start, date_end, region) file_name = '_'.join([product, date_start, date_end, num]) + '.zip' # print(file_name) download_data(url, re_download_dir + file_name) end = time.time() logger.process_bar(flag, len(invalid_list), time_init, start, end, str(flag + 1) + '/' + str(len(invalid_list)) + '\n') flag += 1
def gen_3_months_average_new(): # 用新数据 this_root = os.getcwd() + '\\' # pre = np.load(this_root + 'data\\pre_anomaly_dic.npy').item() temp = np.load(this_root + 'data\\tmp_anomaly_dic.npy').item() # pre = dict(pre) temp = dict(temp) date_list = [] for year in range(2000, 2016): for mon in range(1, 13): date_list.append(str(year) + '%02d' % mon) new_dic = {} time_init = time.time() flag = 0 for key in temp: time_start = time.time() key_split = key.split('_') sta = key_split[0] date_str = key_split[1] year = int(date_str[:4]) mon = int(date_str[-2:]) date = datetime.datetime(year, mon, 15) time_delta = datetime.timedelta(30) date_1 = date - time_delta date_1_str = '%s%02d' % (date_1.year, date_1.month) date_2 = date - time_delta * 2 date_2_str = '%s%02d' % (date_2.year, date_2.month) # print(date_1_str) # print(date_2_str) try: val1 = temp[sta + '_' + date_str] val2 = temp[sta + '_' + date_1_str] val3 = temp[sta + '_' + date_2_str] val_mean = np.mean([val1, val2, val3]) # print(val_mean) new_dic[key] = val_mean except Exception as e: # print(e) pass time_end = time.time() log_process.process_bar(flag, len(temp), time_init, time_start, time_end) flag += 1 print('saving dic...') np.save(this_root + 'data\\temp_3_months_average_new', new_dic)
def gen_line_annotation(): ''' composite shp 将xian_dic_sort生成的shp合成为1个shp 作为annotation :return: ''' gdal.SetConfigOption("SHAPE_ENCODING", "GBK") fdir = this_root + '\\190509\\民权线路cad\\dwg_to_shp\\' flist = os.listdir(fdir) time_init = time.time() time_i = 0 for folder in flist: time_start = time.time() # print(folder.decode('gbk')) shp_dir = fdir + folder + '\\annotation\\' try: shp_list = os.listdir(shp_dir) inlist = [] for shp in shp_list: # print(shp) if shp.endswith('.shp'): # print(shp.decode('gbk')) daShapefile = shp_dir + shp # print(daShapefile.decode('gbk')) driver = ogr.GetDriverByName("ESRI Shapefile") dataSource = driver.Open(daShapefile.decode('gbk'), 0) layer = dataSource.GetLayer() for feature in layer: geom = feature.GetGeometryRef() Points = geom.GetPoints() val1 = feature.GetField("val1") val2 = feature.GetField("val2") val3 = feature.GetField("val3") val4 = feature.GetField("val4") inlist.append( [Points[0], Points[1], val1, val2, val3, val4, '']) # print(fdir.decode('gbk') + folder.decode('gbk') + '\\test.shp') output_fn = fdir.decode('gbk').encode('utf-8') + folder.decode( 'gbk').encode('utf-8') + '\\line_annotation.shp' # print output_fn line_to_shp(inlist, output_fn) time_end = time.time() time_i += 1 log_process.process_bar(time_i, len(flist), time_init, time_start, time_end) except: time_i += 1 pass # exit() pass
def gen_lon_lat_dic_non_normalize(): ''' 生成栅格和经纬度对应的字典 数据格式: dic[str(x.y)] = [lon,lat] :return: ''' # rasterized_tif = this_root + '/conf/my.tif' this_root = r'e:\\MODIS\\' DEM = this_root + 'DEM\\china_1km_dem_wgs1984_resample.tif' _, im_width, im_height, im_geotrans, im_proj = tif_to_array(DEM) array = np.load(this_root + 'DEM\\china_1km_dem_wgs1984_resample_non_normalize.npy') # normalize(array) # exit() # clip_array = np.load(this_root+'\\conf\\china_T_F.npy') # print(array) # array = np.ma.masked_where(array<-9999,array) # plt.imshow(array) # plt.colorbar() # plt.show() print(im_geotrans) lon_lat_dic = {} time_init = time.time() flag = 0 for y in range(len(array)): time_start = time.time() for x in range(len(array[y])): if array[y][x] > -9000: lon_start = im_geotrans[0] lon_step = im_geotrans[1] lon = lon_start + lon_step * x lat_start = im_geotrans[3] lat_step = im_geotrans[5] lat = lat_start + lat_step * y # print('*'*8) # print(str(lon)+'_'+str(lat)) # print(array[y][x]) lon_lat_dic[str(lon) + '_' + str(lat)] = array[y][x] # print(str(lon)+'_'+str(lat),array[y][x]) time_end = time.time() log_process.process_bar(flag, len(array), time_init, time_start, time_end) flag += 1 np.save(this_root + '\\DEM\\DEM_data_transform', lon_lat_dic)
def extract_cci_from_sta(): this_root = 'e:\\MODIS\\' npz = np.load(this_root + 'CCI\\time_interp.npz') sta_pos_dic = np.load(this_root + 'ANN_input_para\\00_PDSI\\sta_pos_dic.npz') date_list = [] for year in range(2003, 2017): for mon in range(1, 13): date_list.append(str(year) + '%02d' % mon) CCI_sta_extract_dic = {} import log_process time_init = time.time() flag = 0 for npy in npz: time_start = time.time() npy_split = npy.split('_') lon = float(npy_split[0]) lat = float(npy_split[1]) vals = npz[npy] if len(vals) < 10: continue for sta in sta_pos_dic: pos = sta_pos_dic[sta] lon_sta = pos[1] lat_sta = pos[0] lon_max = lon_sta + 0.25 lon_min = lon_sta - 0.25 lat_max = lat_sta + 0.25 lat_min = lat_sta - 0.25 if lon_min < lon < lon_max and lat_min < lat < lat_max: for date in range(len(date_list)): key = sta + '_' + date_list[date] CCI_sta_extract_dic[key] = vals[date] time_end = time.time() log_process.process_bar(flag, len(npz), time_init, time_start, time_end) flag += 1 print('saving...') np.save(this_root + 'CCI\\CCI_sta_extract_dic', CCI_sta_extract_dic)
def main(): movie_path = 'E:\\532_all_movie\\movie\\' if not os.path.isdir(movie_path): os.makedirs(movie_path) f = open('movies_url_list.txt', 'r') lines = f.readlines() f.close() import log_process flag = 0 time_init = time.time() for line in lines: time_start = time.time() url = line.split('\n')[0] # print(url) movie_name, urls = get_vedio_url(url) # print(movie_name.decode('utf-8').encode('gbk')) download_movies(url,movie_path) time_end = time.time() log_process.process_bar(flag,len(lines),time_init,time_start,time_end,'\n') flag+=1
def gen_dependent_pdsi1(): # 去除无效值 # key = 2003 pdsi = np.load(this_root + 'PDSI\\project02_pdsi\\self_cal_all_sta_dic.npz') pdsi_out = this_root + 'PDSI\\project02_pdsi\\PDSI_result_filter' valid_year = [] # oneyear_filtered_pdsi = {} for y in range(2000, 2018): # oneyear_filtered_pdsi[y] = [] valid_year.append(y) time_init = time.time() flag = 0 filtered_pdsi = {} for sta in pdsi: oneyear_filtered_pdsi = {} for y in range(2000, 2018): oneyear_filtered_pdsi[y] = [] valid_year.append(y) time_start = time.time() onesta_pdsi_dic = pdsi[sta].item() for year in onesta_pdsi_dic: if year in valid_year: vals = onesta_pdsi_dic[year] for val in vals: if -10 < val < 10: oneyear_filtered_pdsi[year].append(val) # print(oneyear_filtered_pdsi) filtered_pdsi[sta] = oneyear_filtered_pdsi time_end = time.time() log_process.process_bar(flag, len(pdsi), time_init, time_start, time_end, str(flag) + '/' + str(len(pdsi))) flag += 1 print('saving...') np.savez(pdsi_out, **filtered_pdsi)
def trans_nc_to_lon_lat_list(): nc = r'E:\MODIS\CRU_precip\cru_ts4.02.1901.2017.pre.dat.nc' ncin = Dataset(nc, 'r') # print ncin.variables lon = ncin['lon'] lat = ncin['lat'] lon_origin = lon[0] lat_origin = lat[-1] lon_size = lon[1]-lon[0] lat_size = lat[1]-lat[0] print(lon_origin) print(lat_origin) print(lon_size) print(lat_size) # exit() npz = np.load(r'E:\MODIS\CRU_precip\2001.2018.npz') time_init = time.time() flag = 0 for date in npz: # print(date) time_start = time.time() transform_dic = {} arr = npz[date] for i in range(len(arr)): for j in range(len(arr[i])): # print(i,j) lon_i = lon_origin+lon_size*j lat_i = lat_origin-lat_size*i val = arr[i][j] key = str(lon_i)+'_'+str(lat_i) # print(key) transform_dic[key] = val np.save(this_root+r'CRU_precip\transform\\'+date,transform_dic) time_end = time.time() log_process.process_bar(flag,len(npz),time_init,time_start,time_end) flag += 1
# coding=utf-8
def extract_value_from_stations(): # 1、获取站点经纬度,设置50km范围lon_min,lon_max,lat_min,lat_max,存入字典 # 2、建立空字典 sta_pos_dic = np.load(this_root + 'ANN_input_para\\00_PDSI\\sta_pos_dic.npz') sta_pos_dic_range = {} sta_pos_dic_void = {} for sta in sta_pos_dic: # print(sta) sta_pos_dic_void[sta] = [] lat = sta_pos_dic[sta][0] lon = sta_pos_dic[sta][1] lon_min = lon-0.5 lon_max = lon+0.5 lat_min = lat-0.5 lat_max = lat+0.5 sta_pos_dic_range[sta] = [[lon_min,lon_max], [lat_min,lat_max]] #3、读取每个月的CRU数据 fdir = r'E:\MODIS\CRU_precip\transform\\' flist = os.listdir(fdir) CRU_vals_dic = {} time_init = time.time() flag = 0 for f in flist: time_start = time.time() arr = np.load(fdir + f).item() dic = dict(arr) # flag = 0 for key in dic: key_split = key.split('_') lon = float(key_split[0]) lat = float(key_split[1]) for sta in sta_pos_dic_range: sta_pos_dic_range_dic = sta_pos_dic_range[sta] lon_range = sta_pos_dic_range_dic[0] lat_range = sta_pos_dic_range_dic[1] lon_min = lon_range[0] lon_max = lon_range[1] lat_min = lat_range[0] lat_max = lat_range[1] if lon_min < lon < lon_max and lat_min < lat < lat_max: val = dic[key] sta_pos_dic_void[sta].append(val) for sta in sta_pos_dic_void: vals_list = sta_pos_dic_void[sta] if len(vals_list) > 0: key_name = sta+'_'+f[:-4] val = np.mean(vals_list) CRU_vals_dic[key_name] = val time_end = time.time() log_process.process_bar(flag,len(flist),time_init,time_start,time_end) flag += 1 print('\nsaving dic...') np.save(r'E:\MODIS\CRU_precip\CRU_precip_dic',CRU_vals_dic)
def gen_6_months_average(): # 用CRU数据 # this_root = os.getcwd()+'\\' mode = 'tmp' if mode == 'tmp': data = np.load('E:\\MODIS\\CRU_tmp\\CRU_tmp_dic_filter_inf.npy').item() data = dict(data) path = 'CRU_tmp\\tmp' elif mode == 'pre': data = np.load('E:\\MODIS\\CRU_precip\\CRU_precip_dic_filter_inf.npy').item() data = dict(data) path = 'CRU_precip\\pre' else: data = None path = None # temp = dict(temp) date_list = [] for year in range(2000, 2017): for mon in range(1, 13): date_list.append(str(year) + '%02d' % mon) new_dic = {} time_init = time.time() flag = 0 for key in data: time_start = time.time() key_split = key.split('_') sta = key_split[0] date_str = key_split[1] year = int(date_str[:4]) mon = int(date_str[-2:]) date = datetime.datetime(year, mon, 15) time_delta = datetime.timedelta(30) date_1 = date - time_delta * 1 date_1_str = '%s%02d' % (date_1.year, date_1.month) date_2 = date - time_delta * 2 date_2_str = '%s%02d' % (date_2.year, date_2.month) date_3 = date - time_delta * 3 date_3_str = '%s%02d' % (date_2.year, date_3.month) date_4 = date - time_delta * 4 date_4_str = '%s%02d' % (date_2.year, date_4.month) date_5 = date - time_delta * 5 date_5_str = '%s%02d' % (date_2.year, date_5.month) # print(date_1_str) # print(date_2_str) try: val1 = data[sta + '_' + date_str] val2 = data[sta + '_' + date_1_str] val3 = data[sta + '_' + date_2_str] val4 = data[sta + '_' + date_3_str] val5 = data[sta + '_' + date_4_str] val6 = data[sta + '_' + date_5_str] val_mean = np.mean([val1, val2, val3, val4, val5, val6]) # print(val_mean) new_dic[key] = val_mean except Exception as e: # print(e) pass time_end = time.time() log_process.process_bar(flag, len(data), time_init, time_start, time_end) flag += 1 print('saving dic...') np.save('E:\\MODIS\\'+ path +'_6_months_average', new_dic)
def line_annotation(): ''' 生成离散的line shp 1、生成shp dic 根据所有的dwg to shp,生成电线杆的坐标,存入字典 2、打开excel,re筛选汉字,将汉字作为字典键值,生成电线杆字典,将设备名称,所属线路等属性加入字典 3、xian_dic 1-100排序 生成 xian_dic_sort 4、根据xian_dic_sort生成shp 5、下一步合成shp(功能不在此函数) :return: ''' npy = this_root + '190509\\民权线路cad\\shp_dic.npy' shp_dic = np.load(npy).item() shp_dic = dict(shp_dic) f_excel = this_root + '190509\\民权线路cad\\民权台账 - 以此为准.xls' bk = xlrd.open_workbook(f_excel) sh = bk.sheet_by_index(0) nrows = sh.nrows # 筛选汉字 xx = u"([\u4e00-\u9fff]+)" pattern = re.compile(xx) # 建立字典 xian_list = [] for i in range(nrows): shebei_name = sh.cell_value(i, 0) results = pattern.findall(shebei_name) xian_list.append(''.join(results)) xian_list = set(xian_list) xian_dic = {} xian_dic_sort = {} for x in xian_list: xian_dic[x] = [] xian_dic_sort[x] = [] # 读excel 信息 for i in range(nrows): shebei_name = sh.cell_value(i, 0) suoshu_xianlu = sh.cell_value(i, 2) zhixian_mingcheng = sh.cell_value(i, 5) daoxian_xinghao = sh.cell_value(i, 6) results = pattern.findall(shebei_name) hanzi_xian = ''.join(results) xian_dic[hanzi_xian].append( [shebei_name, suoshu_xianlu, zhixian_mingcheng, daoxian_xinghao]) # sort 1-100 error = 0 for key in xian_dic: # line_list = [] for i in range(len(xian_dic[key])): for j in range(len(xian_dic[key])): if u'线'+str(i+1)+u'号杆' in xian_dic[key][j][0] or \ u'支' + str(i+1) + u'号杆' in xian_dic[key][j][0] or \ u'线' + str(i+1) + u'杆塔' in xian_dic[key][j][0]: try: a = shp_dic[xian_dic[key][j][0]] xian_dic_sort[key].append(xian_dic[key][j]) except: error += 1 pass flag = 0 time_init = time.time() for key in xian_dic_sort: time_start = time.time() inlist = [] for i in range(len(xian_dic_sort[key])): if i == len(xian_dic_sort[key]) - 1: continue x1 = shp_dic[xian_dic_sort[key][i][0]][0] y1 = shp_dic[xian_dic_sort[key][i][0]][1] x2 = shp_dic[xian_dic_sort[key][i + 1][0]][0] y2 = shp_dic[xian_dic_sort[key][i + 1][0]][1] start = [x1, y1] end = [x2, y2] # name_gbk = name.decode('utf-8') val1 = xian_dic_sort[key][i][0].encode( 'utf-8') + '-' + xian_dic_sort[key][i + 1][0].encode('utf-8') val2 = xian_dic_sort[key][i][1].encode('utf-8') val3 = xian_dic_sort[key][i][2].encode('utf-8') val4 = xian_dic_sort[key][i][3].encode('utf-8') # print(val4) inlist.append([start, end, val1, val2, val3, val4, '']) # print(key.encode('utf-8')) # for i in inlist: # print(i) dwg_dir = this_root + u'190509\\民权线路cad\\dwg_to_shp\\' + val2.decode( 'utf-8') + '\\annotation\\' mk_dir(dwg_dir) dwg_dir = dwg_dir.encode('utf-8') # print(dwg_dir) # exit() try: line_to_shp( inlist, dwg_dir + '\\' + key.encode('utf-8') + 'line_annotation.shp') except: pass time_end = time.time() log_process.process_bar(flag, len(xian_dic_sort), time_init, time_start, time_end) flag += 1 pass
temp = [] for j in i: if j < -999: val = 255 elif -999 < j < 0: val = 0 elif 0 <= j <=1: val = int(j*100) elif j > 1: val = 100 else: val = 255 temp.append(val) # print val new_array.append(temp) # new_array = np.array(new_array,dtype=int) new_array = np.array(new_array,dtype=int) # print 'ploting' # plt.imshow(new_array) # plt.colorbar() # print 'done' # plt.show() # exit() # print 'saving' newRasterfn = (save_dir+f).decode('gbk').encode('utf-8') to_raster.array2raster(newRasterfn,originX,originY,pixelWidth,pixelHeight,new_array) time_end = time.time() log_process.process_bar(flag,len(flist),time_init,time_start,time_end,f) flag+=1
def download_movies(url, movie_path, selected_episodes=range(1, int(1e5))): ''' :param url: 'http://532movie.bnu.edu.cn/player/3379.html' :return: None download *.ts to RAM write mp4 movie file to local disk from RAM ''' movie_name, urls = get_vedio_url(url) invalid_char = '/\:*"<>|?' for ic in invalid_char: if ic in movie_name: movie_name = movie_name.replace(ic, '.') try: print(movie_name.decode('utf-8').encode('gbk', 'ignore')) except: print(movie_name.split()[0].decode('utf-8').encode('gbk', 'ignore')) try: movie_name_utf8 = movie_name.decode('utf-8').encode('gbk', 'ignore') except: movie_name_utf8 = movie_name.split()[0].decode('utf-8').encode( 'gbk', 'ignore') episode = 0 flag = 0 time_init = time.time() for i in urls: time_start = time.time() if len(urls) == 1: if os.path.isfile(movie_path + movie_name_utf8 + '.mp4'): print(movie_path + movie_name_utf8 + '.mp4 is already existed') return None ts = split_videos(i) pool = ThreadPool(20) bar_fmt = 'Downloading\t' + '|{bar}|{percentage:3.0f}%' results = list( tqdm(pool.imap(download_ts, ts), total=len(ts), ncols=50, bar_format=bar_fmt)) pool.close() pool.join() # print('Writing to disk...') movie = codecs.open(movie_path + movie_name_utf8 + '.mp4', 'wb') bar_fmt1 = 'writing to disk\t' + '|{bar}|{percentage:3.0f}%' for i in tqdm(range(len(results)), bar_format=bar_fmt1, ncols=50): movie.write(results[i]) movie.close() else: episode += 1 if episode not in selected_episodes: continue TV_dir = movie_path + movie_name_utf8 + '\\' if not os.path.isdir(TV_dir): os.makedirs(TV_dir) if os.path.isfile(TV_dir + 'Episode ' + '%02d' % episode + '.mp4'): print(TV_dir + 'Episode ' + '%02d' % episode + '.mp4 is already existed') flag += 1 continue pool = ThreadPool(20) ts = split_videos(i) bar_fmt = 'Episode %02d' % episode + '|{bar}|{percentage:3.0f}%' results = list( tqdm(pool.imap(download_ts, ts), total=len(ts), ncols=50, bar_format=bar_fmt)) pool.close() pool.join() movie = codecs.open( TV_dir + 'Episode ' + '%02d' % episode + '.mp4', 'wb') for r in results: movie.write(r) time_end = time.time() lenurl = len(urls) len_selected = len(selected_episodes) length = min([lenurl, len_selected]) try: name = movie_name.decode('utf-8').encode('gbk', 'ignore') except: name = movie_name.split()[0].decode('utf-8').encode( 'gbk', 'ignore') log_process.process_bar(flag, length, time_init, time_start, time_end, name + '\n') flag += 1
def extract_value_from_stations(): # 1、获取站点经纬度,设置50km范围lon_min,lon_max,lat_min,lat_max,存入字典 # 2、建立空字典 this_root = 'e:\\MODIS\\' sta_pos_dic = np.load(this_root + 'ANN_input_para\\00_PDSI\\sta_pos_dic.npz') sta_pos_dic_range = {} sta_pos_dic_void = {} for sta in sta_pos_dic: # print(sta) sta_pos_dic_void[sta] = [] lat = sta_pos_dic[sta][0] lon = sta_pos_dic[sta][1] lon_min = lon - 0.1 lon_max = lon + 0.1 lat_min = lat - 0.1 lat_max = lat + 0.1 sta_pos_dic_range[sta] = [[lon_min, lon_max], [lat_min, lat_max]] date_list = [] for year in range(2003, 2017): for month in range(1, 13): date_list.append(str(year) + '%02d' % month) DEM_vals_dic = {} f = r'E:\MODIS\DEM\DEM_data_transform.npy' arr = np.load(f).item() dic = dict(arr) flag = 0 time_init = time.time() for key in dic: time_start = time.time() key_split = key.split('_') lon = float(key_split[0]) lat = float(key_split[1]) for sta in sta_pos_dic_range: sta_pos_dic_range_dic = sta_pos_dic_range[sta] lon_range = sta_pos_dic_range_dic[0] lat_range = sta_pos_dic_range_dic[1] lon_min = lon_range[0] lon_max = lon_range[1] lat_min = lat_range[0] lat_max = lat_range[1] if lon_min < lon < lon_max and lat_min < lat < lat_max: val = dic[key] sta_pos_dic_void[sta].append(val) time_end = time.time() log_process.process_bar(flag, len(dic), time_init, time_start, time_end) flag += 1 for sta in sta_pos_dic_void: vals_list = sta_pos_dic_void[sta] if len(vals_list) > 0: for date in date_list: key_name = sta + '_' + date val = np.mean(vals_list) DEM_vals_dic[key_name] = val print(key_name, val) print('saving') np.save(this_root + 'DEM\\DEM_extract_from_sta', DEM_vals_dic)
def merge_line_shp(): ''' composite shp 将xian_dic_sort生成的shp合成为1个shp 作为annotation :return: ''' gdal.SetConfigOption("SHAPE_ENCODING", "GBK") fdir = this_root + '\\190714\\dwg_to_shp\\' flist = os.listdir(fdir) time_init = time.time() time_i = 0 inlist = [] for folder in flist: # print(folder.decode('gbk')) # exit() time_start = time.time() # print(folder.decode('gbk')) shp_dir = fdir + folder + '\\annotation\\' try: shp_list = os.listdir(shp_dir) except: continue for shp in shp_list: # print(shp.decode('gbk')) # exit() if shp.endswith('annotation.shp'): # print(shp.decode('gbk')) # exit() daShapefile = shp_dir + shp print(daShapefile.decode('gbk')) # print(daShapefile.decode('gbk')) driver = ogr.GetDriverByName("ESRI Shapefile") dataSource = driver.Open(daShapefile.decode('gbk'), 0) layer = dataSource.GetLayer() # inlist_i = [] for feature in layer: geom = feature.GetGeometryRef() val1 = feature.GetField("val1") val2 = feature.GetField("val2") val3 = feature.GetField("val3") val4 = feature.GetField("val4") Points = geom.GetPoints() if Points: for i in range(len(Points)): if i == len(Points) - 1: break inlist.append([ Points[i], Points[i + 1], val1, val2, val3, val4, '' ]) # print(Points) # print(inlist_i) # exit() # exit() # inlist.append([Points[0],Points[1],val1,val2,val3,val4,'']) time_end = time.time() time_i += 1 log_process.process_bar(time_i, len(flist), time_init, time_start, time_end) output_fn = this_root + '190714\\line_annotation_merge.shp'.decode( 'gbk').encode('utf-8') print('exporting line shp...') line_to_shp(inlist, output_fn) pass
def gen_temp_anomaly_new_data(): this_root = os.getcwd() + '\\' tmp_dic = dict( np.load(this_root + 'data\\pre_monthly_composite_dic.npy').item()) sta_dic = np.load(this_root + 'data\\sta_num_list.npy').item() sta_dic = list(sta_dic) sta_dic.sort() date_list = [] for year in range(1951, 2018): for mon in range(1, 13): date = str(year) + '%02d' % mon date_list.append(date) # flag = 0 mean_std_dic = {} for sta in sta_dic: # flag+=1 # print(flag,'/',len(sta_dic)) for mon in range(1, 13): mon_str = '%02d' % mon vals = [] for year in range(1951, 2018): year_str = str(year) key = sta + '_' + year_str + mon_str try: val = tmp_dic[key] except Exception as e: val = None pass if val != None: vals.append(val) # print(e) if len(vals) > 10: mean = np.mean(vals) std = np.std(vals) # print(sta+'_'+mon_str) mean_std_dic[sta + '_' + mon_str] = [mean, std] # else: # print(sta,mon_str) # # exit() anomaly_dic = {} flag = 0 time_init = time.time() for key in tmp_dic: # flag+=1 # print(flag,'/',len(tmp_dic)) time_start = time.time() key_split = key.split('_') sta = key_split[0] mon = key_split[1][-2:] val = tmp_dic[key] try: mean, std = mean_std_dic[sta + '_' + mon] anomaly = (val - mean) / std anomaly_dic[key] = anomaly except Exception as e: pass time_end = time.time() log_process.process_bar(flag, len(tmp_dic), time_init, time_start, time_end) flag += 1 print('saving ...') np.save(this_root + 'data\\pre_anomaly_dic', anomaly_dic)
def gen_precip_anomaly(): valid_year = [] for y in range(2003, 2016): valid_year.append(y) date_list = [] for y in valid_year: for m in range(1, 13): date_list.append(str(y) + '%02d' % m) tem_dic = np.load(this_root + 'PDSI\\pre_dic.npz') print('processing temperature...') sta_tem_dic = {} time_init = time.time() flag = 0 for sta in tem_dic: if sta == '56247': continue time_start = time.time() # print(sta) # print(tem_dic[sta]) one_sta_tem_dic = tem_dic[sta].item() one_sta = [] for key in date_list: try: one_sta.append(one_sta_tem_dic[key]) except: pass one_sta = np.array(one_sta) try: one_sta = one_sta.reshape(13, 12) except: continue one_month_average = [] one_month_std = [] for i in one_sta.T: mean_val = np.mean(i) std_val = np.std(i) one_month_average.append(mean_val) one_month_std.append(std_val) anomaly_val = [] for i in one_sta: val = (i - one_month_average) / one_month_std anomaly_val.append(val) anomaly_val = np.array(anomaly_val) # plt.figure() # plt.plot(anomaly_val.flatten()) # # plt.figure() # plt.plot(one_sta.flatten()) # plt.show() anomaly_val_result = anomaly_val.flatten() # if len(one_sta) == 156: sta_tem_dic[sta] = anomaly_val_result time_end = time.time() log_process.process_bar(flag, len(tem_dic), time_init, time_start, time_end, sta) flag += 1 print('\nsaving...') np.save(this_root + 'PDSI\\pre_anomaly_2003-2015', sta_tem_dic)