def set_initial_map(): global d_v, h5, k, dateList, inps, data_lim d_v = h5['timeseries'][inps.epoch_num][:] * inps.unit_fac # Initial Map print(str(dateList)) d_v = readfile.read(inps.timeseries_file, datasetName=dateList[inps.epoch_num])[0] * inps.unit_fac #d_v = h5[k].get(dateList[inps.epoch_num])[:]*inps.unit_fac if inps.ref_date: inps.ref_d_v = readfile.read(inps.timeseries_file, datasetName=inps.ref_date)[0]*inps.unit_fac d_v -= inps.ref_d_v if mask is not None: d_v = mask_matrix(d_v, mask) if inps.ref_yx: d_v -= d_v[inps.ref_yx[0], inps.ref_yx[1]] data_lim = [np.nanmin(d_v), np.nanmax(d_v)] if not inps.ylim_mat: inps.ylim_mat = data_lim print(('Initial data range: '+str(data_lim))) print(('Display data range: '+str(inps.ylim_mat))) print(('Initial data range: ' + str(data_lim))) print(('Display data range: ' + str(inps.ylim)))
def time_slider_update(val): '''Update Displacement Map using Slider''' global inps.tims, tslider, ax_v, d_v, inps, img, fig_v, h5, k, dateList timein = tslider.val idx_nearest = np.argmin(np.abs(np.array(inps.tims) - timein)) ax_v.set_title('N = %d, Time = %s' % (idx_nearest, inps.dates[idx_nearest].strftime('%Y-%m-%d'))) d_v = h5[k][idx_nearest][:] * inps.unit_fac if inps.ref_date: d_v -= inps.ref_d_v if mask is not None: d_v = mask_matrix(d_v, mask) if inps.ref_yx: d_v -= d_v[inps.ref_yx[0], inps.ref_yx[1]] img.set_data(d_v) fig_v.canvas.draw()
def time_slider_update(val): """Update Displacement Map using Slider""" timein = tslider.val idx_nearest = np.argmin(np.abs(np.array(inps.yearList) - timein)) ax_v.set_title( 'N = %d, Time = %s' % (idx_nearest, inps.dates[idx_nearest].strftime('%Y-%m-%d'))) d_v = h5[dateList[idx_nearest]][:] * inps.unit_fac if inps.ref_date: d_v -= inps.ref_d_v if mask is not None: d_v = mask_matrix(d_v, mask) if inps.ref_yx: d_v -= d_v[inps.ref_yx[0], inps.ref_yx[1]] img.set_data(d_v) fig_v.canvas.draw()
def main(iargs=None): # Actual code. inps = cmd_line_parse(iargs) # Time Series Info atr = readfile.read_attribute(inps.timeseries_file) k = atr['FILE_TYPE'] print('input file is ' + k + ': ' + inps.timeseries_file) if not k in ['timeseries', 'GIANT_TS']: raise ValueError('Only timeseries file is supported!') obj = timeseries(inps.timeseries_file) obj.open() h5 = h5py.File(inps.timeseries_file, 'r') if k in ['GIANT_TS']: dateList = [ dt.fromordinal(int(i)).strftime('%Y%m%d') for i in h5['dates'][:].tolist() ] else: dateList = obj.dateList date_num = len(dateList) inps.dates, inps.yearList = ptime.date_list2vector(dateList) # Read exclude dates if inps.ex_date_list: input_ex_date = list(inps.ex_date_list) inps.ex_date_list = [] if input_ex_date: for ex_date in input_ex_date: if os.path.isfile(ex_date): ex_date = ptime.read_date_list(ex_date) else: ex_date = [ptime.yyyymmdd(ex_date)] inps.ex_date_list += list( set(ex_date) - set(inps.ex_date_list)) # delete dates not existed in input file inps.ex_date_list = sorted( list(set(inps.ex_date_list).intersection(dateList))) inps.ex_dates = ptime.date_list2vector(inps.ex_date_list)[0] inps.ex_idx_list = sorted( [dateList.index(i) for i in inps.ex_date_list]) print('exclude date:' + str(inps.ex_date_list)) # Zero displacement for 1st acquisition if inps.zero_first: if inps.ex_date_list: inps.zero_idx = min( list(set(range(date_num)) - set(inps.ex_idx_list))) else: inps.zero_idx = 0 # File Size length = int(atr['LENGTH']) width = int(atr['WIDTH']) print('data size in [y0,y1,x0,x1]: [%d, %d, %d, %d]' % (0, length, 0, width)) try: ullon = float(atr['X_FIRST']) ullat = float(atr['Y_FIRST']) lon_step = float(atr['X_STEP']) lat_step = float(atr['Y_STEP']) lrlon = ullon + width * lon_step lrlat = ullat + length * lat_step print('data size in [lat0,lat1,lon0,lon1]: [%.4f, %.4f, %.4f, %.4f]' % (lrlat, ullat, ullon, lrlon)) except: pass # Initial Pixel Coord if inps.lalo and 'Y_FIRST' in atr.keys(): y = int((inps.lalo[0] - ullat) / lat_step + 0.5) x = int((inps.lalo[1] - ullon) / lon_step + 0.5) inps.yx = [y, x] if inps.ref_lalo and 'Y_FIRST' in atr.keys(): y = int((inps.ref_lalo[0] - ullat) / lat_step + 0.5) x = int((inps.ref_lalo[1] - ullon) / lon_step + 0.5) inps.ref_yx = [y, x] # Display Unit if inps.disp_unit == 'cm': inps.unit_fac = 100.0 elif inps.disp_unit == 'm': inps.unit_fac = 1.0 elif inps.disp_unit == 'dm': inps.unit_fac = 10.0 elif inps.disp_unit == 'mm': inps.unit_fac = 1000.0 elif inps.disp_unit == 'km': inps.unit_fac = 0.001 else: raise ValueError('Un-recognized unit: ' + inps.disp_unit) if k in ['GIANT_TS']: print('data unit: mm') inps.unit_fac *= 0.001 else: print('data unit: m') print('display unit: ' + inps.disp_unit) # Flip up-down / left-right if inps.auto_flip: inps.flip_lr, inps.flip_ud = pp.auto_flip_direction(atr) else: inps.flip_ud = False inps.left_lr = False # Mask file if not inps.mask_file: if os.path.basename(inps.timeseries_file).startswith('geo_'): file_list = ['geo_maskTempCoh.h5'] else: file_list = ['maskTempCoh.h5', 'mask.h5'] try: inps.mask_file = ut.get_file_list(file_list)[0] except: inps.mask_file = None try: mask = readfile.read(inps.mask_file, datasetName='mask')[0] mask[mask != 0] = 1 print('load mask from file: ' + inps.mask_file) except: mask = None print('No mask used.') # Initial Map d_v = readfile.read( inps.timeseries_file, datasetName=dateList[inps.epoch_num])[0] * inps.unit_fac if inps.ref_date: inps.ref_d_v = readfile.read( inps.timeseries_file, datasetName=inps.ref_date)[0] * inps.unit_fac d_v -= inps.ref_d_v if mask is not None: d_v = mask_matrix(d_v, mask) if inps.ref_yx: d_v -= d_v[inps.ref_yx[0], inps.ref_yx[1]] data_lim = [np.nanmin(d_v), np.nanmax(d_v)] if not inps.ylim_mat: inps.ylim_mat = data_lim print('Initial data range: ' + str(data_lim)) print('Display data range: ' + str(inps.ylim_mat)) # Fig 1 - Cumulative Displacement Map if not inps.disp_fig: plt.switch_backend('Agg') fig_v = plt.figure('Cumulative Displacement') # Axes 1 #ax_v = fig_v.add_subplot(111) # ax_v.set_position([0.125,0.25,0.75,0.65]) # This works on OSX. Original worked on Linux. # rect[left, bottom, width, height] ax_v = fig_v.add_axes([0.125, 0.25, 0.75, 0.65]) if inps.dem_file: dem = readfile.read(inps.dem_file, datasetName='height')[0] ax_v = pp.plot_dem_yx(ax_v, dem) img = ax_v.imshow(d_v, cmap=inps.colormap, clim=inps.ylim_mat, interpolation='nearest') # Reference Pixel if inps.ref_yx: d_v -= d_v[inps.ref_yx[0], inps.ref_yx[1]] ax_v.plot(inps.ref_yx[1], inps.ref_yx[0], 'ks', ms=6) else: try: ax_v.plot(int(atr['REF_X']), int(atr['REF_Y']), 'ks', ms=6) except: pass # Initial Pixel if inps.yx: ax_v.plot(inps.yx[1], inps.yx[0], 'ro', markeredgecolor='black') ax_v.set_xlim(0, np.shape(d_v)[1]) ax_v.set_ylim(np.shape(d_v)[0], 0) # Status Bar def format_coord(x, y): col = int(x + 0.5) row = int(y + 0.5) if 0 <= col < width and 0 <= row < length: z = d_v[row, col] try: lon = ullon + x * lon_step lat = ullat + y * lat_step return 'x=%.0f, y=%.0f, value=%.4f, lon=%.4f, lat=%.4f' % ( x, y, z, lon, lat) except: return 'x=%.0f, y=%.0f, value=%.4f' % (x, y, z) ax_v.format_coord = format_coord # Title and Axis Label ax_v.set_title( 'N = %d, Time = %s' % (inps.epoch_num, inps.dates[inps.epoch_num].strftime('%Y-%m-%d'))) if not 'Y_FIRST' in atr.keys(): ax_v.set_xlabel('Range') ax_v.set_ylabel('Azimuth') # Flip axis if inps.flip_lr: ax_v.invert_xaxis() print('flip map left and right') if inps.flip_ud: ax_v.invert_yaxis() print('flip map up and down') # Colorbar cbar = fig_v.colorbar(img, orientation='vertical') cbar.set_label('Displacement [%s]' % inps.disp_unit) # Axes 2 - Time Slider ax_time = fig_v.add_axes([0.125, 0.1, 0.6, 0.07], facecolor='lightgoldenrodyellow', yticks=[]) tslider = Slider(ax_time, 'Years', inps.yearList[0], inps.yearList[-1], valinit=inps.yearList[inps.epoch_num]) tslider.ax.bar(inps.yearList, np.ones(len(inps.yearList)), facecolor='black', width=0.01, ecolor=None) tslider.ax.set_xticks( np.round( np.linspace(inps.yearList[0], inps.yearList[-1], num=5) * 100) / 100) def time_slider_update(val): """Update Displacement Map using Slider""" timein = tslider.val idx_nearest = np.argmin(np.abs(np.array(inps.yearList) - timein)) ax_v.set_title( 'N = %d, Time = %s' % (idx_nearest, inps.dates[idx_nearest].strftime('%Y-%m-%d'))) d_v = h5[dateList[idx_nearest]][:] * inps.unit_fac if inps.ref_date: d_v -= inps.ref_d_v if mask is not None: d_v = mask_matrix(d_v, mask) if inps.ref_yx: d_v -= d_v[inps.ref_yx[0], inps.ref_yx[1]] img.set_data(d_v) fig_v.canvas.draw() tslider.on_changed(time_slider_update) # Fig 2 - Time Series Displacement - Point fig_ts = plt.figure('Time series - point', figsize=inps.fig_size) ax_ts = fig_ts.add_subplot(111) # Read Error List inps.error_ts = None if inps.error_file: error_fileContent = np.loadtxt(inps.error_file, dtype=bytes).astype(str) inps.error_ts = error_fileContent[:, 1].astype( np.float) * inps.unit_fac if inps.ex_date_list: e_ts = inps.error_ts[:] inps.ex_error_ts = np.array([e_ts[i] for i in inps.ex_idx_list]) inps.error_ts = np.array([ e_ts[i] for i in range(date_num) if i not in inps.ex_idx_list ]) def plot_timeseries_errorbar(ax, dis_ts, inps): dates = list(inps.dates) d_ts = dis_ts[:] if inps.ex_date_list: # Update displacement time-series dates = sorted(list(set(inps.dates) - set(inps.ex_dates))) ex_d_ts = np.array([dis_ts[i] for i in inps.ex_idx_list]) d_ts = np.array([ dis_ts[i] for i in range(date_num) if i not in inps.ex_idx_list ]) # Plot excluded dates (_, caps, _) = ax.errorbar(inps.ex_dates, ex_d_ts, yerr=inps.ex_error_ts, fmt='-o', color='gray', ms=inps.marker_size, lw=0, alpha=1, mfc='gray', elinewidth=inps.edge_width, ecolor='black', capsize=inps.marker_size * 0.5) for cap in caps: cap.set_markeredgewidth(inps.edge_width) # Plot kept dates (_, caps, _) = ax.errorbar(dates, d_ts, yerr=inps.error_ts, fmt='-o', ms=inps.marker_size, lw=0, alpha=1, elinewidth=inps.edge_width, ecolor='black', capsize=inps.marker_size * 0.5) for cap in caps: cap.set_markeredgewidth(inps.edge_width) return ax def plot_timeseries_scatter(ax, dis_ts, inps): dates = list(inps.dates) d_ts = dis_ts[:] if inps.ex_date_list: # Update displacement time-series dates = sorted(list(set(inps.dates) - set(inps.ex_dates))) ex_d_ts = np.array([dis_ts[i] for i in inps.ex_idx_list]) d_ts = np.array([ dis_ts[i] for i in range(date_num) if i not in inps.ex_idx_list ]) # Plot excluded dates ax.scatter(inps.ex_dates, ex_d_ts, s=inps.marker_size**2, color='gray') # color='crimson' # Plot kept dates ax.scatter(dates, d_ts, s=inps.marker_size**2) return ax def update_timeseries(ax_ts, y, x): """Plot point time series displacement at pixel [y, x]""" d_ts = read_timeseries_yx( inps.timeseries_file, y, x, ref_yx=inps.ref_yx) * inps.unit_fac # for date in dateList: # d = h5[k].get(date)[y,x] # if inps.ref_yx: # d -= h5[k].get(date)[inps.ref_yx[0], inps.ref_yx[1]] # d_ts.append(d*inps.unit_fac) if inps.zero_first: d_ts -= d_ts[inps.zero_idx] ax_ts.cla() if inps.error_file: ax_ts = plot_timeseries_errorbar(ax_ts, d_ts, inps) else: ax_ts = plot_timeseries_scatter(ax_ts, d_ts, inps) if inps.ylim: ax_ts.set_ylim(inps.ylim) for tick in ax_ts.yaxis.get_major_ticks(): tick.label.set_fontsize(inps.font_size) # Title title_ts = 'Y = %d, X = %d' % (y, x) try: lat = ullat + y * lat_step lon = ullon + x * lon_step title_ts += ', lat = %.4f, lon = %.4f' % (lat, lon) except: pass if inps.disp_title: ax_ts.set_title(title_ts) ax_ts = pp.auto_adjust_xaxis_date(ax_ts, inps.yearList, fontSize=inps.font_size)[0] ax_ts.set_xlabel('Time', fontsize=inps.font_size) ax_ts.set_ylabel('Displacement [%s]' % inps.disp_unit, fontsize=inps.font_size) fig_ts.canvas.draw() # Print to terminal print('\n---------------------------------------') print(title_ts) print(d_ts) # Slope estimation if inps.ex_date_list: inps.yearList_kept = [ inps.yearList[i] for i in range(date_num) if i not in inps.ex_idx_list ] d_ts_kept = [ d_ts[i] for i in range(date_num) if i not in inps.ex_idx_list ] d_slope = stats.linregress(np.array(inps.yearList_kept), np.array(d_ts_kept)) else: d_slope = stats.linregress(np.array(inps.yearList), np.array(d_ts)) print('linear velocity: %.2f +/- %.2f [%s/yr]' % (d_slope[0], d_slope[4], inps.disp_unit)) return d_ts # Initial point time series plot if inps.yx: d_ts = update_timeseries(ax_ts, inps.yx[0], inps.yx[1]) else: d_ts = np.zeros(len(inps.yearList)) ax_ts = plot_timeseries_scatter(ax_ts, d_ts, inps) def plot_timeseries_event(event): """Event function to get y/x from button press""" if event.inaxes != ax_v: return ii = int(event.ydata + 0.5) jj = int(event.xdata + 0.5) d_ts = update_timeseries(ax_ts, ii, jj) # Output if inps.save_fig and inps.yx: print('save info for pixel ' + str(inps.yx)) if not inps.fig_base: inps.fig_base = 'y%d_x%d' % (inps.yx[0], inps.yx[1]) # TXT - point time series outName = inps.fig_base + '_ts.txt' header_info = 'timeseries_file=' + inps.timeseries_file header_info += '\ny=%d, x=%d' % (inps.yx[0], inps.yx[1]) try: lat = ullat + inps.yx[0] * lat_step lon = ullon + inps.yx[1] * lon_step header_info += '\nlat=%.6f, lon=%.6f' % (lat, lon) except: pass if inps.ref_yx: header_info += '\nreference pixel: y=%d, x=%d' % (inps.ref_yx[0], inps.ref_yx[1]) else: header_info += '\nreference pixel: y=%s, x=%s' % (atr['REF_Y'], atr['REF_X']) header_info += '\nunit=m/yr' np.savetxt(outName, list(zip(np.array(dateList), np.array(d_ts) / inps.unit_fac)), fmt='%s', delimiter=' ', header=header_info) print('save time series displacement in meter to ' + outName) # Figure - point time series outName = inps.fig_base + '_ts.pdf' fig_ts.savefig(outName, bbox_inches='tight', transparent=True, dpi=inps.fig_dpi) print('save time series plot to ' + outName) # Figure - map outName = inps.fig_base + '_' + dateList[inps.epoch_num] + '.png' fig_v.savefig(outName, bbox_inches='tight', transparent=True, dpi=inps.fig_dpi) print('save map plot to ' + outName) # Final linking of the canvas to the plots. cid = fig_v.canvas.mpl_connect('button_press_event', plot_timeseries_event) if inps.disp_fig: plt.show() fig_v.canvas.mpl_disconnect(cid)
def estimate_phase_elevation_ratio(dem, ts_data, inps): """Estimate phase/elevation ratio for each acquisition of timeseries Parameters: dem : 2D array in size of ( length, width) ts_data : 3D array in size of (num_date, length, width) inps : Namespace Returns: X : 2D array in size of (poly_num+1, num_date) """ num_date = ts_data.shape[0] # prepare phase and elevation data print('reading mask from file: ' + inps.mask_file) mask = readfile.read(inps.mask_file, datasetName='mask')[0] dem = mask_matrix(np.array(dem), mask) ts_data = mask_matrix(np.array(ts_data), mask) # display # 1. effect of multilooking --> narrow phase range --> better ratio estimation debug_mode = False if debug_mode: import matplotlib.pyplot as plt #d_index = np.argmax(topo_trop_corr) d_index = 47 data = ts_data[d_index, :, :] title = inps.date_list[d_index] fig = plt.figure() plt.plot(dem[~np.isnan(dem)], data[~np.isnan(dem)], '.', label='Number of Looks = 1') mli_dem = multilook_data(dem, 8, 8) mli_data = multilook_data(data, 8, 8) plt.plot(mli_dem[~np.isnan(mli_dem)], mli_data[~np.isnan(mli_dem)], '.', label='Number of Looks = 8') plt.legend() plt.xlabel('Elevation (m)') plt.ylabel('Range Change (m)') plt.title(title) out_file = 'phase_elevation_ratio_{}.png'.format(title) plt.savefig(out_file, bbox_inches='tight', transparent=True, dpi=300) print('save to {}'.format(out_file)) #plt.show() print('----------------------------------------------------------') print( 'Empirical tropospheric delay correction based on phase/elevation ratio (Doin et al., 2009)' ) print('polynomial order: {}'.format(inps.poly_order)) if inps.num_multilook > 1: print('number of multilook: {} (multilook data for estimation only)'. format(inps.num_multilook)) mask = multilook_data(mask, inps.num_multilook, inps.num_multilook) dem = multilook_data(dem, inps.num_multilook, inps.num_multilook) ts_data = multilook_data(ts_data, inps.num_multilook, inps.num_multilook) if inps.threshold > 0.: print('correlation threshold: {}'.format(inps.threshold)) mask_nan = ~np.isnan(dem) dem = dem[mask_nan] ts_data = ts_data[:, mask_nan] # calculate correlation coefficient print('----------------------------------------------------------') print('calculate correlation of DEM with each acquisition') topo_trop_corr = np.zeros(num_date, np.float32) for i in range(num_date): phase = ts_data[i, :] cc = 0. if np.count_nonzero(phase) > 0: comp_data = np.vstack((dem, phase)) cc = np.corrcoef(comp_data)[0, 1] topo_trop_corr[i] = cc print('{}: {:>5.2f}'.format(inps.date_list[i], cc)) topo_trop_corr = np.abs(topo_trop_corr) print('average correlation magnitude: {:>5.2f}'.format( np.nanmean(topo_trop_corr))) # estimate ratio parameter print('----------------------------------------------------------') print('estimate phase/elevation ratio') A = design_matrix(dem=dem, poly_order=inps.poly_order) X = np.dot(np.linalg.pinv(A), ts_data.T) X = np.array(X, dtype=np.float32) X[:, topo_trop_corr < inps.threshold] = 0. return X
def main(): parser = build_parser() parseArgs = parser.parse_args() file_name = parseArgs.file output_folder = parseArgs.outputDir should_mask = True path_name_and_extension = os.path.basename(file_name).split(".") path_name = path_name_and_extension[0] extension = path_name_and_extension[1] # --------------------------------------------------------------------------------------- # start clock to track how long conversion process takes start_time = time.clock() # use h5py to open specified group(s) in the h5 file # then read datasets from h5 file into memory for faster reading of data file = h5py.File(file_name, "r") timeseries_group = file["HDFEOS"]["GRIDS"]["timeseries"] displacement_3d_matrix = timeseries_group["observation"]["displacement"] # get attributes (stored at root) of UNAVCO timeseries file attributes = dict(file.attrs) # in timeseries displacement_3d_matrix, there are datasets # need to get datasets with dates - strings that can be converted to integers dates = displacement_3d_matrix.attrs["DATE_TIMESERIES"].split(" ") # array that stores dates from dates that have been converted to decimal decimal_dates = [] # read datasets in the group into a dictionary of 2d arrays and intialize decimal dates timeseries_datasets = {} i = 0 for displacement_2d_matrix in displacement_3d_matrix: dataset = displacement_2d_matrix[:] if should_mask: print "Masking " + dates[i] mask = timeseries_group["quality"]["mask"][:] dataset = mask_matrix(dataset, mask) timeseries_datasets[dates[i]] = dataset d = get_date(dates[i]) decimal = get_decimal_date(d) decimal_dates.append(decimal) i += 1 # close h5 file file.close() path_list = path_name.split("/") folder_name = path_name.split("/")[len(path_list) - 1] try: # create path for output os.mkdir(output_folder) except: print output_folder + " already exists" # read and convert the datasets, then write them into json files and insert into database convert_data(attributes, decimal_dates, timeseries_datasets, dates, output_folder, folder_name) # run tippecanoe command to get mbtiles file os.chdir(os.path.abspath(output_folder)) os.system( "tippecanoe *.json -l chunk_1 -x d -pf -pk -Bg -d9 -D12 -g12 -r0 -o " + folder_name + ".mbtiles") # --------------------------------------------------------------------------------------- # check how long it took to read h5 file data and create json files end_time = time.clock() print("time elapsed: " + str(end_time - start_time))
inps.mask_file = None try: mask = readfile.read(inps.mask_file)[0] mask[mask != 0] = 1 print 'load mask from file: ' + inps.mask_file except: mask = None print 'No mask used.' # Initial Map d_v = h5[k].get(dateList[inps.epoch_num])[:] * inps.unit_fac if inps.ref_date: inps.ref_d_v = h5[k].get(inps.ref_date)[:] * inps.unit_fac d_v -= inps.ref_d_v if mask is not None: d_v = mask_matrix(d_v, mask) if inps.ref_yx: d_v -= d_v[inps.ref_yx[0], inps.ref_yx[1]] data_lim = [np.nanmin(d_v), np.nanmax(d_v)] if not inps.ylim: inps.ylim = data_lim print 'Initial data range: ' + str(data_lim) print 'Display data range: ' + str(inps.ylim) ########## Fig 1 - Cumulative Displacement Map if not inps.disp_fig: plt.switch_backend('Agg') fig_v = plt.figure('Cumulative Displacement')
def main(): parser = build_parser() parseArgs = parser.parse_args() file_name = parseArgs.file output_folder = parseArgs.outputDir should_mask = True path_name_and_extension = os.path.basename(file_name).split(".") path_name = path_name_and_extension[0] extension = path_name_and_extension[1] # --------------------------------------------------------------------------------------- # start clock to track how long conversion process takes start_time = time.clock() # use h5py to open specified group(s) in the h5 file # then read datasets from h5 file into memory for faster reading of data file = h5py.File(file_name, "r") group = file['timeseries'] # get attributes (stored at root) of UNAVCO timeseries file attributes = dict(group.attrs) # in timeseries group, there are datasets # need to get datasets with dates - strings that can be converted to integers dataset_keys = [] for k in group["GRIDS"].keys(): if k.isdigit(): dataset_keys.append(k) dataset_keys.sort() # array that stores dates from dataset_keys that have been converted to decimal decimal_dates = [] # read datasets in the group into a dictionary of 2d arrays and intialize decimal dates timeseries_datasets = {} for key in dataset_keys: dataset = group["GRIDS"][key][:] if should_mask: print "Masking " + str(key) mask = group["GRIDS"].get('mask')[:] dataset = mask_matrix(dataset, mask) timeseries_datasets[key] = dataset d = get_date(key) decimal = get_decimal_date(d) decimal_dates.append(decimal) # close h5 file file.close() path_list = path_name.split("/") folder_name = path_name.split("/")[len(path_list)-1] try: # create path for output os.mkdir(output_folder) except: print output_folder + " already exists" # read and convert the datasets, then write them into json files and insert into database convert_data(attributes, decimal_dates, timeseries_datasets, dataset_keys, output_folder, folder_name) # run tippecanoe command to get mbtiles file os.chdir(os.path.abspath(output_folder)) os.system("tippecanoe *.json -l chunk_1 -x d -pf -pk -Bg -d9 -D12 -g12 -r0 -o " + folder_name + ".mbtiles") # --------------------------------------------------------------------------------------- # check how long it took to read h5 file data and create json files end_time = time.clock() print ("time elapsed: " + str(end_time - start_time))