def print_date_list(fname, disp_num=False, drop_ifgram=False, print_msg=False):
"""Print time/date info of file"""
atr = readfile.read_attribute(fname)
k = atr['FILE_TYPE']
dateList = None
if k in ['timeseries']:
dateList = timeseries(fname).get_date_list()
elif k == 'HDFEOS':
obj = HDFEOS(fname)
obj.open(print_msg=False)
dateList = obj.dateList
elif k == 'giantTimeseries':
obj = giantTimeseries(fname)
obj.open(print_msg=False)
dateList = obj.dateList
elif k in ['ifgramStack']:
dateList = ifgramStack(fname).get_date12_list(dropIfgram=drop_ifgram)
elif k in ['giantIfgramStack']:
obj = giantIfgramStack(fname)
obj.open(print_msg=False)
dateList = obj.date12List
else:
print('--date option can not be applied to {} file, ignore it.'.format(k))
if print_msg and dateList is not None:
for i in range(len(dateList)):
if disp_num:
print('{}\t{}'.format(dateList[i], i))
else:
print(dateList[i])
return dateList
def print_date_list(fname, disp_ifgram='all', disp_num=False, print_msg=False):
"""Print time/date info of file"""
k = readfile.read_attribute(fname)['FILE_TYPE']
dateList = None
if k in ['timeseries']:
dateList = timeseries(fname).get_date_list()
elif k == 'HDFEOS':
dateList = HDFEOS(fname).get_date_list()
elif k == 'giantTimeseries':
dateList = giantTimeseries(fname).get_date_list()
elif k in ['giantIfgramStack']:
dateList = giantIfgramStack(fname).get_date12_list()
elif k in ['ifgramStack']:
obj = ifgramStack(fname)
obj.open(print_msg=False)
dateListAll = obj.get_date12_list(dropIfgram=False)
dateListKept = obj.get_date12_list(dropIfgram=True)
# show dropped ifgram or not
if disp_ifgram == 'all':
dateList = list(dateListAll)
elif disp_ifgram == 'kept':
dateList = list(dateListKept)
else:
dateList = sorted(list(set(dateListAll) - set(dateListKept)))
else:
print('--date option can not be applied to {} file, ignore it.'.format(
k))
# print list info
if print_msg and dateList is not None:
for d in dateList:
if disp_num:
if k in ['ifgramStack']:
num = dateListAll.index(d)
else:
num = dateList.index(d)
msg = '{}\t{}'.format(d, num)
else:
msg = d
print(msg)
return dateList
def read_date_info(inps):
"""Get inps.excludeDate full list
Inputs:
inps - Namespace,
Output:
inps.excludeDate - list of string for exclude date in YYYYMMDD format
"""
if inps.key == 'timeseries':
tsobj = timeseries(inps.timeseries_file)
elif inps.key == 'giantTimeseries':
tsobj = giantTimeseries(inps.timeseries_file)
elif inps.key == 'HDFEOS':
tsobj = HDFEOS(inps.timeseries_file)
tsobj.open()
inps.excludeDate = read_exclude_date(inps, tsobj.dateList)
# Date used for estimation inps.dateList
inps.dateList = [i for i in tsobj.dateList if i not in inps.excludeDate]
inps.numDate = len(inps.dateList)
print('-' * 50)
print('dates from input file: {}\n{}'.format(tsobj.numDate,
tsobj.dateList))
print('-' * 50)
if len(inps.dateList) == len(tsobj.dateList):
print('using all dates to calculate the velocity')
else:
print('dates used to estimate the velocity: {}\n{}'.format(
inps.numDate, inps.dateList))
print('-' * 50)
# flag array for ts data reading
inps.dropDate = np.array(
[i not in inps.excludeDate for i in tsobj.dateList], dtype=np.bool_)
# output file name
if not inps.outfile:
outname = 'velocity'
if inps.key == 'giantTimeseries':
prefix = os.path.basename(inps.timeseries_file).split('PARAMS')[0]
outname = prefix + outname
outname += '.h5'
inps.outfile = outname
return inps
def get_slice_list(fname):
"""Get list of 2D slice existed in file (for display)"""
fbase, fext = os.path.splitext(os.path.basename(fname))
fext = fext.lower()
atr = read_attribute(fname)
k = atr['FILE_TYPE']
global slice_list
# HDF5 Files
if fext in ['.h5', '.he5']:
with h5py.File(fname, 'r') as f:
d1_list = [i for i in f.keys() if isinstance(f[i], h5py.Dataset)]
if k == 'timeseries' and k in d1_list:
obj = timeseries(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
elif k in ['geometry'] and k not in d1_list:
obj = geometry(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
elif k in ['ifgramStack']:
obj = ifgramStack(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
elif k in ['HDFEOS']:
obj = HDFEOS(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
elif k in ['giantTimeseries']:
obj = giantTimeseries(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
elif k in ['giantIfgramStack']:
obj = giantIfgramStack(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
else:
## Find slice by walking through the file structure
length, width = int(atr['LENGTH']), int(atr['WIDTH'])
def get_hdf5_2d_dataset(name, obj):
global slice_list
if isinstance(obj, h5py.Dataset) and obj.shape[-2:] == (length,
width):
if obj.ndim == 2:
slice_list.append(name)
else:
warnings.warn(
'file has un-defined {}D dataset: {}'.format(
obj.ndim, name))
slice_list = []
with h5py.File(fname, 'r') as f:
f.visititems(get_hdf5_2d_dataset)
# Binary Files
else:
if fext.lower() in ['.trans', '.utm_to_rdc']:
slice_list = ['rangeCoord', 'azimuthCoord']
elif fbase.startswith('los'):
slice_list = ['incidenceAngle', 'azimuthAngle']
elif atr.get('number_bands', '1') == '2' and 'unw' not in k:
slice_list = ['band1', 'band2']
else:
slice_list = ['']
return slice_list
def read_init_info(inps):
# Time Series Info
ts_file0 = inps.timeseries_file[0]
atr = readfile.read_attribute(ts_file0)
inps.key = atr['FILE_TYPE']
if inps.key == 'timeseries':
obj = timeseries(ts_file0)
elif inps.key == 'giantTimeseries':
obj = giantTimeseries(ts_file0)
elif inps.key == 'HDFEOS':
obj = HDFEOS(ts_file0)
else:
raise ValueError('input file is {}, not timeseries.'.format(inps.key))
obj.open()
if not inps.file_label:
inps.file_label = [
str(i) for i in list(range(len(inps.timeseries_file)))
]
# default mask file
if not inps.mask_file and 'masked' not in ts_file0:
dir_name = os.path.dirname(ts_file0)
if 'Y_FIRST' in atr.keys():
inps.mask_file = os.path.join(dir_name, 'geo_maskTempCoh.h5')
else:
inps.mask_file = os.path.join(dir_name, 'maskTempCoh.h5')
if not os.path.isfile(inps.mask_file):
inps.mask_file = None
# date info
inps.date_list = obj.dateList
if inps.start_date:
inps.date_list = [
i for i in inps.date_list if int(i) >= int(inps.start_date)
]
if inps.end_date:
inps.date_list = [
i for i in inps.date_list if int(i) <= int(inps.end_date)
]
inps.num_date = len(inps.date_list)
inps.dates, inps.yearList = ptime.date_list2vector(inps.date_list)
(inps.ex_date_list, inps.ex_dates,
inps.ex_flag) = read_exclude_date(inps.ex_date_list, inps.date_list)
# initial display index
if obj.metadata['REF_DATE'] in inps.date_list:
inps.ref_idx = inps.date_list.index(obj.metadata['REF_DATE'])
else:
inps.ref_idx = 0
if inps.ref_date:
inps.ref_idx = inps.date_list.index(inps.ref_date)
if not inps.init_idx:
if inps.ref_idx < inps.num_date / 2.:
inps.init_idx = -3
else:
inps.init_idx = 3
# Display Unit
(inps.disp_unit,
inps.unit_fac) = pp.scale_data2disp_unit(metadata=atr,
disp_unit=inps.disp_unit)[1:3]
# Read Error List
inps.error_ts = None
inps.ex_error_ts = None
if inps.error_file:
error_fileContent = np.loadtxt(inps.error_file,
dtype=bytes).astype(str)
inps.error_ts = error_fileContent[:, 1].astype(
np.float) * inps.unit_fac
if inps.ex_date_list:
e_ts = inps.error_ts[:]
inps.ex_error_ts = e_ts[inps.ex_flag == 0]
inps.error_ts = e_ts[inps.ex_flag == 1]
# Zero displacement for 1st acquisition
if inps.zero_first:
inps.zero_idx = min(0, np.min(np.where(inps.ex_flag)[0]))
# default lookup table file
if not inps.lookup_file:
inps.lookup_file = ut.get_lookup_file('./INPUTS/geometryRadar.h5')
inps.coord = ut.coordinate(atr, inps.lookup_file)
# size and lalo info
inps.pix_box, inps.geo_box = subset.subset_input_dict2box(vars(inps), atr)
inps.pix_box = inps.coord.check_box_within_data_coverage(inps.pix_box)
inps.geo_box = inps.coord.box_pixel2geo(inps.pix_box)
# Out message
data_box = (0, 0, obj.width, obj.length)
print('data coverage in y/x: ' + str(data_box))
print('subset coverage in y/x: ' + str(inps.pix_box))
print('data coverage in lat/lon: ' +
str(inps.coord.box_pixel2geo(data_box)))
print('subset coverage in lat/lon: ' + str(inps.geo_box))
print(
'------------------------------------------------------------------------'
)
# reference pixel
if not inps.ref_lalo and 'REF_LAT' in atr.keys():
inps.ref_lalo = (float(atr['REF_LAT']), float(atr['REF_LON']))
if inps.ref_lalo:
if inps.ref_lalo[1] > 180.:
inps.ref_lalo[1] -= 360.
inps.ref_yx = inps.coord.geo2radar(inps.ref_lalo[0],
inps.ref_lalo[1],
print_msg=False)[0:2]
if not inps.ref_yx:
inps.ref_yx = [int(atr['REF_Y']), int(atr['REF_X'])]
# Initial Pixel Coord
if inps.lalo:
inps.yx = inps.coord.geo2radar(inps.lalo[0],
inps.lalo[1],
print_msg=False)[0:2]
try:
inps.lalo = inps.coord.radar2geo(inps.yx[0],
inps.yx[1],
print_msg=False)[0:2]
except:
inps.lalo = None
# Flip up-down / left-right
if inps.auto_flip:
inps.flip_lr, inps.flip_ud = pp.auto_flip_direction(atr)
# display unit ans wrap
# if wrap_step == 2*np.pi (default value), set disp_unit_v = radian;
# otherwise set disp_unit_v = disp_unit
inps.disp_unit_v = inps.disp_unit
if inps.wrap:
inps.range2phase = -4. * np.pi / float(atr['WAVELENGTH'])
if 'cm' == inps.disp_unit.split('/')[0]: inps.range2phase /= 100.
elif 'mm' == inps.disp_unit.split('/')[0]: inps.range2phase /= 1000.
elif 'm' == inps.disp_unit.split('/')[0]: inps.range2phase /= 1.
else:
raise ValueError('un-recognized display unit: {}'.format(
inps.disp_unit))
if (inps.wrap_range[1] - inps.wrap_range[0]) == 2 * np.pi:
inps.disp_unit_v = 'radian'
inps.vlim = inps.wrap_range
inps.cbar_label = 'Displacement [{}]'.format(inps.disp_unit_v)
return inps, atr
def read_data(inps):
# metadata
atr = readfile.read_attribute(inps.file)
range2phase = -4 * np.pi / float(atr['WAVELENGTH'])
# change reference pixel
if inps.ref_yx:
atr['REF_Y'] = inps.ref_yx[0]
atr['REF_X'] = inps.ref_yx[1]
print('change reference point to y/x: {}'.format(inps.ref_yx))
# various file types
print('read {} from file {}'.format(inps.dset, inps.file))
k = atr['FILE_TYPE']
if k == 'velocity':
# read/prepare data
data = readfile.read(inps.file)[0] * range2phase
print(
"converting velocity to an interferogram with one year temporal baseline"
)
if inps.ref_yx:
data -= data[inps.ref_yx[0], inps.ref_yx[1]]
# metadata
atr['FILE_TYPE'] = '.unw'
atr['UNIT'] = 'radian'
# output filename
if not inps.outfile:
inps.outfile = '{}{}'.format(
os.path.splitext(inps.file)[0], atr['FILE_TYPE'])
elif k == 'timeseries':
# date1 and date2
if '_' in inps.dset:
date1, date2 = ptime.yyyymmdd(inps.dset.split('_'))
else:
date1 = atr['REF_DATE']
date2 = ptime.yyyymmdd(inps.dset)
# read/prepare data
data = readfile.read(inps.file, datasetName=date2)[0]
data -= readfile.read(inps.file, datasetName=date1)[0]
print('converting range to phase')
data *= range2phase
if inps.ref_yx:
data -= data[inps.ref_yx[0], inps.ref_yx[1]]
# metadata
atr['DATE'] = date1[2:8]
atr['DATE12'] = '{}-{}'.format(date1[2:8], date2[2:8])
atr['FILE_TYPE'] = '.unw'
atr['UNIT'] = 'radian'
# output filename
if not inps.outfile:
inps.outfile = '{}_{}.unw'.format(date1, date2)
if inps.file.startswith('geo_'):
inps.outfile = 'geo_' + inps.outfile
elif k == 'HDFEOS':
dname = inps.dset.split('-')[0]
# date1 and date2
if dname == 'displacement':
if '-' in inps.dset:
suffix = inps.dset.split('-')[1]
if '_' in suffix:
date1, date2 = ptime.yyyymmdd(suffix.split('_'))
else:
date1 = atr['REF_DATE']
date2 = ptime.yyyymmdd(suffix)
else:
raise ValueError(
"No '-' in input dataset! It is required for {}".format(
dname))
else:
date_list = HDFEOS(inps.file).get_date_list()
date1 = date_list[0]
date2 = date_list[-1]
date12 = '{}_{}'.format(date1, date2)
# read / prepare data
slice_list = readfile.get_slice_list(inps.file)
if 'displacement' in inps.dset:
# read/prepare data
slice_name1 = view.check_dataset_input(
slice_list, '{}-{}'.format(dname, date1))[0][0]
slice_name2 = view.check_dataset_input(
slice_list, '{}-{}'.format(dname, date2))[0][0]
data = readfile.read(inps.file, datasetName=slice_name1)[0]
data -= readfile.read(inps.file, datasetName=slice_name2)[0]
print('converting range to phase')
data *= range2phase
if inps.ref_yx:
data -= data[inps.ref_yx[0], inps.ref_yx[1]]
else:
slice_name = view.check_dataset_input(slice_list, inps.dset)[0][0]
data = readfile.read(inps.file, datasetName=slice_name)[0]
# metadata
atr['DATE'] = date1[2:8]
atr['DATE12'] = '{}-{}'.format(date1[2:8], date2[2:8])
if dname == 'displacement':
atr['FILE_TYPE'] = '.unw'
atr['UNIT'] = 'radian'
elif 'coherence' in dname.lower():
atr['FILE_TYPE'] = '.cor'
atr['UNIT'] = '1'
elif dname == 'height':
atr['FILE_TYPE'] = '.dem'
atr['DATA_TYPE'] = 'int16'
else:
raise ValueError('unrecognized input dataset type: {}'.format(
inps.dset))
# output filename
if not inps.outfile:
inps.outfile = '{}{}'.format(date12, atr['FILE_TYPE'])
elif k == 'ifgramStack':
dname, date12 = inps.dset.split('-')
date1, date2 = date12.split('_')
# read / prepare data
data = readfile.read(inps.file, datasetName=inps.dset)[0]
if dname.startswith('unwrapPhase'):
if 'REF_X' in atr.keys():
data -= data[int(atr['REF_Y']), int(atr['REF_X'])]
print('consider reference pixel in y/x: ({}, {})'.format(
atr['REF_Y'], atr['REF_X']))
else:
print('No REF_Y/X found.')
# metadata
atr['DATE'] = date1[2:8]
atr['DATE12'] = '{}-{}'.format(date1[2:8], date2[2:8])
if dname.startswith('unwrapPhase'):
atr['FILE_TYPE'] = '.unw'
atr['UNIT'] = 'radian'
elif dname == 'coherence':
atr['FILE_TYPE'] = '.cor'
atr['UNIT'] = '1'
elif dname == 'wrapPhase':
atr['FILE_TYPE'] = '.int'
atr['UNIT'] = 'radian'
else:
raise ValueError('unrecognized dataset type: {}'.format(inps.dset))
# output filename
if not inps.outfile:
inps.outfile = '{}{}'.format(date12, atr['FILE_TYPE'])
if inps.file.startswith('geo_'):
inps.outfile = 'geo_' + inps.outfile
else:
# temporal coherence
if 'coherence' in k.lower():
atr['FILE_TYPE'] = '.cor'
elif k in ['mask']:
atr['FILE_TYPE'] = '.msk'
atr['DATA_TYPE'] = 'byte'
elif k in ['geometry'] and inps.dset == 'height':
if 'Y_FIRST' in atr.keys():
atr['FILE_TYPE'] = '.dem'
atr['DATA_TYPE'] = 'int16'
else:
atr['FILE_TYPE'] = '.hgt'
atr['UNIT'] = 'm'
else:
atr['FILE_TYPE'] = '.unw'
if not inps.outfile:
inps.outfile = '{}{}'.format(
os.path.splitext(inps.file)[0], atr['FILE_TYPE'])
atr['PROCESSOR'] = 'roipac'
return data, atr, inps.outfile
def main():
parser = build_parser()
parseArgs = parser.parse_args()
file_name = parseArgs.file
output_folder = parseArgs.outputDir
should_mask = True
path_name_and_extension = os.path.basename(file_name).split(".")
path_name = path_name_and_extension[0]
# ---------------------------------------------------------------------------------------
# start clock to track how long conversion process takes
start_time = time.clock()
# use h5py to open specified group(s) in the h5 file
# then read datasets from h5 file into memory for faster reading of data
he_obj = HDFEOS(file_name)
he_obj.open(print_msg=False)
displacement_3d_matrix = he_obj.read(datasetName='displacement')
mask = he_obj.read(datasetName='mask')
if should_mask:
print("Masking displacement")
displacement_3d_matrix = mask_matrix(displacement_3d_matrix, mask)
del mask
dates = he_obj.dateList
attributes = dict(he_obj.metadata)
#file = h5py.File(file_name, "r")
#timeseries_group = file["HDFEOS"]["GRIDS"]["timeseries"]
#displacement_3d_matrix = timeseries_group["observation"]["displacement"]
# get attributes (stored at root) of UNAVCO timeseries file
#attributes = dict(file.attrs)
# in timeseries displacement_3d_matrix, there are datasets
# need to get datasets with dates - strings that can be converted to integers
#dates = displacement_3d_matrix.attrs["DATE_TIMESERIES"].split(" ")
# array that stores dates from dates that have been converted to decimal
decimal_dates = []
# read datasets in the group into a dictionary of 2d arrays and intialize decimal dates
timeseries_datasets = {}
num_date = len(dates)
for i in range(num_date):
timeseries_datasets[dates[i]] = np.squeeze(
displacement_3d_matrix[i, :, :])
d = get_date(dates[i])
decimal = get_decimal_date(d)
decimal_dates.append(decimal)
del displacement_3d_matrix
#for displacement_2d_matrix in displacement_3d_matrix:
# dataset = displacement_2d_matrix[:]
# if should_mask:
# print("Masking " + dates[i])
# mask = timeseries_group["quality"]["mask"][:]
# dataset = mask_matrix(dataset, mask)
# timeseries_datasets[dates[i]] = dataset
# d = get_date(dates[i])
# decimal = get_decimal_date(d)
# decimal_dates.append(decimal)
# i += 1
# close h5 file
#file.close()
path_list = path_name.split("/")
folder_name = path_name.split("/")[len(path_list) - 1]
try: # create path for output
os.mkdir(output_folder)
except:
print(output_folder + " already exists")
# read and convert the datasets, then write them into json files and insert into database
convert_data(attributes, decimal_dates, timeseries_datasets, dates,
output_folder, folder_name)
# run tippecanoe command to get mbtiles file
os.chdir(os.path.abspath(output_folder))
os.system(
"tippecanoe *.json -l chunk_1 -x d -pf -pk -Bg -d9 -D12 -g12 -r0 -o " +
folder_name + ".mbtiles")
# ---------------------------------------------------------------------------------------
# check how long it took to read h5 file data and create json files
end_time = time.clock()
print(("time elapsed: " + str(end_time - start_time)))
return