def run(reference_fname, test_fname, out_fname, compression, save_inputs, filter_opts): """ Run the residuals analysis. """ # note: lower level h5py access is required in order to visit links with h5py.File(reference_fname, 'r') as ref_fid: with h5py.File(test_fname, 'r') as test_fid: with h5py.File(out_fname, 'w') as out_fid: root = h5py.h5g.open(ref_fid.id, b'/') root.links.visit( partial(residuals, ref_fid, test_fid, out_fid, compression, save_inputs, filter_opts)) # create singular TABLES for each Dataset CLASS # IMAGE try: grp = out_fid[ppjoin('RESULTS', 'IMAGE')] image_results(grp, compression, filter_opts) except KeyError: pass # SCALAR try: grp = out_fid[ppjoin('RESULTS', 'SCALAR')] scalar_results(grp) except KeyError: pass # TABLE try: grp = out_fid[ppjoin('RESULTS', 'TABLE')] table_results(grp) except KeyError: pass
def custom_print(path): """ A custom print function for dealing with HDF5 object types, and print formatting. """ try: pathname = normpath(ppjoin("/", path.decode("utf-8"))) except AttributeError: pathname = normpath(ppjoin("/", path)) obj = h5_obj[path] attrs = {k: v for k, v in obj.attrs.items()} if isinstance(obj, h5py.Group): h5_type = "`Group`" elif isinstance(obj, h5py.Dataset): class_name = obj.attrs.get("CLASS") h5_class = "" if class_name is None else class_name fmt = "`{}Dataset`" if class_name is None else "`{} Dataset`" h5_type = fmt.format(h5_class) else: h5_type = "`Other`" # we'll deal with links and references later print("{path}\t{h5_type}".format(path=pathname, h5_type=h5_type)) if verbose: print("Attributes:") pprint(attrs, width=100) print("*" * 80)
def custom_print(path): """ A custom print function for dealing with HDF5 object types, and print formatting. """ try: pathname = normpath(ppjoin('/', path.decode('utf-8'))) except AttributeError: pathname = normpath(ppjoin('/', path)) obj = h5_obj[path] attrs = {k: v for k, v in obj.attrs.items()} if isinstance(obj, h5py.Group): h5_type = '`Group`' elif isinstance(obj, h5py.Dataset): class_name = obj.attrs.get('CLASS') h5_class = '' if class_name is None else class_name fmt = '`{}Dataset`' if class_name is None else '`{} Dataset`' h5_type = fmt.format(h5_class) else: h5_type = '`Other`' # we'll deal with links and references later print('{path}\t{h5_type}'.format(path=pathname, h5_type=h5_type)) if verbose: print('Attributes:') pprint(attrs, width=100) print('*' * 80)
def image_results(image_group, compression=H5CompressionFilter.LZF, filter_opts=None): """ Combine the residual results of each IMAGE Dataset into a single TABLE Dataset. """ # potentially could just use visit... img_paths = find(image_group, 'IMAGE') min_ = [] max_ = [] percent = [] pct_90 = [] pct_99 = [] resid_paths = [] hist_paths = [] chist_paths = [] products = [] name = [] for pth in img_paths: hist_pth = pth.replace('RESIDUALS', 'FREQUENCY-DISTRIBUTIONS') chist_pth = pth.replace('RESIDUALS', 'CUMULATIVE-DISTRIBUTIONS') resid_paths.append(ppjoin(image_group.name, pth)) hist_paths.append(ppjoin(image_group.name, hist_pth)) chist_paths.append(ppjoin(image_group.name, chist_pth)) dset = image_group[pth] min_.append(dset.attrs['min_residual']) max_.append(dset.attrs['max_residual']) percent.append(dset.attrs['percent_difference']) products.append(pbasename(dset.parent.name)) name.append(pbasename(dset.name)) dset = image_group[chist_pth] pct_90.append(dset.attrs['90th_percentile']) pct_99.append(dset.attrs['99th_percentile']) df = pandas.DataFrame({ 'product': products, 'dataset_name': name, 'min_residual': min_, 'max_residual': max_, 'percent_difference': percent, '90th_percentile': pct_90, '99th_percentile': pct_99, 'residual_image_pathname': resid_paths, 'residual_histogram_pathname': hist_paths, 'residual_cumulative_pathname': chist_paths }) # output write_dataframe(df, 'IMAGE-RESIDUALS', image_group, compression, title='RESIDUALS-TABLE', filter_opts=filter_opts)
def convert(aerosol_path, output_filename, compression, filter_opts): """ Converts all the .pix and .cmp files found in `aerosol_path` to a HDF5 file. """ # define a case switch func = {'pix': read_pix, 'cmp': read_cmp} # create the output file with h5py.File(output_filename, 'w') as fid: pattern = ['*.pix', '*.cmp'] for p in pattern: search = pjoin(aerosol_path, p) files = glob.glob(search) for fname in files: pth, ext = splitext(fname) ext = ext.split(".")[-1] grp_name = basename(pth) out_path = ppjoin(ext, grp_name) # read/write df, extents = func[ext](fname) attrs = {'extents': wkt.dumps(extents), 'source filename': fname} write_dataframe(df, out_path, fid, compression=compression, attrs=attrs, filter_opts=filter_opts)
def run(aerosol_path, output_filename): """ Converts all the .pix and .cmp files found in `aerosol_path` to a HDF5 file. """ # define a case switch func = {"pix": read_pix, "cmp": read_cmp} # create the output file fid = h5py.File(output_filename, "w") pattern = ["*.pix", "*.cmp"] for p in pattern: search = pjoin(aerosol_path, p) files = glob.glob(search) for fname in files: pth, ext = splitext(fname) ext = ext.split(".")[-1] grp_name = basename(pth) out_path = ppjoin(ext, grp_name) # read/write df, extents = func[ext](fname) attrs = {"extents": wkt.dumps(extents), "source filename": fname} write_dataframe(df, out_path, fid, attrs=attrs) fid.close()
def browse_for_images(path_to_folder, subfolder, type): matches = [] (_, _, filenames) = folder_walk(ppjoin(path_to_folder, subfolder)) for filename in fnmatch.filter(filenames, ".".join([type, 'svg'])): matches.append(filename) return matches
def process_wn(path_to_folder, fname, data_length): """ Function loads white noise used in stimulation :param path_to_folder: folder containting the experimental subfolders :param fname: white noise file name joined with the subfolder path :param samp_rate: desired sample rate (to match the recording sample rate) :param data_length: number of samples in spike train (must match to length white noise) :param WN_dur: duration of the white noise :return datasWN: vector containing white noise values only. """ # wn_dur /= 1000 # convert to seconds # wn_N_samples = wn_dur*samp_rate # define the input data filenameWN = ppjoin(path_to_folder, fname) # get the file, extract the three data subunits relacs_file = load(filenameWN) # if relacs file is empty or too short due to aborted RePro presentation # "try:" provides normal termination of the loop instead of error try: metasWN, _, datasWN = relacs_file.selectall() except: return None # check if the number of samples matches the number samples in convolved spike train datas_whitenoise = datasWN[0][:,1] if datas_whitenoise.shape != data_length: timepoints_new = np.linspace(datasWN[0][0,0],datasWN[0][-1,0], data_length) datas_interpolated = np.interp(timepoints_new, datasWN[0][:,0], datasWN[0][:,1]) return datas_interpolated
def link_shadow_datasets(self_shadow_fname, cast_shadow_sun_fname, cast_shadow_satellite_fname, out_fname): """ Link the self shadow mask, and the two cast shadow masks into a single file for easier access. """ group_path = GroupName.SHADOW_GROUP.value dname_fmt = DatasetName.CAST_SHADOW_FMT.value dname = ppjoin(group_path, DatasetName.SELF_SHADOW.value) create_external_link(self_shadow_fname, dname, out_fname, dname) dname = ppjoin(group_path, dname_fmt.format(source='SUN')) create_external_link(cast_shadow_sun_fname, dname, out_fname, dname) dname = ppjoin(group_path, dname_fmt.format(source='SATELLITE')) create_external_link(cast_shadow_satellite_fname, dname, out_fname, dname)
def aggregate_ancillary(granule_groups): """ If the acquisition is part of a `tiled` scene such as Sentinel-2a, then we need to average the point measurements gathered from all granules. """ # initialise the mean result ozone = vapour = aerosol = elevation = 0.0 # number of granules in the scene n_tiles = len(granule_groups) for granule in granule_groups: group = granule[GroupName.ANCILLARY_GROUP.value] ozone += group[DatasetName.OZONE.value][()] vapour += group[DatasetName.WATER_VAPOUR.value][()] aerosol += group[DatasetName.AEROSOL.value][()] elevation += group[DatasetName.ELEVATION.value][()] # average ozone /= n_tiles vapour /= n_tiles aerosol /= n_tiles elevation /= n_tiles description = ("The {} value is an average from all the {} values " "retreived for each Granule.") attrs = {"data_source": "granule_average"} # output each average value back into the same granule ancillary group group_name = ppjoin(GroupName.ANCILLARY_GROUP.value, GroupName.ANCILLARY_AVG_GROUP.value) for granule in granule_groups: # for the multifile workflow, we only want to write to one granule try: group = granule.create_group(group_name) except ValueError: continue dset = group.create_dataset(DatasetName.OZONE.value, data=ozone) attrs["description"] = description.format(*(2 * ["Ozone"])) attach_attributes(dset, attrs) dset = group.create_dataset(DatasetName.WATER_VAPOUR.value, data=vapour) attrs["description"] = description.format(*(2 * ["Water Vapour"])) attach_attributes(dset, attrs) dset = group.create_dataset(DatasetName.AEROSOL.value, data=aerosol) attrs["description"] = description.format(*(2 * ["Aerosol"])) attach_attributes(dset, attrs) dset = group.create_dataset(DatasetName.ELEVATION.value, data=elevation) attrs["description"] = description.format(*(2 * ["Elevation"])) attach_attributes(dset, attrs)
def read_info(path_to_folder, subfolder): """ Loads info.dat containing experiment metadata and extracts and returns information of interest :param path_to_folder : path to the subfolders, containg experimental data :param subfolder : subfolder containing experiment/cell data :return meta_info : returns name of the segment in which the recording was performed """ filename = ppjoin(path_to_folder, subfolder, "info.dat") meta_info = load(filename) # print(meta_info) return (meta_info)
def spont_act(path_to_folder, subfolder, info): """ main calls various sub-functions to exctract the data and plot the raster, return map and interval histogram :param path_to_folder: folder containing experiments :param subfolder: folder containing cell experimental data :param info: experiment metadata :return fnames """ # define the input data filename = ppjoin(path_to_folder, subfolder, "saveevents-Spikes-1.dat") # get the file, extract the three data subunits relacs_file = load(filename) # if relacs file is empty or too short due to aborted RePro presentation # "try:" provides normal termination of the loop instead of error try: metas, _, datas = relacs_file.selectall() except: return None # define list for rug filenames fnames_spont = [] # for each iteration of the same RePro for i in range(0, len(metas)): # print processed RePro iteration print("Spont activity ReProIx", i) # sorts spikes aa = [metas[i]] # conversion into miliseconds spikeDict, stim_amps = fi_spikes_sort(aa, datas[i][:, 0] * 1000) # computes instantaneous spike frequencies based on spikes freqDict, timeDict = fi_instant_freq( spikeDict) #, metas[counter[i]:counter[i+1]]) freq_continuous, freq_continuous2, timeLine, freqSD_continuous = fi_inst_freq_continuous( spikeDict, freqDict, metas) # plots fnames = sp_raster_plot(i, path_to_folder, subfolder, spikeDict, freq_continuous, freq_continuous2, timeDict,\ timeLine, metas[i], info, fileN = i) # appends the list of figure file names fnames_spont.append(fnames) return fnames_spont
def convert(aerosol_path, out_h5: h5py.Group, compression, filter_opts): """ Converts all the .pix and .cmp files found in `aerosol_path` to a HDF5 file. """ # define a case switch func = {"pix": read_pix, "cmp": read_cmp} dataset_names = [] metadata = [] pattern = ["*.pix", "*.cmp"] for p in pattern: for search_path in aerosol_path.glob(p): _path = search_path.resolve() fname, ext = _path.stem, _path.suffix[ 1:] # exclude the period from ext out_path = ppjoin(ext, fname) df, extents = func[ext](_path) # read/write df, extents = func[ext](_path) # src checksum; used to help derive fallback uuid with _path.open("rb") as src: src_checksum = generate_md5sum(src).hexdigest() attrs = { "extents": wkt.dumps(extents), "source filename": str(_path) } write_dataframe( df, out_path, out_h5, compression=compression, attrs=attrs, filter_opts=filter_opts, ) dataset_names.append(out_path) metadata.append({ "id": str( generate_fallback_uuid(PRODUCT_HREF, path=str(_path.stem), md5=src_checksum)) }) return metadata, dataset_names
def generate_exp_html(path_to_folder, subfolder, info, figs): # generate page title rec_location = info["Cell"]["Location"] pagetitle = "".join([rec_location, ": ", subfolder]) # creates page instance pg = mup.page() # initialize the page, embed css, titles, header and footer pg.init(title=pagetitle, css=('markup.css', 'two.css'), header="".join( ['<font size="10" color="red" >', pagetitle, '</font>']), footer="The bitter end.") # line break under header pg.br() # loop over the figs ordered dictionary for every type of the figure for k, v in figs.items(): images = [] pg.p("".join( ['<font size="5" color="red" >', '<b>', k, '</b>', '</font>'])) for i in range(0, len(v)): # images.append (".".join([v[i],'svg'])) pg.p() v[i].split("_FI")[0] pg.a(href="".join([v[i].split("_FI")[0], "_FI_rug.html"])) pg.img(src=".".join([v[i], 'svg']), alt="click for rug and return plot", align="middle") pg.br() pg.hr() # line break before footer pg.br() # dump the generated html code into the .html file html_name = ".".join( ["".join([info["Cell"]["Location"], "_", "index"]), "html"]) t = [] f = open(ppjoin(path_to_folder, subfolder, html_name), "w") t.append(str(pg)) f.write(t[0]) f.close() return html_name, rec_location
def extract(output_directory, group, name): """ A simple utility that sends an object to the appropriate extraction utility. """ dataset_name = ppjoin(group.name, name.decode('utf-8')) obj = group[dataset_name] obj_class = obj.attrs.get('CLASS') if obj_class == 'IMAGE': convert_image(obj, output_directory) elif obj_class == 'TABLE': convert_table(group, dataset_name, output_directory) elif obj_class == 'SCALAR': convert_scalar(obj, output_directory) else: return None
def test_lon_array(self): """ Test that the interpolated longitude array has sensible values. :notes: The default image that is used is a 1000 by 1000. This might be too small for a recursive depth of 7 (default within the NBAR framework). As such a depth of 3 is used and produced correct results. If a larger image is used, then the depth level should probably be increased. """ acq = acquisitions(LS5_SCENE1).get_acquisitions()[0] geobox = acq.gridded_geo_box() fid = create_lon_lat_grids(acq, depth=5) dataset_name = ppjoin(GroupName.LON_LAT_GROUP.value, DatasetName.LON.value) lon = fid[dataset_name][:] ids = ut.random_pixel_locations(lon.shape) # We'll transform (reproject) to WGS84 sr = osr.SpatialReference() sr.SetFromUserInput(CRS) # Get a list of x reprojected co-ordinates reprj = [] for i in range(ids[0].shape[0]): # Get pixel (x,y) xy = (ids[1][i], ids[0][i]) # Convert pixel to map mapXY = geobox.convert_coordinates(xy, centre=True) # Transform map to another crs x, _ = geobox.transform_coordinates(mapXY, to_crs=sr) reprj.append(x) reprj = numpy.array(reprj) lon_values = lon[ids] # A decimal degree co-ordinate should be correct up to the 6th dp self.assertIsNone(npt.assert_almost_equal(lon_values, reprj, decimal=5))
def run(self): with self.output().temporary_path() as out_fname: for root, _, files in os.walk(self.work_root): # skip any private files if basename(root)[0] == "_": continue for file_ in files: if splitext(file_)[1] == ".h5": fname = pjoin(root, file_) grp_name = basename(dirname(fname.replace(self.work_root, ""))) with h5py.File(fname, "r") as fid: groups = [g for g in fid] for pth in groups: new_path = ppjoin(self.granule, grp_name, pth) create_external_link(fname, pth, out_fname, new_path) with h5py.File(out_fname, "a") as fid: fid.attrs["level1_uri"] = self.level1
def generate_index(exp_pages, wd, fish_n_chips): pagetitle = "eFISH intracellular in vitro" pg = mup.page() pg.init(title=pagetitle, css=('markup.css', 'two.css'), header="".join( ['<font size="10" color="red" >', pagetitle, '</font>']), footer="The void.") for k, v in exp_pages.items(): pages = [] pg.p("".join( ['<font size="5" color="red" >', '<b>', k, '</b>', '</font>'])) pg.hr() pg.br() for i in range(0, len(v)): if fish_n_chips[v[i].split("/")[-2]] == []: # pg.p(fish_n_chips[v[i]]) continue else: pg.p(fish_n_chips[v[i].split("/")[-2]][0]) pg.a("".join([ v[i], " ", str(fish_n_chips[v[i].split("/")[-2]][1:]) ]), class_='internal', href=v[i]) pg.br() pg.hr() # dump the generated html code into the .html file html_name = "index.html" t = [] f = open(ppjoin(wd, html_name), "w") t.append(str(pg)) # print(t[0]) f.write(t[0]) f.close()
def elevation_provenance(anc_grp): ids = [] # low resolution source dname = DatasetName.ELEVATION.value dset = anc_grp[dname] ids.extend(dset.attrs['id']) # high resolution source (res group is adjacent to ancillary group) parent_group = anc_grp.parent for res_group in res_group_bands: dname = ppjoin(res_group, GroupName.ELEVATION_GROUP.value, DatasetName.DSM_SMOOTHED.value) dset = parent_group[dname] ids.extend(dset.attrs['id']) # unique listing of ids ids = numpy.unique(numpy.array(ids)).tolist() md = { 'id': ids, } return md
def test_modtran_run(self): """ Tests that the interface to modtran (run_modtran) works for known inputs. Used to validate environment configuration/setup """ band_names = [ 'BAND-1', 'BAND-2', 'BAND-3', 'BAND-4', 'BAND-5', 'BAND-6', 'BAND-7', 'BAND-8' ] point = 0 albedo = Albedos.ALBEDO_0 # setup mock acquistions object acquisitions = [] for bandn in band_names: acq = mock.MagicMock() acq.acquisition_datetime = datetime(2001, 1, 1) acq.band_type = BandType.REFLECTIVE acq.spectral_response = mock_spectral_response acquisitions.append(acq) # setup mock atmospherics group attrs = {'lonlat': 'TEST'} atmospherics = mock.MagicMock() atmospherics.attrs = attrs atmospherics_group = {POINT_FMT.format(p=point): atmospherics} # Compute base path -- prefix for hdf5 file base_path = ppjoin(GroupName.ATMOSPHERIC_RESULTS_GRP.value, POINT_FMT.format(p=point)) with tempfile.TemporaryDirectory() as workdir: run_dir = pjoin(workdir, POINT_FMT.format(p=point), ALBEDO_FMT.format(a=albedo.value)) os.makedirs(run_dir) # TODO replace json_input copy with json input generation with open(INPUT_JSON, 'r') as fd: json_data = json.load(fd) for mod_input in json_data['MODTRAN']: mod_input['MODTRANINPUT']['SPECTRAL'][ 'FILTNM'] = SPECTRAL_RESPONSE_LS8 with open( pjoin(run_dir, POINT_ALBEDO_FMT.format(p=point, a=albedo.value)) + ".json", 'w') as fd: json.dump(json_data, fd) fid = run_modtran( acquisitions, atmospherics_group, Workflow.STANDARD, npoints=12, # number of track points point=point, albedos=[albedo], modtran_exe=MODTRAN_EXE, basedir=workdir, out_group=None) assert fid # Test base attrs assert fid[base_path].attrs['lonlat'] == 'TEST' assert fid[base_path].attrs['datetime'] == datetime(2001, 1, 1).isoformat() # test albedo headers? # Summarise modtran results to surface reflectance coefficients test_grp = fid[base_path][ALBEDO_FMT.format(a=albedo.value)] nbar_coefficients, _ = coefficients( read_h5_table( fid, pjoin(base_path, ALBEDO_FMT.format(a=albedo.value), DatasetName.CHANNEL.value)), read_h5_table( fid, pjoin(base_path, ALBEDO_FMT.format(a=albedo.value), DatasetName.SOLAR_ZENITH_CHANNEL.value))) expected = pd.read_csv(EXPECTED_CSV, index_col='band_name') pd.testing.assert_frame_equal(nbar_coefficients, expected, check_less_precise=True)
def format_json(acquisitions, ancillary_group, satellite_solar_group, lon_lat_group, workflow, out_group): """ Creates json files for the albedo (0) and thermal """ # angles data sat_view = satellite_solar_group[DatasetName.SATELLITE_VIEW.value] sat_azi = satellite_solar_group[DatasetName.SATELLITE_AZIMUTH.value] longitude = lon_lat_group[DatasetName.LON.value] latitude = lon_lat_group[DatasetName.LAT.value] # retrieve the averaged ancillary if available anc_grp = ancillary_group.get(GroupName.ANCILLARY_AVG_GROUP.value) if anc_grp is None: anc_grp = ancillary_group # ancillary data coordinator = ancillary_group[DatasetName.COORDINATOR.value] aerosol = anc_grp[DatasetName.AEROSOL.value][()] water_vapour = anc_grp[DatasetName.WATER_VAPOUR.value][()] ozone = anc_grp[DatasetName.OZONE.value][()] elevation = anc_grp[DatasetName.ELEVATION.value][()] npoints = coordinator.shape[0] view = numpy.zeros(npoints, dtype='float32') azi = numpy.zeros(npoints, dtype='float32') lat = numpy.zeros(npoints, dtype='float64') lon = numpy.zeros(npoints, dtype='float64') for i in range(npoints): yidx = coordinator['row_index'][i] xidx = coordinator['col_index'][i] view[i] = sat_view[yidx, xidx] azi[i] = sat_azi[yidx, xidx] lat[i] = latitude[yidx, xidx] lon[i] = longitude[yidx, xidx] view_corrected = 180 - view azi_corrected = azi + 180 rlon = 360 - lon # check if in western hemisphere idx = rlon >= 360 rlon[idx] -= 360 idx = (180 - view_corrected) < 0.1 view_corrected[idx] = 180 azi_corrected[idx] = 0 idx = azi_corrected > 360 azi_corrected[idx] -= 360 # get the modtran profiles to use based on the centre latitude _, centre_lat = acquisitions[0].gridded_geo_box().centre_lonlat if out_group is None: out_group = h5py.File('atmospheric-inputs.h5', 'w') if GroupName.ATMOSPHERIC_INPUTS_GRP.value not in out_group: out_group.create_group(GroupName.ATMOSPHERIC_INPUTS_GRP.value) group = out_group[GroupName.ATMOSPHERIC_INPUTS_GRP.value] iso_time = acquisitions[0].acquisition_datetime.isoformat() group.attrs['acquisition-datetime'] = iso_time json_data = {} # setup the json files required by MODTRAN if workflow in (Workflow.STANDARD, Workflow.NBAR): acqs = [a for a in acquisitions if a.band_type == BandType.REFLECTIVE] for p in range(npoints): for alb in Workflow.NBAR.albedos: input_data = {'name': POINT_ALBEDO_FMT.format(p=p, a=str(alb.value)), 'water': water_vapour, 'ozone': ozone, 'doy': acquisitions[0].julian_day(), 'visibility': -aerosol, 'lat': lat[p], 'lon': rlon[p], 'time': acquisitions[0].decimal_hour(), 'sat_azimuth': azi_corrected[p], 'sat_height': acquisitions[0].altitude / 1000.0, 'elevation': elevation, 'sat_view': view_corrected[p], 'albedo': float(alb.value), 'filter_function': acqs[0].spectral_filter_name, 'binary': False } if centre_lat < -23.0: data = mpjson.midlat_summer_albedo(**input_data) else: data = mpjson.tropical_albedo(**input_data) input_data['description'] = 'Input file for MODTRAN' input_data['file_format'] = 'json' input_data.pop('binary') json_data[(p, alb)] = data data = json.dumps(data, cls=JsonEncoder, indent=4) dname = ppjoin(POINT_FMT.format(p=p), ALBEDO_FMT.format(a=alb.value), DatasetName.MODTRAN_INPUT.value) write_scalar(data, dname, group, input_data) # create json for sbt if it has been collected if ancillary_group.attrs.get('sbt-ancillary'): dname = ppjoin(POINT_FMT, DatasetName.ATMOSPHERIC_PROFILE.value) acqs = [a for a in acquisitions if a.band_type == BandType.THERMAL] for p in range(npoints): atmos_profile = read_h5_table(ancillary_group, dname.format(p=p)) n_layers = atmos_profile.shape[0] + 6 elevation = atmos_profile.iloc[0]['GeoPotential_Height'] input_data = {'name': POINT_ALBEDO_FMT.format(p=p, a='TH'), 'ozone': ozone, 'n': n_layers, 'prof_alt': list(atmos_profile['GeoPotential_Height']), 'prof_pres': list(atmos_profile['Pressure']), 'prof_temp': list(atmos_profile['Temperature']), 'prof_water': list(atmos_profile['Relative_Humidity']), 'visibility': -aerosol, 'sat_height': acquisitions[0].altitude / 1000.0, 'gpheight': elevation, 'sat_view': view_corrected[p], 'filter_function': acqs[0].spectral_filter_name, 'binary': False } data = mpjson.thermal_transmittance(**input_data) input_data['description'] = 'Input File for MODTRAN' input_data['file_format'] = 'json' input_data.pop('binary') json_data[(p, Albedos.ALBEDO_TH)] = data data = json.dumps(data, cls=JsonEncoder, indent=4) out_dname = ppjoin(POINT_FMT.format(p=p), ALBEDO_FMT.format(a=Albedos.ALBEDO_TH.value), DatasetName.MODTRAN_INPUT.value) write_scalar(data, out_dname, group, input_data) # attach location info to each point Group for p in range(npoints): lonlat = (coordinator['longitude'][p], coordinator['latitude'][p]) group[POINT_FMT.format(p=p)].attrs['lonlat'] = lonlat return json_data, out_group
def card4l(level1, granule, workflow, vertices, method, pixel_quality, landsea, tle_path, aerosol, brdf_path, brdf_premodis_path, ozone_path, water_vapour, dem_path, dsm_fname, invariant_fname, modtran_exe, out_fname, ecmwf_path=None, rori=0.52, buffer_distance=8000, compression=H5CompressionFilter.LZF, filter_opts=None, h5_driver=None, acq_parser_hint=None): """ CEOS Analysis Ready Data for Land. A workflow for producing standardised products that meet the CARD4L specification. :param level1: A string containing the full file pathname to the level1 dataset. :param granule: A string containing the granule id to process. :param workflow: An enum from wagl.constants.Workflow representing which workflow workflow to run. :param vertices: An integer 2-tuple indicating the number of rows and columns of sample-locations ("coordinator") to produce. The vertex columns should be an odd number. :param method: An enum from wagl.constants.Method representing the interpolation method to use during the interpolation of the atmospheric coefficients. :param pixel_quality: A bool indicating whether or not to run pixel quality. :param landsea: A string containing the full file pathname to the directory containing the land/sea mask datasets. :param tle_path: A string containing the full file pathname to the directory containing the two line element datasets. :param aerosol: A string containing the full file pathname to the HDF5 file containing the aerosol data. :param brdf_path: A string containing the full file pathname to the directory containing the BRDF data. :param brdf_premodis_path: A string containing the full file pathname to the directory containing the decadal averaged BRDF data used for acquisitions prior to TERRA/AQUA satellite operations, or for near real time applications. :param ozone_path: A string containing the full file pathname to the directory containing the ozone datasets. :param water_vapour: A string containing the full file pathname to the directory containing the water vapour datasets. :param dem_path: A string containing the full file pathname to the directory containing the reduced resolution DEM. :param dsm_path: A string containing the full file pathname to the directory containing the Digital Surface Workflow for use in terrain illumination correction. :param invariant_fname: A string containing the full file pathname to the image file containing the invariant geo-potential data for use within the SBT process. :param modtran_exe: A string containing the full file pathname to the MODTRAN executable. :param out_fname: A string containing the full file pathname that will contain the output data from the data standardisation process. executable. :param ecmwf_path: A string containing the full file pathname to the directory containing the data from the European Centre for Medium Weather Forcast, for use within the SBT process. :param rori: A floating point value for surface reflectance adjustment. TODO Fuqin to add additional documentation for this parameter. Default is 0.52. :param buffer_distance: A number representing the desired distance (in the same units as the acquisition) in which to calculate the extra number of pixels required to buffer an image. Default is 8000, which for an acquisition using metres would equate to 8000 metres. :param compression: An enum from hdf5.compression.H5CompressionFilter representing the desired compression filter to use for writing H5 IMAGE and TABLE class datasets to disk. Default is H5CompressionFilter.LZF. :param filter_opts: A dict containing any additional keyword arguments when generating the configuration for the given compression Filter. Default is None. :param h5_driver: The specific HDF5 file driver to use when creating the output HDF5 file. See http://docs.h5py.org/en/latest/high/file.html#file-drivers for more details. Default is None; which writes direct to disk using the appropriate driver for the underlying OS. :param acq_parser_hint: A string containing any hints to provide the acquisitions loader with. """ tp5_fmt = pjoin(POINT_FMT, ALBEDO_FMT, ''.join([POINT_ALBEDO_FMT, '.tp5'])) nvertices = vertices[0] * vertices[1] container = acquisitions(level1, hint=acq_parser_hint) # TODO: pass through an acquisitions container rather than pathname with h5py.File(out_fname, 'w', driver=h5_driver) as fid: fid.attrs['level1_uri'] = level1 for grp_name in container.supported_groups: log = STATUS_LOGGER.bind(level1=container.label, granule=granule, granule_group=grp_name) # root group for a given granule and resolution group root = fid.create_group(ppjoin(granule, grp_name)) acqs = container.get_acquisitions(granule=granule, group=grp_name) # longitude and latitude log.info('Latitude-Longitude') create_lon_lat_grids(acqs[0], root, compression, filter_opts) # satellite and solar angles log.info('Satellite-Solar-Angles') calculate_angles(acqs[0], root[GroupName.LON_LAT_GROUP.value], root, compression, filter_opts, tle_path) if workflow == Workflow.STANDARD or workflow == Workflow.NBAR: # DEM log.info('DEM-retriveal') get_dsm(acqs[0], dsm_fname, buffer_distance, root, compression, filter_opts) # slope & aspect log.info('Slope-Aspect') slope_aspect_arrays(acqs[0], root[GroupName.ELEVATION_GROUP.value], buffer_distance, root, compression, filter_opts) # incident angles log.info('Incident-Angles') incident_angles(root[GroupName.SAT_SOL_GROUP.value], root[GroupName.SLP_ASP_GROUP.value], root, compression, filter_opts) # exiting angles log.info('Exiting-Angles') exiting_angles(root[GroupName.SAT_SOL_GROUP.value], root[GroupName.SLP_ASP_GROUP.value], root, compression, filter_opts) # relative azimuth slope log.info('Relative-Azimuth-Angles') incident_group_name = GroupName.INCIDENT_GROUP.value exiting_group_name = GroupName.EXITING_GROUP.value relative_azimuth_slope(root[incident_group_name], root[exiting_group_name], root, compression, filter_opts) # self shadow log.info('Self-Shadow') self_shadow(root[incident_group_name], root[exiting_group_name], root, compression, filter_opts) # cast shadow solar source direction log.info('Cast-Shadow-Solar-Direction') dsm_group_name = GroupName.ELEVATION_GROUP.value calculate_cast_shadow(acqs[0], root[dsm_group_name], root[GroupName.SAT_SOL_GROUP.value], buffer_distance, root, compression, filter_opts) # cast shadow satellite source direction log.info('Cast-Shadow-Satellite-Direction') calculate_cast_shadow(acqs[0], root[dsm_group_name], root[GroupName.SAT_SOL_GROUP.value], buffer_distance, root, compression, filter_opts, False) # combined shadow masks log.info('Combined-Shadow') combine_shadow_masks(root[GroupName.SHADOW_GROUP.value], root[GroupName.SHADOW_GROUP.value], root[GroupName.SHADOW_GROUP.value], root, compression, filter_opts) # nbar and sbt ancillary log = STATUS_LOGGER.bind(level1=container.label, granule=granule, granule_group=None) # granule root group root = fid[granule] # get the highest resoltion group cotaining supported bands acqs, grp_name = container.get_highest_resolution(granule=granule) grn_con = container.get_granule(granule=granule, container=True) res_group = root[grp_name] log.info('Ancillary-Retrieval') nbar_paths = {'aerosol_dict': aerosol, 'water_vapour_dict': water_vapour, 'ozone_path': ozone_path, 'dem_path': dem_path, 'brdf_path': brdf_path, 'brdf_premodis_path': brdf_premodis_path} collect_ancillary(grn_con, res_group[GroupName.SAT_SOL_GROUP.value], nbar_paths, ecmwf_path, invariant_fname, vertices, root, compression, filter_opts) # atmospherics log.info('Atmospherics') ancillary_group = root[GroupName.ANCILLARY_GROUP.value] # satellite/solar angles and lon/lat for a resolution group sat_sol_grp = res_group[GroupName.SAT_SOL_GROUP.value] lon_lat_grp = res_group[GroupName.LON_LAT_GROUP.value] # TODO: supported acqs in different groups pointing to different response funcs # tp5 files tp5_data, _ = format_tp5(acqs, ancillary_group, sat_sol_grp, lon_lat_grp, workflow, root) # atmospheric inputs group inputs_grp = root[GroupName.ATMOSPHERIC_INPUTS_GRP.value] # radiative transfer for each point and albedo for key in tp5_data: point, albedo = key log.info('Radiative-Transfer', point=point, albedo=albedo.value) with tempfile.TemporaryDirectory() as tmpdir: prepare_modtran(acqs, point, [albedo], tmpdir, modtran_exe) # tp5 data fname = pjoin(tmpdir, tp5_fmt.format(p=point, a=albedo.value)) with open(fname, 'w') as src: src.writelines(tp5_data[key]) run_modtran(acqs, inputs_grp, workflow, nvertices, point, [albedo], modtran_exe, tmpdir, root, compression, filter_opts) # atmospheric coefficients log.info('Coefficients') results_group = root[GroupName.ATMOSPHERIC_RESULTS_GRP.value] calculate_coefficients(results_group, root, compression, filter_opts) # interpolate coefficients for grp_name in container.supported_groups: log = STATUS_LOGGER.bind(level1=container.label, granule=granule, granule_group=grp_name) log.info('Interpolation') # acquisitions and available bands for the current group level acqs = container.get_acquisitions(granule=granule, group=grp_name) nbar_acqs = [acq for acq in acqs if acq.band_type == BandType.REFLECTIVE] sbt_acqs = [acq for acq in acqs if acq.band_type == BandType.THERMAL] res_group = root[grp_name] sat_sol_grp = res_group[GroupName.SAT_SOL_GROUP.value] comp_grp = root[GroupName.COEFFICIENTS_GROUP.value] for coefficient in workflow.atmos_coefficients: if coefficient in Workflow.NBAR.atmos_coefficients: band_acqs = nbar_acqs else: band_acqs = sbt_acqs for acq in band_acqs: log.info('Interpolate', band_id=acq.band_id, coefficient=coefficient.value) interpolate(acq, coefficient, ancillary_group, sat_sol_grp, comp_grp, res_group, compression, filter_opts, method) # standardised products band_acqs = [] if workflow == Workflow.STANDARD or workflow == Workflow.NBAR: band_acqs.extend(nbar_acqs) if workflow == Workflow.STANDARD or workflow == Workflow.SBT: band_acqs.extend(sbt_acqs) for acq in band_acqs: interp_grp = res_group[GroupName.INTERP_GROUP.value] if acq.band_type == BandType.THERMAL: log.info('SBT', band_id=acq.band_id) surface_brightness_temperature(acq, interp_grp, res_group, compression, filter_opts) else: slp_asp_grp = res_group[GroupName.SLP_ASP_GROUP.value] rel_slp_asp = res_group[GroupName.REL_SLP_GROUP.value] incident_grp = res_group[GroupName.INCIDENT_GROUP.value] exiting_grp = res_group[GroupName.EXITING_GROUP.value] shadow_grp = res_group[GroupName.SHADOW_GROUP.value] log.info('Surface-Reflectance', band_id=acq.band_id) calculate_reflectance(acq, interp_grp, sat_sol_grp, slp_asp_grp, rel_slp_asp, incident_grp, exiting_grp, shadow_grp, ancillary_group, rori, res_group, compression, filter_opts) # metadata yaml's if workflow == Workflow.STANDARD or workflow == Workflow.NBAR: create_ard_yaml(band_acqs, ancillary_group, res_group) if workflow == Workflow.STANDARD or workflow == Workflow.SBT: create_ard_yaml(band_acqs, ancillary_group, res_group, True) # pixel quality sbt_only = workflow == Workflow.SBT if pixel_quality and can_pq(level1, acq_parser_hint) and not sbt_only: run_pq(level1, res_group, landsea, res_group, compression, filter_opts, AP.NBAR, acq_parser_hint) run_pq(level1, res_group, landsea, res_group, compression, filter_opts, AP.NBART, acq_parser_hint)
def convert_file(fname, out_fname, group_name='/', dataset_name='dataset', compression=H5CompressionFilter.LZF, filter_opts=None, attrs=None): """ Convert generic single band image file to HDF5. Processes in a tiled fashion to minimise memory use. Will process all columns by n (default 256) rows at a time, where n can be specified via command line using: --filter-opts '{"chunks": (n, xsize)}' :param fname: A str containing the raster filename. :param out_fname: A str containing the output filename for the HDF5 file. :param dataset_name: A str containing the dataset name to use in the HDF5 file. :param compression: The compression filter to use. Default is H5CompressionFilter.LZF :param filter_opts: A dict of key value pairs available to the given configuration instance of H5CompressionFilter. For example H5CompressionFilter.LZF has the keywords *chunks* and *shuffle* available. Default is None, which will use the default settings for the chosen H5CompressionFilter instance. :param attrs: A dict containing any attribute information to be attached to the HDF5 Dataset. :return: None. Content is written directly to disk. """ # opening as `append` mode allows us to add additional datasets with h5py.File(out_fname) as fid: with rasterio.open(fname) as ds: # create empty or copy the user supplied filter options if not filter_opts: filter_opts = dict() else: filter_opts = filter_opts.copy() # use sds native chunks if none are provided if 'chunks' not in filter_opts: filter_opts['chunks'] = (256, 256) # read all cols for n rows (ytile), as the GA's DEM is BSQ interleaved ytile = filter_opts['chunks'][0] # dataset attributes if attrs: attrs = attrs.copy() else: attrs = {} attrs['geotransform'] = ds.transform.to_gdal() attrs['crs_wkt'] = ds.crs.wkt # dataset creation options kwargs = compression.config( **filter_opts).dataset_compression_kwargs() kwargs['shape'] = (ds.height, ds.width) kwargs['dtype'] = ds.dtypes[0] dataset_name = ppjoin(group_name, dataset_name) dataset = fid.create_dataset(dataset_name, **kwargs) attach_image_attributes(dataset, attrs) # process each tile for tile in generate_tiles(ds.width, ds.height, ds.width, ytile): idx = (slice(tile[0][0], tile[0][1]), slice(tile[1][0], tile[1][1])) data = ds.read(1, window=tile) dataset[idx] = data
def calculate_coefficients(atmospheric_results_group, out_group, compression=H5CompressionFilter.LZF, filter_opts=None): """ Calculate the atmospheric coefficients from the MODTRAN output and used in the BRDF and atmospheric correction. Coefficients are computed for each band for each each coordinate for each atmospheric coefficient. The atmospheric coefficients can be found in `Workflow.STANDARD.atmos_coefficients`. :param atmospheric_results_group: The root HDF5 `Group` that contains the atmospheric results from each MODTRAN run. :param out_group: If set to None (default) then the results will be returned as an in-memory hdf5 file, i.e. the `core` driver. Otherwise, a writeable HDF5 `Group` object. The datasets will be formatted to the HDF5 TABLE specification and the dataset names will be as follows: * DatasetName.NBAR_COEFFICIENTS (if Workflow.STANDARD or Workflow.NBAR) * DatasetName.SBT_COEFFICIENTS (if Workflow.STANDARD or Workflow.SBT) :param compression: The compression filter to use. Default is H5CompressionFilter.LZF :param filter_opts: A dict of key value pairs available to the given configuration instance of H5CompressionFilter. For example H5CompressionFilter.LZF has the keywords *chunks* and *shuffle* available. Default is None, which will use the default settings for the chosen H5CompressionFilter instance. :return: An opened `h5py.File` object, that is either in-memory using the `core` driver, or on disk. """ nbar_coefficients = pd.DataFrame() sbt_coefficients = pd.DataFrame() channel_data = channel_solar_angle = upward = downward = None # Initialise the output group/file if out_group is None: fid = h5py.File('atmospheric-coefficients.h5', driver='core', backing_store=False) else: fid = out_group res = atmospheric_results_group npoints = res.attrs['npoints'] nbar_atmos = res.attrs['nbar_atmospherics'] sbt_atmos = res.attrs['sbt_atmospherics'] for point in range(npoints): point_grp = res[POINT_FMT.format(p=point)] lonlat = point_grp.attrs['lonlat'] timestamp = pd.to_datetime(point_grp.attrs['datetime']) grp_path = ppjoin(POINT_FMT.format(p=point), ALBEDO_FMT) if nbar_atmos: channel_path = ppjoin(grp_path.format(a=Albedos.ALBEDO_0.value), DatasetName.CHANNEL.value) channel_data = read_h5_table(res, channel_path) channel_solar_angle_path = ppjoin( grp_path.format(a=Albedos.ALBEDO_0.value), DatasetName.SOLAR_ZENITH_CHANNEL.value ) channel_solar_angle = read_h5_table(res, channel_solar_angle_path) if sbt_atmos: dname = ppjoin(grp_path.format(a=Albedos.ALBEDO_TH.value), DatasetName.UPWARD_RADIATION_CHANNEL.value) upward = read_h5_table(res, dname) dname = ppjoin(grp_path.format(a=Albedos.ALBEDO_TH.value), DatasetName.DOWNWARD_RADIATION_CHANNEL.value) downward = read_h5_table(res, dname) kwargs = {'channel_data': channel_data, 'solar_zenith_angle': channel_solar_angle, 'upward_radiation': upward, 'downward_radiation': downward} result = coefficients(**kwargs) # insert some datetime/geospatial fields if result[0] is not None: result[0].insert(0, 'POINT', point) result[0].insert(1, 'LONGITUDE', lonlat[0]) result[0].insert(2, 'LATITUDE', lonlat[1]) result[0].insert(3, 'DATETIME', timestamp) nbar_coefficients = nbar_coefficients.append(result[0]) if result[1] is not None: result[1].insert(0, 'POINT', point) result[1].insert(1, 'LONGITUDE', lonlat[0]) result[1].insert(2, 'LATITUDE', lonlat[1]) result[1].insert(3, 'DATETIME', pd.to_datetime(timestamp)) sbt_coefficients = sbt_coefficients.append(result[1]) nbar_coefficients.reset_index(inplace=True) sbt_coefficients.reset_index(inplace=True) attrs = {'npoints': npoints} description = "Coefficients derived from the VNIR solar irradiation." attrs['description'] = description dname = DatasetName.NBAR_COEFFICIENTS.value if GroupName.COEFFICIENTS_GROUP.value not in fid: fid.create_group(GroupName.COEFFICIENTS_GROUP.value) group = fid[GroupName.COEFFICIENTS_GROUP.value] if nbar_atmos: write_dataframe(nbar_coefficients, dname, group, compression, attrs=attrs, filter_opts=filter_opts) description = "Coefficients derived from the THERMAL solar irradiation." attrs['description'] = description dname = DatasetName.SBT_COEFFICIENTS.value if sbt_atmos: write_dataframe(sbt_coefficients, dname, group, compression, attrs=attrs, filter_opts=filter_opts) if out_group is None: return fid
def run_modtran(acquisitions, atmospherics_group, workflow, npoints, point, albedos, modtran_exe, basedir, out_group, compression=H5CompressionFilter.LZF, filter_opts=None): """ Run MODTRAN and channel results. """ lonlat = atmospherics_group[POINT_FMT.format(p=point)].attrs['lonlat'] # determine the output group/file if out_group is None: fid = h5py.File('atmospheric-results.h5', driver='core', backing_store=False) else: fid = out_group # initial attributes base_attrs = {'Point': point, 'lonlat': lonlat, 'datetime': acquisitions[0].acquisition_datetime} base_path = ppjoin(GroupName.ATMOSPHERIC_RESULTS_GRP.value, POINT_FMT.format(p=point)) # what atmospheric calculations have been run and how many points group_name = GroupName.ATMOSPHERIC_RESULTS_GRP.value if group_name not in fid: fid.create_group(group_name) fid[group_name].attrs['npoints'] = npoints applied = workflow in (Workflow.STANDARD, Workflow.NBAR) fid[group_name].attrs['nbar_atmospherics'] = applied applied = workflow in (Workflow.STANDARD, Workflow.SBT) fid[group_name].attrs['sbt_atmospherics'] = applied acqs = acquisitions for albedo in albedos: base_attrs['Albedo'] = albedo.value workpath = pjoin(basedir, POINT_FMT.format(p=point), ALBEDO_FMT.format(a=albedo.value)) json_mod_infile = pjoin(workpath, ''.join( [POINT_ALBEDO_FMT.format(p=point, a=albedo.value), '.json'])) group_path = ppjoin(base_path, ALBEDO_FMT.format(a=albedo.value)) subprocess.check_call([modtran_exe, json_mod_infile], cwd=workpath) chn_fname = glob.glob(pjoin(workpath, '*.chn'))[0] tp6_fname = glob.glob(pjoin(workpath, '*.tp6'))[0] if albedo == Albedos.ALBEDO_TH: acq = [acq for acq in acqs if acq.band_type == BandType.THERMAL][0] channel_data = read_modtran_channel(chn_fname, tp6_fname, acq, albedo) attrs = base_attrs.copy() dataset_name = DatasetName.UPWARD_RADIATION_CHANNEL.value attrs['description'] = ('Upward radiation channel output from ' 'MODTRAN') dset_name = ppjoin(group_path, dataset_name) write_dataframe(channel_data[0], dset_name, fid, compression, attrs=attrs, filter_opts=filter_opts) # downward radiation attrs = base_attrs.copy() dataset_name = DatasetName.DOWNWARD_RADIATION_CHANNEL.value attrs['description'] = ('Downward radiation channel output from ' 'MODTRAN') dset_name = ppjoin(group_path, dataset_name) write_dataframe(channel_data[1], dset_name, fid, compression, attrs=attrs, filter_opts=filter_opts) else: acq = [acq for acq in acqs if acq.band_type == BandType.REFLECTIVE][0] # Will require updating to handle JSON output from modtran channel_data = read_modtran_channel(chn_fname, tp6_fname, acq, albedo) attrs = base_attrs.copy() dataset_name = DatasetName.CHANNEL.value attrs['description'] = 'Channel output from MODTRAN' dset_name = ppjoin(group_path, dataset_name) write_dataframe(channel_data[0], dset_name, fid, compression, attrs=attrs, filter_opts=filter_opts) # solar zenith angle at surface attrs = base_attrs.copy() dataset_name = DatasetName.SOLAR_ZENITH_CHANNEL.value attrs['description'] = 'Solar zenith angle at different atmosphere levels' dset_name = ppjoin(group_path, dataset_name) write_dataframe(channel_data[1], dset_name, fid, compression, attrs=attrs, filter_opts=filter_opts) # metadata for a given point alb_vals = [alb.value for alb in workflow.albedos] fid[base_path].attrs['lonlat'] = lonlat fid[base_path].attrs['datetime'] = acqs[0].acquisition_datetime.isoformat() fid[base_path].attrs.create('albedos', data=alb_vals, dtype=VLEN_STRING) if out_group is None: return fid
def link_atmospheric_results(input_targets, out_fname, npoints, workflow): """ Uses h5py's ExternalLink to combine the atmospheric results into a single file. :param input_targets: A `list` of luigi.LocalTargets. :param out_fname: A `str` containing the output filename. :param npoints: An `int` containing the number of points (vertices) used for evaluating the atmospheric conditions. :param workflow: An Enum given by wagl.constants.Workflow. :return: None. Results from each file in `input_targets` are linked into the output file. """ base_group_name = GroupName.ATMOSPHERIC_RESULTS_GRP.value nbar_atmospherics = False sbt_atmospherics = False attributes = [] for fname in input_targets: with h5py.File(fname.path, 'r') as fid: points = list(fid[base_group_name].keys()) # copy across several attributes on the POINT Group # as the linking we do here links direct to a dataset # which will create the required parent Groups for point in points: group = ppjoin(base_group_name, point) attributes.append((point, fid[group].attrs['lonlat'], fid[group].attrs['datetime'], fid[group].attrs['albedos'])) for point in points: for albedo in workflow.albedos: if albedo == Albedos.ALBEDO_TH: datasets = [DatasetName.UPWARD_RADIATION_CHANNEL.value, DatasetName.DOWNWARD_RADIATION_CHANNEL.value] sbt_atmospherics = True else: datasets = [DatasetName.CHANNEL.value, DatasetName.SOLAR_ZENITH_CHANNEL.value] nbar_atmospherics = True grp_path = ppjoin(base_group_name, point, ALBEDO_FMT.format(a=albedo.value)) for dset in datasets: dname = ppjoin(grp_path, dset) create_external_link(fname.path, dname, out_fname, dname) with h5py.File(out_fname) as fid: group = fid[GroupName.ATMOSPHERIC_RESULTS_GRP.value] group.attrs['npoints'] = npoints group.attrs['nbar_atmospherics'] = nbar_atmospherics group.attrs['sbt_atmospherics'] = sbt_atmospherics # assign the lonlat attribute for each POINT Group for point, lonlat, date_time, albedos in attributes: group[point].attrs['lonlat'] = lonlat group[point].attrs['datetime'] = date_time group[point].attrs.create('albedos', data=albedos, dtype=VLEN_STRING)
def plot_spike_shapes(shape_dict, f_handle, meta, info, my_name): """ Plots spike shapes superimposed, together with the appropriate mean :f_handle: list of figure and axes handles :param shape_dict: dictionary with spike shape for all selected repros and stimuli :return: figure handle, maybe """ color_spread = np.linspace(0.8, 0.4, len(shape_dict)) cmapGrey = [cm.Greys(x) for x in color_spread] counter = 0 for k, v in shape_dict.items(): # for spike in range(shape_dict[k].shape[0]): # peak_voltage = np.max(shape_dict[k][spike,:]) # f_handle[1].plot(shape_dict[k][spike,:]-peak_voltage, color=cmapGrey[counter]) avg_spike = np.mean(shape_dict[k], 0) med_spike = np.median(shape_dict[k], 0) peak_avg_voltage = np.max(avg_spike) peak_med_voltage = np.max(med_spike) q25spike = np.percentile(shape_dict[k], 5, 0) q75spike = np.percentile(shape_dict[k], 95, 0) std_spike = np.std(shape_dict[k], 0) # f_handle[1].fill_between(range(0,med_spike.shape[0]), med_spike-peak_med_voltage-q25spike, med_spike-peak_med_voltage + q75spike, color=cmapGrey[counter], alpha=0.2) # f_handle[1].plot(med_spike-peak_med_voltage, color=cmapGrey[counter]) f_handle[1].fill_between(range(0, avg_spike.shape[0]), avg_spike - peak_med_voltage - std_spike, med_spike - peak_med_voltage + std_spike, color=cmapGrey[counter], alpha=0.1) f_handle[1].plot(avg_spike - peak_avg_voltage, color=cmapGrey[counter]) if "SyncPulse" in meta[k][0]["Status"].keys(): f_handle[1].text(35, -5 - counter * 5, "".join([ r'$g_{Vgate}$', ' = ', meta[k][0]["Status"]["gvgate"].split('.')[0], ' ns' ]), fontsize=10, color=cmapGrey[counter]) f_handle[1].text( 60, -5 - counter * 5, "".join([ r'$\tau_{Vgate}$', ' = ', meta[k][0]["Status"]["vgatetau"].split('.')[0], ' ms' ]), fontsize=10, color=cmapGrey[counter]) counter += 1 f_handle[1].set_title(".".join([ "_".join([info["Cell"]["Location"], expfolder, "spike_shape", my_name]), 'pdf' ])) f_handle[1].set_ylim(-40, 0) f_handle[1].set_xlabel("time [ms]") f_handle[1].set_ylabel("relative voltage [mV]") f_handle[1].set_xticklabels(['-1', '0', '1', '2', '3', '4']) f_handle[0].savefig(ppjoin( '../overviewSpikeShape/', ".".join([ "_".join( [info["Cell"]["Location"], expfolder, "spike_shape", my_name]), 'pdf' ])), transparent=True)
""" Extracts the selected FI traces, detects spikes and plots them one over the other, to get average spike shape. Runs from command line in the current folder Input argument is a dictionary with the RePro indexes which you want to present in the same analysis as key. Stimulus current is used as a value example: report_spike_shape.py -d"{'16':'-0.14nA','34':'-0.14nA','40':'-0.14nA'}" """ # get the current working dir wd = getcwd().split(getcwd().split("/")[-1])[0] expfolder = getcwd().split("/")[-1] # get the command arguments ReProList = command_interpreter(sys.argv[1:]) # get the info.dat (_, _, filenames) = next(walk(ppjoin(wd, expfolder))) if "info.dat" in filenames: exp_info = dict(read_info(wd, expfolder)[0]) print(exp_info) else: exp_info = {"Cell": {"Location": "UNLABELED"}} fn_trace = "ficurve-traces.dat" fn_spike = "ficurve-spikes.dat" # compose list of RePro indices to be analysed ReProDict = ReProList['di'] ReProIndex = sorted([k for k, v in ReProDict.items()]) desired_shapes = OrderedDict() spike_shapes = OrderedDict() metka = OrderedDict()
def wht_noise(path_to_folder, subfolder, info): """ Main engine, opens the spike files. :param path_to_folder: folder with recordings :param subfolder: folder containing the experiment/cell :param info: experimental data from info.dat :return fnames: list of figure names to be used in the html build """ # define sample rate FS = 20000 # define the Gauss kernel width (= 1SD) sigma = 0.001 # seconds, from Berman & Maler 1998 # define the input data filename = ppjoin(path_to_folder, subfolder, "stimulus-whitenoise-spikes.dat") # get the file, extract the three data subunits relacs_file = load(filename) # extract RePro indices try: ReProIx = relacs_file.fields[('ReProIndex',)] except: return None # convert set objet into a list ReProIx = list(ReProIx) ReProIx.sort() # define empty list containing figure names fnames = [] for ix in ReProIx: # if relacs file is empty or too short due to aborted RePro presentation # "try:" provides normal termination of the loop instead of error try: metas, _, datas = relacs_file.select({"ReProIndex": ix}) except: return None print("ReProIx", ix, "Iterations", len(metas)) # determine figure handles fig = figure_handles() # FFT is defined as something + 1, due to mlab reasoning nFFT = 2048 FFT = (nFFT/2)+1 # prepare empty variables coh = np.zeros([len(metas), FFT], ) coh_short = np.zeros([len(metas), FFT], ) P_csd = np.zeros([len(metas), FFT], dtype=complex) P_csd_short = np.zeros([len(metas), FFT], dtype=complex) P_psd = np.zeros([len(metas), FFT]) P_psd_short = np.zeros([len(metas), FFT]) H = np.zeros([len(metas), FFT],)# dtype=complex) H_short = np.zeros([len(metas), FFT],)# dtype=complex) MI = np.zeros([len(metas), FFT], ) MI_short = np.zeros([len(metas), FFT], ) # number of stimulus iterations for i in range(0, len(metas)): color_spread = np.linspace(0.35,0.8, len(metas)) cmap = [ cm.Greys(x) for x in color_spread ] # extract meta infos wnFname = metas[i]["envelope"] wnDur = float(metas[i]["Settings"]["Waveform"]["duration"].split("m")[0]) # duration in miliseconds # conversions spikes = np.array(datas[i]) # conversions wnDur /= 1000 # conversion to miliseconds spikes /= 1000 print(spikes.shape) convolved_Train, _ = train_convolve(spikes, sigma, FS, wnDur) print(sum(convolved_Train)/FS) wNoise = process_wn(path_to_folder, wnFname, len(convolved_Train)) # compute coherence, mutual information, transfer and the power spectra and cross-spectra density freq, coh[i,:], coh_short[i,:], H[i,:], H_short[i,:], MI[i,:], MI_short[i,:], \ P_csd[i,:], P_csd_short[i,:], P_psd[i,:], P_psd_short[i,:] \ = cohere_transfere_MI (convolved_Train, wNoise, nFFT, FS) # plot coherence, mutual information etc.... plot_the_lot(fig, freq, coh, coh_short, MI, MI_short, H, H_short, metas, cmap, np.array(datas)) avgCoh, avgCoh_short, avgH, avgH_short, mut_inf, mut_inf_short = compute_avgs(coh, coh_short, H, H_short, MI, MI_short) plot_the_lot(fig, freq, avgCoh, avgCoh_short, mut_inf, mut_inf_short, avgH, avgH_short, metas, cmap = [cm.Reds(0.6)],raster='empty', annotation=True) fig[2].text(0.05, 0.95, " ".join(['Species:', info["Subject"]["Species"]]), transform=fig[1].transAxes, fontsize=10) fig[2].text(0.05, 0.90, " ".join(['ELL Segment:', info["Cell"]["Location"]]), transform=fig[1].transAxes, fontsize=10) # define file name filename = "".join([str(metas[i]["ReProIndex"]), '_', 'whitenoise']) fnames.append(filename) fig[0].savefig(ppjoin(path_to_folder, subfolder, ".".join([filename, 'png'])), transparent=True) fig[0].savefig(ppjoin(path_to_folder, subfolder, ".".join([filename, 'svg'])), transparent=True) plt.close() return fnames
def noise_transfer(tests, wd, expfolder): """" Plots the transfer and coherence curve into the graphic file (*.png, *.svg) Requires: :param tests: dictionary containing experimental conditions as keys and ReProIx as values :param wd: location of the experimental folder :param expfolder: name of the experimental folder Outputs: graphic files containing coherence, MI and transfer """ # define sample rate FS = 20000 # define the Gauss kernel width (= 1SD) sigma = 0.001 # seconds, from Berman & Maler 1998 # define the input data filename = ppjoin(wd, expfolder, "stimulus-whitenoise-spikes.dat") # read in some experimental data (_, _, filenames) = next(walk(ppjoin(wd, expfolder))) if "info.dat" in filenames: exp_info = dict(read_info(wd, expfolder)[0]) print(exp_info) else: exp_info = {"Cell": {"Location": "UNLABELED"}} # load data relacs_file = load(filename) # four panel figure FHandles = figure_handles() # define colors color_spread = np.linspace(0.5, 0.9, len(tests)) cmap = [cm.Greys(x) for x in color_spread] # cmap = [ cm.viridis(x) for x in color_spread ] # color counter col_count = 0 # FFT is defined as something + 1, due to mlab reasoning nFFT = 1024 FFT = (nFFT / 2) + 1 # define spike dict for the raster plot spike_dict = OrderedDict() for k, v in tests.items(): # get RePro indexes of the same experimental condition ReProIx = tests[k] # define empty variables, containing traces of the same experiments, different RePros coh_repro = np.zeros([len(ReProIx), FFT]) coh_repro_short = np.zeros([len(ReProIx), FFT]) H_repro = np.zeros([len(ReProIx), FFT]) H_repro_short = np.zeros([len(ReProIx), FFT]) MI_repro = np.zeros([len(ReProIx), FFT]) MI_repro_short = np.zeros([len(ReProIx), FFT]) spike_list = [] # define/reset counter counter = 0 # iteration of same experimental condition for ix in ReProIx: try: metas, _, datas = relacs_file.select({"ReProIndex": ix}) except: return None # define empty variables coh = np.zeros([len(metas), FFT]) coh_short = np.zeros([len(metas), FFT]) P_csd = np.zeros([len(metas), FFT], dtype=complex) P_csd_short = np.zeros([len(metas), FFT], dtype=complex) P_psd = np.zeros([len(metas), FFT]) P_psd_short = np.zeros([len(metas), FFT]) H = np.zeros([len(metas), FFT]) H_short = np.zeros([len(metas), FFT]) MI = np.zeros([len(metas), FFT]) MI_short = np.zeros([len(metas), FFT]) meta_repros = [] # number of stimulus iterations for i in range(0, len(metas)): # extract meta infos wnFname = metas[i]["envelope"] wnDur = float( metas[i]["Settings"]["Waveform"]["duration"].split( "m")[0]) # duration in miliseconds # spikes spikes = np.array(datas[i]) # conversions wnDur /= 1000 # conversion to miliseconds spikes /= 1000 print(spikes.shape) convolved_Train, _ = train_convolve(spikes, sigma, FS, wnDur) print(sum(convolved_Train) / FS) wNoise = process_wn(wd, wnFname, len(convolved_Train)) # compute coherence, mutual information, transfer and the power spectra and cross-spectra density freq, coh[i,:], coh_short[i,:], H[i,:], H_short[i,:], MI[i,:], MI_short[i,:], \ P_csd[i,:], P_csd_short[i,:], P_psd[i,:], P_psd_short[i,:] \ = cohere_transfere_MI (convolved_Train, wNoise, nFFT, FS) # compute averages over iterations of the *same* repro coh_repro[counter,:], coh_repro_short[counter,:], \ H_repro[counter,:], H_repro_short[counter,:], \ MI_repro[counter,:], MI_repro_short[counter,:] = compute_avgs(coh, coh_short, H, H_short, MI, MI_short) # store one of the metas meta_repros.append(metas[0]) # store all the spikes from the same type of experiment spike_list.append(datas) counter = counter + 1 # plot the lot plot_the_lot(FHandles, freq, coh_repro, coh_repro_short, MI_repro, MI_repro_short, H_repro, H_repro_short, metas, cmap=[cmap[col_count]], raster='empty', annotation=False, comparison=True) # compute the average of the different repro presentations (with same test conditions) avgCoh, avgCoh_short, avgH, avgH_short, avgMI, avgMI_short = compute_avgs( coh_repro, coh_repro_short, H_repro, H_repro_short, MI_repro, MI_repro_short) # plot the lot plot_the_lot(FHandles, freq, avgCoh, avgCoh_short, avgMI, avgMI_short, avgH, avgH_short, meta_repros, cmap=[cmap[col_count]], raster='empty', annotation=False, comparison=True) # provide gVgate values FHandles[1].text(0.5, 0.8 - 0.05 * col_count, " ".join([ r'$g_{Vgate}$', ' = ', meta_repros[0]["Status"]["gvgate"].split(".")[0], 'nS' ]), color=cmap[col_count], transform=FHandles[1].transAxes, fontsize=10) FHandles[5].text( 0.5, 0.8 - 0.05 * col_count, " ".join([ r'$\tau_{Vgate}$', ' = ', meta_repros[0]["Status"]["vgatetau"].split(".")[0], 'ms' ]), color=cmap[col_count], transform=FHandles[5].transAxes, fontsize=10) # update spike dictionary spike_dict[k] = spike_list # update the color counter col_count += 1 # write FFT value FHandles[1].text(0.05, 0.90, " ".join(['FFT = ', str(nFFT)]), color='k', transform=FHandles[1].transAxes, fontsize=10) # plot raster plot spike_iter_count = 0 for i, k in enumerate(spike_dict): for j in range(len(spike_dict[k])): for gnj in range(len(spike_dict[k][j])): FHandles[7].plot(spike_dict[k][j][gnj], np.zeros(len(spike_dict[k][j][gnj])) + spike_iter_count, '|', color=cmap[i], ms=12) spike_iter_count += 1 FHandles[7].set_title( ppjoin(".".join([ exp_info["Cell"]["Location"], ':', expfolder, "dyn_noise_transfer", '_'.join([k for k, v in tests.items()]), '_'.join([ str(x) for sublist in [v for k, v in tests.items()] for x in sublist ]), "fft", str(nFFT) ]))) # Save figures FHandles[0].savefig(ppjoin(".".join([ expfolder, "coherence_transfer", '.'.join([k for k, v in tests.items()]), '.'.join([ str(x) for sublist in [v for k, v in tests.items()] for x in sublist ]), "fft", str(nFFT), 'svg' ])), transparent=True) FHandles[0].savefig(ppjoin(".".join([ expfolder, "coherence_transfer", '.'.join([k for k, v in tests.items()]), ".".join([ str(x) for sublist in [v for k, v in tests.items()] for x in sublist ]), "fft", str(nFFT), 'png' ])), transparent=True) # save figures into dedicated folder if necessary FHandles[0].savefig(ppjoin( '../overviewTransfer/', ".".join([ "_".join([ exp_info["Cell"]["Location"], expfolder, "coherence_transfer", "".join([k for k, v in tests.items()]), "_".join([ str(x) for sublist in [v for k, v in tests.items()] for x in sublist ]), "fft", str(nFFT) ]), 'pdf' ])), transparent=True)