def import_mpp_data_channel(folder, channel_pattern='ch-<>'): """ Imports MPP data from multiple channels. :param folder: Folder containing channels. :param channel_pattern: Pattern for channel folders. [Default: 'ch-<>'] :returns: tuple of ( voc, jv, mpp ) Pandas DataFrames by cycle and channel. """ # get scan folders channels = os.listdir(folder) # get data for each scan vocs = [] jvs = [] mpps = [] for c_dir in channels: ch_path = os.path.join(folder, c_dir) voc, jv, mpp = import_mpp_datum_channel( # import scan data ch_path) vocs.append(voc) jvs.append(jv) mpps.append(mpp) vocs = std.common_reindex(vocs) jvs = std.common_reindex(jvs) mpps = std.common_reindex(mpps) vocs = pd.concat(vocs, axis=1).sort_index(axis=1) jvs = pd.concat(jvs, axis=1).sort_index(axis=1) mpps = pd.concat(mpps, axis=1).sort_index(axis=1) return (vocs, jvs, mpps)
def import_mpp_cycle_datum_channel(folder, cycle_pattern='cycle-<>', channel_pattern='ch-<>'): """ Imports MPP tracking data from an MPP with JV program, broken in to cycles. :param folder: Folder path containing cycles. :param cycle_pattern: Pattern for cycle folders. [Default: 'cycle-<>'] :param channel_pattern: Pattern for channel folders. [Default: 'ch-<>'] :returns: Tuple of ( voc, jv, mpp ) Pandas DataFrames by cycle. """ # get scan folders cycles = os.listdir(folder) # get data for each scan vocs = [] jvs = [] mpps = [] for cycle in cycles: cycle_path = os.path.join(folder, cycle) dfs = (voc, jv, mpp) = import_mpp_datum_channel(cycle_path) # import scan data # add scan index cycle_id = int( std.metadata_from_file_name(cycle_pattern, cycle_path, full_path=True, is_numeric=True)) for df in dfs: # channel already in headers std.insert_index_levels(df, cycle_id, 'cycle', key_level=1) vocs.append(voc) jvs.append(jv) mpps.append(mpp) vocs = std.common_reindex(vocs) jvs = std.common_reindex(jvs) mpps = std.common_reindex(mpps) vocs = pd.concat(vocs, axis=1).sort_index(axis=1) jvs = pd.concat(jvs, axis=1).sort_index(axis=1) mpps = pd.concat(mpps, axis=1).sort_index(axis=1) return (vocs, jvs, mpps)
def import_data( folder_path, file_pattern = '*.sIV', metadata = None, interpolate = 'linear', fillna = 0 ): """ Imports data from Andor output files :param folder_path: The file path containing the data files :param file_pattern: A glob pattern to filter the imported files [Default: '*'] :param metadata: Metadata from the file name is turned into MultiIndex columns. + If list, use standard keywords to include in index [ 'sample', 'power', 'wavelength', 'time' ] + If Dictionary, keys indicate level name, value is pattern to match + Reseserved key 'standard' can be provided with a list value to get standard metadata :param interpolate: How to interpolate data for a common index [Default: linear] Use None to prevent reindexing :param fillna: Value to fill NaN values with [Default: 0] :returns: A Pandas DataFrame with MultiIndexed columns """ # get dataframes from files df = [] files = std.get_files( folder_path, file_pattern ) for file in files: data = import_datum( file ) # run local import datum function df.append( data ) if interpolate is not None: df = std.common_reindex( df, how = 'interpolate', fillna = fillna, add_values = [ 0 ] ) df = pd.concat( df, axis = 1 ) return df
def import_voc_datum(file, reindex=True): """ Imports a Voc file with channel headers. :param file: Path to file. :param reindex: Reindex by time. [Default: True] :returns: Pandas DataFrame. """ parameters = {'Time [s]': 'time', 'Voltage [V]': 'voltage'} df = import_datum(file, parameters) if reindex: data = [] for ch, datum in df.groupby(level='channel', axis=1): datum = datum.dropna(subset=[(ch, 'time')]) datum.set_index((ch, 'time'), inplace=True) datum.index = datum.index.rename('time') data.append(datum) data = std.common_reindex(data, fillna=np.NaN) df = pd.concat(data, axis=1) df.columns = df.columns.droplevel('metrics') return df
def import_jv_datum(file, skip_rows=0, reindex=True, split_direction=True): """ Imports a JV file with channel headers. :param file: Path to file. :param skip_rows: Skip rows. [Default: 0] :param reindex: Reindex by voltage. [Default: True] :param split_direction: Split scans by direction. [Default: True] :returns: Pandas DataFrame. """ parameters = { 'Voltage [V]': 'voltage', 'Current [A]': 'current', 'Power [W]': 'power' } df = import_datum(file, parameters) df = df[skip_rows:] if reindex: data = [] for ch, datum in df.groupby(level='channel', axis=1): if split_direction: datum = jvdp.split_jv_scan(datum) data.append(datum) data = std.common_reindex(data, fillna=np.NaN) df = pd.concat(data, axis=1) return df
def hysteresis_area(df): """ Computes the hysteresis area for a scan. :param df: A JV scan DataFrame. :returns: A Series of areas. """ areas = [] for name, data in df.groupby(level=['sample', 'index'], axis=1): forward = data.xs('forward', level='direction', axis=1, drop_level=False).dropna() reverse = data.xs( 'reverse', level='direction', axis=1, drop_level=False).dropna().iloc[::1] # reverse index order # save headers index_for = forward.columns index_rev = reverse.columns # strip headers forward = std.keep_levels(forward, 'metric') reverse = std.keep_levels(reverse, 'metric') # create common index for scan directions (forward, reverse) = std.common_reindex([forward, reverse], index='voltage') curr_for = forward.values[:, 0] volt_for = forward.index.values curr_rev = reverse.values[:, 0] volt_rev = reverse.index.values area_for = abs(np.trapz(y=curr_for, x=volt_for)) area_rev = abs(np.trapz(y=curr_rev, x=volt_rev)) area_hist = np.sqrt( np.trapz(y=np.square(curr_rev - curr_for), x=volt_for)) area = pd.Series( { 'forward': area_for, 'reverse': area_rev, 'hyst_abs': area_hist, 'hyst_rel': area_hist / max(area_for, area_rev) }, name=name) areas.append(area) areas = pd.concat(areas, axis=1) areas.columns.set_names(['sample', 'index'], inplace=True) areas.index.set_names('metric', inplace=True) areas = areas.stack(level='index').unstack( 0) # move index as index, metrics as columns return areas
def import_datum(file, reindex=True, sep=','): """ Imports a .txt file from QSense. :param file: The file path to load. :param reindex: Set time as DataFrame Index. [Default: True] :param sep: The data seperator. [Default: ,] :param :returns: A Pandas DataFrame. """ # parse header time_pattern = 'Time_(\d+)' # used to get channel dissipation_pattern = 'D(\d+)_(\d+)' frequency_pattern = 'f(\d+)_(\d+)' with open(file) as f: header = f.readline().split(sep) time_matches = [re.match(time_pattern, head) for head in header] diss_matches = [re.match(dissipation_pattern, head) for head in header] freq_matches = [re.match(frequency_pattern, head) for head in header] headers = [0] * len(header) # df headers: ( channel, parameter, mode ) for i in range(len(header)): time_match = time_matches[i] diss_match = diss_matches[i] freq_match = freq_matches[i] if time_match is not None: headers[i] = (time_match.group(1), 'time', 0) elif diss_match is not None: headers[i] = (diss_match.group(2), 'dissipation', int(diss_match.group(1))) elif freq_match is not None: headers[i] = (freq_match.group(2), 'frequency', int(freq_match.group(1))) df = pd.read_csv(file, skiprows=0) df.columns = pd.MultiIndex.from_tuples( headers, names=['channel', 'parameter', 'mode']) if reindex: channels = [] # split data by channel for name, data in df.groupby(level='channel', axis=1): channel = name[0] data = data.set_index((channel, 'time', 0)) data.index.rename('time', inplace=True) channels.append(data) channels = std.common_reindex(channels) df = pd.concat(channels, axis=1) return df
def import_jv_datum(file, channel_index='ch<>', by_scan=True): """ Imports JV datum from the given file. :param file: File path. :param channel_index: Add channel from file path as index level. Uses value as pattern in standard_functions#metadata_from_file_name. None if channel should be excluded. [Default: 'ch<>'] :param by_scan: Breaks data into forward and reverse scans, and sets the index to voltage. [Default: True] :returns: Pandas DataFrame. :raises ValueError: If multiple sign changes in the scan are detected. """ header = ['voltage', 'current', 'power'] df = pd.read_csv(file, names=header, skiprows=1) if by_scan: # detect direction change in votlage scan dv = df.voltage.diff() change = np.nan_to_num(np.diff(np.sign(dv))) # calculate sign changes change = np.where(change != 0)[0] # get indices of sign changes if change.size > 1: # more than one sign change raise ValueError('Multiple sign changes detected in scan.') change = change[0] # break scans apart forward_first = (dv[0] > 0) df = ([df[:(change + 1)], df[change:]] if forward_first else [df[change:], df[:(change + 1)]]) for index, tdf in enumerate(df): # set index tdf.set_index('voltage', inplace=True) tdf.columns.rename('metrics', inplace=True) # create multi-index name = 'forward' if (index == 0) else 'reverse' tdf = std.add_index_levels(tdf, {name: ['current', 'power']}, names=['direction']) df[index] = tdf # replace with modified # reindex for common voltage values df = std.common_reindex(df) # combine scan directions df = pd.concat(df, axis=1) if channel_index is not None: ch = channel_from_file_path(file, channel_index) df = std.insert_index_levels(df, ch, 'channel') return df
def import_mpp_cycle_data(folder, voc_kwargs={}, jv_kwargs={}, mpp_kwargs={}): """ Import MPP data for multiple cycles. :param folder: Path to folder containing cycle data. :param voc_kwargs: Dictionary of keyword arguments passed to import_voc_datum(). [Default: {}] :param jv_kwargs: Dictionary of keyword arguments passed to import_jv_datum(). [Default: {}] :param mpp_kwargs: Dictionary of keyword arguments passed to import_mpp_datum(). [Default: {}] :returns: Tuple of ( voc, jv, mpp ) DataFrames. """ # get scan folders cycles = os.listdir(folder) # get data for each scan vocs = [] jvs = [] mpps = [] for cy_dir in cycles: cy_path = os.path.join(folder, cy_dir) voc, jv, mpp = import_mpp_cycle_datum(cy_path, voc_kwargs, jv_kwargs, mpp_kwargs) # import cycle data vocs.append(voc) jvs.append(jv) mpps.append(mpp) vocs = std.common_reindex(vocs) jvs = std.common_reindex(jvs) mpps = std.common_reindex(mpps) vocs = pd.concat(vocs, axis=1).sort_index(axis=1) jvs = pd.concat(jvs, axis=1).sort_index(axis=1) mpps = pd.concat(mpps, axis=1).sort_index(axis=1) return (vocs, jvs, mpps)
def import_data( folder_path, file_pattern = '*.txt', metadata = None, time = False, cps = False, interpolate = 'linear', fillna = 0 ): """ Imports data from Andor output files :param folder_path: The file path containing the data files :param file_pattern: A glob pattern to filter the imported files [Default: '*'] :param metadata: Metadata from the file name is turned into MultiIndex columns. + If list, use standard keywords to include in index [ 'sample', 'power', 'wavelength', 'time', 'temperature' ] + If Dictionary, keys indicate level name, value is pattern to match + Reseserved key 'standard' can be provided with a list value to get standard metada :param time: Read the integration time in from the file as a header. [Default: False] :param cps: Converts the data to counts per second. A valid time string of the form XsX must be present. :param interpolate: How to interpolate data for a common index [Default: linear] Use None to prevent reindexing :param fillna: Value to fill NaN values with [Default: 0] :param reindex: Reindex the DataFrame using :returns: A Pandas DataFrame with MultiIndexed columns :raises: RuntimeError if no files are found """ # get dataframes from files df = [] files = std.get_files( folder_path, file_pattern ) if len( files ) == 0: # no files found raise RuntimeError( 'No files found matching {}'.format( os.path.join( folder_path, file_pattern ) ) ) for file in files: data = import_datum( file, metadata = metadata, cps = cps, time = time ) # run local import datum function df.append( data ) if interpolate is not None: df = std.common_reindex( df, how = interpolate, fillna = fillna ) df = pd.concat( df, axis = 1 ) return df
def correct_spectra(df, correction): """ Applies a correction spectral data. :param df: The Pandas DataFrame to correct. :param correction: The correction data to apply. Should be a tuple of ( camera, grating ) Camera values are [ 'idus' ] Grating values are [ '300', '600' ] :returns: The corrected data. """ data_path = os.path.join(os.path.dirname(__file__), 'data', 'andor-corrections.pkl') cdf = pd.read_pickle(data_path) corrections = cdf.xs(('grating', *correction), axis=1) cdf = std.common_reindex([df, corrections]) corrections = cdf[1].reindex(df.index) return df.multiply(corrections, axis=0)
def import_data( folder_path, file_pattern = '*.csv', metadata = None, interpolate = 'linear', fillna = 0 ): """ Imports data from TimeSpec II experiments output files. :param folder_path: The file path containing the data files :param file_pattern: A glob pattern to filter the imported files [Default: '*'] :param metadata: Metadata from the file name is turned into MultiIndex columns. [Default: None] :param interpolate: How to interpolate data for a common index [Default: linear] Use None to prevent reindexing :param fillna: Value to fill NaN values with [Default: 0] :returns: A Pandas DataFrame with MultiIndexed columns :raises: RuntimeError if no files are found """ # get dataframes from files df = [] files = std.get_files( folder_path, file_pattern ) if len( files ) == 0: # no files found raise RuntimeError( 'No files found matching {}'.format( os.path.join( folder_path, file_pattern ) ) ) for file in files: data = import_datum( file, metadata = metadata ) # run local import datum function df.append( data ) if interpolate is not None: df = std.common_reindex( df, how = interpolate, fillna = fillna ) df = pd.concat( df, axis = 1 ).sort_index( axis = 1 ) return df
def import_mpp_tracking_datum(file, reindex=True, drop_cycle=True): """ Imports a MPP Tracking file with channel headers. :param file: Path to file. :param reindex: Reindex by time. [Default: True] :param drop_cycle: Drop cycle columns. [Default: True] :returns: Pandas DataFrame. """ parameters = { 'Time [s]': 'time', 'Voltage [V]': 'voltage', 'Current [A]': 'current', 'Power [W]': 'power', 'Cycle': 'cycle' } df = import_datum(file, parameters) if drop_cycle: df = df.drop('cycle', axis=1, level='metrics') if reindex: data = [] for ch, datum in df.groupby(level='channel', axis=1): datum = datum.dropna(subset=[(ch, 'time')]) datum.set_index((ch, 'time'), inplace=True) datum.index = datum.index.rename('time') datum = datum.mean( level=0) # take mean over duplicate index values data.append(datum) data = std.common_reindex(data, fillna=np.NaN) df = pd.concat(data, axis=1) return df
def import_datum( path, **kwargs ): """ Imports a Cary UV Vis Absorption data file into a Pandas DataFrame. :param path: Path to the file. :returns: Pandas DataFrame. """ data_end = None fields = None metrics = None with open( path ) as f: for ( index, line ) in enumerate( f ): # search for end of data # occurs at first blank line # every line ends with trailing comma if index is 0: fields = line.split( ',' )[ :-1 ] elif index is 1: # get headers metrics = line.split( ',' )[ :-1 ] elif line is '\n': # no data in line data_end = index break # parse metrics for sample width # new samples begin with wavelength sample_indices = [] for index, metric in enumerate( metrics ): if metric == 'Wavelength (nm)': sample_indices.append( index ) # import samples individually df = [] for ( i, sample_index ) in enumerate( sample_indices ): # get sample columns next_sample_index = ( # get next sample index, or end if last sample sample_indices[ i + 1 ] if ( i + 1 ) < len( sample_indices ) else len( metrics ) # final sample ) use_cols = range( sample_index, next_sample_index ) # get names of fields # ignore first as it is wavelength # will be set to index names = map( str.lower, metrics[ sample_index + 1 : next_sample_index ] ) tdf = pd.read_csv( path, header = None, skiprows = 2, nrows = data_end - 2, # account for ignored headers usecols = use_cols, index_col = 0, # wavelength as index names = [ 'wavelength', *names ], **kwargs ).dropna() # add sample name to header tdf = std.insert_index_levels( tdf, fields[ sample_index ] ) df.append( tdf ) std.common_reindex( df ) df = pd.concat( df, axis = 1 ) return df