def import_mpp_data_channel(folder, channel_pattern='ch-<>'):
"""
Imports MPP data from multiple channels.
:param folder: Folder containing channels.
:param channel_pattern: Pattern for channel folders. [Default: 'ch-<>']
:returns: tuple of ( voc, jv, mpp ) Pandas DataFrames by cycle and channel.
"""
# get scan folders
channels = os.listdir(folder)
# get data for each scan
vocs = []
jvs = []
mpps = []
for c_dir in channels:
ch_path = os.path.join(folder, c_dir)
voc, jv, mpp = import_mpp_datum_channel( # import scan data
ch_path)
vocs.append(voc)
jvs.append(jv)
mpps.append(mpp)
vocs = std.common_reindex(vocs)
jvs = std.common_reindex(jvs)
mpps = std.common_reindex(mpps)
vocs = pd.concat(vocs, axis=1).sort_index(axis=1)
jvs = pd.concat(jvs, axis=1).sort_index(axis=1)
mpps = pd.concat(mpps, axis=1).sort_index(axis=1)
return (vocs, jvs, mpps)
def import_mpp_cycle_datum_channel(folder,
cycle_pattern='cycle-<>',
channel_pattern='ch-<>'):
"""
Imports MPP tracking data from an MPP with JV program, broken in to cycles.
:param folder: Folder path containing cycles.
:param cycle_pattern: Pattern for cycle folders. [Default: 'cycle-<>']
:param channel_pattern: Pattern for channel folders. [Default: 'ch-<>']
:returns: Tuple of ( voc, jv, mpp ) Pandas DataFrames by cycle.
"""
# get scan folders
cycles = os.listdir(folder)
# get data for each scan
vocs = []
jvs = []
mpps = []
for cycle in cycles:
cycle_path = os.path.join(folder, cycle)
dfs = (voc, jv,
mpp) = import_mpp_datum_channel(cycle_path) # import scan data
# add scan index
cycle_id = int(
std.metadata_from_file_name(cycle_pattern,
cycle_path,
full_path=True,
is_numeric=True))
for df in dfs:
# channel already in headers
std.insert_index_levels(df, cycle_id, 'cycle', key_level=1)
vocs.append(voc)
jvs.append(jv)
mpps.append(mpp)
vocs = std.common_reindex(vocs)
jvs = std.common_reindex(jvs)
mpps = std.common_reindex(mpps)
vocs = pd.concat(vocs, axis=1).sort_index(axis=1)
jvs = pd.concat(jvs, axis=1).sort_index(axis=1)
mpps = pd.concat(mpps, axis=1).sort_index(axis=1)
return (vocs, jvs, mpps)
def import_data( folder_path, file_pattern = '*.sIV', metadata = None, interpolate = 'linear', fillna = 0 ):
"""
Imports data from Andor output files
:param folder_path: The file path containing the data files
:param file_pattern: A glob pattern to filter the imported files [Default: '*']
:param metadata: Metadata from the file name is turned into MultiIndex columns.
+ If list, use standard keywords to include in index [ 'sample', 'power', 'wavelength', 'time' ]
+ If Dictionary, keys indicate level name, value is pattern to match
+ Reseserved key 'standard' can be provided with a list value to get standard metadata
:param interpolate: How to interpolate data for a common index [Default: linear]
Use None to prevent reindexing
:param fillna: Value to fill NaN values with [Default: 0]
:returns: A Pandas DataFrame with MultiIndexed columns
"""
# get dataframes from files
df = []
files = std.get_files( folder_path, file_pattern )
for file in files:
data = import_datum( file ) # run local import datum function
df.append( data )
if interpolate is not None:
df = std.common_reindex( df, how = 'interpolate', fillna = fillna, add_values = [ 0 ] )
df = pd.concat( df, axis = 1 )
return df
def import_voc_datum(file, reindex=True):
"""
Imports a Voc file with channel headers.
:param file: Path to file.
:param reindex: Reindex by time. [Default: True]
:returns: Pandas DataFrame.
"""
parameters = {'Time [s]': 'time', 'Voltage [V]': 'voltage'}
df = import_datum(file, parameters)
if reindex:
data = []
for ch, datum in df.groupby(level='channel', axis=1):
datum = datum.dropna(subset=[(ch, 'time')])
datum.set_index((ch, 'time'), inplace=True)
datum.index = datum.index.rename('time')
data.append(datum)
data = std.common_reindex(data, fillna=np.NaN)
df = pd.concat(data, axis=1)
df.columns = df.columns.droplevel('metrics')
return df
def import_jv_datum(file, skip_rows=0, reindex=True, split_direction=True):
"""
Imports a JV file with channel headers.
:param file: Path to file.
:param skip_rows: Skip rows. [Default: 0]
:param reindex: Reindex by voltage. [Default: True]
:param split_direction: Split scans by direction. [Default: True]
:returns: Pandas DataFrame.
"""
parameters = {
'Voltage [V]': 'voltage',
'Current [A]': 'current',
'Power [W]': 'power'
}
df = import_datum(file, parameters)
df = df[skip_rows:]
if reindex:
data = []
for ch, datum in df.groupby(level='channel', axis=1):
if split_direction:
datum = jvdp.split_jv_scan(datum)
data.append(datum)
data = std.common_reindex(data, fillna=np.NaN)
df = pd.concat(data, axis=1)
return df
def hysteresis_area(df):
"""
Computes the hysteresis area for a scan.
:param df: A JV scan DataFrame.
:returns: A Series of areas.
"""
areas = []
for name, data in df.groupby(level=['sample', 'index'], axis=1):
forward = data.xs('forward',
level='direction',
axis=1,
drop_level=False).dropna()
reverse = data.xs(
'reverse', level='direction', axis=1,
drop_level=False).dropna().iloc[::1] # reverse index order
# save headers
index_for = forward.columns
index_rev = reverse.columns
# strip headers
forward = std.keep_levels(forward, 'metric')
reverse = std.keep_levels(reverse, 'metric')
# create common index for scan directions
(forward, reverse) = std.common_reindex([forward, reverse],
index='voltage')
curr_for = forward.values[:, 0]
volt_for = forward.index.values
curr_rev = reverse.values[:, 0]
volt_rev = reverse.index.values
area_for = abs(np.trapz(y=curr_for, x=volt_for))
area_rev = abs(np.trapz(y=curr_rev, x=volt_rev))
area_hist = np.sqrt(
np.trapz(y=np.square(curr_rev - curr_for), x=volt_for))
area = pd.Series(
{
'forward': area_for,
'reverse': area_rev,
'hyst_abs': area_hist,
'hyst_rel': area_hist / max(area_for, area_rev)
},
name=name)
areas.append(area)
areas = pd.concat(areas, axis=1)
areas.columns.set_names(['sample', 'index'], inplace=True)
areas.index.set_names('metric', inplace=True)
areas = areas.stack(level='index').unstack(
0) # move index as index, metrics as columns
return areas
def import_datum(file, reindex=True, sep=','):
"""
Imports a .txt file from QSense.
:param file: The file path to load.
:param reindex: Set time as DataFrame Index. [Default: True]
:param sep: The data seperator. [Default: ,]
:param
:returns: A Pandas DataFrame.
"""
# parse header
time_pattern = 'Time_(\d+)' # used to get channel
dissipation_pattern = 'D(\d+)_(\d+)'
frequency_pattern = 'f(\d+)_(\d+)'
with open(file) as f:
header = f.readline().split(sep)
time_matches = [re.match(time_pattern, head) for head in header]
diss_matches = [re.match(dissipation_pattern, head) for head in header]
freq_matches = [re.match(frequency_pattern, head) for head in header]
headers = [0] * len(header) # df headers: ( channel, parameter, mode )
for i in range(len(header)):
time_match = time_matches[i]
diss_match = diss_matches[i]
freq_match = freq_matches[i]
if time_match is not None:
headers[i] = (time_match.group(1), 'time', 0)
elif diss_match is not None:
headers[i] = (diss_match.group(2), 'dissipation',
int(diss_match.group(1)))
elif freq_match is not None:
headers[i] = (freq_match.group(2), 'frequency',
int(freq_match.group(1)))
df = pd.read_csv(file, skiprows=0)
df.columns = pd.MultiIndex.from_tuples(
headers, names=['channel', 'parameter', 'mode'])
if reindex:
channels = []
# split data by channel
for name, data in df.groupby(level='channel', axis=1):
channel = name[0]
data = data.set_index((channel, 'time', 0))
data.index.rename('time', inplace=True)
channels.append(data)
channels = std.common_reindex(channels)
df = pd.concat(channels, axis=1)
return df
def import_jv_datum(file, channel_index='ch<>', by_scan=True):
"""
Imports JV datum from the given file.
:param file: File path.
:param channel_index: Add channel from file path as index level.
Uses value as pattern in standard_functions#metadata_from_file_name.
None if channel should be excluded.
[Default: 'ch<>']
:param by_scan: Breaks data into forward and reverse scans, and sets the index to voltage. [Default: True]
:returns: Pandas DataFrame.
:raises ValueError: If multiple sign changes in the scan are detected.
"""
header = ['voltage', 'current', 'power']
df = pd.read_csv(file, names=header, skiprows=1)
if by_scan:
# detect direction change in votlage scan
dv = df.voltage.diff()
change = np.nan_to_num(np.diff(np.sign(dv))) # calculate sign changes
change = np.where(change != 0)[0] # get indices of sign changes
if change.size > 1:
# more than one sign change
raise ValueError('Multiple sign changes detected in scan.')
change = change[0]
# break scans apart
forward_first = (dv[0] > 0)
df = ([df[:(change + 1)], df[change:]]
if forward_first else [df[change:], df[:(change + 1)]])
for index, tdf in enumerate(df):
# set index
tdf.set_index('voltage', inplace=True)
tdf.columns.rename('metrics', inplace=True)
# create multi-index
name = 'forward' if (index == 0) else 'reverse'
tdf = std.add_index_levels(tdf, {name: ['current', 'power']},
names=['direction'])
df[index] = tdf # replace with modified
# reindex for common voltage values
df = std.common_reindex(df)
# combine scan directions
df = pd.concat(df, axis=1)
if channel_index is not None:
ch = channel_from_file_path(file, channel_index)
df = std.insert_index_levels(df, ch, 'channel')
return df
def import_mpp_cycle_data(folder, voc_kwargs={}, jv_kwargs={}, mpp_kwargs={}):
"""
Import MPP data for multiple cycles.
:param folder: Path to folder containing cycle data.
:param voc_kwargs: Dictionary of keyword arguments passed to import_voc_datum().
[Default: {}]
:param jv_kwargs: Dictionary of keyword arguments passed to import_jv_datum().
[Default: {}]
:param mpp_kwargs: Dictionary of keyword arguments passed to import_mpp_datum().
[Default: {}]
:returns: Tuple of ( voc, jv, mpp ) DataFrames.
"""
# get scan folders
cycles = os.listdir(folder)
# get data for each scan
vocs = []
jvs = []
mpps = []
for cy_dir in cycles:
cy_path = os.path.join(folder, cy_dir)
voc, jv, mpp = import_mpp_cycle_datum(cy_path, voc_kwargs, jv_kwargs,
mpp_kwargs) # import cycle data
vocs.append(voc)
jvs.append(jv)
mpps.append(mpp)
vocs = std.common_reindex(vocs)
jvs = std.common_reindex(jvs)
mpps = std.common_reindex(mpps)
vocs = pd.concat(vocs, axis=1).sort_index(axis=1)
jvs = pd.concat(jvs, axis=1).sort_index(axis=1)
mpps = pd.concat(mpps, axis=1).sort_index(axis=1)
return (vocs, jvs, mpps)
def import_data(
folder_path,
file_pattern = '*.txt',
metadata = None,
time = False,
cps = False,
interpolate = 'linear',
fillna = 0
):
"""
Imports data from Andor output files
:param folder_path: The file path containing the data files
:param file_pattern: A glob pattern to filter the imported files [Default: '*']
:param metadata: Metadata from the file name is turned into MultiIndex columns.
+ If list, use standard keywords to include in index
[ 'sample', 'power', 'wavelength', 'time', 'temperature' ]
+ If Dictionary, keys indicate level name, value is pattern to match
+ Reseserved key 'standard' can be provided with a list value to get standard metada
:param time: Read the integration time in from the file as a header. [Default: False]
:param cps: Converts the data to counts per second.
A valid time string of the form XsX must be present.
:param interpolate: How to interpolate data for a common index [Default: linear]
Use None to prevent reindexing
:param fillna: Value to fill NaN values with [Default: 0]
:param reindex: Reindex the DataFrame using
:returns: A Pandas DataFrame with MultiIndexed columns
:raises: RuntimeError if no files are found
"""
# get dataframes from files
df = []
files = std.get_files( folder_path, file_pattern )
if len( files ) == 0:
# no files found
raise RuntimeError( 'No files found matching {}'.format( os.path.join( folder_path, file_pattern ) ) )
for file in files:
data = import_datum(
file,
metadata = metadata,
cps = cps,
time = time
) # run local import datum function
df.append( data )
if interpolate is not None:
df = std.common_reindex( df, how = interpolate, fillna = fillna )
df = pd.concat( df, axis = 1 )
return df
def correct_spectra(df, correction):
"""
Applies a correction spectral data.
:param df: The Pandas DataFrame to correct.
:param correction: The correction data to apply.
Should be a tuple of ( camera, grating )
Camera values are [ 'idus' ]
Grating values are [ '300', '600' ]
:returns: The corrected data.
"""
data_path = os.path.join(os.path.dirname(__file__), 'data',
'andor-corrections.pkl')
cdf = pd.read_pickle(data_path)
corrections = cdf.xs(('grating', *correction), axis=1)
cdf = std.common_reindex([df, corrections])
corrections = cdf[1].reindex(df.index)
return df.multiply(corrections, axis=0)
def import_data(
folder_path,
file_pattern = '*.csv',
metadata = None,
interpolate = 'linear',
fillna = 0
):
"""
Imports data from TimeSpec II experiments output files.
:param folder_path: The file path containing the data files
:param file_pattern: A glob pattern to filter the imported files [Default: '*']
:param metadata: Metadata from the file name is turned into MultiIndex columns.
[Default: None]
:param interpolate: How to interpolate data for a common index [Default: linear]
Use None to prevent reindexing
:param fillna: Value to fill NaN values with [Default: 0]
:returns: A Pandas DataFrame with MultiIndexed columns
:raises: RuntimeError if no files are found
"""
# get dataframes from files
df = []
files = std.get_files( folder_path, file_pattern )
if len( files ) == 0:
# no files found
raise RuntimeError( 'No files found matching {}'.format( os.path.join( folder_path, file_pattern ) ) )
for file in files:
data = import_datum( file, metadata = metadata ) # run local import datum function
df.append( data )
if interpolate is not None:
df = std.common_reindex( df, how = interpolate, fillna = fillna )
df = pd.concat( df, axis = 1 ).sort_index( axis = 1 )
return df
def import_mpp_tracking_datum(file, reindex=True, drop_cycle=True):
"""
Imports a MPP Tracking file with channel headers.
:param file: Path to file.
:param reindex: Reindex by time. [Default: True]
:param drop_cycle: Drop cycle columns. [Default: True]
:returns: Pandas DataFrame.
"""
parameters = {
'Time [s]': 'time',
'Voltage [V]': 'voltage',
'Current [A]': 'current',
'Power [W]': 'power',
'Cycle': 'cycle'
}
df = import_datum(file, parameters)
if drop_cycle:
df = df.drop('cycle', axis=1, level='metrics')
if reindex:
data = []
for ch, datum in df.groupby(level='channel', axis=1):
datum = datum.dropna(subset=[(ch, 'time')])
datum.set_index((ch, 'time'), inplace=True)
datum.index = datum.index.rename('time')
datum = datum.mean(
level=0) # take mean over duplicate index values
data.append(datum)
data = std.common_reindex(data, fillna=np.NaN)
df = pd.concat(data, axis=1)
return df
def import_datum( path, **kwargs ):
"""
Imports a Cary UV Vis Absorption data file into a Pandas DataFrame.
:param path: Path to the file.
:returns: Pandas DataFrame.
"""
data_end = None
fields = None
metrics = None
with open( path ) as f:
for ( index, line ) in enumerate( f ):
# search for end of data
# occurs at first blank line
# every line ends with trailing comma
if index is 0:
fields = line.split( ',' )[ :-1 ]
elif index is 1:
# get headers
metrics = line.split( ',' )[ :-1 ]
elif line is '\n':
# no data in line
data_end = index
break
# parse metrics for sample width
# new samples begin with wavelength
sample_indices = []
for index, metric in enumerate( metrics ):
if metric == 'Wavelength (nm)':
sample_indices.append( index )
# import samples individually
df = []
for ( i, sample_index ) in enumerate( sample_indices ):
# get sample columns
next_sample_index = ( # get next sample index, or end if last sample
sample_indices[ i + 1 ]
if ( i + 1 ) < len( sample_indices ) else
len( metrics ) # final sample
)
use_cols = range( sample_index, next_sample_index )
# get names of fields
# ignore first as it is wavelength
# will be set to index
names = map(
str.lower,
metrics[ sample_index + 1 : next_sample_index ]
)
tdf = pd.read_csv(
path,
header = None,
skiprows = 2,
nrows = data_end - 2, # account for ignored headers
usecols = use_cols,
index_col = 0, # wavelength as index
names = [ 'wavelength', *names ],
**kwargs
).dropna()
# add sample name to header
tdf = std.insert_index_levels( tdf, fields[ sample_index ] )
df.append( tdf )
std.common_reindex( df )
df = pd.concat( df, axis = 1 )
return df