def load_from_stats(self):
'''loads stats data and converts to timeseries without saving'''
stats = pd.read_csv(self.stats_filename, parse_dates=['timestamp'])
u = stats['timestamp'].unique()
u = pd.to_datetime(u)
sample_volume = scpp.get_sample_volume(
self.settings.PostProcess.pix_size,
path_length=self.settings.PostProcess.path_length)
dias, bin_lims = scpp.get_size_bins()
vd_oil = np.zeros((len(u), len(dias)))
vd_gas = np.zeros_like(vd_oil)
vd_total = np.zeros_like(vd_oil)
d50_gas = np.zeros(len(u))
d50_oil = np.zeros_like(d50_gas)
d50_total = np.zeros_like(d50_gas)
self.cos = np.zeros_like(d50_total)
# @todo make this number of particles per image, and sum according to index later
nparticles_all = 0
nparticles_total = 0
nparticles_oil = 0
nparticles_gas = 0
for i, s in enumerate(tqdm(u)):
substats = stats[stats['timestamp'] == s]
nparticles_all += len(substats)
nims = scpp.count_images_in_stats(substats)
sv = sample_volume * nims
oil = scog.extract_oil(substats)
nparticles_oil += len(oil)
dias, vd_oil_ = scpp.vd_from_stats(oil, self.settings.PostProcess)
vd_oil_ /= sv
vd_oil[i, :] = vd_oil_
gas = scog.extract_gas(substats)
nparticles_gas += len(gas)
dias, vd_gas_ = scpp.vd_from_stats(gas, self.settings.PostProcess)
vd_gas_ /= sv
vd_gas[i, :] = vd_gas_
d50_gas[i] = scpp.d50_from_vd(vd_gas_, dias)
nparticles_total += len(oil) + len(gas)
vd_total_ = vd_oil_ + vd_gas_
d50_total[i] = scpp.d50_from_vd(vd_total_, dias)
vd_total[i, :] = vd_total_
self.cos[i] = scog.cos_check(dias, vd_total[i, :])
self.vd_total = vd_total
self.vd_gas = vd_gas
self.vd_oil = vd_oil
self.d50_total = d50_total
self.d50_oil = d50_oil
self.d50_gas = d50_gas
self.u = u.tz_localize('UTC')
self.dias = dias
def stats_to_xls_png(config_file, stats_filename, oilgas=outputPartType.all):
'''summarises stats in two excel sheets of time-series PSD and averaged
PSD.
Args:
config_file (string) : Path of the config file for this data
stats_filename (string) : Path of the stats csv file
oilgas=oc_pp.outputPartType.all : the oilgas enum if you want to just make the figure for oil, or just gas (defaults to all particles)
Returns:
dataframe: of time series
files: in the proc folder)
'''
settings = PySilcamSettings(config_file)
stats = pd.read_csv(stats_filename)
stats.sort_values(by='timestamp', inplace=True)
oilgasTxt = ''
if oilgas == outputPartType.oil:
from pysilcam.oilgas import extract_oil
stats = extract_oil(stats)
oilgasTxt = 'oil'
elif oilgas == outputPartType.gas:
from pysilcam.oilgas import extract_gas
stats = extract_gas(stats)
oilgasTxt = 'gas'
df = make_timeseries_vd(stats, settings)
df.to_excel(
stats_filename.replace('-STATS.csv', '') + '-TIMESERIES' + oilgasTxt +
'.xlsx')
sample_volume = get_sample_volume(
settings.PostProcess.pix_size,
path_length=settings.PostProcess.path_length)
dias, vd = vd_from_stats(stats, settings.PostProcess)
nims = count_images_in_stats(stats)
sv = sample_volume * nims
vd /= sv
d50 = d50_from_vd(vd, dias)
dfa = pd.DataFrame(data=[vd], columns=dias)
dfa['d50'] = d50
timestamp = np.min(pd.to_datetime(df['Time']))
dfa['Time'] = timestamp
dfa.to_excel(
stats_filename.replace('-STATS.csv', '') + '-AVERAGE' + oilgasTxt +
'.xlsx')
return df
def make_montage(stats_csv_file,
pixel_size,
roidir,
auto_scaler=500,
msize=1024,
maxlength=100000,
oilgas=outputPartType.all):
''' wrapper function for montage_maker
Args:
stats_csv_file : location of the stats_csv file that comes from silcam process
pixel_size : pixel size of system defined by settings.PostProcess.pix_size
roidir : location of roifiles usually defined by settings.ExportParticles.outputpath
auto_scaler=500 : approximate number of particle that are attempted to be pack into montage
msize=1024 : size of canvas in pixels
maxlength=100000 : maximum length in microns of particles to be included in montage
oilgas=outputPartType.all : enum defining which type of particle to be selected for use in the montage
Returns:
montage (uint8) : a nicely-made montage in the form of an image, which can be plotted using plotting.montage_plot(montage, settings.PostProcess.pix_size)
'''
# obtain particle statistics from the csv file
stats = pd.read_csv(stats_csv_file)
# remove nans because concentrations are not important here
stats = stats[~np.isnan(stats['major_axis_length'])]
stats = stats[(stats['major_axis_length'] * pixel_size) < maxlength]
# extract only wanted particle stats
if oilgas == outputPartType.oil:
from pysilcam.oilgas import extract_oil
stats = extract_oil(stats)
elif oilgas == outputPartType.gas:
from pysilcam.oilgas import extract_gas
stats = extract_gas(stats)
# sort the particles based on their length
stats.sort_values(by=['major_axis_length'], ascending=False, inplace=True)
roifiles = gen_roifiles(stats, auto_scaler=auto_scaler)
eyecandy = True
if not (oilgas == outputPartType.all):
eyecandy = False
montage = montage_maker(roifiles,
roidir,
pixel_size,
msize,
eyecandy=eyecandy)
return montage
def plot_trimmed_stats(self):
start_time = pd.to_datetime(
self.ui.dateTimeStart.dateTime().toPyDateTime())
end_time = pd.to_datetime(
self.ui.dateTimeEnd.dateTime().toPyDateTime())
self.trimmed_stats, self.output_filename = scpp.trim_stats(
self.stats_filename,
start_time,
end_time,
write_new=False,
stats=self.stats)
if np.isnan(self.trimmed_stats.equivalent_diameter.max()) or len(
self.trimmed_stats) == 0:
QMessageBox.warning(
self, "No data in this segment!",
'No data was found within the specified time range.',
QMessageBox.Ok)
return
settings = PySilcamSettings(self.config_file)
plt.figure(self.figure.number)
plt.clf()
stats_oil = scog.extract_oil(self.trimmed_stats)
stats_gas = scog.extract_gas(self.trimmed_stats)
sample_volume = scpp.get_sample_volume(
settings.PostProcess.pix_size,
path_length=settings.PostProcess.path_length)
nims = scpp.count_images_in_stats(self.trimmed_stats)
dias, vd = scpp.vd_from_stats(self.trimmed_stats, settings.PostProcess)
dias, vd_oil = scpp.vd_from_stats(stats_oil, settings.PostProcess)
dias, vd_gas = scpp.vd_from_stats(stats_gas, settings.PostProcess)
sv = sample_volume * nims
vd /= sv
vd_oil /= sv
vd_gas /= sv
plt.plot(dias, vd_oil + vd_gas, 'k', label='TOTAL')
plt.plot(dias, vd_oil, 'r', label='OIL')
plt.plot(dias, vd_gas, 'b', label='GAS')
plt.xscale('log')
plt.xlabel('Equiv. diam (um)')
plt.ylabel('Volume concentration (uL/L)')
plt.xlim(10, 12000)
plt.legend()
self.canvas.draw()
def export_summary_data_slow(self):
if self.configfile == '':
self.status_update('Asking user for config file')
self.load_sc_config()
if self.configfile == '':
self.status_update('Did not get STATS file')
return
settings = PySilcamSettings(self.configfile)
self.stats_filename = ''
self.status_update('Asking user for *-STATS.csv file')
self.load_stats_filename()
if self.stats_filename == '':
self.status_update('Did not get STATS file')
return
stats = pd.read_csv(self.stats_filename)
stats.sort_values(by=['timestamp'], inplace=True)
self.status_update('Exporting all data....')
df = scpp.stats_to_xls_png(self.configfile, self.stats_filename)
plt.figure(figsize=(20, 10))
self.status_update('Exporting oil data....')
df = scpp.stats_to_xls_png(self.configfile,
self.stats_filename,
oilgas=scpp.outputPartType.oil)
plt.plot(df['Time'], df['D50'], 'ro')
d50, time = scpp.d50_timeseries(scog.extract_oil(stats),
settings.PostProcess)
lns1 = plt.plot(time, d50, 'r-', label='OIL')
self.status_update('Exporting gas data....')
df = scpp.stats_to_xls_png(self.configfile,
self.stats_filename,
oilgas=scpp.outputPartType.gas)
plt.plot(df['Time'], df['D50'], 'bo')
d50, time = scpp.d50_timeseries(scog.extract_gas(stats),
settings.PostProcess)
lns2 = plt.plot(time, d50, 'b-', label='GAS')
plt.ylabel('d50 [um]')
plt.ylim(0, max(plt.gca().get_ylim()))
self.status_update('Calculating GOR time series....')
gor, time = scog.gor_timeseries(stats, settings.PostProcess)
ax = plt.gca().twinx()
plt.sca(ax)
plt.ylabel('GOR')
plt.ylim(0, max([max(gor), max(plt.gca().get_ylim())]))
lns3 = ax.plot(time, gor, 'k', label='GOR')
lns = lns1 + lns2 + lns3
labs = [l.get_label() for l in lns]
plt.legend(lns, labs)
plt.savefig(self.stats_filename.replace('-STATS.csv', '') +
'-d50_TimeSeries.png',
dpi=600,
bbox_inches='tight')
self.status_update('Export finished.')
plt.figure(self.fig_main.number)
def export_timeseries(configfile, statsfile):
settings = PySilcamSettings(configfile)
print('Loading STATS data: ', statsfile)
stats = pd.read_csv(statsfile)
stats['timestamp'] = pd.to_datetime(stats['timestamp'])
stats.sort_values(by='timestamp', inplace=True)
print('Extracting oil and gas')
stats_oil = scog.extract_oil(stats)
stats_gas = scog.extract_gas(stats)
print('Calculating timeseries')
u = pd.to_datetime(stats['timestamp']).unique()
sample_volume = sc_pp.get_sample_volume(settings.PostProcess.pix_size, path_length=settings.PostProcess.path_length)
td = pd.to_timedelta('00:00:' + str(settings.PostProcess.window_size / 2.))
vdts_all = []
vdts_oil = []
vdts_gas = []
d50_all = []
d50_oil = []
d50_gas = []
timestamp = []
d50_av_all = []
d50_av_oil = []
d50_av_gas = []
gor = []
for s in tqdm(u):
timestamp.append(pd.to_datetime(s))
dt = pd.to_datetime(s)
dias, vd_all = sc_pp.vd_from_stats(stats[stats['timestamp'] == s],
settings.PostProcess)
dias, vd_oil = sc_pp.vd_from_stats(stats_oil[stats_oil['timestamp'] == s],
settings.PostProcess)
dias, vd_gas = sc_pp.vd_from_stats(stats_gas[stats_gas['timestamp'] == s],
settings.PostProcess)
nims = sc_pp.count_images_in_stats(stats[stats['timestamp'] == s])
sv = sample_volume * nims
vd_all /= sv
vd_oil /= sv
vd_gas /= sv
d50_all.append(sc_pp.d50_from_vd(vd_all, dias))
d50_oil.append(sc_pp.d50_from_vd(vd_oil, dias))
d50_gas.append(sc_pp.d50_from_vd(vd_gas, dias))
vdts_all.append(vd_all)
vdts_oil.append(vd_oil)
vdts_gas.append(vd_gas)
stats_av = stats[(stats['timestamp']<(dt+td)) & (stats['timestamp']>(dt-td))]
stats_av_oil = scog.extract_oil(stats_av)
stats_av_gas = scog.extract_gas(stats_av)
d50_av_all.append(sc_pp.d50_from_stats(stats_av, settings.PostProcess))
d50_av_oil.append(sc_pp.d50_from_stats(stats_av_oil, settings.PostProcess))
d50_av_gas.append(sc_pp.d50_from_stats(stats_av_gas, settings.PostProcess))
dias, vdts_av = sc_pp.vd_from_stats(stats_av, settings.PostProcess)
dias, vdts_av_oil = sc_pp.vd_from_stats(stats_av_oil, settings.PostProcess)
dias, vdts_av_gas = sc_pp.vd_from_stats(stats_av_gas, settings.PostProcess)
nims = sc_pp.count_images_in_stats(stats_av)
sv = sample_volume * nims
vdts_av /= sv
vdts_av_oil /= sv
vdts_av_gas /= sv
gor.append(np.sum(vdts_av_gas)/np.sum(vdts_av_oil))
outpath, outfile = os.path.split(statsfile)
outfile = outfile.replace('-STATS.csv','')
outfile = os.path.join(outpath, outfile)
time_series = pd.DataFrame(data=np.squeeze(vdts_all), columns=dias)
time_series['D50'] = d50_all
time_series['Time'] = timestamp
time_series.to_excel(outfile +
'-TIMESERIES' + '' + '.xlsx')
time_series = pd.DataFrame(data=np.squeeze(vdts_oil), columns=dias)
time_series['D50'] = d50_oil
time_series['Time'] = timestamp
time_series.to_excel(outfile +
'-TIMESERIES' + 'oil' + '.xlsx')
time_series = pd.DataFrame(data=np.squeeze(vdts_gas), columns=dias)
time_series['D50'] = d50_gas
time_series['Time'] = timestamp
time_series.to_excel(outfile +
'-TIMESERIES' + 'gas' + '.xlsx')
plt.figure(figsize=(20, 10))
if not np.min(np.isnan(d50_oil)):
plt.plot(timestamp, d50_oil, 'ro')
if not np.min(np.isnan(d50_av_oil)):
plt.plot(timestamp, d50_av_oil, 'r-')
lns1 = plt.plot(np.nan, np.nan, 'r-', label='OIL')
if not np.min(np.isnan(d50_gas)):
plt.plot(timestamp, d50_gas, 'bo')
if not np.min(np.isnan(d50_av_gas)):
plt.plot(timestamp, d50_av_gas, 'b-')
lns2 = plt.plot(np.nan, np.nan, 'b-', label='GAS')
plt.ylabel('d50 [um]')
plt.ylim(0, max(plt.gca().get_ylim()))
ax = plt.gca().twinx()
plt.sca(ax)
plt.ylabel('GOR')
if not np.min(np.isnan(gor)):
plt.plot(timestamp, gor, 'k')
lns3 = plt.plot(np.nan, np.nan, 'k', label='GOR')
plt.ylim(0, max(plt.gca().get_ylim()))
lns = lns1 + lns2 + lns3
labs = [l.get_label() for l in lns]
plt.legend(lns, labs)
plt.savefig(outfile +
'-d50_TimeSeries.png', dpi=600, bbox_inches='tight')
plt.close()
print('Export figure made. ')
print('Exporting averages... ')
# average all
dias, vd = sc_pp.vd_from_stats(stats,
settings.PostProcess)
nims = sc_pp.count_images_in_stats(stats)
sv = sample_volume * nims
vd /= sv
d50 = sc_pp.d50_from_vd(vd, dias)
dfa = pd.DataFrame(data=[vd], columns=dias)
dfa['d50'] = d50
timestamp = np.min(pd.to_datetime(stats['timestamp']))
dfa['Time'] = timestamp
dfa.to_excel(statsfile.replace('-STATS.csv', '') +
'-AVERAGE' + '' + '.xlsx')
#average oil
dias, vd = sc_pp.vd_from_stats(stats_oil,
settings.PostProcess)
vd /= sv # sample volume remains the same as 'all'
d50 = sc_pp.d50_from_vd(vd, dias)
dfa = pd.DataFrame(data=[vd], columns=dias)
dfa['d50'] = d50
timestamp = np.min(pd.to_datetime(stats['timestamp'])) # still use total stats for this time
dfa['Time'] = timestamp
dfa.to_excel(statsfile.replace('-STATS.csv', '') +
'-AVERAGE' + 'oil' + '.xlsx')
#average gas
dias, vd = sc_pp.vd_from_stats(stats_gas,
settings.PostProcess)
vd /= sv # sample volume remains the same as 'all'
d50 = sc_pp.d50_from_vd(vd, dias)
dfa = pd.DataFrame(data=[vd], columns=dias)
dfa['d50'] = d50
timestamp = np.min(pd.to_datetime(stats['timestamp'])) # still use total stats for this time
dfa['Time'] = timestamp
dfa.to_excel(statsfile.replace('-STATS.csv', '') +
'-AVERAGE' + 'gas' + '.xlsx')
print('Export done: ', outfile)
def summarise_fancy_stats(stats_csv_file, config_file, monitor=False,
maxlength=100000, msize=2048, oilgas=sc_pp.outputPartType.all):
'''
Plots a summary figure of a dataset which shows
the volume distribution, number distribution and a montage of randomly selected particles
Args:
stats_csv_file (str) : path of the *-STATS.csv file created by silcam process
config_file (str) : path of the config ini file associated with the data
monitor=False (Bool) : if True then this function will run forever, continuously reading the stats_csv_file and plotting the data
might be useful in monitoring the progress of processing, for example
maxlength=100000 (int) : particles longer than this number will not be put in the montage
msize=2048 (int) : the montage created will have a canvas size of msize x msize pixels
oilgas=oc_pp.outputPartType.all : the oilgas enum if you want to just make the figure for oil, or just gas (defulats to all particles)
'''
sns.set_style('ticks')
settings = PySilcamSettings(config_file)
min_length = settings.ExportParticles.min_length + 1
#f,a = plt.subplots(2,2)
ax1 = plt.subplot2grid((2,2),(0, 0))
ax2 = plt.subplot2grid((2,2),(1, 0))
ax3 = plt.subplot2grid((2,2), (0, 1), rowspan=2)
logger = logging.getLogger(__name__)
while True:
try:
montage = sc_pp.make_montage(stats_csv_file,
settings.PostProcess.pix_size,
roidir=settings.ExportParticles.outputpath,
auto_scaler=msize*2, msize=msize,
maxlength=maxlength,
oilgas=oilgas)
except:
montage = np.zeros((msize, msize, 3), dtype=np.uint8) + 255
logger.warning('Unable to make montage. Check: {0} folder for h5 files'.format(settings.ExportParticles.outputpath))
logger.warning(' in config file ExportParticles.export_images is {0}'.format(settings.ExportParticles.export_images))
stats = pd.read_csv(stats_csv_file)
stats = stats[(stats['major_axis_length'] *
settings.PostProcess.pix_size) < maxlength]
# average numer and volume concentrations
nc, vc, sv_total, junge = sc_pp.nc_vc_from_stats(stats,
settings.PostProcess, oilgas=oilgas)
# extract only wanted particle stats
if oilgas==sc_pp.outputPartType.oil:
from pysilcam.oilgas import extract_oil
stats = extract_oil(stats)
elif oilgas==sc_pp.outputPartType.gas:
from pysilcam.oilgas import extract_gas
stats = extract_gas(stats)
d50 = sc_pp.d50_from_stats(stats, settings.PostProcess)
total_measured_particles = len(stats['major_axis_length'])
plt.sca(ax1)
plt.cla()
psd(stats, settings.PostProcess, plt.gca())
plt.title('Volume conc.: {0:.2f}uL/L d50: {1:.0f}um'.format(vc, d50))
plt.sca(ax2)
plt.cla()
nd(stats, settings.PostProcess, plt.gca(), sample_volume=sv_total)
plt.title('Number conc.: {0:.0f}#/L Junge exp.: {1:.2f}'.format(nc,
junge))
plt.sca(ax3)
plt.cla()
montage_plot(montage, settings.PostProcess.pix_size)
plt.title('Volume processed: {0:.1f}L {1:.0f} particles measured'.format(sv_total,
total_measured_particles))
plt.draw()
if monitor:
plt.pause(1)
else:
break
def nc_vc_from_stats(stats, settings, oilgas=outputPartType.all):
'''
Calculates important summary statistics from a stats DataFrame
Args:
stats (DataFrame) : particle statistics from silcam process
settings (PySilcamSettings) : settings associated with the data, loaded with PySilcamSettings
oilgas=oc_pp.outputPartType.all : the oilgas enum if you want to just make the figure for oil, or just gas (defaults to all particles)
Returns:
nc (float) : the total number concentration in #/L
vc (float) : the total volume concentration in uL/L
sample_volume (float) : the total volume of water sampled in L
junge (float) : the slope of a fitted juge distribution between 150-300um
'''
# get the path length from the config file
path_length = settings.path_length
# get pixel_size from config file
pix_size = settings.pix_size
# calculate the sample volume per image
sample_volume = get_sample_volume(pix_size, path_length=path_length, imx=2048, imy=2448)
# count the number of images analysed
nims = count_images_in_stats(stats)
# scale the sample volume by the number of images recorded
sample_volume *= nims
# extract only wanted particle stats
if oilgas==outputPartType.oil:
from pysilcam.oilgas import extract_oil
stats = extract_oil(stats)
elif oilgas==outputPartType.gas:
from pysilcam.oilgas import extract_gas
stats = extract_gas(stats)
# calculate the number distribution
dias, necd = nd_from_stats(stats, settings)
# calculate the volume distribution from the number distribution
vd = vd_from_nd(necd, dias, sample_volume)
# calculate the volume concentration
vc = np.sum(vd)
# calculate the number concentration
nc = nc_from_nd(necd, sample_volume)
# convert nd to units of nc per micron per litre
nd = nd_rescale(dias, necd, sample_volume)
# remove data from first bin which will be part-full
ind = np.argwhere(nd>0)
nd[ind[0]] = np.nan
# calcualte the junge distirbution slope
junge = get_j(dias,nd)
return nc, vc, sample_volume, junge