def load_from_stats(self):
'''loads stats data and converts to timeseries without saving'''
stats = pd.read_csv(self.stats_filename, parse_dates=['timestamp'])
u = stats['timestamp'].unique()
u = pd.to_datetime(u)
sample_volume = scpp.get_sample_volume(
self.settings.PostProcess.pix_size,
path_length=self.settings.PostProcess.path_length)
dias, bin_lims = scpp.get_size_bins()
vd_oil = np.zeros((len(u), len(dias)))
vd_gas = np.zeros_like(vd_oil)
vd_total = np.zeros_like(vd_oil)
d50_gas = np.zeros(len(u))
d50_oil = np.zeros_like(d50_gas)
d50_total = np.zeros_like(d50_gas)
self.cos = np.zeros_like(d50_total)
# @todo make this number of particles per image, and sum according to index later
nparticles_all = 0
nparticles_total = 0
nparticles_oil = 0
nparticles_gas = 0
for i, s in enumerate(tqdm(u)):
substats = stats[stats['timestamp'] == s]
nparticles_all += len(substats)
nims = scpp.count_images_in_stats(substats)
sv = sample_volume * nims
oil = scog.extract_oil(substats)
nparticles_oil += len(oil)
dias, vd_oil_ = scpp.vd_from_stats(oil, self.settings.PostProcess)
vd_oil_ /= sv
vd_oil[i, :] = vd_oil_
gas = scog.extract_gas(substats)
nparticles_gas += len(gas)
dias, vd_gas_ = scpp.vd_from_stats(gas, self.settings.PostProcess)
vd_gas_ /= sv
vd_gas[i, :] = vd_gas_
d50_gas[i] = scpp.d50_from_vd(vd_gas_, dias)
nparticles_total += len(oil) + len(gas)
vd_total_ = vd_oil_ + vd_gas_
d50_total[i] = scpp.d50_from_vd(vd_total_, dias)
vd_total[i, :] = vd_total_
self.cos[i] = scog.cos_check(dias, vd_total[i, :])
self.vd_total = vd_total
self.vd_gas = vd_gas
self.vd_oil = vd_oil
self.d50_total = d50_total
self.d50_oil = d50_oil
self.d50_gas = d50_gas
self.u = u.tz_localize('UTC')
self.dias = dias
def convert_to_pj_format(stats_csv_file, config_file):
'''converts stats files into a total, and gas-only time-series csvfile which can be read by the old matlab
SummaryPlot exe'''
settings = PySilcamSettings(config_file)
logger.info('Loading stats....')
stats = pd.read_csv(stats_csv_file)
base_name = stats_csv_file.replace('-STATS.csv', '-PJ.csv')
gas_name = base_name.replace('-PJ.csv', '-PJ-GAS.csv')
ogdatafile = DataLogger(base_name, ogdataheader())
ogdatafile_gas = DataLogger(gas_name, ogdataheader())
stats['timestamp'] = pd.to_datetime(stats['timestamp'])
u = stats['timestamp'].unique()
sample_volume = sc_pp.get_sample_volume(
settings.PostProcess.pix_size,
path_length=settings.PostProcess.path_length)
logger.info('Analysing time-series')
for s in tqdm(u):
substats = stats[stats['timestamp'] == s]
nims = sc_pp.count_images_in_stats(substats)
sv = sample_volume * nims
oil = extract_oil(substats)
dias, vd_oil = sc_pp.vd_from_stats(oil, settings.PostProcess)
vd_oil /= sv
gas = extract_gas(substats)
dias, vd_gas = sc_pp.vd_from_stats(gas, settings.PostProcess)
vd_gas /= sv
d50_gas = sc_pp.d50_from_vd(vd_gas, dias)
vd_total = vd_oil + vd_gas
d50_total = sc_pp.d50_from_vd(vd_total, dias)
data_total = cat_data_pj(s, vd_total, d50_total, len(oil) + len(gas))
ogdatafile.append_data(data_total)
data_gas = cat_data_pj(s, vd_gas, d50_gas, len(gas))
ogdatafile_gas.append_data(data_gas)
logger.info(' OK.')
logger.info('Deleting header!')
with open(base_name, 'r') as fin:
data = fin.read().splitlines(True)
with open(base_name, 'w') as fout:
fout.writelines(data[1:])
with open(gas_name, 'r') as fin:
data = fin.read().splitlines(True)
with open(gas_name, 'w') as fout:
fout.writelines(data[1:])
logger.info('Conversion complete.')
def plot_trimmed_stats(self):
start_time = pd.to_datetime(
self.ui.dateTimeStart.dateTime().toPyDateTime())
end_time = pd.to_datetime(
self.ui.dateTimeEnd.dateTime().toPyDateTime())
self.trimmed_stats, self.output_filename = scpp.trim_stats(
self.stats_filename,
start_time,
end_time,
write_new=False,
stats=self.stats)
if np.isnan(self.trimmed_stats.equivalent_diameter.max()) or len(
self.trimmed_stats) == 0:
QMessageBox.warning(
self, "No data in this segment!",
'No data was found within the specified time range.',
QMessageBox.Ok)
return
settings = PySilcamSettings(self.config_file)
plt.figure(self.figure.number)
plt.clf()
stats_oil = scog.extract_oil(self.trimmed_stats)
stats_gas = scog.extract_gas(self.trimmed_stats)
sample_volume = scpp.get_sample_volume(
settings.PostProcess.pix_size,
path_length=settings.PostProcess.path_length)
nims = scpp.count_images_in_stats(self.trimmed_stats)
dias, vd = scpp.vd_from_stats(self.trimmed_stats, settings.PostProcess)
dias, vd_oil = scpp.vd_from_stats(stats_oil, settings.PostProcess)
dias, vd_gas = scpp.vd_from_stats(stats_gas, settings.PostProcess)
sv = sample_volume * nims
vd /= sv
vd_oil /= sv
vd_gas /= sv
plt.plot(dias, vd_oil + vd_gas, 'k', label='TOTAL')
plt.plot(dias, vd_oil, 'r', label='OIL')
plt.plot(dias, vd_gas, 'b', label='GAS')
plt.xscale('log')
plt.xlabel('Equiv. diam (um)')
plt.ylabel('Volume concentration (uL/L)')
plt.xlim(10, 12000)
plt.legend()
self.canvas.draw()
def psd(stats, settings, ax, line=None, c='k'):
'''
Plot a normalised particle volume distribution
Args:
stats (DataFrame) : particle statistics from silcam process
settings (PySilcamSettings) : settings associated with the data, loaded with PySilcamSettings
ax () : axis to plot data on
line=None () : ?? possibly an option to rapidly replace a proviously plotted line??
c='k' (str) : color of the line to be plotted
Returns:
line () : ?? the plotted line, possible returned incase of future adjustment??
'''
dias, vd = sc_pp.vd_from_stats(stats, settings)
if line:
line.set_data(dias, vd/np.sum(vd)*100)
else:
line, = ax.plot(dias,vd/np.sum(vd)*100, color=c)
ax.set_xscale('log')
ax.set_xlabel('Equiv. diam (um)')
ax.set_ylabel('Volume concentration (%/sizebin)')
ax.set_xlim(10, 10000)
ax.set_ylim(0, max(vd/np.sum(vd)*100))
#ax.axvline(sc_pp.d50_from_vd(vd,dias), color=c)
return line
def gor_timeseries(stats, settings):
from tqdm import tqdm
u = stats['timestamp'].unique()
td = pd.to_timedelta('00:00:' + str(settings.window_size / 2.))
sample_volume = sc_pp.get_sample_volume(settings.pix_size,
path_length=settings.path_length)
gor = []
time = []
for t in tqdm(u):
dt = pd.to_datetime(t)
stats_ = stats[(pd.to_datetime(stats['timestamp']) < (dt + td))
& (pd.to_datetime(stats['timestamp']) > (dt - td))]
oilstats = extract_oil(stats_)
dias, vd_oil = sc_pp.vd_from_stats(oilstats, settings)
nims = sc_pp.count_images_in_stats(oilstats)
sv = sample_volume * nims
vd_oil /= sv
gasstats = extract_gas(stats_)
dias, vd_gas = sc_pp.vd_from_stats(gasstats, settings)
nims = sc_pp.count_images_in_stats(gasstats)
sv = sample_volume * nims
vd_gas /= sv
gor_ = sum(vd_gas) / sum(vd_oil)
time.append(pd.to_datetime(t))
gor.append(gor_)
if (len(gor) == 0) or (np.isnan(max(gor))):
gor = np.nan
time = np.nan
return gor, time
def cat_data(timestamp, stats, settings):
'''
Possibly a redundant function....
'''
dias, vd = sc_pp.vd_from_stats(stats, settings.PostProcess)
d50 = sc_pp.d50_from_vd(vd, dias)
data = [[
timestamp.year, timestamp.month, timestamp.day, timestamp.hour,
timestamp.minute, timestamp.second + timestamp.microsecond / 1e6
], vd, [d50, len(stats)]]
data = list(itertools.chain.from_iterable(data))
return data
def update(self):
'''
Updates the rt_stats to remove data from before the specified window of seconds
given in the config ini file, here: settings.PostProcess.window_size
'''
self.stats = sc_pp.extract_latest_stats(
self.stats, self.settings.PostProcess.window_size)
#extract seperate stats on oil and gas
self.oil_stats = extract_oil(self.stats)
self.gas_stats = extract_gas(self.stats)
#calculate d50
self.oil_d50 = sc_pp.d50_from_stats(self.oil_stats,
self.settings.PostProcess)
self.gas_d50 = sc_pp.d50_from_stats(self.gas_stats,
self.settings.PostProcess)
self.dias, self.vd_oil = sc_pp.vd_from_stats(self.oil_stats,
self.settings.PostProcess)
self.dias, self.vd_gas = sc_pp.vd_from_stats(self.gas_stats,
self.settings.PostProcess)
self.saturation = np.max(self.stats.saturation)
def test_analysis(img, PIX_SIZE, PATH_LENGTH, config_file=''):
'''wrapper for pysilcam processing
Args:
img (unit8) : image to be processed (equivalent to the background-corrected image obtained from the SilCam)
PIX_SIZE (float) : pixel size of the setup [um]
PATH_LENGTH (float) : path length of the setup [mm]
Returns:
dias (array) : mid points of the size distribution bins [um]
vd (array) : volume concentration in each size bin [uL/L/bin]
imbw (uint8) : segmented image
stat_extract_time (timestamp) : time taken to run statextract
'''
# administer configuration settings according to specified setup
if config_file=='':
testconfig = os.path.split(sccf.default_config_path())[0]
testconfig = os.path.join(testconfig, 'tests/config.ini')
conf.set('Process', 'real_time_stats', 'True')
conf.set('Process', 'threshold', '0.85')
conf.set('ExportParticles', 'export_images', 'False')
else:
testconfig = config_file
conf = sccf.load_config(testconfig)
conf.set('PostProcess', 'pix_size', str(PIX_SIZE))
conf.set('PostProcess', 'path_length', str(PATH_LENGTH))
settings = sccf.PySilcamSettings(conf) # pass these settings without saving a config file to disc
# load tensorflow model
nnmodel, class_labels = sccl.load_model(model_path=settings.NNClassify.model_path)
start_time = pd.Timestamp.now() # time statextract
# process the image
stats, imbw, saturation = scpr.statextract(img, settings, pd.Timestamp.now(),
nnmodel, class_labels)
end_time = pd.Timestamp.now() # time statextract
stat_extract_time = end_time - start_time
dias, vd = scpp.vd_from_stats(stats, settings.PostProcess) # calculate the volume distribution from the processed stats
# scale the volume distribution according to the SilCam setup specified
sv = scpp.get_sample_volume(settings.PostProcess.pix_size,
path_length=settings.PostProcess.path_length)
vd /= sv
return dias, vd, imbw, stat_extract_time
def export_timeseries(configfile, statsfile):
settings = PySilcamSettings(configfile)
print('Loading STATS data: ', statsfile)
stats = pd.read_csv(statsfile)
stats['timestamp'] = pd.to_datetime(stats['timestamp'])
stats.sort_values(by='timestamp', inplace=True)
print('Extracting oil and gas')
stats_oil = scog.extract_oil(stats)
stats_gas = scog.extract_gas(stats)
print('Calculating timeseries')
u = pd.to_datetime(stats['timestamp']).unique()
sample_volume = sc_pp.get_sample_volume(settings.PostProcess.pix_size, path_length=settings.PostProcess.path_length)
td = pd.to_timedelta('00:00:' + str(settings.PostProcess.window_size / 2.))
vdts_all = []
vdts_oil = []
vdts_gas = []
d50_all = []
d50_oil = []
d50_gas = []
timestamp = []
d50_av_all = []
d50_av_oil = []
d50_av_gas = []
gor = []
for s in tqdm(u):
timestamp.append(pd.to_datetime(s))
dt = pd.to_datetime(s)
dias, vd_all = sc_pp.vd_from_stats(stats[stats['timestamp'] == s],
settings.PostProcess)
dias, vd_oil = sc_pp.vd_from_stats(stats_oil[stats_oil['timestamp'] == s],
settings.PostProcess)
dias, vd_gas = sc_pp.vd_from_stats(stats_gas[stats_gas['timestamp'] == s],
settings.PostProcess)
nims = sc_pp.count_images_in_stats(stats[stats['timestamp'] == s])
sv = sample_volume * nims
vd_all /= sv
vd_oil /= sv
vd_gas /= sv
d50_all.append(sc_pp.d50_from_vd(vd_all, dias))
d50_oil.append(sc_pp.d50_from_vd(vd_oil, dias))
d50_gas.append(sc_pp.d50_from_vd(vd_gas, dias))
vdts_all.append(vd_all)
vdts_oil.append(vd_oil)
vdts_gas.append(vd_gas)
stats_av = stats[(stats['timestamp']<(dt+td)) & (stats['timestamp']>(dt-td))]
stats_av_oil = scog.extract_oil(stats_av)
stats_av_gas = scog.extract_gas(stats_av)
d50_av_all.append(sc_pp.d50_from_stats(stats_av, settings.PostProcess))
d50_av_oil.append(sc_pp.d50_from_stats(stats_av_oil, settings.PostProcess))
d50_av_gas.append(sc_pp.d50_from_stats(stats_av_gas, settings.PostProcess))
dias, vdts_av = sc_pp.vd_from_stats(stats_av, settings.PostProcess)
dias, vdts_av_oil = sc_pp.vd_from_stats(stats_av_oil, settings.PostProcess)
dias, vdts_av_gas = sc_pp.vd_from_stats(stats_av_gas, settings.PostProcess)
nims = sc_pp.count_images_in_stats(stats_av)
sv = sample_volume * nims
vdts_av /= sv
vdts_av_oil /= sv
vdts_av_gas /= sv
gor.append(np.sum(vdts_av_gas)/np.sum(vdts_av_oil))
outpath, outfile = os.path.split(statsfile)
outfile = outfile.replace('-STATS.csv','')
outfile = os.path.join(outpath, outfile)
time_series = pd.DataFrame(data=np.squeeze(vdts_all), columns=dias)
time_series['D50'] = d50_all
time_series['Time'] = timestamp
time_series.to_excel(outfile +
'-TIMESERIES' + '' + '.xlsx')
time_series = pd.DataFrame(data=np.squeeze(vdts_oil), columns=dias)
time_series['D50'] = d50_oil
time_series['Time'] = timestamp
time_series.to_excel(outfile +
'-TIMESERIES' + 'oil' + '.xlsx')
time_series = pd.DataFrame(data=np.squeeze(vdts_gas), columns=dias)
time_series['D50'] = d50_gas
time_series['Time'] = timestamp
time_series.to_excel(outfile +
'-TIMESERIES' + 'gas' + '.xlsx')
plt.figure(figsize=(20, 10))
if not np.min(np.isnan(d50_oil)):
plt.plot(timestamp, d50_oil, 'ro')
if not np.min(np.isnan(d50_av_oil)):
plt.plot(timestamp, d50_av_oil, 'r-')
lns1 = plt.plot(np.nan, np.nan, 'r-', label='OIL')
if not np.min(np.isnan(d50_gas)):
plt.plot(timestamp, d50_gas, 'bo')
if not np.min(np.isnan(d50_av_gas)):
plt.plot(timestamp, d50_av_gas, 'b-')
lns2 = plt.plot(np.nan, np.nan, 'b-', label='GAS')
plt.ylabel('d50 [um]')
plt.ylim(0, max(plt.gca().get_ylim()))
ax = plt.gca().twinx()
plt.sca(ax)
plt.ylabel('GOR')
if not np.min(np.isnan(gor)):
plt.plot(timestamp, gor, 'k')
lns3 = plt.plot(np.nan, np.nan, 'k', label='GOR')
plt.ylim(0, max(plt.gca().get_ylim()))
lns = lns1 + lns2 + lns3
labs = [l.get_label() for l in lns]
plt.legend(lns, labs)
plt.savefig(outfile +
'-d50_TimeSeries.png', dpi=600, bbox_inches='tight')
plt.close()
print('Export figure made. ')
print('Exporting averages... ')
# average all
dias, vd = sc_pp.vd_from_stats(stats,
settings.PostProcess)
nims = sc_pp.count_images_in_stats(stats)
sv = sample_volume * nims
vd /= sv
d50 = sc_pp.d50_from_vd(vd, dias)
dfa = pd.DataFrame(data=[vd], columns=dias)
dfa['d50'] = d50
timestamp = np.min(pd.to_datetime(stats['timestamp']))
dfa['Time'] = timestamp
dfa.to_excel(statsfile.replace('-STATS.csv', '') +
'-AVERAGE' + '' + '.xlsx')
#average oil
dias, vd = sc_pp.vd_from_stats(stats_oil,
settings.PostProcess)
vd /= sv # sample volume remains the same as 'all'
d50 = sc_pp.d50_from_vd(vd, dias)
dfa = pd.DataFrame(data=[vd], columns=dias)
dfa['d50'] = d50
timestamp = np.min(pd.to_datetime(stats['timestamp'])) # still use total stats for this time
dfa['Time'] = timestamp
dfa.to_excel(statsfile.replace('-STATS.csv', '') +
'-AVERAGE' + 'oil' + '.xlsx')
#average gas
dias, vd = sc_pp.vd_from_stats(stats_gas,
settings.PostProcess)
vd /= sv # sample volume remains the same as 'all'
d50 = sc_pp.d50_from_vd(vd, dias)
dfa = pd.DataFrame(data=[vd], columns=dias)
dfa['d50'] = d50
timestamp = np.min(pd.to_datetime(stats['timestamp'])) # still use total stats for this time
dfa['Time'] = timestamp
dfa.to_excel(statsfile.replace('-STATS.csv', '') +
'-AVERAGE' + 'gas' + '.xlsx')
print('Export done: ', outfile)