def test_small_standards():
'''Testing that the small standards are sized correctly'''
path = os.path.dirname(__file__)
conf_file = os.path.join(ROOTPATH, 'STANDARDS',
'config_glass_standards.ini')
conf_file_out = os.path.join(ROOTPATH, 'STANDARDS',
'config_glass_standards_generated.ini')
conf = load_config(conf_file)
data_file = os.path.join(ROOTPATH, 'STANDARDS/StandardsSmall')
conf.set('General', 'datafile', os.path.join(ROOTPATH, 'STANDARDS',
'proc'))
conf.set('General', 'logfile',
os.path.join(ROOTPATH, 'STANDARDS', 'log.log'))
if MODEL_PATH is not None:
conf.set('NNClassify', 'model_path', MODEL_PATH)
conf_file_hand = open(conf_file_out, 'w')
conf.write(conf_file_hand)
conf_file_hand.close()
stats_file = os.path.join(ROOTPATH,
'STANDARDS/proc/StandardsSmall-STATS.csv')
# if csv file already exists, it has to be deleted
if (os.path.isfile(stats_file)):
os.remove(stats_file)
# call process function
silcam_process(conf_file_out, data_file, multiProcess=False, nbImages=10)
# check that csv file has been created
assert os.path.isfile(stats_file), 'stats_file not created'
# check that csv file has been properly built
csvfile = open(stats_file)
lines = csvfile.readlines()
numline = len(lines)
assert numline > 1, 'csv file empty'
# check the columns
assert lines[0] == 'particle index,major_axis_length,minor_axis_length,equivalent_diameter,solidity,minr,minc,maxr,maxc,'\
'probability_oil,probability_other,probability_bubble,probability_faecal_pellets,probability_copepod,'\
'probability_diatom_chain,probability_oily_gas,export name,timestamp,saturation\n', 'columns not properly built'
settings = PySilcamSettings(conf_file_out)
stats = pd.read_csv(stats_file)
d50 = scpp.d50_from_stats(stats, settings.PostProcess)
print('Small d50:', d50)
assert (d50 > 70 and d50 < 90), 'incorrect d50'
def update(self):
'''
Updates the rt_stats to remove data from before the specified window of seconds
given in the config ini file, here: settings.PostProcess.window_size
'''
self.stats = sc_pp.extract_latest_stats(
self.stats, self.settings.PostProcess.window_size)
#extract seperate stats on oil and gas
self.oil_stats = extract_oil(self.stats)
self.gas_stats = extract_gas(self.stats)
#calculate d50
self.oil_d50 = sc_pp.d50_from_stats(self.oil_stats,
self.settings.PostProcess)
self.gas_d50 = sc_pp.d50_from_stats(self.gas_stats,
self.settings.PostProcess)
self.dias, self.vd_oil = sc_pp.vd_from_stats(self.oil_stats,
self.settings.PostProcess)
self.dias, self.vd_gas = sc_pp.vd_from_stats(self.gas_stats,
self.settings.PostProcess)
self.saturation = np.max(self.stats.saturation)
def export_timeseries(configfile, statsfile):
settings = PySilcamSettings(configfile)
print('Loading STATS data: ', statsfile)
stats = pd.read_csv(statsfile)
stats['timestamp'] = pd.to_datetime(stats['timestamp'])
stats.sort_values(by='timestamp', inplace=True)
print('Extracting oil and gas')
stats_oil = scog.extract_oil(stats)
stats_gas = scog.extract_gas(stats)
print('Calculating timeseries')
u = pd.to_datetime(stats['timestamp']).unique()
sample_volume = sc_pp.get_sample_volume(settings.PostProcess.pix_size, path_length=settings.PostProcess.path_length)
td = pd.to_timedelta('00:00:' + str(settings.PostProcess.window_size / 2.))
vdts_all = []
vdts_oil = []
vdts_gas = []
d50_all = []
d50_oil = []
d50_gas = []
timestamp = []
d50_av_all = []
d50_av_oil = []
d50_av_gas = []
gor = []
for s in tqdm(u):
timestamp.append(pd.to_datetime(s))
dt = pd.to_datetime(s)
dias, vd_all = sc_pp.vd_from_stats(stats[stats['timestamp'] == s],
settings.PostProcess)
dias, vd_oil = sc_pp.vd_from_stats(stats_oil[stats_oil['timestamp'] == s],
settings.PostProcess)
dias, vd_gas = sc_pp.vd_from_stats(stats_gas[stats_gas['timestamp'] == s],
settings.PostProcess)
nims = sc_pp.count_images_in_stats(stats[stats['timestamp'] == s])
sv = sample_volume * nims
vd_all /= sv
vd_oil /= sv
vd_gas /= sv
d50_all.append(sc_pp.d50_from_vd(vd_all, dias))
d50_oil.append(sc_pp.d50_from_vd(vd_oil, dias))
d50_gas.append(sc_pp.d50_from_vd(vd_gas, dias))
vdts_all.append(vd_all)
vdts_oil.append(vd_oil)
vdts_gas.append(vd_gas)
stats_av = stats[(stats['timestamp']<(dt+td)) & (stats['timestamp']>(dt-td))]
stats_av_oil = scog.extract_oil(stats_av)
stats_av_gas = scog.extract_gas(stats_av)
d50_av_all.append(sc_pp.d50_from_stats(stats_av, settings.PostProcess))
d50_av_oil.append(sc_pp.d50_from_stats(stats_av_oil, settings.PostProcess))
d50_av_gas.append(sc_pp.d50_from_stats(stats_av_gas, settings.PostProcess))
dias, vdts_av = sc_pp.vd_from_stats(stats_av, settings.PostProcess)
dias, vdts_av_oil = sc_pp.vd_from_stats(stats_av_oil, settings.PostProcess)
dias, vdts_av_gas = sc_pp.vd_from_stats(stats_av_gas, settings.PostProcess)
nims = sc_pp.count_images_in_stats(stats_av)
sv = sample_volume * nims
vdts_av /= sv
vdts_av_oil /= sv
vdts_av_gas /= sv
gor.append(np.sum(vdts_av_gas)/np.sum(vdts_av_oil))
outpath, outfile = os.path.split(statsfile)
outfile = outfile.replace('-STATS.csv','')
outfile = os.path.join(outpath, outfile)
time_series = pd.DataFrame(data=np.squeeze(vdts_all), columns=dias)
time_series['D50'] = d50_all
time_series['Time'] = timestamp
time_series.to_excel(outfile +
'-TIMESERIES' + '' + '.xlsx')
time_series = pd.DataFrame(data=np.squeeze(vdts_oil), columns=dias)
time_series['D50'] = d50_oil
time_series['Time'] = timestamp
time_series.to_excel(outfile +
'-TIMESERIES' + 'oil' + '.xlsx')
time_series = pd.DataFrame(data=np.squeeze(vdts_gas), columns=dias)
time_series['D50'] = d50_gas
time_series['Time'] = timestamp
time_series.to_excel(outfile +
'-TIMESERIES' + 'gas' + '.xlsx')
plt.figure(figsize=(20, 10))
if not np.min(np.isnan(d50_oil)):
plt.plot(timestamp, d50_oil, 'ro')
if not np.min(np.isnan(d50_av_oil)):
plt.plot(timestamp, d50_av_oil, 'r-')
lns1 = plt.plot(np.nan, np.nan, 'r-', label='OIL')
if not np.min(np.isnan(d50_gas)):
plt.plot(timestamp, d50_gas, 'bo')
if not np.min(np.isnan(d50_av_gas)):
plt.plot(timestamp, d50_av_gas, 'b-')
lns2 = plt.plot(np.nan, np.nan, 'b-', label='GAS')
plt.ylabel('d50 [um]')
plt.ylim(0, max(plt.gca().get_ylim()))
ax = plt.gca().twinx()
plt.sca(ax)
plt.ylabel('GOR')
if not np.min(np.isnan(gor)):
plt.plot(timestamp, gor, 'k')
lns3 = plt.plot(np.nan, np.nan, 'k', label='GOR')
plt.ylim(0, max(plt.gca().get_ylim()))
lns = lns1 + lns2 + lns3
labs = [l.get_label() for l in lns]
plt.legend(lns, labs)
plt.savefig(outfile +
'-d50_TimeSeries.png', dpi=600, bbox_inches='tight')
plt.close()
print('Export figure made. ')
print('Exporting averages... ')
# average all
dias, vd = sc_pp.vd_from_stats(stats,
settings.PostProcess)
nims = sc_pp.count_images_in_stats(stats)
sv = sample_volume * nims
vd /= sv
d50 = sc_pp.d50_from_vd(vd, dias)
dfa = pd.DataFrame(data=[vd], columns=dias)
dfa['d50'] = d50
timestamp = np.min(pd.to_datetime(stats['timestamp']))
dfa['Time'] = timestamp
dfa.to_excel(statsfile.replace('-STATS.csv', '') +
'-AVERAGE' + '' + '.xlsx')
#average oil
dias, vd = sc_pp.vd_from_stats(stats_oil,
settings.PostProcess)
vd /= sv # sample volume remains the same as 'all'
d50 = sc_pp.d50_from_vd(vd, dias)
dfa = pd.DataFrame(data=[vd], columns=dias)
dfa['d50'] = d50
timestamp = np.min(pd.to_datetime(stats['timestamp'])) # still use total stats for this time
dfa['Time'] = timestamp
dfa.to_excel(statsfile.replace('-STATS.csv', '') +
'-AVERAGE' + 'oil' + '.xlsx')
#average gas
dias, vd = sc_pp.vd_from_stats(stats_gas,
settings.PostProcess)
vd /= sv # sample volume remains the same as 'all'
d50 = sc_pp.d50_from_vd(vd, dias)
dfa = pd.DataFrame(data=[vd], columns=dias)
dfa['d50'] = d50
timestamp = np.min(pd.to_datetime(stats['timestamp'])) # still use total stats for this time
dfa['Time'] = timestamp
dfa.to_excel(statsfile.replace('-STATS.csv', '') +
'-AVERAGE' + 'gas' + '.xlsx')
print('Export done: ', outfile)
def summarise_fancy_stats(stats_csv_file, config_file, monitor=False,
maxlength=100000, msize=2048, oilgas=sc_pp.outputPartType.all):
'''
Plots a summary figure of a dataset which shows
the volume distribution, number distribution and a montage of randomly selected particles
Args:
stats_csv_file (str) : path of the *-STATS.csv file created by silcam process
config_file (str) : path of the config ini file associated with the data
monitor=False (Bool) : if True then this function will run forever, continuously reading the stats_csv_file and plotting the data
might be useful in monitoring the progress of processing, for example
maxlength=100000 (int) : particles longer than this number will not be put in the montage
msize=2048 (int) : the montage created will have a canvas size of msize x msize pixels
oilgas=oc_pp.outputPartType.all : the oilgas enum if you want to just make the figure for oil, or just gas (defulats to all particles)
'''
sns.set_style('ticks')
settings = PySilcamSettings(config_file)
min_length = settings.ExportParticles.min_length + 1
#f,a = plt.subplots(2,2)
ax1 = plt.subplot2grid((2,2),(0, 0))
ax2 = plt.subplot2grid((2,2),(1, 0))
ax3 = plt.subplot2grid((2,2), (0, 1), rowspan=2)
logger = logging.getLogger(__name__)
while True:
try:
montage = sc_pp.make_montage(stats_csv_file,
settings.PostProcess.pix_size,
roidir=settings.ExportParticles.outputpath,
auto_scaler=msize*2, msize=msize,
maxlength=maxlength,
oilgas=oilgas)
except:
montage = np.zeros((msize, msize, 3), dtype=np.uint8) + 255
logger.warning('Unable to make montage. Check: {0} folder for h5 files'.format(settings.ExportParticles.outputpath))
logger.warning(' in config file ExportParticles.export_images is {0}'.format(settings.ExportParticles.export_images))
stats = pd.read_csv(stats_csv_file)
stats = stats[(stats['major_axis_length'] *
settings.PostProcess.pix_size) < maxlength]
# average numer and volume concentrations
nc, vc, sv_total, junge = sc_pp.nc_vc_from_stats(stats,
settings.PostProcess, oilgas=oilgas)
# extract only wanted particle stats
if oilgas==sc_pp.outputPartType.oil:
from pysilcam.oilgas import extract_oil
stats = extract_oil(stats)
elif oilgas==sc_pp.outputPartType.gas:
from pysilcam.oilgas import extract_gas
stats = extract_gas(stats)
d50 = sc_pp.d50_from_stats(stats, settings.PostProcess)
total_measured_particles = len(stats['major_axis_length'])
plt.sca(ax1)
plt.cla()
psd(stats, settings.PostProcess, plt.gca())
plt.title('Volume conc.: {0:.2f}uL/L d50: {1:.0f}um'.format(vc, d50))
plt.sca(ax2)
plt.cla()
nd(stats, settings.PostProcess, plt.gca(), sample_volume=sv_total)
plt.title('Number conc.: {0:.0f}#/L Junge exp.: {1:.2f}'.format(nc,
junge))
plt.sca(ax3)
plt.cla()
montage_plot(montage, settings.PostProcess.pix_size)
plt.title('Volume processed: {0:.1f}L {1:.0f} particles measured'.format(sv_total,
total_measured_particles))
plt.draw()
if monitor:
plt.pause(1)
else:
break