def db_saver(self):
"""Autoclosing db_saver yield_fixture"""
# Init db_saver
db_saver = DataSetSaver("measurements_dummy", "xy_values_dummy", "dummy", "dummy")
db_saver.start()
yield db_saver
db_saver.stop()
def __init__(self):
# Initialize power supply
# Initialize flow meter driver
"""Try to connect to the COM port of the flow meter and ask for the serial number"""
print("Try and connect to red flow meter")
try:
self.red_flow_meter = RedFlowMeter(ramp.RED_FLOW_METER_COM_PORT,
slave_address=2)
except SerialException:
message = 'Cannot find red flow meter on {}'.format(
ramp.RED_FLOW_METER_COM_PORT)
raise RuntimeError(message)
# Test serial number reply
if self.red_flow_meter.read_value('serial') != 181787:
raise RuntimeError('Incorrect reply to serial number')
# Print actual flow value
print(self.red_flow_meter.read_value('fluid_name'),
self.red_flow_meter.read_value('flow'))
self.red_flow_meter.set_address(247)
print(self.red_flow_meter.read_value('fluid_name'),
self.red_flow_meter.read_value('flow'))
# Try to conect to power supply
print("Try and connect to power supply")
try:
self.cpx = CPX400DPDriver(output=1,
interface='serial',
device=ramp.POWER_SUPPLY_COM_PORT)
except SerialException:
message = 'Cannot find power supply on {}'.format(
ramp.POWER_SUPPLY_COM_PORT)
raise RuntimeError(message)
if 'CPX400DP' not in self.cpx.read_software_version():
raise RuntimeError('Incorrect software version reply')
pass
# Print the set point voltage
print("The voltage set point is {}".format(
self.cpx.read_set_voltage()))
self.ramp = ramp.RAMP
self.metadata = ramp.metadata
self.verify_ramp()
self.data_set_saver = DataSetSaver(
'measurements_large_CO2_MEA',
'xy_values_large_CO2_MEA',
credentials.USERNAME,
credentials.PASSWORD,
)
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sock.settimeout(0.1)
self.host = ramp.RASPPI_HOST
self.port = ramp.RASSPI_PORT
self.message = 'json_wn#' + json.dumps({'no_voltages_to_set': True})
def __init__(self, mux, smu, comment, user):
self.data_set_saver = DataSetSaver("measurements_" + user,
"xy_values_" + user, user, user)
self.data_set_saver.start()
self.data_set_time = time.time() # Update for each scan...
self.comment = comment
self.mux = mux
self.smu = smu
mux.scpi_comm(
'INSTRUMENT:DMM OFF') # Turn off DMM to allow use as mux device
self.smu.set_current_limit(0.01) # Current limit, 10mA
self.smu.set_current_measure_range(
current_range=0.01) # Measurement range 10mA
self.smu.set_integration_time(1)
def start_measurement(self, comment):
""" Start a measurement """
data_set_saver = DataSetSaver("measurements_dummy", "xy_values_dummy",
"dummy", "dummy")
data_set_saver.start()
data_set_time = time.time()
metadata = {
"Time": CustomColumn(data_set_time, "FROM_UNIXTIME(%s)"),
"comment": comment,
"type": 65,
"preamp_range": 9,
"sem_voltage": -1
}
labels = [
"2W Heater 1", "2W Heater 2", "2W RTD Source", "2W RTD Sense",
"Short circuit check", "RTD Source Sense Left",
"RTD Source Sense Right", "4W RTD", "Temperature", "Temperature 2"
]
for label in labels:
metadata["mass_label"] = label
data_set_saver.add_measurement(label, metadata)
for i in range(0, 99999999):
self.update_values()
for label in labels:
print(
str(i) + ': Channel: ' + label + ': ' +
str(self.data[label]))
data_set_saver.save_point(
label, (time.time() - data_set_time, self.data[label]))
time.sleep(1)
def init_logging(self):
if self.log_data:
self.comment = 'Run'
self.data_set_saver = DataSetSaver('measurements_mailscan',
'xy_values_mailscan',
credentials.user, credentials.passwd)
self.data_set_saver.start()
# PyExpLabSys does does not excell in db-normalization - add
# metadata to all channels
metadata = {"Time": CustomColumn(self.start_time, "FROM_UNIXTIME(%s)"),
"comment": self.comment, "type": 1, "label": None,
"processes": self.number_of_threads()[1]}
metadata['label'] = 'Avg export speed'
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = 'Total export size'
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = 'Total users'
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = 'Total memory'
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = 'Total CPU'
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = 'Total Mails'
self.data_set_saver.add_measurement(metadata['label'], metadata)
for scanner in self.amqp_messages.keys():
if scanner == 'global':
continue
metadata['processes'] = 1
metadata['label'] = '{} memory'.format(scanner)
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = '{} exported users'.format(scanner)
self.data_set_saver.add_measurement(metadata['label'], metadata)
def __init__(self, hostname, anode):
threading.Thread.__init__(self)
self.agilent = agilent_34972A.Agilent34972ADriver(interface='lan',
hostname=hostname)
self.calib = 500 # Analyser voltage pr. input voltage
self.chamber_name = "dummy"
self.data_set_saver = DataSetSaver("measurements_" + self.chamber_name,
"xy_values_" + self.chamber_name,
credentials.user,
credentials.passwd)
self.data_set_saver.start()
if anode == 'Mg':
self.x_ray = 1253.44
if anode == 'Al':
self.x_ray = 1487.0
def start_measurement(self, comment):
""" Start a measurement """
data_set_saver = DataSetSaver("measurements_dummy", "xy_values_dummy",
"dummy", "dummy")
data_set_saver.start()
data_set_time = time.time()
metadata = {"Time": CustomColumn(data_set_time, "FROM_UNIXTIME(%s)"),
"comment": comment, "type": 65,
"preamp_range": 9, "sem_voltage": -1}
labels = ["2W Heater 1", "2W Heater 2", "2W RTD Source", "2W RTD Sense",
"Short circuit check", "RTD Source Sense Left",
"RTD Source Sense Right", "4W RTD", "Temperature", "Temperature 2"]
for label in labels:
metadata["mass_label"] = label
data_set_saver.add_measurement(label, metadata)
for i in range(0, 99999999):
self.update_values()
for label in labels:
print(str(i) + ': Channel: ' + label + ': ' + str(self.data[label]))
data_set_saver.save_point(label, (time.time() - data_set_time,
self.data[label]))
time.sleep(1)
def __init__(self, mux, smu, comment, user):
self.data_set_saver = DataSetSaver("measurements_" + user,
"xy_values_" + user, user, user)
self.data_set_saver.start()
self.data_set_time = time.time() # Update for each scan...
self.comment = comment
self.mux = mux
self.smu = smu
mux.scpi_comm('INSTRUMENT:DMM OFF') # Turn off DMM to allow use as mux device
self.smu.set_current_limit(0.01) # Current limit, 10mA
self.smu.set_current_measure_range(current_range=0.01) # Measurement range 10mA
self.smu.set_integration_time(1)
def __init__(self):
# Initialize power supply
# Initialize flow meter driver
"""Try to connect to the COM port of the flow meter and ask for the serial number"""
print("Try and connect to red flow meter")
try:
self.red_flow_meter = RedFlowMeter(ramp.RED_FLOW_METER_COM_PORT, slave_address=42)
except SerialException:
message = 'Cannot find red flow meter on {}'.format(ramp.RED_FLOW_METER_COM_PORT)
raise RuntimeError(message)
# Test serial number reply
if self.red_flow_meter.read_value('serial') != 210059:
raise RuntimeError('Incorrect reply to serial number')
# Print actual flow value
print(self.red_flow_meter.read_value('fluid_name'), self.red_flow_meter.read_value('flow'))
# If more flow meters are used try them here
#self.red_flow_meter.set_address(247)
#print(self.red_flow_meter.read_value('fluid_name'), self.red_flow_meter.read_value('flow'))
# Try to conect to power supply
print("Try and connect to power supply")
try:
self.cpx = CPX400DPDriver(output=1, interface='serial', device=ramp.POWER_SUPPLY_COM_PORT)
except SerialException:
message = 'Cannot find power supply on {}'.format(ramp.POWER_SUPPLY_COM_PORT)
raise RuntimeError(message)
if 'CPX400DP' not in self.cpx.read_software_version():
raise RuntimeError('Incorrect software version reply')
pass
# Print the set point voltage
print("The voltage set point is {}".format(self.cpx.read_set_voltage()))
self.ramp = ramp.RAMP
self.metadata = ramp.metadata
self.area = ramp.area
self.setpoint_pressure = ramp.setpoint_pressure
self.setpoint_gas_flow = ramp.setpoint_gas_flow
self.verify_ramp()
self.data_set_saver = DataSetSaver(
'measurements_large_CO2_MEA',
'xy_values_large_CO2_MEA',
credentials.USERNAME,
credentials.PASSWORD,
)
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sock.settimeout(0.1)
self.host = ramp.RASPPI_HOST
self.port = ramp.RASSPI_PORT
self.message = 'json_wn#' + json.dumps({'no_voltages_to_set':True})
def init_logging(self):
if self.log_data:
self.comment = 'Run'
self.data_set_saver = DataSetSaver('measurements_mailscan',
'xy_values_mailscan',
credentials.user,
credentials.passwd)
self.data_set_saver.start()
# PyExpLabSys does does not excell in db-normalization - add
# metadata to all channels
metadata = {
"Time": CustomColumn(self.start_time, "FROM_UNIXTIME(%s)"),
"comment": self.comment,
"type": 1,
"label": None,
"processes": self.number_of_threads()[1]
}
metadata['label'] = 'Avg export speed'
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = 'Total export size'
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = 'Total users'
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = 'Total memory'
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = 'Total CPU'
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = 'Total Mails'
self.data_set_saver.add_measurement(metadata['label'], metadata)
for scanner in self.amqp_messages.keys():
if scanner == 'global':
continue
metadata['processes'] = 1
metadata['label'] = '{} memory'.format(scanner)
self.data_set_saver.add_measurement(metadata['label'],
metadata)
metadata['label'] = '{} exported users'.format(scanner)
self.data_set_saver.add_measurement(metadata['label'],
metadata)
def db_saver(self):
"""Autoclosing db_saver yield_fixture"""
# Init db_saver
db_saver = DataSetSaver('measurements_dummy', 'xy_values_dummy',
'dummy', 'dummy')
db_saver.start()
yield db_saver
db_saver.stop()
class SolarCellTester(object):
""" Performs an IV test """
def __init__(self, mux, smu, comment, user):
self.data_set_saver = DataSetSaver("measurements_" + user,
"xy_values_" + user, user, user)
self.data_set_saver.start()
self.data_set_time = time.time() # Update for each scan...
self.comment = comment
self.mux = mux
self.smu = smu
mux.scpi_comm('INSTRUMENT:DMM OFF') # Turn off DMM to allow use as mux device
self.smu.set_current_limit(0.01) # Current limit, 10mA
self.smu.set_current_measure_range(current_range=0.01) # Measurement range 10mA
self.smu.set_integration_time(1)
def run_measurement(self, v_from, v_to, scan_both_ways):
""" Perform the actual measurement """
metadata = {"Time": CustomColumn(self.data_set_time, "FROM_UNIXTIME(%s)"),
"comment": self.comment, "type": 1}
for label in ["Ch1", "Ch2", "Ch3", "Ch4", "Ch5", "Ch6", "Ch7", "Ch8",]:
metadata["mass_label"] = label
self.data_set_saver.add_measurement(label, metadata)
self.smu.set_voltage(0.0)
self.smu.output_state(True)
for channel in range(1, 9):
print('Channel: ' + str(channel))
self.mux.scpi_comm('ROUTE:CLOSE (@10' + str(channel) + ')')
time.sleep(0.2)
voltage, current = self.smu.iv_scan(v_from=v_from, v_to=v_to,
steps=10, settle_time=0)
for i in range(0, len(current)):
self.data_set_saver.save_point('Ch' + str(channel), (voltage[i], current[i]))
if scan_both_ways:
voltage, current = self.smu.iv_scan(v_from=v_to, v_to=v_from,
steps=50, settle_time=0)
for i in range(0, len(current)):
self.data_set_saver.save_point('Ch' + str(channel),
(voltage[i], current[i]))
self.mux.scpi_comm('ROUTE:OPEN (@10' + str(channel) + ')')
self.smu.set_voltage(0.0)
self.smu.output_state(False)
time.sleep(0.2)
def __init__(self, hostname, anode):
threading.Thread.__init__(self)
self.agilent = agilent_34972A.Agilent34972ADriver(interface='lan', hostname=hostname)
self.calib = 500 # Analyser voltage pr. input voltage
self.chamber_name = "dummy"
self.data_set_saver = DataSetSaver("measurements_" + self.chamber_name,
"xy_values_" + self.chamber_name,
credentials.user, credentials.passwd)
self.data_set_saver.start()
if anode == 'Mg':
self.x_ray = 1253.44
if anode == 'Al':
self.x_ray = 1487.0
class XPS(threading.Thread):
""" Perform a single XPS scan """
def __init__(self, hostname, anode):
threading.Thread.__init__(self)
self.agilent = agilent_34972A.Agilent34972ADriver(interface='lan',
hostname=hostname)
self.calib = 500 # Analyser voltage pr. input voltage
self.chamber_name = "dummy"
self.data_set_saver = DataSetSaver("measurements_" + self.chamber_name,
"xy_values_" + self.chamber_name,
credentials.user,
credentials.passwd)
self.data_set_saver.start()
if anode == 'Mg':
self.x_ray = 1253.44
if anode == 'Al':
self.x_ray = 1487.0
def scan(self, start_energy, end_energy, step, integration_time):
""" Perform a scan """
metadata = {
"Time": CustomColumn(time.time(), "FROM_UNIXTIME(%s)"),
"comment": 'Test',
"type": 2
}
label = 'XPS signal'
self.data_set_saver.add_measurement(label, metadata)
count_string = self.agilent.scpi_comm("SENS:TOT:DATA? (@203)")
meas_time = time.time()
for binding_energy in np.arange(end_energy, start_energy, -1 * step):
kin_energy = str((self.x_ray - binding_energy) / self.calib)
string = "SOURCE:VOLT " + kin_energy + ", (@205)"
self.agilent.scpi_comm(string)
time.sleep(integration_time)
count_string = self.agilent.scpi_comm("SENS:TOT:DATA? (@203)")
count = int(float(count_string.strip()))
int_time = time.time() - meas_time
meas_time = time.time()
count_rate = count / int_time
self.data_set_saver.save_point(label, (binding_energy, count_rate))
class XPS(threading.Thread):
""" Perform a single XPS scan """
def __init__(self, hostname, anode):
threading.Thread.__init__(self)
self.agilent = agilent_34972A.Agilent34972ADriver(interface='lan', hostname=hostname)
self.calib = 500 # Analyser voltage pr. input voltage
self.chamber_name = "dummy"
self.data_set_saver = DataSetSaver("measurements_" + self.chamber_name,
"xy_values_" + self.chamber_name,
credentials.user, credentials.passwd)
self.data_set_saver.start()
if anode == 'Mg':
self.x_ray = 1253.44
if anode == 'Al':
self.x_ray = 1487.0
def scan(self, start_energy, end_energy, step, integration_time):
""" Perform a scan """
metadata = {"Time": CustomColumn(time.time(), "FROM_UNIXTIME(%s)"),
"comment": 'Test', "type": 2}
label = 'XPS signal'
self.data_set_saver.add_measurement(label, metadata)
count_string = self.agilent.scpi_comm("SENS:TOT:DATA? (@203)")
meas_time = time.time()
for binding_energy in np.arange(end_energy, start_energy, -1 * step):
kin_energy = str((self.x_ray - binding_energy) / self.calib)
string = "SOURCE:VOLT " + kin_energy + ", (@205)"
self.agilent.scpi_comm(string)
time.sleep(integration_time)
count_string = self.agilent.scpi_comm("SENS:TOT:DATA? (@203)")
count = int(float(count_string.strip()))
int_time = time.time() - meas_time
meas_time = time.time()
count_rate = count / int_time
self.data_set_saver.save_point(label, (binding_energy, count_rate))
from __future__ import print_function
import os
import time
import logging
# logging.basicConfig(level=logging.DEBUG) # Comment in for more logging output
from collections import defaultdict
from PyExpLabSys.file_parsers.chemstation import Sequence
from PyExpLabSys.common.database_saver import DataSetSaver
from PyExpLabSys.common.database_saver import CustomColumn
import credentials
# Instantiate the data set saver
data_set_saver = DataSetSaver('measurements_vhp_setup', 'xy_values_vhp_setup',
credentials.USERNAME, credentials.PASSWORD)
data_set_saver.start()
# Get the set of aleady uploaded files
already_uploaded = data_set_saver.get_unique_values_from_measurements('relative_path')
print('Fetched relative paths for {} known sequences'.format(len(already_uploaded)))
# This is the measurement path, should be generated somehow
basefolder = '/home/cinf/Desktop/Shared_folder'
sequence_identifyer = 'sequence.acaml'
for root, dirs, files in os.walk(basefolder):
if sequence_identifyer in files:
# Check if file is known
relative_path = root.replace(basefolder, '').strip(os.sep)
if relative_path in already_uploaded:
continue
# -*- coding: utf-8 -*-
"""
Spyder Editor
This is a temporary script file.
"""
import time
from PyExpLabSys.common.database_saver import DataSetSaver, CustomColumn
import credentials
data_set_saver = DataSetSaver(
'measurements_large_CO2_MEA',
'xy_values_large_CO2_MEA',
credentials.USERNAME,
credentials.PASSWORD,
)
data_set_saver.start()
now = time.time()
now_custom_column = CustomColumn(now, 'FROM_UNIXTIME(%s)')
#CREATE TABLE `measurements_large_CO2_MEA` (
# `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
# `time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
# `type` int(10) unsigned NOT NULL COMMENT 'Type of measurement, as found in the types table',
# `comment` varchar(128) NOT NULL COMMENT 'Comment',
# `cathode_gas_type` varchar(128) NOT NULL COMMENT 'Cathode gas type',
# `anode_gas_type` varchar(128) NOT NULL COMMENT 'Anode gas type',
# `cathode_catalyst_description` varchar(128) NOT NULL COMMENT 'Cathode catalyst description',
# `anode_catalyst_description` varchar(128) NOT NULL COMMENT 'Anode catalyst description',
if database_saver:
self.data_set_saver.save_point(self.codename, (t, reading))
print(reading, t, status)
except KeyboardInterrupt:
break
print('Emptying buffer..')
time.sleep(1)
if self.ser.inWaiting() > 0:
self.ser.readline()
print('Done')
if database_saver:
self.data_set_saver.stop()
if __name__ == '__main__':
from PyExpLabSys.common.database_saver import DataSetSaver
import credentials2 as cred
port = '/dev/serial/by-id/usb-Prolific_Technology_Inc._USB-Serial_Controller-if00-port0'
data_set_saver = DataSetSaver('measurements_dummy', 'xy_values_dummy',
cred.user, cred.passwd)
dmm = Keithley(port,
data_set_saver=data_set_saver,
codename='keithley_voltage')
time.sleep(1)
print(dmm.comm('*IDN?'))
dmm.read_voltage(meas_range=20, database_saver=False, count=10)
class RampRunner(object):
"""The RampRunner. Program to run current and voltage measurements and log measurements
of voltage, current, temperature, pressure and flow to the surfcat database.
"""
def __init__(self):
# Initialize power supply
# Initialize flow meter driver
"""Try to connect to the COM port of the flow meter and ask for the serial number"""
print("Try and connect to red flow meter")
try:
self.red_flow_meter = RedFlowMeter(ramp.RED_FLOW_METER_COM_PORT, slave_address=42)
except SerialException:
message = 'Cannot find red flow meter on {}'.format(ramp.RED_FLOW_METER_COM_PORT)
raise RuntimeError(message)
# Test serial number reply
if self.red_flow_meter.read_value('serial') != 210059:
raise RuntimeError('Incorrect reply to serial number')
# Print actual flow value
print(self.red_flow_meter.read_value('fluid_name'), self.red_flow_meter.read_value('flow'))
# If more flow meters are used try them here
#self.red_flow_meter.set_address(247)
#print(self.red_flow_meter.read_value('fluid_name'), self.red_flow_meter.read_value('flow'))
# Try to conect to power supply
print("Try and connect to power supply")
try:
self.cpx = CPX400DPDriver(output=1, interface='serial', device=ramp.POWER_SUPPLY_COM_PORT)
except SerialException:
message = 'Cannot find power supply on {}'.format(ramp.POWER_SUPPLY_COM_PORT)
raise RuntimeError(message)
if 'CPX400DP' not in self.cpx.read_software_version():
raise RuntimeError('Incorrect software version reply')
pass
# Print the set point voltage
print("The voltage set point is {}".format(self.cpx.read_set_voltage()))
self.ramp = ramp.RAMP
self.metadata = ramp.metadata
self.area = ramp.area
self.setpoint_pressure = ramp.setpoint_pressure
self.setpoint_gas_flow = ramp.setpoint_gas_flow
self.verify_ramp()
self.data_set_saver = DataSetSaver(
'measurements_large_CO2_MEA',
'xy_values_large_CO2_MEA',
credentials.USERNAME,
credentials.PASSWORD,
)
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sock.settimeout(0.1)
self.host = ramp.RASPPI_HOST
self.port = ramp.RASSPI_PORT
self.message = 'json_wn#' + json.dumps({'no_voltages_to_set':True})
def verify_ramp(self):
"""Verify the ramp for consistency and safety"""
for step in self.ramp:
# Check if a contant voltage step has the right keys
if step['type'] == 'constant_voltage':
required_keys = {'type','duration','end_voltage', 'save_rate'}
if step.keys() != required_keys:
raise RuntimeError('The keys must in a constant voltage step must be\
{}'.format(required_keys))
# Check if a voltage ramp step has the right keys
elif step['type'] == 'voltage_ramp':
required_keys = {'type', 'duration',\
'start_voltage', 'end_voltage', 'voltage_step', 'save_rate'}
if step.keys() != required_keys:
raise RuntimeError('The keys must in a constant voltage step must be\
{}'.format(required_keys))
elif step['start_voltage'] < 0:
raise RuntimeError('Voltage must be must be positive and less than 5V, got: {}V'\
.format(step['end_voltage']))
elif step['end_voltage'] > 5:
raise RuntimeError('Voltage must be must be positive and less than 5V, got: {}V'\
.format(step['end_voltage']))
elif step['type'] == 'constant_current':
required_keys = {'type', 'duration',\
'current_density', 'save_rate'}
if step.keys() != required_keys:
raise RuntimeError('The keys must in a constant voltage step must be\
{}'.format(required_keys))
elif step['current_density'] < 0:
raise RuntimeError('Current density must be positive, got: {}mA/cm2'\
.format(step['current_density']))
# Do checks on the common keys
elif step['duration'] <= 0:
raise RuntimeError('Duration must be positive, got: {}'.format(step['duration']))
# Check that the voltage is greater than 0
elif step['end_voltage'] < 0:
raise RuntimeError('Voltage must be must be positive and less than 5V, got: {}V'\
.format(step['end_voltage']))
#...and less than 5V
elif step['end_voltage'] > 5:
raise RuntimeError('Voltage must be must be positive and less than 5V, got: {}V'\
.format(step['end_voltage']))
# Check if the rate at which the data should be saved is positive.
elif step['save_rate'] <=0:
raise RuntimeError('The save rate must be positive, got: {}'.format(step['save_rate']))
def run(self):
"""Run the ramp
This is the main function that contains a loop over the steps in the ramp and the
method that does all the work
"""
self.data_set_saver.start()
# Specify the start time for database
now = time()
now_custom_column = CustomColumn(now, 'FROM_UNIXTIME(%s)')
# Add all measurements
self.metadata['label'] = 'voltage'
self.data_set_saver.add_measurement('voltage', self.metadata)
self.metadata['label'] = 'current'
self.data_set_saver.add_measurement('current', self.metadata)
self.metadata['label'] = 'CO2 flow'
self.data_set_saver.add_measurement('CO2 flow', self.metadata)
# If another flow meter is used collect the flow here
#self.metadata['label'] = 'cell outlet flow'
#self.data_set_saver.add_measurement('cell outlet flow', self.metadata)
# Collect the pressure reading from the pressure controller
#self.metadata['label'] = 'pressure'
#self.data_set_saver.add_measurement('pressure', self.metadata)
# Add the one custom column
self.metadata['time'] = now_custom_column
# Set the current limit on the power supply to 10A
self.cpx.set_current_limit(10)
# Change address to CO2 flow controller
self.red_flow_meter.set_address(42)
# Set the CO2 flow to the specified value
self.red_flow_meter.write_value('setpoint_gas_flow', self.setpoint_gas_flow)
# Set the pressure on the pressure controller
#self.message = 'json_wn#' + json.dumps({'A': 0.0, 'B': self.setpoint_pressure })
#self.sock.sendto(message.encode('ascii'), (self.host, self.port))
try:
for step in self.ramp:
# Go through the steps in the ramp file
if step['type'] == 'constant_voltage':
self.run_constant_voltage_step(step)
elif step['type'] == 'voltage_ramp':
self.run_voltage_ramp_step(step)
elif step['type'] == 'constant_current':
self.run_constant_current_step(step)
except KeyboardInterrupt:
print('Program interupted by user')
finally:
# End the data collection and turn off power supply after run or interruption
print('The program has ended')
self.cpx.output_status(False)
self.red_flow_meter.set_address(42)
self.red_flow_meter.write_value('setpoint_gas_flow', 0.0)
#self.message = 'json_wn#' + json.dumps({'A': 0.0, 'B': 0.0})
#self.sock.sendto(message.encode('ascii'), (self.host, self.port))
self.data_set_saver.wait_for_queue_to_empty()
self.data_set_saver.stop()
def run_constant_voltage_step(self, step):
"""Run a single step"""
print('Run constant voltage step with parameters', step)
start_time = time() # Read from step
# Set voltage on power supply
voltage_difference_limit = 0.01
print("Setting voltage to {} V".format(step["end_voltage"]))
self.cpx.set_voltage(step["end_voltage"])
while abs(step["end_voltage"]-self.cpx.read_set_voltage())>voltage_difference_limit:
print(self.cpx.read_set_voltage())
pass
# Turn on the power supply
self.cpx.output_status(True)
last_save = 0
# Read of the flow and current as long as the duration lasts
while (time() - start_time) < step['duration']*3600:
# Log results to database
# Send communication to datacard
self.sock.sendto(self.message.encode('ascii'), (self.host, self.port))
try:
reply = self.sock.recv(1024).decode('ascii')
except socket.timeout:
pass
else:
# This is the data collection rate right now
sleep(step['save_rate'])
now = time()
# Read voltage on power supply
power_supply_voltage = self.cpx.read_actual_voltage()
print('The voltage is: {}V on the power supply'.format(power_supply_voltage))
# Read actual voltage on raspberry pi
reply_data = json.loads(reply[4:])
actual_voltage = reply_data['3']
print('The voltage is: {}V on the cell'.format(actual_voltage))
#pressure = reply_data['1']
# Read the current from the power supply
actual_current = self.cpx.read_actual_current()
print('The current is: {}A'.format(actual_current))
# Change address to CO2 flow meter
self.red_flow_meter.set_address(42)
# Get the CO2 flow from the MFC
CO2_flow = self.red_flow_meter.read_value('flow')
# If another flow meter is used collect the flow here
#self.red_flow_meter.set_address(247)
# Get the flow from the cell into the GC from the MFM
#cell_outlet_flow = self.red_flow_meter.read_value('flow')
# time between data is saved
print('Time since last save: {}s'.format(time()-last_save))
last_save = time()
# Save the measurements to the database
self.data_set_saver.save_point('voltage', (now, actual_voltage))
self.data_set_saver.save_point('current', (now, actual_current))
self.data_set_saver.save_point('CO2 flow', (now, CO2_flow))
#self.data_set_saver.save_point('CO2 flow', (now, pressure))
#self.data_set_saver.save_point('cell outlet flow', (now, cell_outlet_flow))
def run_constant_current_step(self, step):
"""Run a single step"""
print('Run constant current step with parameters', step)
start_time = time() # Read from step
# Change address to CO2 flow meter
self.red_flow_meter.set_address(42)
# Set current limit on power supply and set a high voltage to hit the limit
# and let the power supply control the voltage
current_density = step['current_density'] # in mA
# Calculate the current from the corresponding area and corrent density
current = current_density/1000*self.area
print("Setting current to {}A".format(current))
self.cpx.set_current_limit(current)
self.cpx.set_voltage(4)
# Turn on the power supply
self.cpx.output_status(True)
#self.red_flow_meter.write_value('setpoint_gas_flow', step['setpoint_gas_flow'])
last_save = 0
# Read of the flow and current as long as the duration lasts
while (time() - start_time) < step['duration']*3600:
# Log results to database
# Send communication to datacard
self.sock.sendto(self.message.encode('ascii'), (self.host, self.port))
try:
reply = self.sock.recv(1024).decode('ascii')
except socket.timeout:
pass
else:
# This is the data collection rate right now
sleep(step['save_rate'])
now = time()
# Read voltage on power supply
power_supply_voltage = self.cpx.read_actual_voltage()
print('The voltage is: {}V on the power supply'.format(power_supply_voltage))
# Read actual voltage on raspberry pi
reply_data = json.loads(reply[4:])
actual_voltage = reply_data['3']
print('The voltage is: {}V on the cell'.format(actual_voltage))
#pressure = reply_data['1']
# Read the current from the power supply
actual_current = self.cpx.read_actual_current()
print('The current is: {}A'.format(actual_current))
# Change address to CO2 flow controller
self.red_flow_meter.set_address(42)
# Get the CO2 flow from the MFM
CO2_flow = self.red_flow_meter.read_value('flow')
# If another flow meter is used collect the flow here
# Change adress to the cell outlet MFM
#self.red_flow_meter.set_address(247)
# Get the CO2 flow from the MFM
#cell_outlet_flow = self.red_flow_meter.read_value('flow')
# time between data is saved
print('Time since last save: {}s'.format(time()-last_save))
last_save = time()
# Save the measurements to the database
self.data_set_saver.save_point('voltage', (now, actual_voltage))
self.data_set_saver.save_point('current', (now, actual_current))
self.data_set_saver.save_point('CO2 flow', (now, CO2_flow))
#self.data_set_saver.save_point('pressure', (now, pressure))
#self.data_set_saver.save_point('cell outlet flow', (now, cell_outlet_flow))
def run_voltage_ramp_step(self, step):
""" Run voltage ramp step"""
print('Run voltage ramp step with parameters', step)
# Start time of this step for reference
start_time = time()
# Set voltage to the start voltage in the step and turn on the voltage
self.cpx.set_voltage(step['start_voltage'])
print('Starting voltage ramp at: {}'.format(step['start_voltage']))
self.cpx.output_status(True)
voltage_set_point = step['start_voltage']
while (time() - start_time) < step['duration']*3600:
now = time()
# Change address to CO2 flow meter
self.red_flow_meter.set_address(247)
# Read flow value
CO2_flow = self.red_flow_meter.read_value('flow')
print(self.red_flow_meter.read_value('fluid_name'), CO2_flow)
# Change address to N2 flow meter
self.red_flow_meter.set_address(2)
# Read flow value
N2_flow = self.red_flow_meter.read_value('flow')
print(self.red_flow_meter.read_value('fluid_name'), N2_flow)
# Set the voltage on the power supply
self.cpx.set_voltage(voltage_set_point)
# Read actual voltage
actual_voltage = self.cpx.read_actual_voltage()
print('The current voltage is: {}V'.format(actual_voltage))
print('The voltage set point is {}V'.format(voltage_set_point))
# Read the current from the power supply
actual_current = self.cpx.read_actual_current()
print('The current current is: {}A'.format(actual_current))
# This is the data collection rate right now
sleep(step['save_rate'])
# Save the measurements to the database
self.data_set_saver.save_point('voltage', (now, actual_voltage))
self.data_set_saver.save_point('current', (now, actual_current))
self.data_set_saver.save_point('CO2 flow', (now, CO2_flow))
# Ramp the voltage by the voltage step in step
voltage_set_point = voltage_set_point + step['voltage_step']
# Terminate the ramp when the end voltage is reached
if voltage_set_point >= step['end_voltage']:
print('end voltage, {} reached'.format(step['end_voltage']))
sleep(1)
# Turn off the power supply
#self.cpx.output_status(False)
break
# Tell when the time loop is done and turn of power supply
print('voltage ramp step has ended')
class SolarCellTester(object):
""" Performs an IV test """
def __init__(self, mux, smu, comment, user):
self.data_set_saver = DataSetSaver("measurements_" + user,
"xy_values_" + user, user, user)
self.data_set_saver.start()
self.data_set_time = time.time() # Update for each scan...
self.comment = comment
self.mux = mux
self.smu = smu
mux.scpi_comm(
'INSTRUMENT:DMM OFF') # Turn off DMM to allow use as mux device
self.smu.set_current_limit(0.01) # Current limit, 10mA
self.smu.set_current_measure_range(
current_range=0.01) # Measurement range 10mA
self.smu.set_integration_time(1)
def run_measurement(self, v_from, v_to, scan_both_ways):
""" Perform the actual measurement """
metadata = {
"Time": CustomColumn(self.data_set_time, "FROM_UNIXTIME(%s)"),
"comment": self.comment,
"type": 1
}
for label in [
"Ch1",
"Ch2",
"Ch3",
"Ch4",
"Ch5",
"Ch6",
"Ch7",
"Ch8",
]:
metadata["mass_label"] = label
self.data_set_saver.add_measurement(label, metadata)
self.smu.set_voltage(0.0)
self.smu.output_state(True)
for channel in range(1, 9):
print('Channel: ' + str(channel))
self.mux.scpi_comm('ROUTE:CLOSE (@10' + str(channel) + ')')
time.sleep(0.2)
voltage, current = self.smu.iv_scan(v_from=v_from,
v_to=v_to,
steps=10,
settle_time=0)
for i in range(0, len(current)):
self.data_set_saver.save_point('Ch' + str(channel),
(voltage[i], current[i]))
if scan_both_ways:
voltage, current = self.smu.iv_scan(v_from=v_to,
v_to=v_from,
steps=50,
settle_time=0)
for i in range(0, len(current)):
self.data_set_saver.save_point('Ch' + str(channel),
(voltage[i], current[i]))
self.mux.scpi_comm('ROUTE:OPEN (@10' + str(channel) + ')')
self.smu.set_voltage(0.0)
self.smu.output_state(False)
time.sleep(0.2)
class VoltageCurrentProgram(Thread): # pylint: disable=too-many-instance-attributes
"""The Voltage Current Program
This program uses the generic stepped program runner as a GUI and
serves information to that
"""
def __init__(self, args):
super(VoltageCurrentProgram, self).__init__()
# Form channel_id e.g: EA1
self.channel_id = args.power_supply + args.output
### Required by the stepped program runner
# Accepted capabilities are: can_edit, can_play,
# can_stop, can_quit
self.capabilities = ('can_stop', 'can_start', 'can_edit')
# Status fields (in order)
self.status_fields = (
# Status
{
'codename': 'status_field',
'title': 'Status'
},
# Voltage
{
'codename': self.channel_id + '_voltage',
'title': 'Voltage',
'formatter': '{:.3f}',
'unit': 'V'
},
# Voltage setpoint
{
'codename': self.channel_id + '_voltage_setpoint',
'title': 'Voltage Setpoint',
'formatter': '{:.3f}',
'unit': 'V'
},
# Current
{
'codename': self.channel_id + '_current',
'title': 'Current',
'formatter': '{:.3f}',
'unit': 'A'
},
# Current limit
{
'codename': self.channel_id + '_current_limit',
'title': 'Current limit',
'formatter': '{:.3f}',
'unit': 'A'
},
# Charge
{
'codename': self.channel_id + '_accum_charge',
'title': 'Accumulated charge',
'formatter': '{:.3f}',
'unit': 'C'
},
# Time elapsed (step)
{
'codename': 'elapsed',
'title': 'Time elapsed (step)',
'formatter': '{:.1f}',
'unit': 's'
},
# Time remaining (step)
{
'codename': 'remaining',
'title': 'Time remaining (step)',
'formatter': '{:.1f}',
'unit': 's'
},
# Time elapsed (total)
{
'codename': 'elapsed_total',
'title': 'Time elapsed (total)',
'formatter': '{:.1f}',
'unit': 's'
},
# Time remaining (total)
{
'codename': 'remaining_total',
'title': 'Time remaining (total)',
'formatter': '{:.1f}',
'unit': 's'
},
# Iteration time
{
'codename': 'iteration_time',
'title': 'Iteration time',
'formatter': '{:.2f}',
'unit': 's'
},
)
self.extra_capabilities = {
'psuchannel': {
'help_text': ('Used for simple PSU control when not on\n'
'a ramp. Possibly usages are:\n'
' psuchannel voltage=1.23\n'
'which will set the voltage and\n'
' psuchannel off\n'
'which will set the output off'),
'completions': [
'psuchannel',
'psuchannel voltage=',
'psuchannel off',
]
}
}
# Queue for GUI updates
self.message_queue = Queue()
# The GUI also looks in self.config, see below
### Normal program
# Setup my program
with open(path.join(THIS_DIR, args.program_file)) as file__:
self.config, self.steps = parse_ramp(file__)
# The GUI will look for keys: program_title in config
self.say('Using power supply channel: ' + self.channel_id)
self.say('Loaded with config:\n' + pformat(self.config))
self.active_step = 0
self.send_steps()
# Add completions for the edits
self._completion_additions = []
for number, step in enumerate(self.steps):
base = 'edit {} '.format(number)
self._completion_additions.append(base)
for field in sorted(step.fields):
self._completion_additions.append('{}{}='.format(base, field))
# Base for the status
self.status = {'status_field': 'Initialized'}
# General variables
self.stop = False
self.ok_to_start = False
# Create a partial function with the output substitued in
self.send_command = partial(
_send_command,
args.output,
args.power_supply,
)
# Setup power supply
self.power_supply_on_off(True, self.config['maxcurrent_start'])
# Power supply commands, must match order with self.codenames
self.power_supply_commands = ('read_actual_current',
'read_actual_voltage',
'read_set_voltage', 'read_current_limit')
# Setup dataset saver and live socket
self.codenames = [
self.channel_id + id_
for id_ in ('_current', '_voltage', '_voltage_setpoint',
'_current_limit')
]
self.live_socket = LiveSocket('H2O2_proactive_' + self.channel_id,
self.codenames +
[self.channel_id + '_accum_charge'],
no_internal_data_pull_socket=True)
self.live_socket.reset(self.codenames)
self.live_socket.start()
self.data_set_saver = DataSetSaver(credentials.measurements,
credentials.xy_values,
username=credentials.username,
password=credentials.password)
self.data_set_saver.start()
# Done with init, send status
self.send_status()
def command(self, command, args_str):
"""Process commands from the GUI"""
if command == 'stop': # stop is sent on quit
self.stop = True
elif command == 'start':
self.ok_to_start = True
elif command == 'edit':
# Parse the edit line, start by splitting up in step_num, field and value
try:
num_step, field_value = [
arg.strip() for arg in args_str.split(' ')
]
field, value = [arg.strip() for arg in field_value.split('=')]
except ValueError:
message = ('Bad edit command, must be on the form:\n'
'edit step_num field=value')
self.say(message, message_type='error')
return
# Try to get the correct step
try:
step = self.steps[int(num_step)]
except (ValueError, IndexError):
message = 'Unable to convert step number {} to integer or nu such step '\
'exists'
self.say(message.format(num_step), message_type='error')
return
# Edit the value
try:
step.edit_value(field, value)
except (ValueError, AttributeError) as exception:
self.say(str(exception.args[0]), message_type='error')
return
# Finally send the new steps to the GUI
self.send_steps()
elif command == "psuchannel":
if self.ok_to_start and not self.stop:
message = "Using psuchannel during ramp not allowed"
self.say(message, message_type='error')
return
if args_str.startswith('voltage='):
voltage_str = args_str.replace('voltage=', '')
try:
voltage = float(voltage_str)
except ValueError:
message = 'Invalid voltage for psuchannel {}'.format(
voltage_str)
self.say(message, message_type='error')
return
self.send_command('set_voltage', voltage)
self.say('PSU channel set to {}.\n'
'NOTE: The GUI will not update to show\n'
'this, only the actual PSU.'.format(voltage))
elif args_str == 'off':
self.power_supply_on_off(False)
self.say('PSU channel set to off')
else:
message = 'Invalid argument. psuchannel can do "voltage=" and "off"'
self.say(message, message_type='error')
return
def send_status(self, update_dict=None):
"""Send the status to the GUI"""
if update_dict:
self.status.update(update_dict)
self.message_queue.put(('status', self.status.copy()))
def send_steps(self):
"""Send the steps list to the GUI"""
steps = [(index == self.active_step, str(step))
for index, step in enumerate(self.steps)]
self.message_queue.put(('steps', steps))
def say(self, text, message_type='message'):
"""Send a ordinary text message to the gui"""
self.message_queue.put((message_type, text))
def run(self):
"""The MAIN run method"""
# Wait for start
while not self.ok_to_start:
if self.stop:
self.send_status({'status_field': 'Stopped'})
return
sleep(0.1)
# Start
self.send_status({'status_field': 'Starting'})
self.setup_data_set_saver()
# Run the MAIN measurement loop
# (This is where most of the time is spent)
self.main_measure()
# Shutdown powersupply, livesocket and possibly server
self.send_status({'status_field': 'Stopping'})
self.stop_everything()
self.send_status({'status_field': 'Stopped'})
sleep(0.1)
self.say("I have stopped")
def setup_data_set_saver(self):
"""Setup the data set saver"""
sql_time = CustomColumn(time(), 'FROM_UNIXTIME(%s)')
for codename in self.codenames:
metadata = {
'time': sql_time,
'comment': self.config['comment'],
'label': codename[3:],
'type': 1,
'power_supply_channel': self.channel_id,
}
self.data_set_saver.add_measurement(codename, metadata)
def main_measure(self): # pylint: disable=too-many-locals
"""The main measurement loop"""
self.send_status({'status_field': 'Running'})
# Initial setup
last_set_voltage = None
last_set_max_current = None
last_time = time()
iteration_time = 'N/A'
self.status['elapsed'] = 0.0
accum_charge_codename = self.channel_id + '_accum_charge'
self.status[accum_charge_codename] = 0.0
current_id = self.channel_id + '_current'
last_measured_current = 0.0
self.say('I started on step 0')
for self.active_step, current_step in enumerate(self.steps):
self.send_status(
{'status_field': 'Running step {}'.format(self.active_step)})
# Also give the step an instance name (for steps list)
if self.active_step > 0:
self.say('Switched to step: {}'.format(self.active_step))
self.send_steps()
current_step.start()
# While the step hasn't completed yet
while current_step.elapsed() < current_step.duration:
# Check if we should stop
if self.stop:
self.say('I have been asked to stop')
return
iteration_start = now = time()
# Calculate the time for one iteration and update times in status
iteration_time = now - last_time
last_time = now
self.status.update({
'elapsed':
current_step.elapsed(),
'remaining':
current_step.remaining(),
'iteration_time':
iteration_time,
'elapsed_total':
sum(step.elapsed() for step in self.steps),
'remaining_total':
sum(step.remaining() for step in self.steps),
})
# Ask the power supply to set a new voltage if needed
required_voltage, required_max_current = current_step.values()
if required_max_current != last_set_max_current:
self.send_command('set_current_limit',
required_max_current)
last_set_max_current = required_max_current
if required_voltage != last_set_voltage:
self.send_command('set_voltage', required_voltage)
last_set_voltage = required_voltage
# Read value from the power supply
self._read_values_from_power_supply()
# Calculate, set and send accumulated charge
charge_addition = \
(last_measured_current + self.status[current_id])\
/ 2 * iteration_time
last_measured_current = self.status[current_id]
self.status[accum_charge_codename] += charge_addition
point = (self.status['elapsed_total'],
self.status[accum_charge_codename])
self.live_socket.set_point(accum_charge_codename, point)
# Send the new status
self.send_status()
# Calculate time to sleep to use the proper probe interval
time_to_sleep = current_step.probe_interval - (time() -
iteration_start)
if time_to_sleep > 0:
sleep(time_to_sleep)
# Stop the step(s own time keeping)
current_step.stop()
# For loop over steps ended
self.send_status({'status_field': 'Program Complete'})
self.say('Stepped program completed')
def _read_values_from_power_supply(self):
"""Read all required values from the power supply (used only from run)"""
for command, codename in zip(self.power_supply_commands,
self.codenames):
# Get a value for the current command
value = self.send_command(command)
# Set/save it on the live_socket, database and in the GUI
point = (self.status['elapsed_total'], value)
self.live_socket.set_point(codename, point)
self.data_set_saver.save_point(codename, point)
self.status[codename] = value
def stop_everything(self):
"""Stop power supply and live socket"""
self.live_socket.stop()
self.data_set_saver.stop()
def power_supply_on_off(self, state, current_limit=None):
"""Set power supply on off"""
LOG.debug('Stop power supply')
# Set current limit
if current_limit is not None:
self.send_command('set_current_limit', current_limit)
read_current_limit = self.send_command('read_current_limit')
if not isclose(read_current_limit, current_limit):
raise RuntimeError('Unable to set current limit')
# Set state
self.send_command('output_status', state)
read_state = self.send_command('read_output_status').strip() == '1'
if not read_state is state:
raise RuntimeError('Could not set output state')
def main():
"""Main function"""
parser = argparse.ArgumentParser(description='Run the infamous ph control script.')
# ph_setpoint, ph_direction (string [up, down]), kick_in_rate (float)
# Main (constant), secondary (regulation)
# python ph_increment.py 10.0 --direction up --main-offset=24.0 --second-min=10.0 --second-max=40.0 --second-offset=17.0
parser.add_argument('setpoint', type=float, help='The Ph setpoint.')
parser.add_argument('--direction', default=None, help='Pumping direction. Must be up or down.', required=True)
parser.add_argument('--main-offset', type=float, default=None, help='Start pumping speed for the main pump.', required=True)
parser.add_argument('--second-min', type=float, default=None, help='Minimum pumping speed for the secondary pump.', required=True)
parser.add_argument('--second-max', type=float, default=None, help='Maximum pumping speed for the secondary pump.', required=True)
parser.add_argument('--second-offset', type=float, default=None, help='Start pumping value for the secondary pump.', required=True)
parser.add_argument('--comment', default=None, help='Optional comment', required=True)
args = parser.parse_args()
## Edit comment
comment ="pH "+ str(args.setpoint) + " // p: 0.02 // start rate " + str(args.second_offset) + " // " + args.comment
## Edit comment
# raise SystemExit('All good')
# Init pumps, Omegabus, PID and data saver
pump_main = AL1000("/dev/serial/by-id/usb-FTDI_USB-RS232_Cable_FTV9X9TM-if00-port0")
pump_second = AL1000("/dev/serial/by-id/usb-FTDI_USB-RS232_Cable_FTWZCJLU-if00-port0")
obus = OmegaBus(
"/dev/serial/by-id/usb-FTDI_USB-RS232_Cable_FTWZCGSW-if00-port0",
baud=9600,
)
data_set_saver = DataSetSaver("measurements_dummy", "xy_values_dummy", "dummy", "dummy")
pid = PID(pid_p=0.04, pid_i=0.0, pid_d=0, p_max=9999, p_min=-9)
pid.update_setpoint(args.setpoint)
# time_start = time.time()
#Pre flight check.
pump_main_check = pump_main.get_firmware()
print("main: "+pump_main_check)
pump_second_check = pump_second.get_firmware()
print("second: " +pump_second_check)
obus_check = obus.read_value(1)
obus_check = current_to_ph(obus_check)
if pump_main_check != "NE1000V3.923":
print("Main pump failed")
raise SystemExit(1)
if pump_second_check != "NE1000V3.923":
print("Secondary pump failed")
raise SystemExit(1)
if obus_check < 0:
print("OmegaBus failed")
raise SystemExit(1)
#Set initial condition for pumps
pump_second.set_direction("INF")
pump_main.set_direction("INF")
if args.direction == "up":
pump_second.set_rate(args.second_offset)
rate_second = args.second_offset
pump_main.set_rate(0)
rate_main = 0
if args.direction == "down":
pump_second.set_rate(0)
rate_second = 0
pump_main.set_rate(args.main_offset)
rate_main = args.main_offset
pump_second.set_vol(9999)
pump_main.set_vol(9999)
pump_second.start_program()
pump_main.start_program()
# Setup database saver
data_set_saver.start()
data_set_time = time.time()
metadata = {"Time": CustomColumn(data_set_time, "FROM_UNIXTIME(%s)"), "comment": comment,
"type": 64, "preamp_range": 0, "sem_voltage": 0}
for label in ["p_korr", "i_korr", "total_korr", "ph_setpoint", "ph_value", "pump_rate"]:
metadata["mass_label"] = label
data_set_saver.add_measurement(label, metadata)
# Run PID
try:
time_start = time.time()
run(pump_second, pump_main, obus, pid, data_set_saver, time_start, args.setpoint, rate_second, rate_main, args.second_offset, args.main_offset, args.second_min , args.second_max, args.direction, minutes)
except KeyboardInterrupt:
# Clean up
pump_main.set_rate(0)
pump_second.set_rate(0)
pump_main.stop_program()
pump_second.stop_program()
data_set_saver.stop()
raise SystemExit(" Sequence stopped.")
# Clean up
pump_main.set_rate(0)
pump_second.set_rate(0)
pump_main.stop_program()
pump_second.stop_program()
# Log pH during ageing
try:
while True:
value = obus.read_value(1)
value = current_to_ph(value)
print("pH: " + str(round(value,3)))
data_set_saver.save_point("ph_value", (time.time()-time_start, value))
data_set_saver.save_point("ph_setpoint",(time.time()-time_start, args.setpoint))
time.sleep(8)
except KeyboardInterrupt:
data_set_saver.stop()
pass
class MyProgram(Thread): # pylint: disable=too-many-instance-attributes
"""My fancy program"""
def __init__(self, args):
super(MyProgram, self).__init__()
# Form channel_id e.g: A1
self.channel_id = args.power_supply + args.output
### Required by the stepped program runner
# Accepted capabilities are: can_edit, can_play,
# can_stop, can_quit
self.capabilities = ('can_stop', 'can_start', 'can_edit')
# Status fields (in order)
self.status_fields = (
# Status
{'codename': 'status_field', 'title': 'Status'},
# Voltage
{'codename': self.channel_id + '_voltage',
'title': 'Voltage', 'formatter': '{:.3f}', 'unit': 'V'},
# Voltage setpoint
{'codename': self.channel_id + '_voltage_setpoint',
'title': 'Voltage Setpoint', 'formatter': '{:.3f}', 'unit': 'V'},
# Current
{'codename': self.channel_id + '_current',
'title': 'Current', 'formatter': '{:.3f}', 'unit': 'A'},
# Current limit
{'codename': self.channel_id + '_current_limit',
'title': 'Current limit', 'formatter': '{:.3f}', 'unit': 'A'},
# Charge
{'codename': self.channel_id + '_accum_charge',
'title': 'Accumulated charge', 'formatter': '{:.3f}', 'unit': 'C'},
# Time elapsed (step)
{'codename': 'elapsed',
'title': 'Time elapsed (step)', 'formatter': '{:.1f}', 'unit': 's'},
# Time remaining (step)
{'codename': 'remaining',
'title': 'Time remaining (step)', 'formatter': '{:.1f}', 'unit': 's'},
# Time elapsed (total)
{'codename': 'elapsed_total',
'title': 'Time elapsed (total)', 'formatter': '{:.1f}', 'unit': 's'},
# Time remaining (total)
{'codename': 'remaining_total',
'title': 'Time remaining (total)', 'formatter': '{:.1f}', 'unit': 's'},
# Iteration time
{'codename': 'iteration_time',
'title': 'Iteration time', 'formatter': '{:.2f}', 'unit': 's'},
)
# Queue for GUI updates
self.message_queue = Queue()
# The GUI also looks in self.config, see below
### Normal program
# Setup my program
with open(path.join(THIS_DIR, args.program_file)) as file__:
self.config, self.steps = parse_ramp(file__)
# The GUI will look for keys: program_title in config
self.say('Using power supply channel: ' + self.channel_id)
self.say('Loaded with config:\n' + pformat(self.config))
self.active_step = 0
self.send_steps()
# Add completions for the edits
self._completion_additions = []
for number, step in enumerate(self.steps):
base = 'edit {} '.format(number)
self._completion_additions.append(base)
for field in sorted(step.fields):
self._completion_additions.append('{}{}='.format(base, field))
# Base for the status
self.status = {'status_field': 'Initialized'}
# General variables
self.stop = False
self.ok_to_start = False
# Setup power supply
# Create a partial function with the output substitued in
self.send_command = partial(_send_command, args.output, args.power_supply)
self.power_supply_on_off(True, self.config['maxcurrent_start'])
# Power supply commands, must match order with self.codenames
self.power_supply_commands = (
'read_actual_current', 'read_actual_voltage', 'read_set_voltage',
'read_current_limit'
)
# Setup dataset saver and live socket
self.codenames = [self.channel_id + id_ for id_ in
('_current', '_voltage', '_voltage_setpoint',
'_current_limit')]
self.live_socket = LiveSocket(
'H2O2_proactive_' + self.channel_id,
self.codenames + [self.channel_id + '_accum_charge'],
no_internal_data_pull_socket=True
)
self.live_socket.reset(self.codenames)
self.live_socket.start()
self.data_set_saver = DataSetSaver(
credentials.measurements, credentials.xy_values,
username=credentials.username, password=credentials.password
)
self.data_set_saver.start()
# Done with init, send status
self.send_status()
def command(self, command, args_str):
"""Process commands from the GUI"""
if command == 'stop': # stop is sent on quit
self.stop = True
elif command == 'start':
self.ok_to_start = True
elif command == 'edit':
# Parse the edit line, start by splitting up in step_num, field and value
try:
num_step, field_value = [arg.strip() for arg in args_str.split(' ')]
field, value = [arg.strip() for arg in field_value.split('=')]
except ValueError:
message = ('Bad edit command, must be on the form:\n'
'edit step_num field=value')
self.say(message, message_type='error')
return
# Try to get the correct step
try:
step = self.steps[int(num_step)]
except (ValueError, IndexError):
message = 'Unable to convert step number {} to integer or nu such step '\
'exists'
self.say(message.format(num_step), message_type='error')
return
# Edit the value
try:
step.edit_value(field, value)
except (ValueError, AttributeError) as exception:
self.say(str(exception.args[0]), message_type='error')
return
# Finally send the new steps to the GUI
self.send_steps()
def send_status(self, update_dict=None):
"""Send the status to the GUI"""
if update_dict:
self.status.update(update_dict)
self.message_queue.put(('status', self.status.copy()))
def send_steps(self):
"""Send the steps list to the GUI"""
steps = [(index == self.active_step, str(step))
for index, step in enumerate(self.steps)]
self.message_queue.put(('steps', steps))
def say(self, text, message_type='message'):
"""Send a ordinary text message to the gui"""
self.message_queue.put((message_type, text))
def run(self):
"""The MAIN run method"""
# Wait for start
while not self.ok_to_start:
if self.stop:
self.send_status({'status_field': 'Stopping'})
self.power_supply_on_off(False)
self.send_status({'status_field': 'Stopped'})
return
sleep(0.1)
# Start
self.send_status({'status_field': 'Starting'})
self.setup_data_set_saver()
# Run the MAIN measurement loop
# (This is where most of the time is spent)
self.main_measure()
# Shutdown powersupply, livesocket and possibly server
self.send_status({'status_field': 'Stopping'})
self.stop_everything()
self.send_status({'status_field': 'Stopped'})
sleep(0.1)
self.say("I have stopped")
def setup_data_set_saver(self):
"""Setup the data set saver"""
sql_time = CustomColumn(time(), 'FROM_UNIXTIME(%s)')
for codename in self.codenames:
metadata = {
'time': sql_time, 'comment': self.config['comment'],
'label': codename[3:], 'type': 1,
'power_supply_channel': self.channel_id,
}
self.data_set_saver.add_measurement(codename, metadata)
def main_measure(self): # pylint: disable=too-many-locals
"""The main measurement loop"""
self.send_status({'status_field': 'Running'})
# Initial setup
last_set_voltage = None
last_set_max_current = None
last_time = time()
iteration_time = 'N/A'
self.status['elapsed'] = 0.0
accum_charge_codename = self.channel_id + '_accum_charge'
self.status[accum_charge_codename] = 0.0
current_id = self.channel_id + '_current'
last_measured_current = 0.0
self.say('I started on step 0')
for self.active_step, current_step in enumerate(self.steps):
self.send_status({'status_field': 'Running step {}'.format(self.active_step)})
# Also give the step an instance name (for steps list)
if self.active_step > 0:
self.say('Switched to step: {}'.format(self.active_step))
self.send_steps()
current_step.start()
# While the step hasn't completed yet
while current_step.elapsed() < current_step.duration:
# Check if we should stop
if self.stop:
self.say('I have been asked to stop')
return
iteration_start = now = time()
# Calculate the time for one iteration and update times in status
iteration_time = now - last_time
last_time = now
self.status.update({
'elapsed': current_step.elapsed(),
'remaining': current_step.remaining(),
'iteration_time': iteration_time,
'elapsed_total': sum(step.elapsed() for step in self.steps),
'remaining_total': sum(step.remaining() for step in self.steps),
})
# Ask the power supply to set a new voltage if needed
required_voltage, required_max_current = current_step.values()
if required_max_current != last_set_max_current:
self.send_command('set_current_limit', required_max_current)
last_set_max_current = required_max_current
if required_voltage != last_set_voltage:
self.send_command('set_voltage', required_voltage)
last_set_voltage = required_voltage
# Read value from the power supply
self._read_values_from_power_supply()
# Calculate, set and send accumulated charge
charge_addition = \
(last_measured_current + self.status[current_id])\
/ 2 * iteration_time
last_measured_current = self.status[current_id]
self.status[accum_charge_codename] += charge_addition
point = (self.status['elapsed_total'], self.status[accum_charge_codename])
self.live_socket.set_point(accum_charge_codename, point)
# Send the new status
self.send_status()
# Calculate time to sleep to use the proper probe interval
time_to_sleep = current_step.probe_interval - (time() - iteration_start)
if time_to_sleep > 0:
sleep(time_to_sleep)
# Stop the step(s own time keeping)
current_step.stop()
# For loop over steps ended
self.send_status({'status_field': 'Program Complete'})
self.say('Stepped program completed')
def _read_values_from_power_supply(self):
"""Read all required values from the power supply (used only from run)"""
for command, codename in zip(self.power_supply_commands, self.codenames):
# Get a value for the current command
value = self.send_command(command)
# Set/save it on the live_socket, database and in the GUI
point = (self.status['elapsed_total'], value)
self.live_socket.set_point(codename, point)
self.data_set_saver.save_point(codename, point)
self.status[codename] = value
def stop_everything(self):
"""Stop power supply and live socket"""
self.power_supply_on_off(False)
self.live_socket.stop()
self.data_set_saver.stop()
def power_supply_on_off(self, state, current_limit=0.0):
"""Set power supply on off"""
# Set voltage to 0
LOG.debug('Stopping everything. Set voltage to 0.0')
self.send_command('set_voltage', 0.0)
start = time()
# Anything < 1.0 Volt is OK
while self.send_command('read_actual_voltage') > 1.0:
if time() - start > 60:
LOG.error('Unable to set voltage to 0')
if state:
raise RuntimeError('Unable to set voltage to 0')
else:
self.say('Unable to set voltage to 0')
break
sleep(1)
# Set current limit
self.send_command('set_current_limit', current_limit)
read_current_limit = self.send_command('read_current_limit')
if not isclose(read_current_limit, current_limit):
raise RuntimeError('Unable to set current limit')
# Set state
self.send_command('output_status', state)
read_state = self.send_command('read_output_status').strip() == '1'
if not read_state is state:
raise RuntimeError('Could not set output state')
import time
from PyExpLabSys.common.database_saver import DataSetSaver, CustomColumn
import credentials
## EDIT HERE
comment = "First run"
## EDIT HERE
# Create data set saver object
data_set_saver = DataSetSaver(
"measurements_dummy",
"xy_values_dummy",
credentials.USERNAME,
credentials.PASSWORD,
)
data_set_saver.start()
# Create measurement specs i.e. entires entries in the metadata table
data_set_time = time.time()
metadata = {
"time": CustomColumn(data_set_time, "FROM_UNIXTIME(%s)"),
"comment": comment,
"type": 64,
"preamp_range": 0,
"sem_voltage": 0
}
# Only the labels differ, so we generate the metadata with a loop
for label in [
"p_korr", "i_korr", "total_korr", "ph_setpoint", "ph_value",
"pump_rate"
import time
from PyExpLabSys.common.database_saver import DataSetSaver, CustomColumn
import credentials
## EDIT HERE
comment = "First run"
## EDIT HERE
# Create data set saver object
data_set_saver = DataSetSaver(
"measurements_dummy", "xy_values_dummy", credentials.USERNAME, credentials.PASSWORD,
)
data_set_saver.start()
# Create measurement specs i.e. entires entries in the metadata table
data_set_time = time.time()
metadata = {
"time": CustomColumn(data_set_time, "FROM_UNIXTIME(%s)"), "comment": comment,
"type" : 64, "preamp_range": 0, "sem_voltage": 0}
# Only the labels differ, so we generate the metadata with a loop
for label in ["p_korr", "i_korr", "total_korr", "ph_setpoint", "ph_value", "pump_rate"]:
metadata["mass_label"] = label
data_set_saver.add_measurement(label, metadata)
def __init__(self, args):
super(VoltageCurrentProgram, self).__init__()
# Form channel_id e.g: EA1
self.channel_id = args.power_supply + args.output
### Required by the stepped program runner
# Accepted capabilities are: can_edit, can_play,
# can_stop, can_quit
self.capabilities = ('can_stop', 'can_start', 'can_edit')
# Status fields (in order)
self.status_fields = (
# Status
{
'codename': 'status_field',
'title': 'Status'
},
# Voltage
{
'codename': self.channel_id + '_voltage',
'title': 'Voltage',
'formatter': '{:.3f}',
'unit': 'V'
},
# Voltage setpoint
{
'codename': self.channel_id + '_voltage_setpoint',
'title': 'Voltage Setpoint',
'formatter': '{:.3f}',
'unit': 'V'
},
# Current
{
'codename': self.channel_id + '_current',
'title': 'Current',
'formatter': '{:.3f}',
'unit': 'A'
},
# Current limit
{
'codename': self.channel_id + '_current_limit',
'title': 'Current limit',
'formatter': '{:.3f}',
'unit': 'A'
},
# Charge
{
'codename': self.channel_id + '_accum_charge',
'title': 'Accumulated charge',
'formatter': '{:.3f}',
'unit': 'C'
},
# Time elapsed (step)
{
'codename': 'elapsed',
'title': 'Time elapsed (step)',
'formatter': '{:.1f}',
'unit': 's'
},
# Time remaining (step)
{
'codename': 'remaining',
'title': 'Time remaining (step)',
'formatter': '{:.1f}',
'unit': 's'
},
# Time elapsed (total)
{
'codename': 'elapsed_total',
'title': 'Time elapsed (total)',
'formatter': '{:.1f}',
'unit': 's'
},
# Time remaining (total)
{
'codename': 'remaining_total',
'title': 'Time remaining (total)',
'formatter': '{:.1f}',
'unit': 's'
},
# Iteration time
{
'codename': 'iteration_time',
'title': 'Iteration time',
'formatter': '{:.2f}',
'unit': 's'
},
)
self.extra_capabilities = {
'psuchannel': {
'help_text': ('Used for simple PSU control when not on\n'
'a ramp. Possibly usages are:\n'
' psuchannel voltage=1.23\n'
'which will set the voltage and\n'
' psuchannel off\n'
'which will set the output off'),
'completions': [
'psuchannel',
'psuchannel voltage=',
'psuchannel off',
]
}
}
# Queue for GUI updates
self.message_queue = Queue()
# The GUI also looks in self.config, see below
### Normal program
# Setup my program
with open(path.join(THIS_DIR, args.program_file)) as file__:
self.config, self.steps = parse_ramp(file__)
# The GUI will look for keys: program_title in config
self.say('Using power supply channel: ' + self.channel_id)
self.say('Loaded with config:\n' + pformat(self.config))
self.active_step = 0
self.send_steps()
# Add completions for the edits
self._completion_additions = []
for number, step in enumerate(self.steps):
base = 'edit {} '.format(number)
self._completion_additions.append(base)
for field in sorted(step.fields):
self._completion_additions.append('{}{}='.format(base, field))
# Base for the status
self.status = {'status_field': 'Initialized'}
# General variables
self.stop = False
self.ok_to_start = False
# Create a partial function with the output substitued in
self.send_command = partial(
_send_command,
args.output,
args.power_supply,
)
# Setup power supply
self.power_supply_on_off(True, self.config['maxcurrent_start'])
# Power supply commands, must match order with self.codenames
self.power_supply_commands = ('read_actual_current',
'read_actual_voltage',
'read_set_voltage', 'read_current_limit')
# Setup dataset saver and live socket
self.codenames = [
self.channel_id + id_
for id_ in ('_current', '_voltage', '_voltage_setpoint',
'_current_limit')
]
self.live_socket = LiveSocket('H2O2_proactive_' + self.channel_id,
self.codenames +
[self.channel_id + '_accum_charge'],
no_internal_data_pull_socket=True)
self.live_socket.reset(self.codenames)
self.live_socket.start()
self.data_set_saver = DataSetSaver(credentials.measurements,
credentials.xy_values,
username=credentials.username,
password=credentials.password)
self.data_set_saver.start()
# Done with init, send status
self.send_status()
def main():
"""Main function"""
parser = argparse.ArgumentParser(
description='Run the infamous ph control script.')
# ph_setpoint, ph_direction (string [up, down]), kick_in_rate (float)
# Main (constant), secondary (regulation)
# python ph_increment.py 10.0 --direction up --main-offset=24.0 --second-min=10.0 --second-max=40.0 --second-offset=17.0
parser.add_argument('setpoint', type=float, help='The Ph setpoint.')
parser.add_argument('--direction',
default=None,
help='Pumping direction. Must be up or down.',
required=True)
parser.add_argument('--main-offset',
type=float,
default=None,
help='Start pumping speed for the main pump.',
required=True)
parser.add_argument('--second-min',
type=float,
default=None,
help='Minimum pumping speed for the secondary pump.',
required=True)
parser.add_argument('--second-max',
type=float,
default=None,
help='Maximum pumping speed for the secondary pump.',
required=True)
parser.add_argument('--second-offset',
type=float,
default=None,
help='Start pumping value for the secondary pump.',
required=True)
parser.add_argument('--comment',
default=None,
help='Optional comment',
required=True)
args = parser.parse_args()
## Edit comment
comment = "pH " + str(args.setpoint) + " // p: 0.02 // start rate " + str(
args.second_offset) + " // " + args.comment
## Edit comment
# raise SystemExit('All good')
# Init pumps, Omegabus, PID and data saver
pump_main = AL1000(
"/dev/serial/by-id/usb-FTDI_USB-RS232_Cable_FTV9X9TM-if00-port0")
pump_second = AL1000(
"/dev/serial/by-id/usb-FTDI_USB-RS232_Cable_FTWZCJLU-if00-port0")
obus = OmegaBus(
"/dev/serial/by-id/usb-FTDI_USB-RS232_Cable_FTWZCGSW-if00-port0",
baud=9600,
)
data_set_saver = DataSetSaver("measurements_dummy", "xy_values_dummy",
"dummy", "dummy")
pid = PID(pid_p=0.04, pid_i=0.0, pid_d=0, p_max=9999, p_min=-9)
pid.update_setpoint(args.setpoint)
# time_start = time.time()
#Pre flight check.
pump_main_check = pump_main.get_firmware()
print("main: " + pump_main_check)
pump_second_check = pump_second.get_firmware()
print("second: " + pump_second_check)
obus_check = obus.read_value(1)
obus_check = current_to_ph(obus_check)
if pump_main_check != "NE1000V3.923":
print("Main pump failed")
raise SystemExit(1)
if pump_second_check != "NE1000V3.923":
print("Secondary pump failed")
raise SystemExit(1)
if obus_check < 0:
print("OmegaBus failed")
raise SystemExit(1)
#Set initial condition for pumps
pump_second.set_direction("INF")
pump_main.set_direction("INF")
if args.direction == "up":
pump_second.set_rate(args.second_offset)
rate_second = args.second_offset
pump_main.set_rate(0)
rate_main = 0
if args.direction == "down":
pump_second.set_rate(0)
rate_second = 0
pump_main.set_rate(args.main_offset)
rate_main = args.main_offset
pump_second.set_vol(9999)
pump_main.set_vol(9999)
pump_second.start_program()
pump_main.start_program()
# Setup database saver
data_set_saver.start()
data_set_time = time.time()
metadata = {
"Time": CustomColumn(data_set_time, "FROM_UNIXTIME(%s)"),
"comment": comment,
"type": 64,
"preamp_range": 0,
"sem_voltage": 0
}
for label in [
"p_korr", "i_korr", "total_korr", "ph_setpoint", "ph_value",
"pump_rate"
]:
metadata["mass_label"] = label
data_set_saver.add_measurement(label, metadata)
# Run PID
try:
time_start = time.time()
run(pump_second, pump_main, obus, pid, data_set_saver, time_start,
args.setpoint, rate_second, rate_main, args.second_offset,
args.main_offset, args.second_min, args.second_max, args.direction,
minutes)
except KeyboardInterrupt:
# Clean up
pump_main.set_rate(0)
pump_second.set_rate(0)
pump_main.stop_program()
pump_second.stop_program()
data_set_saver.stop()
raise SystemExit(" Sequence stopped.")
# Clean up
pump_main.set_rate(0)
pump_second.set_rate(0)
pump_main.stop_program()
pump_second.stop_program()
# Log pH during ageing
try:
while True:
value = obus.read_value(1)
value = current_to_ph(value)
print("pH: " + str(round(value, 3)))
data_set_saver.save_point("ph_value",
(time.time() - time_start, value))
data_set_saver.save_point(
"ph_setpoint", (time.time() - time_start, args.setpoint))
time.sleep(8)
except KeyboardInterrupt:
data_set_saver.stop()
pass
class Stats(multiprocessing.Process):
def __init__(self, user_queue, log_data=False):
psutil.cpu_percent(percpu=True) # Initial dummy readout
multiprocessing.Process.__init__(self)
conn_params = pika.ConnectionParameters('localhost')
connection = pika.BlockingConnection(conn_params)
self.channel = connection.channel()
self.channel.queue_declare('global')
self.amqp_messages = {}
self.amqp_messages['global'] = {}
self.amqp_messages['global']['children'] = -1
self.amqp_messages['global']['amqp_time'] = -1
self.user_queue = user_queue
self.scanners = []
self.screen = curses.initscr()
curses.noecho()
curses.cbreak()
curses.curs_set(False)
self.screen.keypad(1)
self.screen.nodelay(1)
# Measure initial values while we have the chance
self.start_time = time.time()
self.total_users = self.user_queue.qsize()
self.init_du = self.disk_usage()
self.log_data = log_data
def _amqp_single_update(self, queue_name):
method, header, body = self.channel.basic_get(queue_name)
while method: # Always empty queue, do now show old data
body_dict = pickle.loads(body)
self.amqp_messages[queue_name] = body_dict
method, header, body = self.channel.basic_get(queue_name)
def amqp_update(self):
t = time.time()
self._amqp_single_update('global')
for pid in self.scanners:
self._amqp_single_update(str(pid))
self.amqp_messages['global']['amqp_time'] = time.time() - t
def number_of_threads(self):
""" Number of threads
:return: Tuple with Number of threads, and nuber of active threads
"""
return (len(self.scanners), self.amqp_messages['global']['children'])
def add_scanner(self, scanner):
""" Add a scanner to the internal list of scanners
:param scanner: The scanner object to be added
:return: The new number of scanners
"""
self.scanners.append(scanner)
return len(self.scanners)
def disk_usage(self):
""" Return the current disk usage
:return: Disk usage in MB
"""
error = True
while error:
try:
du_output = subprocess.check_output(['du', '-s',
export_path],
stderr=subprocess.DEVNULL)
error = False
except subprocess.CalledProcessError:
# Happens if du is called while folder is being marked done
# logger.warn('du-error') Warning ends up in the terminal...
# will need to investiate
time.sleep(1)
size = float(du_output.decode('utf-8').split('\t')[0]) / 1024
return size
def exported_users(self):
""" Returns the number of exported users
:return: Tuple with finished exports and total users
"""
finished_users = (self.total_users - self.user_queue.qsize() -
self.number_of_threads()[1])
return (finished_users, self.total_users)
def amount_of_exported_data(self):
""" Return the total amount of exported data (MB)
:return: The total amount of exported data sinze start
"""
return self.disk_usage() - self.init_du
def memory_info(self):
""" Returns the memory consumption (in MB) of all threads
:return: List of memory consumptions
"""
mem_list = {}
for pid in self.scanners:
try:
process = psutil.Process(pid)
mem_info = process.memory_full_info()
used_memory = mem_info.uss/1024**2
if self.log_data:
label = '{} memory'.format(pid)
dt = (time.time() - self.start_time)
self.data_set_saver.save_point(label, (dt, used_memory))
except psutil._exceptions.NoSuchProcess:
used_memory = -1
mem_list[str(pid)] = used_memory
return mem_list
def status(self):
template = ('Threads: {}. ' +
'Queue: {}. ' +
'Export: {:.3f}GB. ' +
'Time: {:.2f}min. ' +
'Speed: {:.2f}MB/s. ' +
'Memory consumption: {:.3f}GB. ' +
'Scanned users: {} / {}')
memory = sum(self.memory_info().values()) / 1024
processes = self.number_of_threads()[1]
dt = (time.time() - self.start_time)
users = self.exported_users()
ret_str = template.format(processes,
self.user_queue.qsize(),
self.disk_usage() / 1024,
dt / 60.0,
self.amount_of_exported_data() / dt,
memory,
users[0], users[1])
return ret_str
def init_logging(self):
if self.log_data:
self.comment = 'Run'
self.data_set_saver = DataSetSaver('measurements_mailscan',
'xy_values_mailscan',
credentials.user, credentials.passwd)
self.data_set_saver.start()
# PyExpLabSys does does not excell in db-normalization - add
# metadata to all channels
metadata = {"Time": CustomColumn(self.start_time, "FROM_UNIXTIME(%s)"),
"comment": self.comment, "type": 1, "label": None,
"processes": self.number_of_threads()[1]}
metadata['label'] = 'Avg export speed'
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = 'Total export size'
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = 'Total users'
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = 'Total memory'
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = 'Total CPU'
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = 'Total Mails'
self.data_set_saver.add_measurement(metadata['label'], metadata)
for scanner in self.amqp_messages.keys():
if scanner == 'global':
continue
metadata['processes'] = 1
metadata['label'] = '{} memory'.format(scanner)
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = '{} exported users'.format(scanner)
self.data_set_saver.add_measurement(metadata['label'], metadata)
def run(self):
self.amqp_update()
self.init_logging()
processes = self.number_of_threads()[1]
while processes is not 0:
self.amqp_update()
thread_info = self.number_of_threads()
processes = thread_info[1]
status = self.status()
logger.info(status)
# print(status)
dt = int((time.time() - self.start_time))
msg = 'Run-time: {}min {}s '.format(int(dt / 60),
int(dt % 60))
self.screen.addstr(2, 3, msg)
users = self.exported_users()
msg = 'Exported users: {}/{} '.format(users[0], users[1])
self.screen.addstr(3, 3, msg)
total_export_size = self.amount_of_exported_data()
msg = 'Total export: {:.3f}MB '.format(total_export_size)
self.screen.addstr(4, 3, msg)
speed = total_export_size / dt
msg = 'Avg eksport speed: {:.3f}MB/s '.format(speed)
self.screen.addstr(5, 3, msg)
mem_info = self.memory_info()
msg = 'Memory usage: {:.1f}MB'
self.screen.addstr(6, 3, msg.format(sum(mem_info.values())))
msg = 'amqp update time: {:.1f}ms '
update_time = self.amqp_messages['global']['amqp_time'] * 1000
self.screen.addstr(7, 3, msg.format(update_time))
cpu_usage = psutil.cpu_percent(percpu=True)
msg = 'CPU{} usage: {}% '
for i in range(0, len(cpu_usage)):
self.screen.addstr(9 + i, 3, msg.format(i, cpu_usage[i]))
if self.log_data:
dt = (time.time() - self.start_time)
label = 'Total users'
self.data_set_saver.save_point(label, (dt, users[0]))
label = 'Total export size'
self.data_set_saver.save_point(label, (dt, total_export_size))
label = 'Avg export speed'
self.data_set_saver.save_point(label, (dt, speed))
label = 'Total CPU'
self.data_set_saver.save_point(label, (dt, sum(cpu_usage)))
label = 'Total memory'
self.data_set_saver.save_point(label,
(dt, sum(mem_info.values())))
i = i + 2
msg = 'Total threads: {}. '
self.screen.addstr(9 + i, 3, msg.format(thread_info[0]))
i = i + 1
msg = 'Active threads: {} '
self.screen.addstr(9 + i, 3, msg.format(thread_info[1]))
i = 0
exported_grand_total = 0
self.screen.addstr(2 + i, 50, 'Scan status:')
i = i + 1
for key, data in self.amqp_messages.items():
if key == 'global':
continue
msg = 'ID {}'.format(key)
self.screen.addstr(2 + i, 50, msg)
i = i + 1
try:
if mem_info[key] > 0:
msg = 'Current path: {}/{}'.format(data['rel_path'],
data['folder'])
else:
msg = 'Current path: {}'.format(data['rel_path'])
self.screen.addstr(2 + i, 50, msg)
self.screen.clrtoeol()
i = i + 1
if mem_info[key] > 0:
run_time = (time.time() - data['start_time']) / 60.0
else:
run_time = 0
msg = 'Progress: {} of {} mails. Export time: {:.1f}min'
self.screen.addstr(2 + i, 50, msg.format(data['total_scanned'],
data['total_count'],
run_time))
self.screen.clrtoeol()
i = i + 1
msg = 'Exported users: {}. Exported mails: {} Total mails: {}. Memory consumption: {:.1f}MB '
if self.log_data:
label = '{} exported users'.format(key)
dt = (time.time() - self.start_time)
eu = data['exported_users']
self.data_set_saver.save_point(label, (dt, eu))
msg = msg.format(data['exported_users'],
data['exported_mails'],
data['total_mails'],
mem_info[key])
exported_grand_total += data['exported_mails']
exported_grand_total += data['total_mails']
self.screen.addstr(2 + i, 50, msg)
i = i + 1
msg = 'Last amqp update: {}'.format(data['latest_update'])
self.screen.addstr(2 + i, 50, msg)
self.screen.clrtoeol()
except KeyError:
self.screen.addstr(2 + i, 50, str(data))
i = i + 1
self.screen.addstr(2 + i, 50, str(mem_info))
i = i + 2
self.screen.refresh()
if self.log_data:
dt = (time.time() - self.start_time)
label = 'Total Mails'
self.data_set_saver.save_point(label,
(dt, exported_grand_total))
key = self.screen.getch()
if key == ord('q'):
# Quit program
pass
time.sleep(1)
curses.nocbreak()
self.screen.keypad(0)
curses.echo()
curses.endwin()
class Stats(multiprocessing.Process):
def __init__(self, user_queue, log_data=False):
psutil.cpu_percent(percpu=True) # Initial dummy readout
multiprocessing.Process.__init__(self)
conn_params = pika.ConnectionParameters('localhost')
connection = pika.BlockingConnection(conn_params)
self.channel = connection.channel()
self.channel.queue_declare('global')
self.amqp_messages = {}
self.amqp_messages['global'] = {}
self.amqp_messages['global']['children'] = -1
self.amqp_messages['global']['amqp_time'] = -1
self.user_queue = user_queue
self.scanners = []
self.screen = curses.initscr()
curses.noecho()
curses.cbreak()
curses.curs_set(False)
self.screen.keypad(1)
self.screen.nodelay(1)
# Measure initial values while we have the chance
self.start_time = time.time()
self.total_users = self.user_queue.qsize()
self.init_du = self.disk_usage()
self.log_data = log_data
def _amqp_single_update(self, queue_name):
method, header, body = self.channel.basic_get(queue_name)
while method: # Always empty queue, do now show old data
body_dict = pickle.loads(body)
self.amqp_messages[queue_name] = body_dict
method, header, body = self.channel.basic_get(queue_name)
def amqp_update(self):
t = time.time()
self._amqp_single_update('global')
for pid in self.scanners:
self._amqp_single_update(str(pid))
self.amqp_messages['global']['amqp_time'] = time.time() - t
def number_of_threads(self):
""" Number of threads
:return: Tuple with Number of threads, and nuber of active threads
"""
return (len(self.scanners), self.amqp_messages['global']['children'])
def add_scanner(self, scanner):
""" Add a scanner to the internal list of scanners
:param scanner: The scanner object to be added
:return: The new number of scanners
"""
self.scanners.append(scanner)
return len(self.scanners)
def disk_usage(self):
""" Return the current disk usage
:return: Disk usage in MB
"""
error = True
while error:
try:
du_output = subprocess.check_output(['du', '-s', export_path],
stderr=subprocess.DEVNULL)
error = False
except subprocess.CalledProcessError:
# Happens if du is called while folder is being marked done
# logger.warn('du-error') Warning ends up in the terminal...
# will need to investiate
time.sleep(1)
size = float(du_output.decode('utf-8').split('\t')[0]) / 1024
return size
def exported_users(self):
""" Returns the number of exported users
:return: Tuple with finished exports and total users
"""
finished_users = (self.total_users - self.user_queue.qsize() -
self.number_of_threads()[1])
return (finished_users, self.total_users)
def amount_of_exported_data(self):
""" Return the total amount of exported data (MB)
:return: The total amount of exported data sinze start
"""
return self.disk_usage() - self.init_du
def memory_info(self):
""" Returns the memory consumption (in MB) of all threads
:return: List of memory consumptions
"""
mem_list = {}
for pid in self.scanners:
try:
process = psutil.Process(pid)
mem_info = process.memory_full_info()
used_memory = mem_info.uss / 1024**2
if self.log_data:
label = '{} memory'.format(pid)
dt = (time.time() - self.start_time)
self.data_set_saver.save_point(label, (dt, used_memory))
except psutil._exceptions.NoSuchProcess:
used_memory = -1
mem_list[str(pid)] = used_memory
return mem_list
def status(self):
template = ('Threads: {}. ' + 'Queue: {}. ' + 'Export: {:.3f}GB. ' +
'Time: {:.2f}min. ' + 'Speed: {:.2f}MB/s. ' +
'Memory consumption: {:.3f}GB. ' +
'Scanned users: {} / {}')
memory = sum(self.memory_info().values()) / 1024
processes = self.number_of_threads()[1]
dt = (time.time() - self.start_time)
users = self.exported_users()
ret_str = template.format(processes, self.user_queue.qsize(),
self.disk_usage() / 1024, dt / 60.0,
self.amount_of_exported_data() / dt, memory,
users[0], users[1])
return ret_str
def init_logging(self):
if self.log_data:
self.comment = 'Run'
self.data_set_saver = DataSetSaver('measurements_mailscan',
'xy_values_mailscan',
credentials.user,
credentials.passwd)
self.data_set_saver.start()
# PyExpLabSys does does not excell in db-normalization - add
# metadata to all channels
metadata = {
"Time": CustomColumn(self.start_time, "FROM_UNIXTIME(%s)"),
"comment": self.comment,
"type": 1,
"label": None,
"processes": self.number_of_threads()[1]
}
metadata['label'] = 'Avg export speed'
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = 'Total export size'
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = 'Total users'
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = 'Total memory'
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = 'Total CPU'
self.data_set_saver.add_measurement(metadata['label'], metadata)
metadata['label'] = 'Total Mails'
self.data_set_saver.add_measurement(metadata['label'], metadata)
for scanner in self.amqp_messages.keys():
if scanner == 'global':
continue
metadata['processes'] = 1
metadata['label'] = '{} memory'.format(scanner)
self.data_set_saver.add_measurement(metadata['label'],
metadata)
metadata['label'] = '{} exported users'.format(scanner)
self.data_set_saver.add_measurement(metadata['label'],
metadata)
def run(self):
self.amqp_update()
self.init_logging()
processes = self.number_of_threads()[1]
while processes is not 0:
self.amqp_update()
thread_info = self.number_of_threads()
processes = thread_info[1]
status = self.status()
logger.info(status)
# print(status)
dt = int((time.time() - self.start_time))
msg = 'Run-time: {}min {}s '.format(int(dt / 60), int(dt % 60))
self.screen.addstr(2, 3, msg)
users = self.exported_users()
msg = 'Exported users: {}/{} '.format(users[0], users[1])
self.screen.addstr(3, 3, msg)
total_export_size = self.amount_of_exported_data()
msg = 'Total export: {:.3f}MB '.format(total_export_size)
self.screen.addstr(4, 3, msg)
speed = total_export_size / dt
msg = 'Avg eksport speed: {:.3f}MB/s '.format(speed)
self.screen.addstr(5, 3, msg)
mem_info = self.memory_info()
msg = 'Memory usage: {:.1f}MB'
self.screen.addstr(6, 3, msg.format(sum(mem_info.values())))
msg = 'amqp update time: {:.1f}ms '
update_time = self.amqp_messages['global']['amqp_time'] * 1000
self.screen.addstr(7, 3, msg.format(update_time))
cpu_usage = psutil.cpu_percent(percpu=True)
msg = 'CPU{} usage: {}% '
for i in range(0, len(cpu_usage)):
self.screen.addstr(9 + i, 3, msg.format(i, cpu_usage[i]))
if self.log_data:
dt = (time.time() - self.start_time)
label = 'Total users'
self.data_set_saver.save_point(label, (dt, users[0]))
label = 'Total export size'
self.data_set_saver.save_point(label, (dt, total_export_size))
label = 'Avg export speed'
self.data_set_saver.save_point(label, (dt, speed))
label = 'Total CPU'
self.data_set_saver.save_point(label, (dt, sum(cpu_usage)))
label = 'Total memory'
self.data_set_saver.save_point(label,
(dt, sum(mem_info.values())))
i = i + 2
msg = 'Total threads: {}. '
self.screen.addstr(9 + i, 3, msg.format(thread_info[0]))
i = i + 1
msg = 'Active threads: {} '
self.screen.addstr(9 + i, 3, msg.format(thread_info[1]))
i = 0
exported_grand_total = 0
self.screen.addstr(2 + i, 50, 'Scan status:')
i = i + 1
for key, data in self.amqp_messages.items():
if key == 'global':
continue
msg = 'ID {}'.format(key)
self.screen.addstr(2 + i, 50, msg)
i = i + 1
try:
if mem_info[key] > 0:
msg = 'Current path: {}/{}'.format(
data['rel_path'], data['folder'])
else:
msg = 'Current path: {}'.format(data['rel_path'])
self.screen.addstr(2 + i, 50, msg)
self.screen.clrtoeol()
i = i + 1
if mem_info[key] > 0:
run_time = (time.time() - data['start_time']) / 60.0
else:
run_time = 0
msg = 'Progress: {} of {} mails. Export time: {:.1f}min'
self.screen.addstr(
2 + i, 50,
msg.format(data['total_scanned'], data['total_count'],
run_time))
self.screen.clrtoeol()
i = i + 1
msg = 'Exported users: {}. Exported mails: {} Total mails: {}. Memory consumption: {:.1f}MB '
if self.log_data:
label = '{} exported users'.format(key)
dt = (time.time() - self.start_time)
eu = data['exported_users']
self.data_set_saver.save_point(label, (dt, eu))
msg = msg.format(data['exported_users'],
data['exported_mails'],
data['total_mails'], mem_info[key])
exported_grand_total += data['exported_mails']
exported_grand_total += data['total_mails']
self.screen.addstr(2 + i, 50, msg)
i = i + 1
msg = 'Last amqp update: {}'.format(data['latest_update'])
self.screen.addstr(2 + i, 50, msg)
self.screen.clrtoeol()
except KeyError:
self.screen.addstr(2 + i, 50, str(data))
i = i + 1
self.screen.addstr(2 + i, 50, str(mem_info))
i = i + 2
self.screen.refresh()
if self.log_data:
dt = (time.time() - self.start_time)
label = 'Total Mails'
self.data_set_saver.save_point(label,
(dt, exported_grand_total))
key = self.screen.getch()
if key == ord('q'):
# Quit program
pass
time.sleep(1)
curses.nocbreak()
self.screen.keypad(0)
curses.echo()
curses.endwin()
def __init__(self, args):
super(MyProgram, self).__init__()
# Form channel_id e.g: A1
self.channel_id = args.power_supply + args.output
### Required by the stepped program runner
# Accepted capabilities are: can_edit, can_play,
# can_stop, can_quit
self.capabilities = ('can_stop', 'can_start', 'can_edit')
# Status fields (in order)
self.status_fields = (
# Status
{'codename': 'status_field', 'title': 'Status'},
# Voltage
{'codename': self.channel_id + '_voltage',
'title': 'Voltage', 'formatter': '{:.3f}', 'unit': 'V'},
# Voltage setpoint
{'codename': self.channel_id + '_voltage_setpoint',
'title': 'Voltage Setpoint', 'formatter': '{:.3f}', 'unit': 'V'},
# Current
{'codename': self.channel_id + '_current',
'title': 'Current', 'formatter': '{:.3f}', 'unit': 'A'},
# Current limit
{'codename': self.channel_id + '_current_limit',
'title': 'Current limit', 'formatter': '{:.3f}', 'unit': 'A'},
# Charge
{'codename': self.channel_id + '_accum_charge',
'title': 'Accumulated charge', 'formatter': '{:.3f}', 'unit': 'C'},
# Time elapsed (step)
{'codename': 'elapsed',
'title': 'Time elapsed (step)', 'formatter': '{:.1f}', 'unit': 's'},
# Time remaining (step)
{'codename': 'remaining',
'title': 'Time remaining (step)', 'formatter': '{:.1f}', 'unit': 's'},
# Time elapsed (total)
{'codename': 'elapsed_total',
'title': 'Time elapsed (total)', 'formatter': '{:.1f}', 'unit': 's'},
# Time remaining (total)
{'codename': 'remaining_total',
'title': 'Time remaining (total)', 'formatter': '{:.1f}', 'unit': 's'},
# Iteration time
{'codename': 'iteration_time',
'title': 'Iteration time', 'formatter': '{:.2f}', 'unit': 's'},
)
# Queue for GUI updates
self.message_queue = Queue()
# The GUI also looks in self.config, see below
### Normal program
# Setup my program
with open(path.join(THIS_DIR, args.program_file)) as file__:
self.config, self.steps = parse_ramp(file__)
# The GUI will look for keys: program_title in config
self.say('Using power supply channel: ' + self.channel_id)
self.say('Loaded with config:\n' + pformat(self.config))
self.active_step = 0
self.send_steps()
# Add completions for the edits
self._completion_additions = []
for number, step in enumerate(self.steps):
base = 'edit {} '.format(number)
self._completion_additions.append(base)
for field in sorted(step.fields):
self._completion_additions.append('{}{}='.format(base, field))
# Base for the status
self.status = {'status_field': 'Initialized'}
# General variables
self.stop = False
self.ok_to_start = False
# Setup power supply
# Create a partial function with the output substitued in
self.send_command = partial(_send_command, args.output, args.power_supply)
self.power_supply_on_off(True, self.config['maxcurrent_start'])
# Power supply commands, must match order with self.codenames
self.power_supply_commands = (
'read_actual_current', 'read_actual_voltage', 'read_set_voltage',
'read_current_limit'
)
# Setup dataset saver and live socket
self.codenames = [self.channel_id + id_ for id_ in
('_current', '_voltage', '_voltage_setpoint',
'_current_limit')]
self.live_socket = LiveSocket(
'H2O2_proactive_' + self.channel_id,
self.codenames + [self.channel_id + '_accum_charge'],
no_internal_data_pull_socket=True
)
self.live_socket.reset(self.codenames)
self.live_socket.start()
self.data_set_saver = DataSetSaver(
credentials.measurements, credentials.xy_values,
username=credentials.username, password=credentials.password
)
self.data_set_saver.start()
# Done with init, send status
self.send_status()
from PyExpLabSys.file_parsers.chemstation import Sequence
### REMOVE after database move is complete
from PyExpLabSys.common import database_saver
database_saver.HOSTNAME = 'cinfsql'
### REMOVE after database move is complete
from PyExpLabSys.common.database_saver import DataSetSaver
from PyExpLabSys.common.database_saver import CustomColumn
import credentials
# Instantiate the data set saver
data_set_saver = DataSetSaver('measurements_vhp_setup', 'xy_values_vhp_setup',
credentials.USERNAME, credentials.PASSWORD)
data_set_saver.start()
# Get the set of aleady uploaded files
already_uploaded = data_set_saver.get_unique_values_from_measurements('relative_path')
print('Fetched relative paths for {} known sequences'.format(len(already_uploaded)))
# This is the measurement path, should be generated somehow
basefolder = '/home/cinf/o/FYSIK/list-SurfCat/setups/vhp-setup'
sequence_identifyer = 'sequence.acaml'
# Find the active month
newest = None
highest_value = 0
for dir_ in os.listdir(basefolder):
dir_split = dir_.split(' ')
