def db_saver(self): """Autoclosing db_saver yield_fixture""" # Init db_saver db_saver = DataSetSaver("measurements_dummy", "xy_values_dummy", "dummy", "dummy") db_saver.start() yield db_saver db_saver.stop()
def __init__(self): # Initialize power supply # Initialize flow meter driver """Try to connect to the COM port of the flow meter and ask for the serial number""" print("Try and connect to red flow meter") try: self.red_flow_meter = RedFlowMeter(ramp.RED_FLOW_METER_COM_PORT, slave_address=2) except SerialException: message = 'Cannot find red flow meter on {}'.format( ramp.RED_FLOW_METER_COM_PORT) raise RuntimeError(message) # Test serial number reply if self.red_flow_meter.read_value('serial') != 181787: raise RuntimeError('Incorrect reply to serial number') # Print actual flow value print(self.red_flow_meter.read_value('fluid_name'), self.red_flow_meter.read_value('flow')) self.red_flow_meter.set_address(247) print(self.red_flow_meter.read_value('fluid_name'), self.red_flow_meter.read_value('flow')) # Try to conect to power supply print("Try and connect to power supply") try: self.cpx = CPX400DPDriver(output=1, interface='serial', device=ramp.POWER_SUPPLY_COM_PORT) except SerialException: message = 'Cannot find power supply on {}'.format( ramp.POWER_SUPPLY_COM_PORT) raise RuntimeError(message) if 'CPX400DP' not in self.cpx.read_software_version(): raise RuntimeError('Incorrect software version reply') pass # Print the set point voltage print("The voltage set point is {}".format( self.cpx.read_set_voltage())) self.ramp = ramp.RAMP self.metadata = ramp.metadata self.verify_ramp() self.data_set_saver = DataSetSaver( 'measurements_large_CO2_MEA', 'xy_values_large_CO2_MEA', credentials.USERNAME, credentials.PASSWORD, ) self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.sock.settimeout(0.1) self.host = ramp.RASPPI_HOST self.port = ramp.RASSPI_PORT self.message = 'json_wn#' + json.dumps({'no_voltages_to_set': True})
def __init__(self, mux, smu, comment, user): self.data_set_saver = DataSetSaver("measurements_" + user, "xy_values_" + user, user, user) self.data_set_saver.start() self.data_set_time = time.time() # Update for each scan... self.comment = comment self.mux = mux self.smu = smu mux.scpi_comm( 'INSTRUMENT:DMM OFF') # Turn off DMM to allow use as mux device self.smu.set_current_limit(0.01) # Current limit, 10mA self.smu.set_current_measure_range( current_range=0.01) # Measurement range 10mA self.smu.set_integration_time(1)
def start_measurement(self, comment): """ Start a measurement """ data_set_saver = DataSetSaver("measurements_dummy", "xy_values_dummy", "dummy", "dummy") data_set_saver.start() data_set_time = time.time() metadata = { "Time": CustomColumn(data_set_time, "FROM_UNIXTIME(%s)"), "comment": comment, "type": 65, "preamp_range": 9, "sem_voltage": -1 } labels = [ "2W Heater 1", "2W Heater 2", "2W RTD Source", "2W RTD Sense", "Short circuit check", "RTD Source Sense Left", "RTD Source Sense Right", "4W RTD", "Temperature", "Temperature 2" ] for label in labels: metadata["mass_label"] = label data_set_saver.add_measurement(label, metadata) for i in range(0, 99999999): self.update_values() for label in labels: print( str(i) + ': Channel: ' + label + ': ' + str(self.data[label])) data_set_saver.save_point( label, (time.time() - data_set_time, self.data[label])) time.sleep(1)
def init_logging(self): if self.log_data: self.comment = 'Run' self.data_set_saver = DataSetSaver('measurements_mailscan', 'xy_values_mailscan', credentials.user, credentials.passwd) self.data_set_saver.start() # PyExpLabSys does does not excell in db-normalization - add # metadata to all channels metadata = {"Time": CustomColumn(self.start_time, "FROM_UNIXTIME(%s)"), "comment": self.comment, "type": 1, "label": None, "processes": self.number_of_threads()[1]} metadata['label'] = 'Avg export speed' self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = 'Total export size' self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = 'Total users' self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = 'Total memory' self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = 'Total CPU' self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = 'Total Mails' self.data_set_saver.add_measurement(metadata['label'], metadata) for scanner in self.amqp_messages.keys(): if scanner == 'global': continue metadata['processes'] = 1 metadata['label'] = '{} memory'.format(scanner) self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = '{} exported users'.format(scanner) self.data_set_saver.add_measurement(metadata['label'], metadata)
def __init__(self, hostname, anode): threading.Thread.__init__(self) self.agilent = agilent_34972A.Agilent34972ADriver(interface='lan', hostname=hostname) self.calib = 500 # Analyser voltage pr. input voltage self.chamber_name = "dummy" self.data_set_saver = DataSetSaver("measurements_" + self.chamber_name, "xy_values_" + self.chamber_name, credentials.user, credentials.passwd) self.data_set_saver.start() if anode == 'Mg': self.x_ray = 1253.44 if anode == 'Al': self.x_ray = 1487.0
def start_measurement(self, comment): """ Start a measurement """ data_set_saver = DataSetSaver("measurements_dummy", "xy_values_dummy", "dummy", "dummy") data_set_saver.start() data_set_time = time.time() metadata = {"Time": CustomColumn(data_set_time, "FROM_UNIXTIME(%s)"), "comment": comment, "type": 65, "preamp_range": 9, "sem_voltage": -1} labels = ["2W Heater 1", "2W Heater 2", "2W RTD Source", "2W RTD Sense", "Short circuit check", "RTD Source Sense Left", "RTD Source Sense Right", "4W RTD", "Temperature", "Temperature 2"] for label in labels: metadata["mass_label"] = label data_set_saver.add_measurement(label, metadata) for i in range(0, 99999999): self.update_values() for label in labels: print(str(i) + ': Channel: ' + label + ': ' + str(self.data[label])) data_set_saver.save_point(label, (time.time() - data_set_time, self.data[label])) time.sleep(1)
def __init__(self, mux, smu, comment, user): self.data_set_saver = DataSetSaver("measurements_" + user, "xy_values_" + user, user, user) self.data_set_saver.start() self.data_set_time = time.time() # Update for each scan... self.comment = comment self.mux = mux self.smu = smu mux.scpi_comm('INSTRUMENT:DMM OFF') # Turn off DMM to allow use as mux device self.smu.set_current_limit(0.01) # Current limit, 10mA self.smu.set_current_measure_range(current_range=0.01) # Measurement range 10mA self.smu.set_integration_time(1)
def __init__(self): # Initialize power supply # Initialize flow meter driver """Try to connect to the COM port of the flow meter and ask for the serial number""" print("Try and connect to red flow meter") try: self.red_flow_meter = RedFlowMeter(ramp.RED_FLOW_METER_COM_PORT, slave_address=42) except SerialException: message = 'Cannot find red flow meter on {}'.format(ramp.RED_FLOW_METER_COM_PORT) raise RuntimeError(message) # Test serial number reply if self.red_flow_meter.read_value('serial') != 210059: raise RuntimeError('Incorrect reply to serial number') # Print actual flow value print(self.red_flow_meter.read_value('fluid_name'), self.red_flow_meter.read_value('flow')) # If more flow meters are used try them here #self.red_flow_meter.set_address(247) #print(self.red_flow_meter.read_value('fluid_name'), self.red_flow_meter.read_value('flow')) # Try to conect to power supply print("Try and connect to power supply") try: self.cpx = CPX400DPDriver(output=1, interface='serial', device=ramp.POWER_SUPPLY_COM_PORT) except SerialException: message = 'Cannot find power supply on {}'.format(ramp.POWER_SUPPLY_COM_PORT) raise RuntimeError(message) if 'CPX400DP' not in self.cpx.read_software_version(): raise RuntimeError('Incorrect software version reply') pass # Print the set point voltage print("The voltage set point is {}".format(self.cpx.read_set_voltage())) self.ramp = ramp.RAMP self.metadata = ramp.metadata self.area = ramp.area self.setpoint_pressure = ramp.setpoint_pressure self.setpoint_gas_flow = ramp.setpoint_gas_flow self.verify_ramp() self.data_set_saver = DataSetSaver( 'measurements_large_CO2_MEA', 'xy_values_large_CO2_MEA', credentials.USERNAME, credentials.PASSWORD, ) self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.sock.settimeout(0.1) self.host = ramp.RASPPI_HOST self.port = ramp.RASSPI_PORT self.message = 'json_wn#' + json.dumps({'no_voltages_to_set':True})
def init_logging(self): if self.log_data: self.comment = 'Run' self.data_set_saver = DataSetSaver('measurements_mailscan', 'xy_values_mailscan', credentials.user, credentials.passwd) self.data_set_saver.start() # PyExpLabSys does does not excell in db-normalization - add # metadata to all channels metadata = { "Time": CustomColumn(self.start_time, "FROM_UNIXTIME(%s)"), "comment": self.comment, "type": 1, "label": None, "processes": self.number_of_threads()[1] } metadata['label'] = 'Avg export speed' self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = 'Total export size' self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = 'Total users' self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = 'Total memory' self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = 'Total CPU' self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = 'Total Mails' self.data_set_saver.add_measurement(metadata['label'], metadata) for scanner in self.amqp_messages.keys(): if scanner == 'global': continue metadata['processes'] = 1 metadata['label'] = '{} memory'.format(scanner) self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = '{} exported users'.format(scanner) self.data_set_saver.add_measurement(metadata['label'], metadata)
def db_saver(self): """Autoclosing db_saver yield_fixture""" # Init db_saver db_saver = DataSetSaver('measurements_dummy', 'xy_values_dummy', 'dummy', 'dummy') db_saver.start() yield db_saver db_saver.stop()
class SolarCellTester(object): """ Performs an IV test """ def __init__(self, mux, smu, comment, user): self.data_set_saver = DataSetSaver("measurements_" + user, "xy_values_" + user, user, user) self.data_set_saver.start() self.data_set_time = time.time() # Update for each scan... self.comment = comment self.mux = mux self.smu = smu mux.scpi_comm('INSTRUMENT:DMM OFF') # Turn off DMM to allow use as mux device self.smu.set_current_limit(0.01) # Current limit, 10mA self.smu.set_current_measure_range(current_range=0.01) # Measurement range 10mA self.smu.set_integration_time(1) def run_measurement(self, v_from, v_to, scan_both_ways): """ Perform the actual measurement """ metadata = {"Time": CustomColumn(self.data_set_time, "FROM_UNIXTIME(%s)"), "comment": self.comment, "type": 1} for label in ["Ch1", "Ch2", "Ch3", "Ch4", "Ch5", "Ch6", "Ch7", "Ch8",]: metadata["mass_label"] = label self.data_set_saver.add_measurement(label, metadata) self.smu.set_voltage(0.0) self.smu.output_state(True) for channel in range(1, 9): print('Channel: ' + str(channel)) self.mux.scpi_comm('ROUTE:CLOSE (@10' + str(channel) + ')') time.sleep(0.2) voltage, current = self.smu.iv_scan(v_from=v_from, v_to=v_to, steps=10, settle_time=0) for i in range(0, len(current)): self.data_set_saver.save_point('Ch' + str(channel), (voltage[i], current[i])) if scan_both_ways: voltage, current = self.smu.iv_scan(v_from=v_to, v_to=v_from, steps=50, settle_time=0) for i in range(0, len(current)): self.data_set_saver.save_point('Ch' + str(channel), (voltage[i], current[i])) self.mux.scpi_comm('ROUTE:OPEN (@10' + str(channel) + ')') self.smu.set_voltage(0.0) self.smu.output_state(False) time.sleep(0.2)
class XPS(threading.Thread): """ Perform a single XPS scan """ def __init__(self, hostname, anode): threading.Thread.__init__(self) self.agilent = agilent_34972A.Agilent34972ADriver(interface='lan', hostname=hostname) self.calib = 500 # Analyser voltage pr. input voltage self.chamber_name = "dummy" self.data_set_saver = DataSetSaver("measurements_" + self.chamber_name, "xy_values_" + self.chamber_name, credentials.user, credentials.passwd) self.data_set_saver.start() if anode == 'Mg': self.x_ray = 1253.44 if anode == 'Al': self.x_ray = 1487.0 def scan(self, start_energy, end_energy, step, integration_time): """ Perform a scan """ metadata = { "Time": CustomColumn(time.time(), "FROM_UNIXTIME(%s)"), "comment": 'Test', "type": 2 } label = 'XPS signal' self.data_set_saver.add_measurement(label, metadata) count_string = self.agilent.scpi_comm("SENS:TOT:DATA? (@203)") meas_time = time.time() for binding_energy in np.arange(end_energy, start_energy, -1 * step): kin_energy = str((self.x_ray - binding_energy) / self.calib) string = "SOURCE:VOLT " + kin_energy + ", (@205)" self.agilent.scpi_comm(string) time.sleep(integration_time) count_string = self.agilent.scpi_comm("SENS:TOT:DATA? (@203)") count = int(float(count_string.strip())) int_time = time.time() - meas_time meas_time = time.time() count_rate = count / int_time self.data_set_saver.save_point(label, (binding_energy, count_rate))
class XPS(threading.Thread): """ Perform a single XPS scan """ def __init__(self, hostname, anode): threading.Thread.__init__(self) self.agilent = agilent_34972A.Agilent34972ADriver(interface='lan', hostname=hostname) self.calib = 500 # Analyser voltage pr. input voltage self.chamber_name = "dummy" self.data_set_saver = DataSetSaver("measurements_" + self.chamber_name, "xy_values_" + self.chamber_name, credentials.user, credentials.passwd) self.data_set_saver.start() if anode == 'Mg': self.x_ray = 1253.44 if anode == 'Al': self.x_ray = 1487.0 def scan(self, start_energy, end_energy, step, integration_time): """ Perform a scan """ metadata = {"Time": CustomColumn(time.time(), "FROM_UNIXTIME(%s)"), "comment": 'Test', "type": 2} label = 'XPS signal' self.data_set_saver.add_measurement(label, metadata) count_string = self.agilent.scpi_comm("SENS:TOT:DATA? (@203)") meas_time = time.time() for binding_energy in np.arange(end_energy, start_energy, -1 * step): kin_energy = str((self.x_ray - binding_energy) / self.calib) string = "SOURCE:VOLT " + kin_energy + ", (@205)" self.agilent.scpi_comm(string) time.sleep(integration_time) count_string = self.agilent.scpi_comm("SENS:TOT:DATA? (@203)") count = int(float(count_string.strip())) int_time = time.time() - meas_time meas_time = time.time() count_rate = count / int_time self.data_set_saver.save_point(label, (binding_energy, count_rate))
from __future__ import print_function import os import time import logging # logging.basicConfig(level=logging.DEBUG) # Comment in for more logging output from collections import defaultdict from PyExpLabSys.file_parsers.chemstation import Sequence from PyExpLabSys.common.database_saver import DataSetSaver from PyExpLabSys.common.database_saver import CustomColumn import credentials # Instantiate the data set saver data_set_saver = DataSetSaver('measurements_vhp_setup', 'xy_values_vhp_setup', credentials.USERNAME, credentials.PASSWORD) data_set_saver.start() # Get the set of aleady uploaded files already_uploaded = data_set_saver.get_unique_values_from_measurements('relative_path') print('Fetched relative paths for {} known sequences'.format(len(already_uploaded))) # This is the measurement path, should be generated somehow basefolder = '/home/cinf/Desktop/Shared_folder' sequence_identifyer = 'sequence.acaml' for root, dirs, files in os.walk(basefolder): if sequence_identifyer in files: # Check if file is known relative_path = root.replace(basefolder, '').strip(os.sep) if relative_path in already_uploaded: continue
# -*- coding: utf-8 -*- """ Spyder Editor This is a temporary script file. """ import time from PyExpLabSys.common.database_saver import DataSetSaver, CustomColumn import credentials data_set_saver = DataSetSaver( 'measurements_large_CO2_MEA', 'xy_values_large_CO2_MEA', credentials.USERNAME, credentials.PASSWORD, ) data_set_saver.start() now = time.time() now_custom_column = CustomColumn(now, 'FROM_UNIXTIME(%s)') #CREATE TABLE `measurements_large_CO2_MEA` ( # `id` int(10) unsigned NOT NULL AUTO_INCREMENT, # `time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, # `type` int(10) unsigned NOT NULL COMMENT 'Type of measurement, as found in the types table', # `comment` varchar(128) NOT NULL COMMENT 'Comment', # `cathode_gas_type` varchar(128) NOT NULL COMMENT 'Cathode gas type', # `anode_gas_type` varchar(128) NOT NULL COMMENT 'Anode gas type', # `cathode_catalyst_description` varchar(128) NOT NULL COMMENT 'Cathode catalyst description', # `anode_catalyst_description` varchar(128) NOT NULL COMMENT 'Anode catalyst description',
if database_saver: self.data_set_saver.save_point(self.codename, (t, reading)) print(reading, t, status) except KeyboardInterrupt: break print('Emptying buffer..') time.sleep(1) if self.ser.inWaiting() > 0: self.ser.readline() print('Done') if database_saver: self.data_set_saver.stop() if __name__ == '__main__': from PyExpLabSys.common.database_saver import DataSetSaver import credentials2 as cred port = '/dev/serial/by-id/usb-Prolific_Technology_Inc._USB-Serial_Controller-if00-port0' data_set_saver = DataSetSaver('measurements_dummy', 'xy_values_dummy', cred.user, cred.passwd) dmm = Keithley(port, data_set_saver=data_set_saver, codename='keithley_voltage') time.sleep(1) print(dmm.comm('*IDN?')) dmm.read_voltage(meas_range=20, database_saver=False, count=10)
class RampRunner(object): """The RampRunner. Program to run current and voltage measurements and log measurements of voltage, current, temperature, pressure and flow to the surfcat database. """ def __init__(self): # Initialize power supply # Initialize flow meter driver """Try to connect to the COM port of the flow meter and ask for the serial number""" print("Try and connect to red flow meter") try: self.red_flow_meter = RedFlowMeter(ramp.RED_FLOW_METER_COM_PORT, slave_address=42) except SerialException: message = 'Cannot find red flow meter on {}'.format(ramp.RED_FLOW_METER_COM_PORT) raise RuntimeError(message) # Test serial number reply if self.red_flow_meter.read_value('serial') != 210059: raise RuntimeError('Incorrect reply to serial number') # Print actual flow value print(self.red_flow_meter.read_value('fluid_name'), self.red_flow_meter.read_value('flow')) # If more flow meters are used try them here #self.red_flow_meter.set_address(247) #print(self.red_flow_meter.read_value('fluid_name'), self.red_flow_meter.read_value('flow')) # Try to conect to power supply print("Try and connect to power supply") try: self.cpx = CPX400DPDriver(output=1, interface='serial', device=ramp.POWER_SUPPLY_COM_PORT) except SerialException: message = 'Cannot find power supply on {}'.format(ramp.POWER_SUPPLY_COM_PORT) raise RuntimeError(message) if 'CPX400DP' not in self.cpx.read_software_version(): raise RuntimeError('Incorrect software version reply') pass # Print the set point voltage print("The voltage set point is {}".format(self.cpx.read_set_voltage())) self.ramp = ramp.RAMP self.metadata = ramp.metadata self.area = ramp.area self.setpoint_pressure = ramp.setpoint_pressure self.setpoint_gas_flow = ramp.setpoint_gas_flow self.verify_ramp() self.data_set_saver = DataSetSaver( 'measurements_large_CO2_MEA', 'xy_values_large_CO2_MEA', credentials.USERNAME, credentials.PASSWORD, ) self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.sock.settimeout(0.1) self.host = ramp.RASPPI_HOST self.port = ramp.RASSPI_PORT self.message = 'json_wn#' + json.dumps({'no_voltages_to_set':True}) def verify_ramp(self): """Verify the ramp for consistency and safety""" for step in self.ramp: # Check if a contant voltage step has the right keys if step['type'] == 'constant_voltage': required_keys = {'type','duration','end_voltage', 'save_rate'} if step.keys() != required_keys: raise RuntimeError('The keys must in a constant voltage step must be\ {}'.format(required_keys)) # Check if a voltage ramp step has the right keys elif step['type'] == 'voltage_ramp': required_keys = {'type', 'duration',\ 'start_voltage', 'end_voltage', 'voltage_step', 'save_rate'} if step.keys() != required_keys: raise RuntimeError('The keys must in a constant voltage step must be\ {}'.format(required_keys)) elif step['start_voltage'] < 0: raise RuntimeError('Voltage must be must be positive and less than 5V, got: {}V'\ .format(step['end_voltage'])) elif step['end_voltage'] > 5: raise RuntimeError('Voltage must be must be positive and less than 5V, got: {}V'\ .format(step['end_voltage'])) elif step['type'] == 'constant_current': required_keys = {'type', 'duration',\ 'current_density', 'save_rate'} if step.keys() != required_keys: raise RuntimeError('The keys must in a constant voltage step must be\ {}'.format(required_keys)) elif step['current_density'] < 0: raise RuntimeError('Current density must be positive, got: {}mA/cm2'\ .format(step['current_density'])) # Do checks on the common keys elif step['duration'] <= 0: raise RuntimeError('Duration must be positive, got: {}'.format(step['duration'])) # Check that the voltage is greater than 0 elif step['end_voltage'] < 0: raise RuntimeError('Voltage must be must be positive and less than 5V, got: {}V'\ .format(step['end_voltage'])) #...and less than 5V elif step['end_voltage'] > 5: raise RuntimeError('Voltage must be must be positive and less than 5V, got: {}V'\ .format(step['end_voltage'])) # Check if the rate at which the data should be saved is positive. elif step['save_rate'] <=0: raise RuntimeError('The save rate must be positive, got: {}'.format(step['save_rate'])) def run(self): """Run the ramp This is the main function that contains a loop over the steps in the ramp and the method that does all the work """ self.data_set_saver.start() # Specify the start time for database now = time() now_custom_column = CustomColumn(now, 'FROM_UNIXTIME(%s)') # Add all measurements self.metadata['label'] = 'voltage' self.data_set_saver.add_measurement('voltage', self.metadata) self.metadata['label'] = 'current' self.data_set_saver.add_measurement('current', self.metadata) self.metadata['label'] = 'CO2 flow' self.data_set_saver.add_measurement('CO2 flow', self.metadata) # If another flow meter is used collect the flow here #self.metadata['label'] = 'cell outlet flow' #self.data_set_saver.add_measurement('cell outlet flow', self.metadata) # Collect the pressure reading from the pressure controller #self.metadata['label'] = 'pressure' #self.data_set_saver.add_measurement('pressure', self.metadata) # Add the one custom column self.metadata['time'] = now_custom_column # Set the current limit on the power supply to 10A self.cpx.set_current_limit(10) # Change address to CO2 flow controller self.red_flow_meter.set_address(42) # Set the CO2 flow to the specified value self.red_flow_meter.write_value('setpoint_gas_flow', self.setpoint_gas_flow) # Set the pressure on the pressure controller #self.message = 'json_wn#' + json.dumps({'A': 0.0, 'B': self.setpoint_pressure }) #self.sock.sendto(message.encode('ascii'), (self.host, self.port)) try: for step in self.ramp: # Go through the steps in the ramp file if step['type'] == 'constant_voltage': self.run_constant_voltage_step(step) elif step['type'] == 'voltage_ramp': self.run_voltage_ramp_step(step) elif step['type'] == 'constant_current': self.run_constant_current_step(step) except KeyboardInterrupt: print('Program interupted by user') finally: # End the data collection and turn off power supply after run or interruption print('The program has ended') self.cpx.output_status(False) self.red_flow_meter.set_address(42) self.red_flow_meter.write_value('setpoint_gas_flow', 0.0) #self.message = 'json_wn#' + json.dumps({'A': 0.0, 'B': 0.0}) #self.sock.sendto(message.encode('ascii'), (self.host, self.port)) self.data_set_saver.wait_for_queue_to_empty() self.data_set_saver.stop() def run_constant_voltage_step(self, step): """Run a single step""" print('Run constant voltage step with parameters', step) start_time = time() # Read from step # Set voltage on power supply voltage_difference_limit = 0.01 print("Setting voltage to {} V".format(step["end_voltage"])) self.cpx.set_voltage(step["end_voltage"]) while abs(step["end_voltage"]-self.cpx.read_set_voltage())>voltage_difference_limit: print(self.cpx.read_set_voltage()) pass # Turn on the power supply self.cpx.output_status(True) last_save = 0 # Read of the flow and current as long as the duration lasts while (time() - start_time) < step['duration']*3600: # Log results to database # Send communication to datacard self.sock.sendto(self.message.encode('ascii'), (self.host, self.port)) try: reply = self.sock.recv(1024).decode('ascii') except socket.timeout: pass else: # This is the data collection rate right now sleep(step['save_rate']) now = time() # Read voltage on power supply power_supply_voltage = self.cpx.read_actual_voltage() print('The voltage is: {}V on the power supply'.format(power_supply_voltage)) # Read actual voltage on raspberry pi reply_data = json.loads(reply[4:]) actual_voltage = reply_data['3'] print('The voltage is: {}V on the cell'.format(actual_voltage)) #pressure = reply_data['1'] # Read the current from the power supply actual_current = self.cpx.read_actual_current() print('The current is: {}A'.format(actual_current)) # Change address to CO2 flow meter self.red_flow_meter.set_address(42) # Get the CO2 flow from the MFC CO2_flow = self.red_flow_meter.read_value('flow') # If another flow meter is used collect the flow here #self.red_flow_meter.set_address(247) # Get the flow from the cell into the GC from the MFM #cell_outlet_flow = self.red_flow_meter.read_value('flow') # time between data is saved print('Time since last save: {}s'.format(time()-last_save)) last_save = time() # Save the measurements to the database self.data_set_saver.save_point('voltage', (now, actual_voltage)) self.data_set_saver.save_point('current', (now, actual_current)) self.data_set_saver.save_point('CO2 flow', (now, CO2_flow)) #self.data_set_saver.save_point('CO2 flow', (now, pressure)) #self.data_set_saver.save_point('cell outlet flow', (now, cell_outlet_flow)) def run_constant_current_step(self, step): """Run a single step""" print('Run constant current step with parameters', step) start_time = time() # Read from step # Change address to CO2 flow meter self.red_flow_meter.set_address(42) # Set current limit on power supply and set a high voltage to hit the limit # and let the power supply control the voltage current_density = step['current_density'] # in mA # Calculate the current from the corresponding area and corrent density current = current_density/1000*self.area print("Setting current to {}A".format(current)) self.cpx.set_current_limit(current) self.cpx.set_voltage(4) # Turn on the power supply self.cpx.output_status(True) #self.red_flow_meter.write_value('setpoint_gas_flow', step['setpoint_gas_flow']) last_save = 0 # Read of the flow and current as long as the duration lasts while (time() - start_time) < step['duration']*3600: # Log results to database # Send communication to datacard self.sock.sendto(self.message.encode('ascii'), (self.host, self.port)) try: reply = self.sock.recv(1024).decode('ascii') except socket.timeout: pass else: # This is the data collection rate right now sleep(step['save_rate']) now = time() # Read voltage on power supply power_supply_voltage = self.cpx.read_actual_voltage() print('The voltage is: {}V on the power supply'.format(power_supply_voltage)) # Read actual voltage on raspberry pi reply_data = json.loads(reply[4:]) actual_voltage = reply_data['3'] print('The voltage is: {}V on the cell'.format(actual_voltage)) #pressure = reply_data['1'] # Read the current from the power supply actual_current = self.cpx.read_actual_current() print('The current is: {}A'.format(actual_current)) # Change address to CO2 flow controller self.red_flow_meter.set_address(42) # Get the CO2 flow from the MFM CO2_flow = self.red_flow_meter.read_value('flow') # If another flow meter is used collect the flow here # Change adress to the cell outlet MFM #self.red_flow_meter.set_address(247) # Get the CO2 flow from the MFM #cell_outlet_flow = self.red_flow_meter.read_value('flow') # time between data is saved print('Time since last save: {}s'.format(time()-last_save)) last_save = time() # Save the measurements to the database self.data_set_saver.save_point('voltage', (now, actual_voltage)) self.data_set_saver.save_point('current', (now, actual_current)) self.data_set_saver.save_point('CO2 flow', (now, CO2_flow)) #self.data_set_saver.save_point('pressure', (now, pressure)) #self.data_set_saver.save_point('cell outlet flow', (now, cell_outlet_flow)) def run_voltage_ramp_step(self, step): """ Run voltage ramp step""" print('Run voltage ramp step with parameters', step) # Start time of this step for reference start_time = time() # Set voltage to the start voltage in the step and turn on the voltage self.cpx.set_voltage(step['start_voltage']) print('Starting voltage ramp at: {}'.format(step['start_voltage'])) self.cpx.output_status(True) voltage_set_point = step['start_voltage'] while (time() - start_time) < step['duration']*3600: now = time() # Change address to CO2 flow meter self.red_flow_meter.set_address(247) # Read flow value CO2_flow = self.red_flow_meter.read_value('flow') print(self.red_flow_meter.read_value('fluid_name'), CO2_flow) # Change address to N2 flow meter self.red_flow_meter.set_address(2) # Read flow value N2_flow = self.red_flow_meter.read_value('flow') print(self.red_flow_meter.read_value('fluid_name'), N2_flow) # Set the voltage on the power supply self.cpx.set_voltage(voltage_set_point) # Read actual voltage actual_voltage = self.cpx.read_actual_voltage() print('The current voltage is: {}V'.format(actual_voltage)) print('The voltage set point is {}V'.format(voltage_set_point)) # Read the current from the power supply actual_current = self.cpx.read_actual_current() print('The current current is: {}A'.format(actual_current)) # This is the data collection rate right now sleep(step['save_rate']) # Save the measurements to the database self.data_set_saver.save_point('voltage', (now, actual_voltage)) self.data_set_saver.save_point('current', (now, actual_current)) self.data_set_saver.save_point('CO2 flow', (now, CO2_flow)) # Ramp the voltage by the voltage step in step voltage_set_point = voltage_set_point + step['voltage_step'] # Terminate the ramp when the end voltage is reached if voltage_set_point >= step['end_voltage']: print('end voltage, {} reached'.format(step['end_voltage'])) sleep(1) # Turn off the power supply #self.cpx.output_status(False) break # Tell when the time loop is done and turn of power supply print('voltage ramp step has ended')
class SolarCellTester(object): """ Performs an IV test """ def __init__(self, mux, smu, comment, user): self.data_set_saver = DataSetSaver("measurements_" + user, "xy_values_" + user, user, user) self.data_set_saver.start() self.data_set_time = time.time() # Update for each scan... self.comment = comment self.mux = mux self.smu = smu mux.scpi_comm( 'INSTRUMENT:DMM OFF') # Turn off DMM to allow use as mux device self.smu.set_current_limit(0.01) # Current limit, 10mA self.smu.set_current_measure_range( current_range=0.01) # Measurement range 10mA self.smu.set_integration_time(1) def run_measurement(self, v_from, v_to, scan_both_ways): """ Perform the actual measurement """ metadata = { "Time": CustomColumn(self.data_set_time, "FROM_UNIXTIME(%s)"), "comment": self.comment, "type": 1 } for label in [ "Ch1", "Ch2", "Ch3", "Ch4", "Ch5", "Ch6", "Ch7", "Ch8", ]: metadata["mass_label"] = label self.data_set_saver.add_measurement(label, metadata) self.smu.set_voltage(0.0) self.smu.output_state(True) for channel in range(1, 9): print('Channel: ' + str(channel)) self.mux.scpi_comm('ROUTE:CLOSE (@10' + str(channel) + ')') time.sleep(0.2) voltage, current = self.smu.iv_scan(v_from=v_from, v_to=v_to, steps=10, settle_time=0) for i in range(0, len(current)): self.data_set_saver.save_point('Ch' + str(channel), (voltage[i], current[i])) if scan_both_ways: voltage, current = self.smu.iv_scan(v_from=v_to, v_to=v_from, steps=50, settle_time=0) for i in range(0, len(current)): self.data_set_saver.save_point('Ch' + str(channel), (voltage[i], current[i])) self.mux.scpi_comm('ROUTE:OPEN (@10' + str(channel) + ')') self.smu.set_voltage(0.0) self.smu.output_state(False) time.sleep(0.2)
class VoltageCurrentProgram(Thread): # pylint: disable=too-many-instance-attributes """The Voltage Current Program This program uses the generic stepped program runner as a GUI and serves information to that """ def __init__(self, args): super(VoltageCurrentProgram, self).__init__() # Form channel_id e.g: EA1 self.channel_id = args.power_supply + args.output ### Required by the stepped program runner # Accepted capabilities are: can_edit, can_play, # can_stop, can_quit self.capabilities = ('can_stop', 'can_start', 'can_edit') # Status fields (in order) self.status_fields = ( # Status { 'codename': 'status_field', 'title': 'Status' }, # Voltage { 'codename': self.channel_id + '_voltage', 'title': 'Voltage', 'formatter': '{:.3f}', 'unit': 'V' }, # Voltage setpoint { 'codename': self.channel_id + '_voltage_setpoint', 'title': 'Voltage Setpoint', 'formatter': '{:.3f}', 'unit': 'V' }, # Current { 'codename': self.channel_id + '_current', 'title': 'Current', 'formatter': '{:.3f}', 'unit': 'A' }, # Current limit { 'codename': self.channel_id + '_current_limit', 'title': 'Current limit', 'formatter': '{:.3f}', 'unit': 'A' }, # Charge { 'codename': self.channel_id + '_accum_charge', 'title': 'Accumulated charge', 'formatter': '{:.3f}', 'unit': 'C' }, # Time elapsed (step) { 'codename': 'elapsed', 'title': 'Time elapsed (step)', 'formatter': '{:.1f}', 'unit': 's' }, # Time remaining (step) { 'codename': 'remaining', 'title': 'Time remaining (step)', 'formatter': '{:.1f}', 'unit': 's' }, # Time elapsed (total) { 'codename': 'elapsed_total', 'title': 'Time elapsed (total)', 'formatter': '{:.1f}', 'unit': 's' }, # Time remaining (total) { 'codename': 'remaining_total', 'title': 'Time remaining (total)', 'formatter': '{:.1f}', 'unit': 's' }, # Iteration time { 'codename': 'iteration_time', 'title': 'Iteration time', 'formatter': '{:.2f}', 'unit': 's' }, ) self.extra_capabilities = { 'psuchannel': { 'help_text': ('Used for simple PSU control when not on\n' 'a ramp. Possibly usages are:\n' ' psuchannel voltage=1.23\n' 'which will set the voltage and\n' ' psuchannel off\n' 'which will set the output off'), 'completions': [ 'psuchannel', 'psuchannel voltage=', 'psuchannel off', ] } } # Queue for GUI updates self.message_queue = Queue() # The GUI also looks in self.config, see below ### Normal program # Setup my program with open(path.join(THIS_DIR, args.program_file)) as file__: self.config, self.steps = parse_ramp(file__) # The GUI will look for keys: program_title in config self.say('Using power supply channel: ' + self.channel_id) self.say('Loaded with config:\n' + pformat(self.config)) self.active_step = 0 self.send_steps() # Add completions for the edits self._completion_additions = [] for number, step in enumerate(self.steps): base = 'edit {} '.format(number) self._completion_additions.append(base) for field in sorted(step.fields): self._completion_additions.append('{}{}='.format(base, field)) # Base for the status self.status = {'status_field': 'Initialized'} # General variables self.stop = False self.ok_to_start = False # Create a partial function with the output substitued in self.send_command = partial( _send_command, args.output, args.power_supply, ) # Setup power supply self.power_supply_on_off(True, self.config['maxcurrent_start']) # Power supply commands, must match order with self.codenames self.power_supply_commands = ('read_actual_current', 'read_actual_voltage', 'read_set_voltage', 'read_current_limit') # Setup dataset saver and live socket self.codenames = [ self.channel_id + id_ for id_ in ('_current', '_voltage', '_voltage_setpoint', '_current_limit') ] self.live_socket = LiveSocket('H2O2_proactive_' + self.channel_id, self.codenames + [self.channel_id + '_accum_charge'], no_internal_data_pull_socket=True) self.live_socket.reset(self.codenames) self.live_socket.start() self.data_set_saver = DataSetSaver(credentials.measurements, credentials.xy_values, username=credentials.username, password=credentials.password) self.data_set_saver.start() # Done with init, send status self.send_status() def command(self, command, args_str): """Process commands from the GUI""" if command == 'stop': # stop is sent on quit self.stop = True elif command == 'start': self.ok_to_start = True elif command == 'edit': # Parse the edit line, start by splitting up in step_num, field and value try: num_step, field_value = [ arg.strip() for arg in args_str.split(' ') ] field, value = [arg.strip() for arg in field_value.split('=')] except ValueError: message = ('Bad edit command, must be on the form:\n' 'edit step_num field=value') self.say(message, message_type='error') return # Try to get the correct step try: step = self.steps[int(num_step)] except (ValueError, IndexError): message = 'Unable to convert step number {} to integer or nu such step '\ 'exists' self.say(message.format(num_step), message_type='error') return # Edit the value try: step.edit_value(field, value) except (ValueError, AttributeError) as exception: self.say(str(exception.args[0]), message_type='error') return # Finally send the new steps to the GUI self.send_steps() elif command == "psuchannel": if self.ok_to_start and not self.stop: message = "Using psuchannel during ramp not allowed" self.say(message, message_type='error') return if args_str.startswith('voltage='): voltage_str = args_str.replace('voltage=', '') try: voltage = float(voltage_str) except ValueError: message = 'Invalid voltage for psuchannel {}'.format( voltage_str) self.say(message, message_type='error') return self.send_command('set_voltage', voltage) self.say('PSU channel set to {}.\n' 'NOTE: The GUI will not update to show\n' 'this, only the actual PSU.'.format(voltage)) elif args_str == 'off': self.power_supply_on_off(False) self.say('PSU channel set to off') else: message = 'Invalid argument. psuchannel can do "voltage=" and "off"' self.say(message, message_type='error') return def send_status(self, update_dict=None): """Send the status to the GUI""" if update_dict: self.status.update(update_dict) self.message_queue.put(('status', self.status.copy())) def send_steps(self): """Send the steps list to the GUI""" steps = [(index == self.active_step, str(step)) for index, step in enumerate(self.steps)] self.message_queue.put(('steps', steps)) def say(self, text, message_type='message'): """Send a ordinary text message to the gui""" self.message_queue.put((message_type, text)) def run(self): """The MAIN run method""" # Wait for start while not self.ok_to_start: if self.stop: self.send_status({'status_field': 'Stopped'}) return sleep(0.1) # Start self.send_status({'status_field': 'Starting'}) self.setup_data_set_saver() # Run the MAIN measurement loop # (This is where most of the time is spent) self.main_measure() # Shutdown powersupply, livesocket and possibly server self.send_status({'status_field': 'Stopping'}) self.stop_everything() self.send_status({'status_field': 'Stopped'}) sleep(0.1) self.say("I have stopped") def setup_data_set_saver(self): """Setup the data set saver""" sql_time = CustomColumn(time(), 'FROM_UNIXTIME(%s)') for codename in self.codenames: metadata = { 'time': sql_time, 'comment': self.config['comment'], 'label': codename[3:], 'type': 1, 'power_supply_channel': self.channel_id, } self.data_set_saver.add_measurement(codename, metadata) def main_measure(self): # pylint: disable=too-many-locals """The main measurement loop""" self.send_status({'status_field': 'Running'}) # Initial setup last_set_voltage = None last_set_max_current = None last_time = time() iteration_time = 'N/A' self.status['elapsed'] = 0.0 accum_charge_codename = self.channel_id + '_accum_charge' self.status[accum_charge_codename] = 0.0 current_id = self.channel_id + '_current' last_measured_current = 0.0 self.say('I started on step 0') for self.active_step, current_step in enumerate(self.steps): self.send_status( {'status_field': 'Running step {}'.format(self.active_step)}) # Also give the step an instance name (for steps list) if self.active_step > 0: self.say('Switched to step: {}'.format(self.active_step)) self.send_steps() current_step.start() # While the step hasn't completed yet while current_step.elapsed() < current_step.duration: # Check if we should stop if self.stop: self.say('I have been asked to stop') return iteration_start = now = time() # Calculate the time for one iteration and update times in status iteration_time = now - last_time last_time = now self.status.update({ 'elapsed': current_step.elapsed(), 'remaining': current_step.remaining(), 'iteration_time': iteration_time, 'elapsed_total': sum(step.elapsed() for step in self.steps), 'remaining_total': sum(step.remaining() for step in self.steps), }) # Ask the power supply to set a new voltage if needed required_voltage, required_max_current = current_step.values() if required_max_current != last_set_max_current: self.send_command('set_current_limit', required_max_current) last_set_max_current = required_max_current if required_voltage != last_set_voltage: self.send_command('set_voltage', required_voltage) last_set_voltage = required_voltage # Read value from the power supply self._read_values_from_power_supply() # Calculate, set and send accumulated charge charge_addition = \ (last_measured_current + self.status[current_id])\ / 2 * iteration_time last_measured_current = self.status[current_id] self.status[accum_charge_codename] += charge_addition point = (self.status['elapsed_total'], self.status[accum_charge_codename]) self.live_socket.set_point(accum_charge_codename, point) # Send the new status self.send_status() # Calculate time to sleep to use the proper probe interval time_to_sleep = current_step.probe_interval - (time() - iteration_start) if time_to_sleep > 0: sleep(time_to_sleep) # Stop the step(s own time keeping) current_step.stop() # For loop over steps ended self.send_status({'status_field': 'Program Complete'}) self.say('Stepped program completed') def _read_values_from_power_supply(self): """Read all required values from the power supply (used only from run)""" for command, codename in zip(self.power_supply_commands, self.codenames): # Get a value for the current command value = self.send_command(command) # Set/save it on the live_socket, database and in the GUI point = (self.status['elapsed_total'], value) self.live_socket.set_point(codename, point) self.data_set_saver.save_point(codename, point) self.status[codename] = value def stop_everything(self): """Stop power supply and live socket""" self.live_socket.stop() self.data_set_saver.stop() def power_supply_on_off(self, state, current_limit=None): """Set power supply on off""" LOG.debug('Stop power supply') # Set current limit if current_limit is not None: self.send_command('set_current_limit', current_limit) read_current_limit = self.send_command('read_current_limit') if not isclose(read_current_limit, current_limit): raise RuntimeError('Unable to set current limit') # Set state self.send_command('output_status', state) read_state = self.send_command('read_output_status').strip() == '1' if not read_state is state: raise RuntimeError('Could not set output state')
def main(): """Main function""" parser = argparse.ArgumentParser(description='Run the infamous ph control script.') # ph_setpoint, ph_direction (string [up, down]), kick_in_rate (float) # Main (constant), secondary (regulation) # python ph_increment.py 10.0 --direction up --main-offset=24.0 --second-min=10.0 --second-max=40.0 --second-offset=17.0 parser.add_argument('setpoint', type=float, help='The Ph setpoint.') parser.add_argument('--direction', default=None, help='Pumping direction. Must be up or down.', required=True) parser.add_argument('--main-offset', type=float, default=None, help='Start pumping speed for the main pump.', required=True) parser.add_argument('--second-min', type=float, default=None, help='Minimum pumping speed for the secondary pump.', required=True) parser.add_argument('--second-max', type=float, default=None, help='Maximum pumping speed for the secondary pump.', required=True) parser.add_argument('--second-offset', type=float, default=None, help='Start pumping value for the secondary pump.', required=True) parser.add_argument('--comment', default=None, help='Optional comment', required=True) args = parser.parse_args() ## Edit comment comment ="pH "+ str(args.setpoint) + " // p: 0.02 // start rate " + str(args.second_offset) + " // " + args.comment ## Edit comment # raise SystemExit('All good') # Init pumps, Omegabus, PID and data saver pump_main = AL1000("/dev/serial/by-id/usb-FTDI_USB-RS232_Cable_FTV9X9TM-if00-port0") pump_second = AL1000("/dev/serial/by-id/usb-FTDI_USB-RS232_Cable_FTWZCJLU-if00-port0") obus = OmegaBus( "/dev/serial/by-id/usb-FTDI_USB-RS232_Cable_FTWZCGSW-if00-port0", baud=9600, ) data_set_saver = DataSetSaver("measurements_dummy", "xy_values_dummy", "dummy", "dummy") pid = PID(pid_p=0.04, pid_i=0.0, pid_d=0, p_max=9999, p_min=-9) pid.update_setpoint(args.setpoint) # time_start = time.time() #Pre flight check. pump_main_check = pump_main.get_firmware() print("main: "+pump_main_check) pump_second_check = pump_second.get_firmware() print("second: " +pump_second_check) obus_check = obus.read_value(1) obus_check = current_to_ph(obus_check) if pump_main_check != "NE1000V3.923": print("Main pump failed") raise SystemExit(1) if pump_second_check != "NE1000V3.923": print("Secondary pump failed") raise SystemExit(1) if obus_check < 0: print("OmegaBus failed") raise SystemExit(1) #Set initial condition for pumps pump_second.set_direction("INF") pump_main.set_direction("INF") if args.direction == "up": pump_second.set_rate(args.second_offset) rate_second = args.second_offset pump_main.set_rate(0) rate_main = 0 if args.direction == "down": pump_second.set_rate(0) rate_second = 0 pump_main.set_rate(args.main_offset) rate_main = args.main_offset pump_second.set_vol(9999) pump_main.set_vol(9999) pump_second.start_program() pump_main.start_program() # Setup database saver data_set_saver.start() data_set_time = time.time() metadata = {"Time": CustomColumn(data_set_time, "FROM_UNIXTIME(%s)"), "comment": comment, "type": 64, "preamp_range": 0, "sem_voltage": 0} for label in ["p_korr", "i_korr", "total_korr", "ph_setpoint", "ph_value", "pump_rate"]: metadata["mass_label"] = label data_set_saver.add_measurement(label, metadata) # Run PID try: time_start = time.time() run(pump_second, pump_main, obus, pid, data_set_saver, time_start, args.setpoint, rate_second, rate_main, args.second_offset, args.main_offset, args.second_min , args.second_max, args.direction, minutes) except KeyboardInterrupt: # Clean up pump_main.set_rate(0) pump_second.set_rate(0) pump_main.stop_program() pump_second.stop_program() data_set_saver.stop() raise SystemExit(" Sequence stopped.") # Clean up pump_main.set_rate(0) pump_second.set_rate(0) pump_main.stop_program() pump_second.stop_program() # Log pH during ageing try: while True: value = obus.read_value(1) value = current_to_ph(value) print("pH: " + str(round(value,3))) data_set_saver.save_point("ph_value", (time.time()-time_start, value)) data_set_saver.save_point("ph_setpoint",(time.time()-time_start, args.setpoint)) time.sleep(8) except KeyboardInterrupt: data_set_saver.stop() pass
class MyProgram(Thread): # pylint: disable=too-many-instance-attributes """My fancy program""" def __init__(self, args): super(MyProgram, self).__init__() # Form channel_id e.g: A1 self.channel_id = args.power_supply + args.output ### Required by the stepped program runner # Accepted capabilities are: can_edit, can_play, # can_stop, can_quit self.capabilities = ('can_stop', 'can_start', 'can_edit') # Status fields (in order) self.status_fields = ( # Status {'codename': 'status_field', 'title': 'Status'}, # Voltage {'codename': self.channel_id + '_voltage', 'title': 'Voltage', 'formatter': '{:.3f}', 'unit': 'V'}, # Voltage setpoint {'codename': self.channel_id + '_voltage_setpoint', 'title': 'Voltage Setpoint', 'formatter': '{:.3f}', 'unit': 'V'}, # Current {'codename': self.channel_id + '_current', 'title': 'Current', 'formatter': '{:.3f}', 'unit': 'A'}, # Current limit {'codename': self.channel_id + '_current_limit', 'title': 'Current limit', 'formatter': '{:.3f}', 'unit': 'A'}, # Charge {'codename': self.channel_id + '_accum_charge', 'title': 'Accumulated charge', 'formatter': '{:.3f}', 'unit': 'C'}, # Time elapsed (step) {'codename': 'elapsed', 'title': 'Time elapsed (step)', 'formatter': '{:.1f}', 'unit': 's'}, # Time remaining (step) {'codename': 'remaining', 'title': 'Time remaining (step)', 'formatter': '{:.1f}', 'unit': 's'}, # Time elapsed (total) {'codename': 'elapsed_total', 'title': 'Time elapsed (total)', 'formatter': '{:.1f}', 'unit': 's'}, # Time remaining (total) {'codename': 'remaining_total', 'title': 'Time remaining (total)', 'formatter': '{:.1f}', 'unit': 's'}, # Iteration time {'codename': 'iteration_time', 'title': 'Iteration time', 'formatter': '{:.2f}', 'unit': 's'}, ) # Queue for GUI updates self.message_queue = Queue() # The GUI also looks in self.config, see below ### Normal program # Setup my program with open(path.join(THIS_DIR, args.program_file)) as file__: self.config, self.steps = parse_ramp(file__) # The GUI will look for keys: program_title in config self.say('Using power supply channel: ' + self.channel_id) self.say('Loaded with config:\n' + pformat(self.config)) self.active_step = 0 self.send_steps() # Add completions for the edits self._completion_additions = [] for number, step in enumerate(self.steps): base = 'edit {} '.format(number) self._completion_additions.append(base) for field in sorted(step.fields): self._completion_additions.append('{}{}='.format(base, field)) # Base for the status self.status = {'status_field': 'Initialized'} # General variables self.stop = False self.ok_to_start = False # Setup power supply # Create a partial function with the output substitued in self.send_command = partial(_send_command, args.output, args.power_supply) self.power_supply_on_off(True, self.config['maxcurrent_start']) # Power supply commands, must match order with self.codenames self.power_supply_commands = ( 'read_actual_current', 'read_actual_voltage', 'read_set_voltage', 'read_current_limit' ) # Setup dataset saver and live socket self.codenames = [self.channel_id + id_ for id_ in ('_current', '_voltage', '_voltage_setpoint', '_current_limit')] self.live_socket = LiveSocket( 'H2O2_proactive_' + self.channel_id, self.codenames + [self.channel_id + '_accum_charge'], no_internal_data_pull_socket=True ) self.live_socket.reset(self.codenames) self.live_socket.start() self.data_set_saver = DataSetSaver( credentials.measurements, credentials.xy_values, username=credentials.username, password=credentials.password ) self.data_set_saver.start() # Done with init, send status self.send_status() def command(self, command, args_str): """Process commands from the GUI""" if command == 'stop': # stop is sent on quit self.stop = True elif command == 'start': self.ok_to_start = True elif command == 'edit': # Parse the edit line, start by splitting up in step_num, field and value try: num_step, field_value = [arg.strip() for arg in args_str.split(' ')] field, value = [arg.strip() for arg in field_value.split('=')] except ValueError: message = ('Bad edit command, must be on the form:\n' 'edit step_num field=value') self.say(message, message_type='error') return # Try to get the correct step try: step = self.steps[int(num_step)] except (ValueError, IndexError): message = 'Unable to convert step number {} to integer or nu such step '\ 'exists' self.say(message.format(num_step), message_type='error') return # Edit the value try: step.edit_value(field, value) except (ValueError, AttributeError) as exception: self.say(str(exception.args[0]), message_type='error') return # Finally send the new steps to the GUI self.send_steps() def send_status(self, update_dict=None): """Send the status to the GUI""" if update_dict: self.status.update(update_dict) self.message_queue.put(('status', self.status.copy())) def send_steps(self): """Send the steps list to the GUI""" steps = [(index == self.active_step, str(step)) for index, step in enumerate(self.steps)] self.message_queue.put(('steps', steps)) def say(self, text, message_type='message'): """Send a ordinary text message to the gui""" self.message_queue.put((message_type, text)) def run(self): """The MAIN run method""" # Wait for start while not self.ok_to_start: if self.stop: self.send_status({'status_field': 'Stopping'}) self.power_supply_on_off(False) self.send_status({'status_field': 'Stopped'}) return sleep(0.1) # Start self.send_status({'status_field': 'Starting'}) self.setup_data_set_saver() # Run the MAIN measurement loop # (This is where most of the time is spent) self.main_measure() # Shutdown powersupply, livesocket and possibly server self.send_status({'status_field': 'Stopping'}) self.stop_everything() self.send_status({'status_field': 'Stopped'}) sleep(0.1) self.say("I have stopped") def setup_data_set_saver(self): """Setup the data set saver""" sql_time = CustomColumn(time(), 'FROM_UNIXTIME(%s)') for codename in self.codenames: metadata = { 'time': sql_time, 'comment': self.config['comment'], 'label': codename[3:], 'type': 1, 'power_supply_channel': self.channel_id, } self.data_set_saver.add_measurement(codename, metadata) def main_measure(self): # pylint: disable=too-many-locals """The main measurement loop""" self.send_status({'status_field': 'Running'}) # Initial setup last_set_voltage = None last_set_max_current = None last_time = time() iteration_time = 'N/A' self.status['elapsed'] = 0.0 accum_charge_codename = self.channel_id + '_accum_charge' self.status[accum_charge_codename] = 0.0 current_id = self.channel_id + '_current' last_measured_current = 0.0 self.say('I started on step 0') for self.active_step, current_step in enumerate(self.steps): self.send_status({'status_field': 'Running step {}'.format(self.active_step)}) # Also give the step an instance name (for steps list) if self.active_step > 0: self.say('Switched to step: {}'.format(self.active_step)) self.send_steps() current_step.start() # While the step hasn't completed yet while current_step.elapsed() < current_step.duration: # Check if we should stop if self.stop: self.say('I have been asked to stop') return iteration_start = now = time() # Calculate the time for one iteration and update times in status iteration_time = now - last_time last_time = now self.status.update({ 'elapsed': current_step.elapsed(), 'remaining': current_step.remaining(), 'iteration_time': iteration_time, 'elapsed_total': sum(step.elapsed() for step in self.steps), 'remaining_total': sum(step.remaining() for step in self.steps), }) # Ask the power supply to set a new voltage if needed required_voltage, required_max_current = current_step.values() if required_max_current != last_set_max_current: self.send_command('set_current_limit', required_max_current) last_set_max_current = required_max_current if required_voltage != last_set_voltage: self.send_command('set_voltage', required_voltage) last_set_voltage = required_voltage # Read value from the power supply self._read_values_from_power_supply() # Calculate, set and send accumulated charge charge_addition = \ (last_measured_current + self.status[current_id])\ / 2 * iteration_time last_measured_current = self.status[current_id] self.status[accum_charge_codename] += charge_addition point = (self.status['elapsed_total'], self.status[accum_charge_codename]) self.live_socket.set_point(accum_charge_codename, point) # Send the new status self.send_status() # Calculate time to sleep to use the proper probe interval time_to_sleep = current_step.probe_interval - (time() - iteration_start) if time_to_sleep > 0: sleep(time_to_sleep) # Stop the step(s own time keeping) current_step.stop() # For loop over steps ended self.send_status({'status_field': 'Program Complete'}) self.say('Stepped program completed') def _read_values_from_power_supply(self): """Read all required values from the power supply (used only from run)""" for command, codename in zip(self.power_supply_commands, self.codenames): # Get a value for the current command value = self.send_command(command) # Set/save it on the live_socket, database and in the GUI point = (self.status['elapsed_total'], value) self.live_socket.set_point(codename, point) self.data_set_saver.save_point(codename, point) self.status[codename] = value def stop_everything(self): """Stop power supply and live socket""" self.power_supply_on_off(False) self.live_socket.stop() self.data_set_saver.stop() def power_supply_on_off(self, state, current_limit=0.0): """Set power supply on off""" # Set voltage to 0 LOG.debug('Stopping everything. Set voltage to 0.0') self.send_command('set_voltage', 0.0) start = time() # Anything < 1.0 Volt is OK while self.send_command('read_actual_voltage') > 1.0: if time() - start > 60: LOG.error('Unable to set voltage to 0') if state: raise RuntimeError('Unable to set voltage to 0') else: self.say('Unable to set voltage to 0') break sleep(1) # Set current limit self.send_command('set_current_limit', current_limit) read_current_limit = self.send_command('read_current_limit') if not isclose(read_current_limit, current_limit): raise RuntimeError('Unable to set current limit') # Set state self.send_command('output_status', state) read_state = self.send_command('read_output_status').strip() == '1' if not read_state is state: raise RuntimeError('Could not set output state')
import time from PyExpLabSys.common.database_saver import DataSetSaver, CustomColumn import credentials ## EDIT HERE comment = "First run" ## EDIT HERE # Create data set saver object data_set_saver = DataSetSaver( "measurements_dummy", "xy_values_dummy", credentials.USERNAME, credentials.PASSWORD, ) data_set_saver.start() # Create measurement specs i.e. entires entries in the metadata table data_set_time = time.time() metadata = { "time": CustomColumn(data_set_time, "FROM_UNIXTIME(%s)"), "comment": comment, "type": 64, "preamp_range": 0, "sem_voltage": 0 } # Only the labels differ, so we generate the metadata with a loop for label in [ "p_korr", "i_korr", "total_korr", "ph_setpoint", "ph_value", "pump_rate"
import time from PyExpLabSys.common.database_saver import DataSetSaver, CustomColumn import credentials ## EDIT HERE comment = "First run" ## EDIT HERE # Create data set saver object data_set_saver = DataSetSaver( "measurements_dummy", "xy_values_dummy", credentials.USERNAME, credentials.PASSWORD, ) data_set_saver.start() # Create measurement specs i.e. entires entries in the metadata table data_set_time = time.time() metadata = { "time": CustomColumn(data_set_time, "FROM_UNIXTIME(%s)"), "comment": comment, "type" : 64, "preamp_range": 0, "sem_voltage": 0} # Only the labels differ, so we generate the metadata with a loop for label in ["p_korr", "i_korr", "total_korr", "ph_setpoint", "ph_value", "pump_rate"]: metadata["mass_label"] = label data_set_saver.add_measurement(label, metadata)
def __init__(self, args): super(VoltageCurrentProgram, self).__init__() # Form channel_id e.g: EA1 self.channel_id = args.power_supply + args.output ### Required by the stepped program runner # Accepted capabilities are: can_edit, can_play, # can_stop, can_quit self.capabilities = ('can_stop', 'can_start', 'can_edit') # Status fields (in order) self.status_fields = ( # Status { 'codename': 'status_field', 'title': 'Status' }, # Voltage { 'codename': self.channel_id + '_voltage', 'title': 'Voltage', 'formatter': '{:.3f}', 'unit': 'V' }, # Voltage setpoint { 'codename': self.channel_id + '_voltage_setpoint', 'title': 'Voltage Setpoint', 'formatter': '{:.3f}', 'unit': 'V' }, # Current { 'codename': self.channel_id + '_current', 'title': 'Current', 'formatter': '{:.3f}', 'unit': 'A' }, # Current limit { 'codename': self.channel_id + '_current_limit', 'title': 'Current limit', 'formatter': '{:.3f}', 'unit': 'A' }, # Charge { 'codename': self.channel_id + '_accum_charge', 'title': 'Accumulated charge', 'formatter': '{:.3f}', 'unit': 'C' }, # Time elapsed (step) { 'codename': 'elapsed', 'title': 'Time elapsed (step)', 'formatter': '{:.1f}', 'unit': 's' }, # Time remaining (step) { 'codename': 'remaining', 'title': 'Time remaining (step)', 'formatter': '{:.1f}', 'unit': 's' }, # Time elapsed (total) { 'codename': 'elapsed_total', 'title': 'Time elapsed (total)', 'formatter': '{:.1f}', 'unit': 's' }, # Time remaining (total) { 'codename': 'remaining_total', 'title': 'Time remaining (total)', 'formatter': '{:.1f}', 'unit': 's' }, # Iteration time { 'codename': 'iteration_time', 'title': 'Iteration time', 'formatter': '{:.2f}', 'unit': 's' }, ) self.extra_capabilities = { 'psuchannel': { 'help_text': ('Used for simple PSU control when not on\n' 'a ramp. Possibly usages are:\n' ' psuchannel voltage=1.23\n' 'which will set the voltage and\n' ' psuchannel off\n' 'which will set the output off'), 'completions': [ 'psuchannel', 'psuchannel voltage=', 'psuchannel off', ] } } # Queue for GUI updates self.message_queue = Queue() # The GUI also looks in self.config, see below ### Normal program # Setup my program with open(path.join(THIS_DIR, args.program_file)) as file__: self.config, self.steps = parse_ramp(file__) # The GUI will look for keys: program_title in config self.say('Using power supply channel: ' + self.channel_id) self.say('Loaded with config:\n' + pformat(self.config)) self.active_step = 0 self.send_steps() # Add completions for the edits self._completion_additions = [] for number, step in enumerate(self.steps): base = 'edit {} '.format(number) self._completion_additions.append(base) for field in sorted(step.fields): self._completion_additions.append('{}{}='.format(base, field)) # Base for the status self.status = {'status_field': 'Initialized'} # General variables self.stop = False self.ok_to_start = False # Create a partial function with the output substitued in self.send_command = partial( _send_command, args.output, args.power_supply, ) # Setup power supply self.power_supply_on_off(True, self.config['maxcurrent_start']) # Power supply commands, must match order with self.codenames self.power_supply_commands = ('read_actual_current', 'read_actual_voltage', 'read_set_voltage', 'read_current_limit') # Setup dataset saver and live socket self.codenames = [ self.channel_id + id_ for id_ in ('_current', '_voltage', '_voltage_setpoint', '_current_limit') ] self.live_socket = LiveSocket('H2O2_proactive_' + self.channel_id, self.codenames + [self.channel_id + '_accum_charge'], no_internal_data_pull_socket=True) self.live_socket.reset(self.codenames) self.live_socket.start() self.data_set_saver = DataSetSaver(credentials.measurements, credentials.xy_values, username=credentials.username, password=credentials.password) self.data_set_saver.start() # Done with init, send status self.send_status()
def main(): """Main function""" parser = argparse.ArgumentParser( description='Run the infamous ph control script.') # ph_setpoint, ph_direction (string [up, down]), kick_in_rate (float) # Main (constant), secondary (regulation) # python ph_increment.py 10.0 --direction up --main-offset=24.0 --second-min=10.0 --second-max=40.0 --second-offset=17.0 parser.add_argument('setpoint', type=float, help='The Ph setpoint.') parser.add_argument('--direction', default=None, help='Pumping direction. Must be up or down.', required=True) parser.add_argument('--main-offset', type=float, default=None, help='Start pumping speed for the main pump.', required=True) parser.add_argument('--second-min', type=float, default=None, help='Minimum pumping speed for the secondary pump.', required=True) parser.add_argument('--second-max', type=float, default=None, help='Maximum pumping speed for the secondary pump.', required=True) parser.add_argument('--second-offset', type=float, default=None, help='Start pumping value for the secondary pump.', required=True) parser.add_argument('--comment', default=None, help='Optional comment', required=True) args = parser.parse_args() ## Edit comment comment = "pH " + str(args.setpoint) + " // p: 0.02 // start rate " + str( args.second_offset) + " // " + args.comment ## Edit comment # raise SystemExit('All good') # Init pumps, Omegabus, PID and data saver pump_main = AL1000( "/dev/serial/by-id/usb-FTDI_USB-RS232_Cable_FTV9X9TM-if00-port0") pump_second = AL1000( "/dev/serial/by-id/usb-FTDI_USB-RS232_Cable_FTWZCJLU-if00-port0") obus = OmegaBus( "/dev/serial/by-id/usb-FTDI_USB-RS232_Cable_FTWZCGSW-if00-port0", baud=9600, ) data_set_saver = DataSetSaver("measurements_dummy", "xy_values_dummy", "dummy", "dummy") pid = PID(pid_p=0.04, pid_i=0.0, pid_d=0, p_max=9999, p_min=-9) pid.update_setpoint(args.setpoint) # time_start = time.time() #Pre flight check. pump_main_check = pump_main.get_firmware() print("main: " + pump_main_check) pump_second_check = pump_second.get_firmware() print("second: " + pump_second_check) obus_check = obus.read_value(1) obus_check = current_to_ph(obus_check) if pump_main_check != "NE1000V3.923": print("Main pump failed") raise SystemExit(1) if pump_second_check != "NE1000V3.923": print("Secondary pump failed") raise SystemExit(1) if obus_check < 0: print("OmegaBus failed") raise SystemExit(1) #Set initial condition for pumps pump_second.set_direction("INF") pump_main.set_direction("INF") if args.direction == "up": pump_second.set_rate(args.second_offset) rate_second = args.second_offset pump_main.set_rate(0) rate_main = 0 if args.direction == "down": pump_second.set_rate(0) rate_second = 0 pump_main.set_rate(args.main_offset) rate_main = args.main_offset pump_second.set_vol(9999) pump_main.set_vol(9999) pump_second.start_program() pump_main.start_program() # Setup database saver data_set_saver.start() data_set_time = time.time() metadata = { "Time": CustomColumn(data_set_time, "FROM_UNIXTIME(%s)"), "comment": comment, "type": 64, "preamp_range": 0, "sem_voltage": 0 } for label in [ "p_korr", "i_korr", "total_korr", "ph_setpoint", "ph_value", "pump_rate" ]: metadata["mass_label"] = label data_set_saver.add_measurement(label, metadata) # Run PID try: time_start = time.time() run(pump_second, pump_main, obus, pid, data_set_saver, time_start, args.setpoint, rate_second, rate_main, args.second_offset, args.main_offset, args.second_min, args.second_max, args.direction, minutes) except KeyboardInterrupt: # Clean up pump_main.set_rate(0) pump_second.set_rate(0) pump_main.stop_program() pump_second.stop_program() data_set_saver.stop() raise SystemExit(" Sequence stopped.") # Clean up pump_main.set_rate(0) pump_second.set_rate(0) pump_main.stop_program() pump_second.stop_program() # Log pH during ageing try: while True: value = obus.read_value(1) value = current_to_ph(value) print("pH: " + str(round(value, 3))) data_set_saver.save_point("ph_value", (time.time() - time_start, value)) data_set_saver.save_point( "ph_setpoint", (time.time() - time_start, args.setpoint)) time.sleep(8) except KeyboardInterrupt: data_set_saver.stop() pass
class Stats(multiprocessing.Process): def __init__(self, user_queue, log_data=False): psutil.cpu_percent(percpu=True) # Initial dummy readout multiprocessing.Process.__init__(self) conn_params = pika.ConnectionParameters('localhost') connection = pika.BlockingConnection(conn_params) self.channel = connection.channel() self.channel.queue_declare('global') self.amqp_messages = {} self.amqp_messages['global'] = {} self.amqp_messages['global']['children'] = -1 self.amqp_messages['global']['amqp_time'] = -1 self.user_queue = user_queue self.scanners = [] self.screen = curses.initscr() curses.noecho() curses.cbreak() curses.curs_set(False) self.screen.keypad(1) self.screen.nodelay(1) # Measure initial values while we have the chance self.start_time = time.time() self.total_users = self.user_queue.qsize() self.init_du = self.disk_usage() self.log_data = log_data def _amqp_single_update(self, queue_name): method, header, body = self.channel.basic_get(queue_name) while method: # Always empty queue, do now show old data body_dict = pickle.loads(body) self.amqp_messages[queue_name] = body_dict method, header, body = self.channel.basic_get(queue_name) def amqp_update(self): t = time.time() self._amqp_single_update('global') for pid in self.scanners: self._amqp_single_update(str(pid)) self.amqp_messages['global']['amqp_time'] = time.time() - t def number_of_threads(self): """ Number of threads :return: Tuple with Number of threads, and nuber of active threads """ return (len(self.scanners), self.amqp_messages['global']['children']) def add_scanner(self, scanner): """ Add a scanner to the internal list of scanners :param scanner: The scanner object to be added :return: The new number of scanners """ self.scanners.append(scanner) return len(self.scanners) def disk_usage(self): """ Return the current disk usage :return: Disk usage in MB """ error = True while error: try: du_output = subprocess.check_output(['du', '-s', export_path], stderr=subprocess.DEVNULL) error = False except subprocess.CalledProcessError: # Happens if du is called while folder is being marked done # logger.warn('du-error') Warning ends up in the terminal... # will need to investiate time.sleep(1) size = float(du_output.decode('utf-8').split('\t')[0]) / 1024 return size def exported_users(self): """ Returns the number of exported users :return: Tuple with finished exports and total users """ finished_users = (self.total_users - self.user_queue.qsize() - self.number_of_threads()[1]) return (finished_users, self.total_users) def amount_of_exported_data(self): """ Return the total amount of exported data (MB) :return: The total amount of exported data sinze start """ return self.disk_usage() - self.init_du def memory_info(self): """ Returns the memory consumption (in MB) of all threads :return: List of memory consumptions """ mem_list = {} for pid in self.scanners: try: process = psutil.Process(pid) mem_info = process.memory_full_info() used_memory = mem_info.uss/1024**2 if self.log_data: label = '{} memory'.format(pid) dt = (time.time() - self.start_time) self.data_set_saver.save_point(label, (dt, used_memory)) except psutil._exceptions.NoSuchProcess: used_memory = -1 mem_list[str(pid)] = used_memory return mem_list def status(self): template = ('Threads: {}. ' + 'Queue: {}. ' + 'Export: {:.3f}GB. ' + 'Time: {:.2f}min. ' + 'Speed: {:.2f}MB/s. ' + 'Memory consumption: {:.3f}GB. ' + 'Scanned users: {} / {}') memory = sum(self.memory_info().values()) / 1024 processes = self.number_of_threads()[1] dt = (time.time() - self.start_time) users = self.exported_users() ret_str = template.format(processes, self.user_queue.qsize(), self.disk_usage() / 1024, dt / 60.0, self.amount_of_exported_data() / dt, memory, users[0], users[1]) return ret_str def init_logging(self): if self.log_data: self.comment = 'Run' self.data_set_saver = DataSetSaver('measurements_mailscan', 'xy_values_mailscan', credentials.user, credentials.passwd) self.data_set_saver.start() # PyExpLabSys does does not excell in db-normalization - add # metadata to all channels metadata = {"Time": CustomColumn(self.start_time, "FROM_UNIXTIME(%s)"), "comment": self.comment, "type": 1, "label": None, "processes": self.number_of_threads()[1]} metadata['label'] = 'Avg export speed' self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = 'Total export size' self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = 'Total users' self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = 'Total memory' self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = 'Total CPU' self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = 'Total Mails' self.data_set_saver.add_measurement(metadata['label'], metadata) for scanner in self.amqp_messages.keys(): if scanner == 'global': continue metadata['processes'] = 1 metadata['label'] = '{} memory'.format(scanner) self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = '{} exported users'.format(scanner) self.data_set_saver.add_measurement(metadata['label'], metadata) def run(self): self.amqp_update() self.init_logging() processes = self.number_of_threads()[1] while processes is not 0: self.amqp_update() thread_info = self.number_of_threads() processes = thread_info[1] status = self.status() logger.info(status) # print(status) dt = int((time.time() - self.start_time)) msg = 'Run-time: {}min {}s '.format(int(dt / 60), int(dt % 60)) self.screen.addstr(2, 3, msg) users = self.exported_users() msg = 'Exported users: {}/{} '.format(users[0], users[1]) self.screen.addstr(3, 3, msg) total_export_size = self.amount_of_exported_data() msg = 'Total export: {:.3f}MB '.format(total_export_size) self.screen.addstr(4, 3, msg) speed = total_export_size / dt msg = 'Avg eksport speed: {:.3f}MB/s '.format(speed) self.screen.addstr(5, 3, msg) mem_info = self.memory_info() msg = 'Memory usage: {:.1f}MB' self.screen.addstr(6, 3, msg.format(sum(mem_info.values()))) msg = 'amqp update time: {:.1f}ms ' update_time = self.amqp_messages['global']['amqp_time'] * 1000 self.screen.addstr(7, 3, msg.format(update_time)) cpu_usage = psutil.cpu_percent(percpu=True) msg = 'CPU{} usage: {}% ' for i in range(0, len(cpu_usage)): self.screen.addstr(9 + i, 3, msg.format(i, cpu_usage[i])) if self.log_data: dt = (time.time() - self.start_time) label = 'Total users' self.data_set_saver.save_point(label, (dt, users[0])) label = 'Total export size' self.data_set_saver.save_point(label, (dt, total_export_size)) label = 'Avg export speed' self.data_set_saver.save_point(label, (dt, speed)) label = 'Total CPU' self.data_set_saver.save_point(label, (dt, sum(cpu_usage))) label = 'Total memory' self.data_set_saver.save_point(label, (dt, sum(mem_info.values()))) i = i + 2 msg = 'Total threads: {}. ' self.screen.addstr(9 + i, 3, msg.format(thread_info[0])) i = i + 1 msg = 'Active threads: {} ' self.screen.addstr(9 + i, 3, msg.format(thread_info[1])) i = 0 exported_grand_total = 0 self.screen.addstr(2 + i, 50, 'Scan status:') i = i + 1 for key, data in self.amqp_messages.items(): if key == 'global': continue msg = 'ID {}'.format(key) self.screen.addstr(2 + i, 50, msg) i = i + 1 try: if mem_info[key] > 0: msg = 'Current path: {}/{}'.format(data['rel_path'], data['folder']) else: msg = 'Current path: {}'.format(data['rel_path']) self.screen.addstr(2 + i, 50, msg) self.screen.clrtoeol() i = i + 1 if mem_info[key] > 0: run_time = (time.time() - data['start_time']) / 60.0 else: run_time = 0 msg = 'Progress: {} of {} mails. Export time: {:.1f}min' self.screen.addstr(2 + i, 50, msg.format(data['total_scanned'], data['total_count'], run_time)) self.screen.clrtoeol() i = i + 1 msg = 'Exported users: {}. Exported mails: {} Total mails: {}. Memory consumption: {:.1f}MB ' if self.log_data: label = '{} exported users'.format(key) dt = (time.time() - self.start_time) eu = data['exported_users'] self.data_set_saver.save_point(label, (dt, eu)) msg = msg.format(data['exported_users'], data['exported_mails'], data['total_mails'], mem_info[key]) exported_grand_total += data['exported_mails'] exported_grand_total += data['total_mails'] self.screen.addstr(2 + i, 50, msg) i = i + 1 msg = 'Last amqp update: {}'.format(data['latest_update']) self.screen.addstr(2 + i, 50, msg) self.screen.clrtoeol() except KeyError: self.screen.addstr(2 + i, 50, str(data)) i = i + 1 self.screen.addstr(2 + i, 50, str(mem_info)) i = i + 2 self.screen.refresh() if self.log_data: dt = (time.time() - self.start_time) label = 'Total Mails' self.data_set_saver.save_point(label, (dt, exported_grand_total)) key = self.screen.getch() if key == ord('q'): # Quit program pass time.sleep(1) curses.nocbreak() self.screen.keypad(0) curses.echo() curses.endwin()
class Stats(multiprocessing.Process): def __init__(self, user_queue, log_data=False): psutil.cpu_percent(percpu=True) # Initial dummy readout multiprocessing.Process.__init__(self) conn_params = pika.ConnectionParameters('localhost') connection = pika.BlockingConnection(conn_params) self.channel = connection.channel() self.channel.queue_declare('global') self.amqp_messages = {} self.amqp_messages['global'] = {} self.amqp_messages['global']['children'] = -1 self.amqp_messages['global']['amqp_time'] = -1 self.user_queue = user_queue self.scanners = [] self.screen = curses.initscr() curses.noecho() curses.cbreak() curses.curs_set(False) self.screen.keypad(1) self.screen.nodelay(1) # Measure initial values while we have the chance self.start_time = time.time() self.total_users = self.user_queue.qsize() self.init_du = self.disk_usage() self.log_data = log_data def _amqp_single_update(self, queue_name): method, header, body = self.channel.basic_get(queue_name) while method: # Always empty queue, do now show old data body_dict = pickle.loads(body) self.amqp_messages[queue_name] = body_dict method, header, body = self.channel.basic_get(queue_name) def amqp_update(self): t = time.time() self._amqp_single_update('global') for pid in self.scanners: self._amqp_single_update(str(pid)) self.amqp_messages['global']['amqp_time'] = time.time() - t def number_of_threads(self): """ Number of threads :return: Tuple with Number of threads, and nuber of active threads """ return (len(self.scanners), self.amqp_messages['global']['children']) def add_scanner(self, scanner): """ Add a scanner to the internal list of scanners :param scanner: The scanner object to be added :return: The new number of scanners """ self.scanners.append(scanner) return len(self.scanners) def disk_usage(self): """ Return the current disk usage :return: Disk usage in MB """ error = True while error: try: du_output = subprocess.check_output(['du', '-s', export_path], stderr=subprocess.DEVNULL) error = False except subprocess.CalledProcessError: # Happens if du is called while folder is being marked done # logger.warn('du-error') Warning ends up in the terminal... # will need to investiate time.sleep(1) size = float(du_output.decode('utf-8').split('\t')[0]) / 1024 return size def exported_users(self): """ Returns the number of exported users :return: Tuple with finished exports and total users """ finished_users = (self.total_users - self.user_queue.qsize() - self.number_of_threads()[1]) return (finished_users, self.total_users) def amount_of_exported_data(self): """ Return the total amount of exported data (MB) :return: The total amount of exported data sinze start """ return self.disk_usage() - self.init_du def memory_info(self): """ Returns the memory consumption (in MB) of all threads :return: List of memory consumptions """ mem_list = {} for pid in self.scanners: try: process = psutil.Process(pid) mem_info = process.memory_full_info() used_memory = mem_info.uss / 1024**2 if self.log_data: label = '{} memory'.format(pid) dt = (time.time() - self.start_time) self.data_set_saver.save_point(label, (dt, used_memory)) except psutil._exceptions.NoSuchProcess: used_memory = -1 mem_list[str(pid)] = used_memory return mem_list def status(self): template = ('Threads: {}. ' + 'Queue: {}. ' + 'Export: {:.3f}GB. ' + 'Time: {:.2f}min. ' + 'Speed: {:.2f}MB/s. ' + 'Memory consumption: {:.3f}GB. ' + 'Scanned users: {} / {}') memory = sum(self.memory_info().values()) / 1024 processes = self.number_of_threads()[1] dt = (time.time() - self.start_time) users = self.exported_users() ret_str = template.format(processes, self.user_queue.qsize(), self.disk_usage() / 1024, dt / 60.0, self.amount_of_exported_data() / dt, memory, users[0], users[1]) return ret_str def init_logging(self): if self.log_data: self.comment = 'Run' self.data_set_saver = DataSetSaver('measurements_mailscan', 'xy_values_mailscan', credentials.user, credentials.passwd) self.data_set_saver.start() # PyExpLabSys does does not excell in db-normalization - add # metadata to all channels metadata = { "Time": CustomColumn(self.start_time, "FROM_UNIXTIME(%s)"), "comment": self.comment, "type": 1, "label": None, "processes": self.number_of_threads()[1] } metadata['label'] = 'Avg export speed' self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = 'Total export size' self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = 'Total users' self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = 'Total memory' self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = 'Total CPU' self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = 'Total Mails' self.data_set_saver.add_measurement(metadata['label'], metadata) for scanner in self.amqp_messages.keys(): if scanner == 'global': continue metadata['processes'] = 1 metadata['label'] = '{} memory'.format(scanner) self.data_set_saver.add_measurement(metadata['label'], metadata) metadata['label'] = '{} exported users'.format(scanner) self.data_set_saver.add_measurement(metadata['label'], metadata) def run(self): self.amqp_update() self.init_logging() processes = self.number_of_threads()[1] while processes is not 0: self.amqp_update() thread_info = self.number_of_threads() processes = thread_info[1] status = self.status() logger.info(status) # print(status) dt = int((time.time() - self.start_time)) msg = 'Run-time: {}min {}s '.format(int(dt / 60), int(dt % 60)) self.screen.addstr(2, 3, msg) users = self.exported_users() msg = 'Exported users: {}/{} '.format(users[0], users[1]) self.screen.addstr(3, 3, msg) total_export_size = self.amount_of_exported_data() msg = 'Total export: {:.3f}MB '.format(total_export_size) self.screen.addstr(4, 3, msg) speed = total_export_size / dt msg = 'Avg eksport speed: {:.3f}MB/s '.format(speed) self.screen.addstr(5, 3, msg) mem_info = self.memory_info() msg = 'Memory usage: {:.1f}MB' self.screen.addstr(6, 3, msg.format(sum(mem_info.values()))) msg = 'amqp update time: {:.1f}ms ' update_time = self.amqp_messages['global']['amqp_time'] * 1000 self.screen.addstr(7, 3, msg.format(update_time)) cpu_usage = psutil.cpu_percent(percpu=True) msg = 'CPU{} usage: {}% ' for i in range(0, len(cpu_usage)): self.screen.addstr(9 + i, 3, msg.format(i, cpu_usage[i])) if self.log_data: dt = (time.time() - self.start_time) label = 'Total users' self.data_set_saver.save_point(label, (dt, users[0])) label = 'Total export size' self.data_set_saver.save_point(label, (dt, total_export_size)) label = 'Avg export speed' self.data_set_saver.save_point(label, (dt, speed)) label = 'Total CPU' self.data_set_saver.save_point(label, (dt, sum(cpu_usage))) label = 'Total memory' self.data_set_saver.save_point(label, (dt, sum(mem_info.values()))) i = i + 2 msg = 'Total threads: {}. ' self.screen.addstr(9 + i, 3, msg.format(thread_info[0])) i = i + 1 msg = 'Active threads: {} ' self.screen.addstr(9 + i, 3, msg.format(thread_info[1])) i = 0 exported_grand_total = 0 self.screen.addstr(2 + i, 50, 'Scan status:') i = i + 1 for key, data in self.amqp_messages.items(): if key == 'global': continue msg = 'ID {}'.format(key) self.screen.addstr(2 + i, 50, msg) i = i + 1 try: if mem_info[key] > 0: msg = 'Current path: {}/{}'.format( data['rel_path'], data['folder']) else: msg = 'Current path: {}'.format(data['rel_path']) self.screen.addstr(2 + i, 50, msg) self.screen.clrtoeol() i = i + 1 if mem_info[key] > 0: run_time = (time.time() - data['start_time']) / 60.0 else: run_time = 0 msg = 'Progress: {} of {} mails. Export time: {:.1f}min' self.screen.addstr( 2 + i, 50, msg.format(data['total_scanned'], data['total_count'], run_time)) self.screen.clrtoeol() i = i + 1 msg = 'Exported users: {}. Exported mails: {} Total mails: {}. Memory consumption: {:.1f}MB ' if self.log_data: label = '{} exported users'.format(key) dt = (time.time() - self.start_time) eu = data['exported_users'] self.data_set_saver.save_point(label, (dt, eu)) msg = msg.format(data['exported_users'], data['exported_mails'], data['total_mails'], mem_info[key]) exported_grand_total += data['exported_mails'] exported_grand_total += data['total_mails'] self.screen.addstr(2 + i, 50, msg) i = i + 1 msg = 'Last amqp update: {}'.format(data['latest_update']) self.screen.addstr(2 + i, 50, msg) self.screen.clrtoeol() except KeyError: self.screen.addstr(2 + i, 50, str(data)) i = i + 1 self.screen.addstr(2 + i, 50, str(mem_info)) i = i + 2 self.screen.refresh() if self.log_data: dt = (time.time() - self.start_time) label = 'Total Mails' self.data_set_saver.save_point(label, (dt, exported_grand_total)) key = self.screen.getch() if key == ord('q'): # Quit program pass time.sleep(1) curses.nocbreak() self.screen.keypad(0) curses.echo() curses.endwin()
def __init__(self, args): super(MyProgram, self).__init__() # Form channel_id e.g: A1 self.channel_id = args.power_supply + args.output ### Required by the stepped program runner # Accepted capabilities are: can_edit, can_play, # can_stop, can_quit self.capabilities = ('can_stop', 'can_start', 'can_edit') # Status fields (in order) self.status_fields = ( # Status {'codename': 'status_field', 'title': 'Status'}, # Voltage {'codename': self.channel_id + '_voltage', 'title': 'Voltage', 'formatter': '{:.3f}', 'unit': 'V'}, # Voltage setpoint {'codename': self.channel_id + '_voltage_setpoint', 'title': 'Voltage Setpoint', 'formatter': '{:.3f}', 'unit': 'V'}, # Current {'codename': self.channel_id + '_current', 'title': 'Current', 'formatter': '{:.3f}', 'unit': 'A'}, # Current limit {'codename': self.channel_id + '_current_limit', 'title': 'Current limit', 'formatter': '{:.3f}', 'unit': 'A'}, # Charge {'codename': self.channel_id + '_accum_charge', 'title': 'Accumulated charge', 'formatter': '{:.3f}', 'unit': 'C'}, # Time elapsed (step) {'codename': 'elapsed', 'title': 'Time elapsed (step)', 'formatter': '{:.1f}', 'unit': 's'}, # Time remaining (step) {'codename': 'remaining', 'title': 'Time remaining (step)', 'formatter': '{:.1f}', 'unit': 's'}, # Time elapsed (total) {'codename': 'elapsed_total', 'title': 'Time elapsed (total)', 'formatter': '{:.1f}', 'unit': 's'}, # Time remaining (total) {'codename': 'remaining_total', 'title': 'Time remaining (total)', 'formatter': '{:.1f}', 'unit': 's'}, # Iteration time {'codename': 'iteration_time', 'title': 'Iteration time', 'formatter': '{:.2f}', 'unit': 's'}, ) # Queue for GUI updates self.message_queue = Queue() # The GUI also looks in self.config, see below ### Normal program # Setup my program with open(path.join(THIS_DIR, args.program_file)) as file__: self.config, self.steps = parse_ramp(file__) # The GUI will look for keys: program_title in config self.say('Using power supply channel: ' + self.channel_id) self.say('Loaded with config:\n' + pformat(self.config)) self.active_step = 0 self.send_steps() # Add completions for the edits self._completion_additions = [] for number, step in enumerate(self.steps): base = 'edit {} '.format(number) self._completion_additions.append(base) for field in sorted(step.fields): self._completion_additions.append('{}{}='.format(base, field)) # Base for the status self.status = {'status_field': 'Initialized'} # General variables self.stop = False self.ok_to_start = False # Setup power supply # Create a partial function with the output substitued in self.send_command = partial(_send_command, args.output, args.power_supply) self.power_supply_on_off(True, self.config['maxcurrent_start']) # Power supply commands, must match order with self.codenames self.power_supply_commands = ( 'read_actual_current', 'read_actual_voltage', 'read_set_voltage', 'read_current_limit' ) # Setup dataset saver and live socket self.codenames = [self.channel_id + id_ for id_ in ('_current', '_voltage', '_voltage_setpoint', '_current_limit')] self.live_socket = LiveSocket( 'H2O2_proactive_' + self.channel_id, self.codenames + [self.channel_id + '_accum_charge'], no_internal_data_pull_socket=True ) self.live_socket.reset(self.codenames) self.live_socket.start() self.data_set_saver = DataSetSaver( credentials.measurements, credentials.xy_values, username=credentials.username, password=credentials.password ) self.data_set_saver.start() # Done with init, send status self.send_status()
from PyExpLabSys.file_parsers.chemstation import Sequence ### REMOVE after database move is complete from PyExpLabSys.common import database_saver database_saver.HOSTNAME = 'cinfsql' ### REMOVE after database move is complete from PyExpLabSys.common.database_saver import DataSetSaver from PyExpLabSys.common.database_saver import CustomColumn import credentials # Instantiate the data set saver data_set_saver = DataSetSaver('measurements_vhp_setup', 'xy_values_vhp_setup', credentials.USERNAME, credentials.PASSWORD) data_set_saver.start() # Get the set of aleady uploaded files already_uploaded = data_set_saver.get_unique_values_from_measurements('relative_path') print('Fetched relative paths for {} known sequences'.format(len(already_uploaded))) # This is the measurement path, should be generated somehow basefolder = '/home/cinf/o/FYSIK/list-SurfCat/setups/vhp-setup' sequence_identifyer = 'sequence.acaml' # Find the active month newest = None highest_value = 0 for dir_ in os.listdir(basefolder): dir_split = dir_.split(' ')