def dado_campbell(): # Conecta com CR1000 e coleta os dados
try:
print('Conectando com Campbell CR1000... ', end='')
device = CR1000.from_url('serial:/dev/ttyUSB0:38400')
print('OK')
except Exception:
print(
'ERRO\n---------------------------- 60 min ---------------------------'
)
time.sleep(1200)
print('Conectando com Campbell CR1000', end='')
time.sleep(3)
print('.', end='')
time.sleep(3)
print('.', end='')
time.sleep(3)
print('. ', end='')
try:
device = CR1000.from_url('serial:/dev/ttyUSB0:38400')
print('OK')
except Exception:
print(
'ERRO\n---------------------------- EXIT ----------------------------'
)
sys.exit()
return device
def UpdateNow(self):
addr = "tcp:ip:port"
print("UNFINISHED: TODO")
return
try:
start = SensorData.objects.latest('timestamp').timestamp
except ObjectDoesNotExist:
print("No prevouse data records found.")
print("Downloading all logger data...")
device = CR1000.from_url(addr)
dataList = device.get_data('Ten_Min')
else:
print("Beginning download of all data recorded since " +
str(start))
device = CR1000.from_url(addr)
dataList = device.get_data('Ten_Min', start)
toSaveList = []
for record in dataList:
ts = record['TIMESTAMP']
del record['TIMESTAMP']
for val, col in record.items():
try:
dat = SensorData(sensor=None,
timestamp=ts,
val_type=None,
val=float(val))
dat.full_clean()
toSaveList.append(dat)
except:
continue
for dat in toSaveList:
dat.save()
def __init__(self, client, sensordict, confdict):
self.client = client
self.sensordict = sensordict
self.confdict = confdict
# variables for broadcasting via mqtt:
self.count=0
self.datalst = []
#self.datacnt = 0
self.metacnt = 10
###
port = confdict['serialport']+sensordict.get('port')
baudrate = sensordict.get('baudrate')
# string should look like that: device = CR1000.from_url('serial:/dev/ttyUSB3:38400')
#try:
log.msg('connecting to device...')
self.device = CR1000.from_url('serial:{}:{}'.format(port,baudrate))
#except:
# TODO no CR1000 present... "Can not access to device"
#log.msg('Can not access to device')
tables = self.device.list_tables()
if not tables == ['Status', 'SamplesEvery2s', 'ValuesEveryMinute', 'Public']:
# TODO log-Befehl, Abbruch!
log.msg('CR1000 not configured for Judd JC')
else:
self.device.settime(datetime.utcnow())
# TODO folgende Ausgabe nur fuers Programmieren!!
log.msg('++++++++ Information von den Konfig-Dateien +++++++++++++')
log.msg('++ client:')
log.msg(client)
log.msg('++ sensordict:')
log.msg(sensordict)
log.msg('++ confdict:')
log.msg(confdict)
def connection():
device = CR1000.from_url(CRX_URL)
if device.ping_node():
yield device
else:
raise Exception("CRX did not respond to ping")
device.bye()
def get_data(url, start_date, end_date, dir_path):
"""
Extract data from the campbell data logger for each specified table and save to a daily csv file between the date ranges specified.
Default tables are: Housekeeping, GPS_datetime, SoilTemperature, SoilMoisture, SoilHeatFlux and Radiation
:param url: (str) URL for connection with logger in format 'tcp:iphost:port' or 'serial:/dev/ttyUSB0:19200:8N1'
:param start_date: (datetime.datetime) The start date from which to collect data
:param end_date: (datetime.datetime) The end date after which to stop collecting data. (the end date will be included in the data.)
:param dir_path: (str) The path to the top level directory in which to create the csv files and folders.
:returns: None
"""
device = CR1000.from_url(url)
# device.list_tables():
# ['Status', 'Housekeeping', 'GPS_datetime', 'SoilTemperature', 'SoilMoisture', 'SoilHeatFlux', 'Radiation', 'DataTableInfo', 'Public']
tables = CONFIG['common']['logger_tables']
while start_date <= end_date:
for table in tables:
end_of_day = start_date + timedelta(hours=23, minutes=59, seconds=59, microseconds=59)
csv_dirs = os.path.join(dir_path, table)
csv_name = f"{table}_{start_date.strftime('%Y-%m-%d')}.csv"
csv_path = os.path.join(csv_dirs, csv_name)
# create csv path
if not os.path.exists(csv_dirs):
os.makedirs(csv_dirs)
# if file doesn't exist - make it
if not os.path.isfile(csv_path):
open(csv_path, 'w').close()
# open the csv file
try:
df = pd.read_csv(csv_path)
except pd.errors.EmptyDataError:
get_data_from_range(device, table, csv_path, start_date, end_of_day, header=True)
continue
if df.empty:
get_data_from_range(device, table, csv_path, start_date, end_of_day, header=False)
else:
# if the file already has data in it, it will just update the existing data from latest entry onwards
# get lastest date before updating - add a microsecond on, so no issues with duplication of record
latest = datetime.strptime(df['Datetime'].iloc[-1], "%Y-%m-%d %H:%M:%S") + timedelta(microseconds=1)
get_data_from_range(device, table, csv_path, latest, end_of_day, header=False)
print(f"Completed for {table} for {start_date.strftime('%Y-%m-%d')}")
time.sleep(3)
start_date = start_date + timedelta(1)
def UpdateNow(self):
addr = "tcp:ip:port"
print("UNFINISHED: TODO")
return
try:
start = SensorData.objects.latest('timestamp').timestamp
except ObjectDoesNotExist:
print("No prevouse data records found.")
print("Downloading all logger data...")
device = CR1000.from_url(addr)
dataList = device.get_data('Ten_Min')
else:
print("Beginning download of all data recorded since " +
str(start))
device = CR1000.from_url(addr)
dataList = device.get_data('Ten_Min', start)
toSaveList = []
for record in dataList:
ts = record['TIMESTAMP']
del record['TIMESTAMP']
for val, col in record.items():
try:
dat = SensorData(
sensor=None,
timestamp=ts,
val_type=None,
val=float(val))
dat.full_clean()
toSaveList.append(dat)
except:
continue
for dat in toSaveList:
dat.save()
def finddata(self):
modem_on(1)
cr1000_on(1)
sleep(90)
device = CR1000.from_url('tcp:192.168.0.30:6785')
data = device.get_data('Public')
# print(data[0])
# finds strings inbetween parenthesis
Timestamp = str(data[0]['Datetime'])
RecNbr = str(data[0]['RecNbr'])
Batt_volt = str(data[0]['Batt_volt'])
Ptemp_C = str(data[0]['Ptemp_C'])
R6 = str(data[0]['R6'])
R10 = str(data[0]['R10'])
R20 = str(data[0]['R20'])
R40 = str(data[0]['R40'])
R2_5 = str(data[0]['R2_5'])
R4_5 = str(data[0]['R4_5'])
R6_5 = str(data[0]['R6_5'])
R8_5 = str(data[0]['R8_5'])
T6 = str(data[0]['T6'])
T10 = str(data[0]['T10'])
T20 = str(data[0]['T20'])
T40 = str(data[0]['T40'])
T2_5 = str(data[0]['T2_5'])
T4_5 = str(data[0]['T4_5'])
T6_5 = str(data[0]['T6_5'])
T8_5 = str(data[0]['T8_5'])
dt = str(data[0]['DT'])
Q = str(data[0]['Q'])
tcdt = str(data[0]['TCDT'])
labels = [
'Timestamp', 'RecNbr', 'Batt_volt', 'Ptemp_C', 'R40', 'R6', 'R10',
'R20', 'R2_5', 'R4_5', 'R6_5', 'R8_5', 'T6,', 'T10', 'T20', 'T40',
'T2_5', 'T4_5', 'T6_5', 'T8_5', 'DT', 'Q', 'TCDT'
]
values = [
Timestamp, RecNbr, Batt_volt, Ptemp_C, R6, R10, R20, R2_5, R4_5,
R6_5, R8_5, R40, T6, T6_5, T10, T20, T40, T2_5, T4_5, T8_5, dt, Q,
tcdt
]
return labels, values
def crpull(ip, port, tstart, tend):
print('Connecting to device...')
device = CR1000.from_url('tcp:' + str(ip) + ':' + str(port), timeout=10)
if len(tstart) > 0:
tstart = datetime.strptime(tstart, '%Y-%m-%d %H:%M:%S')
tend = datetime.strptime(tend, '%Y-%m-%d %H:%M:%S')
print('Pulling data from the Dat table after ' + str(tstart) + '...')
rawdata = device.get_data('Dat', tstart, tend)
else:
print('Pulling all data from the Dat table...')
rawdata = device.get_data('Dat')
device.bye()
if len(rawdata) > 0:
orderedcol = rawdata.filter(('Datetime', 'RecNbr', 'Year', 'jDay', 'HH', 'MM', 'SS',
'batt_volt_Min', 'PTemp_Avg', 'Room_T_Avg', 'IRGA_T_Avg',
'IRGA_P_Avg', 'MF_Controller_mLmin_Avg', 'PressureVolt_Avg',
'RH_voltage_Avg', 'Gas_T_Avg', 'rawCO2_Voltage_Avg',
'rawCO2_Avg', 'rawH2O_Avg', 'ID', 'Program'))
newdata = orderedcol.to_csv(header=False).split('\r\n')
return(newdata)
else:
return('')
def cr_read(self):
try:
modem_on(1)
is_on = is_on_checker(0, 5)
if not is_on:
# Turn on CR1000x
cr1000_on(1)
sleep(15)
except:
printf("Problem with port or problem with power to the CR1000")
traceback.print_exc(
file=open("/media/mmcblk0p1/logs/system.log", "a+"))
else:
# Read data
device = CR1000.from_url('tcp:192.168.0.30:6785')
data = device.get_data('Public')
finally:
if not is_on:
# Turn off CR1000
cr1000_off(1)
modem_off(1)
return data
NTU2_15_MedColm = 'TurbNTU_Med'
NTU3_15_MedColm = 'TurbNTU2_Med'
NTU2_24_MedColm = 'TurbNTU_Med'
NTU3_24_MedColm = 'TurbNTU2_Med'
# Initialize field variables
NTU2_15_Med = 0
NTU3_15_Med = 0
NTU2_24_Med = 0
NTU3_24_Med = 0
# Holds the table that contains the data we're plotting
dataTable = 'TableEachScan'
# The device we're connecting to,
device = CR1000.from_url('serial:/' + location + ":" + port)
# Get all tables from device
tables = device.list_tables()
# Get Streaming Tokens from plot.ly
stream_ids = tls.get_credentials_file()['stream_ids']
# Grab first Token in list
stream_id = stream_ids[0]
# Used to check if header should be used
has_ran = False
# File descriptor for log file
log_file = os.open("logfile.txt", os.O_RDWR | os.O_APPEND | os.O_CREAT)
# Set up traces for plot
dailyTurbidMed = dict(
x=[],
y=[],
def sendRequest(self):
# TODO wohin mit debug?
debug = False
if self.reconnect.is_set():
log.msg('exiting, mutex locked!')
return
t = datetime.utcnow()
past = t - timedelta(seconds=3)
vals = self.device.get_data('SamplesEvery2s', past, t)
# vals[0] because we grap no older data, there is only one value in 2 seconds
# timestamp directly from datetime into array
# TODO Roman fragen, ob oder wie Vergleich mit Computerzeit
try:
darray = datetime2array(vals[0]['Datetime'])
# TODO "again" ist Provisorium
again = False
except:
again = True
try:
if again:
t = datetime.utcnow()
past = t - timedelta(seconds=3)
darray = datetime2array(vals[0]['Datetime'])
log.msg("IT TOOK A SECOND TIME TO GET DATA PROPERLY!")
except:
# there will be no log messages when the logger is turned off
return
# TODO reconnect Loesung, nur wenn sie sauber funktioniert!
log.msg('NO DATA FROM CR1000 !!! - vals:')
log.msg(vals)
port = self.confdict['serialport'] + self.sensordict.get(
'port')
baudrate = self.sensordict.get('baudrate')
self.reconnect.set()
connected = False
while not connected:
self.device.bye()
log.msg('reconnecting to device...')
time.sleep(5)
try:
self.device = CR1000.from_url('serial:{}:{}'.format(
port, baudrate))
tables = self.device.list_tables()
if tables == [
'Status', 'SamplesEvery2s',
'ValuesEveryMinute', 'Public'
]:
connected = True
log.msg('schaut ok aus...')
time.sleep(2)
except:
log.msg('reconnect wohl missglueckt!')
try:
past = t - timedelta(seconds=3)
vals = self.device.get_data('SamplesEvery2s',
past,
t,
debug=True)
log.msg(SENSOR_HEIGHT * 1000. -
vals[0]['DiffVolt'] * 250.)
except:
log.msg('...wohl doch nicht!')
connected = False
self.reconnect.clear()
log.msg('mutex released...')
return
if debug:
log.msg('getting data...')
# snowheight (1000mV is 250cm) - values from CR1000 in mV, factor 1000 for packing
snowheight = SENSOR_HEIGHT * 1000. - vals[0]['DiffVolt'] * 250.
darray.append(int(round(snowheight)))
# TODO weg
if debug:
log.msg(darray)
# preparations for file save
# date 6 short microsecond unsigned long snowheight signed long
# TODO alter packcode!
packcode = "6hLl"
#packcode = "<6hLl"
# header
sensorid = self.sensordict['sensorid']
header = "# MagPyBin %s %s %s %s %s %s %d" % (
sensorid, '[f]', '[JC]', '[cm]', '[1000]', packcode,
struct.calcsize(packcode))
data_bin = struct.pack(packcode, *darray)
# date of dataloggers timestamp
filedate = datetime.strftime(
datetime(darray[0], darray[1], darray[2]), "%Y-%m-%d")
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filedate, data_bin, header)
if debug:
log.msg('Daten gesichert...')
# sending via MQTT
data = ','.join(list(map(str, darray)))
head = header
topic = self.confdict.get('station') + '/' + self.sensordict.get(
'sensorid')
coll = int(self.sensordict.get('stack'))
if coll > 1:
self.metacnt = 1 # send meta data with every block
if self.datacnt < coll:
self.datalst.append(data)
self.datacnt += 1
else:
senddata = True
data = ';'.join(self.datalst)
self.datalst = []
self.datacnt = 0
else:
senddata = True
if senddata:
self.client.publish(topic + "/data", data)
if self.count == 0:
self.client.publish(topic + "/meta", head)
self.count += 1
if self.count >= self.metacnt:
self.count = 0
# right now auxiliary data only in the log file
if t.second < 2:
# every minute aux data (battery voltage and logger temperature) will be available
# going 61s into the past to make sure there are already data
past = t - timedelta(seconds=61)
aux = self.device.get_data('ValuesEveryMinute', past, t)
log.msg('----- aux every minute:')
# timestamp directly from datetime into array
try:
darray = datetime2array(aux[0]['Datetime'])
except:
# following should never happen...
log.msg('AUXILIARY DATA NOT GOT PROPERLY! - aux:')
log.msg(aux)
log.msg('trying again...')
past = t - timedelta(seconds=62)
aux = self.device.get_data('ValuesEveryMinute', past, t)
try:
darray = datetime2array(aux[0]['Datetime'])
except:
log.msg('giving up...')
return
# battery voltage - factor 1000 for packing
BattV_Min = int(round(aux[0]['BattV_Min'] * 1000))
PTemp_C_Avg = int(round(aux[0]['PTemp_C_Avg'] * 1000))
darray.extend([BattV_Min, PTemp_C_Avg])
# alernative reading:
#aux = (aux.filter(('Datetime', 'BattV_Min','PTemp_C_Avg')).to_csv(header=False))
log.msg(darray)
packcode = "<6hLLl"
# Check system platform, if windows, we need to open files in binary mode
platform = platform.system()
# Holds the device's mapped location
if platform == 'Linux':
location = "dev/ttyO4"
elif platform == 'Windows':
location = "COM1"
else:
location = "COM1"
# Holds the port on which we're communicating with the device
port = "115200"
# The device we're connecting to,
device = CR1000.from_url('serial:/' + location + ":" + port)
# Get all tables from device
tables = device.list_tables()
# Start date for data collection, should be fifteen minutes in the past
start_date_form = datetime.now() - timedelta(minutes=15)
# End date for data collection, should be now, to complete our 15 minute interval
end_date_form = datetime.now()
"""
" Function which takes in a table name, gathers its data and exports it as a CSV file for analysis.
" @:param table_name - name of table to collect data and export
"""
#logger_address='serial:/dev/serial/by-id/usb-Prolific_Technology_Inc._USB-Serial_Controller_D-if00-port0:38400'
#device = CR1000.from_url('serial:/dev/serial/by-id/usb-Prolific_Technology_Inc._USB-Serial_Controller_D-if00-port0:38400')
#time.sleep(5)
#device = CR1000.from_url(logger_address)
file_name = 'campbell_output.csv'
fid = open(file_name, 'a', 0)
while True:
log_attempt = 1
while log_attempt < 10:
try:
device = CR1000.from_url(
'serial:/dev/serial/by-id/usb-Prolific_Technology_Inc._USB-Serial_Controller_D-if00-port0:38400'
)
data = device.get_data('WPS_Tab_noarray', time_previous,
time_current)
break
except Exception, e:
# https://stackoverflow.com/questions/1483429/how-to-print-an-error-in-python
print time_current.strftime(
"%d/%b/%Y %H:%M:%S") + " extracting failed " + str(
log_attempt) + ' ' + str(e)
log_attempt += 1
time.sleep(10)
#device = CR1000.from_url('serial:/dev/serial/by-id/usb-Prolific_Technology_Inc._USB-Serial_Controller_D-if00-port0:38400')
continue
data[0]['Datetime'] = data[0]['Datetime'].strftime("%d/%b/%Y %H:%M:%S")
#data['Datetime'] =data['Datetime'].strftime("%d/%b/%Y %H:%M:%S")
def log(url, dir_path, set_time=False):
"""
Extract the data from the campbell data logger for each specified table and save to a daily csv file.
This will backfill the file to get all data from the start of the day or update from the latest data entry if data already exists in the file.
Default tables are: Housekeeping, GPS_datetime, SoilTemperature, SoilMoisture, SoilHeatFlux and Radiation
If set_time=True the logger time will be updated when the script runs at midnight. Default is False.
:param url: (str) URL for connection with logger in format 'tcp:iphost:port' or 'serial:/dev/ttyUSB0:19200:8N1'
:param dir_path: (str) The path to the top level directory in which to create the csv files and folders.
:param set_time: (boolean) If True, the logger time will be updated when the script runs at midnight. Default is False.
:returns: None
"""
device = CR1000.from_url(url)
# first check if it's midnight (utc) & sync time, if set_time=True
if set_time:
try:
c = ntplib.NTPClient()
start_time = datetime.utcfromtimestamp(
c.request('pool.ntp.org').tx_time)
if start_time.hour == "0" and start_time.minute == "0":
device.set_time(
datetime.utcfromtimestamp(
c.request('pool.ntp.org').tx_time))
except:
print("Could not sync with time server.")
# device.list_tables():
# ['Status', 'Housekeeping', 'GPS_datetime', 'SoilTemperature', 'SoilMoisture', 'SoilHeatFlux', 'Radiation', 'DataTableInfo', 'Public']
tables = CONFIG['common']['logger_tables']
date = datetime.utcnow().strftime("%Y-%m-%d")
for table in tables:
csv_dirs = os.path.join(dir_path, table)
csv_name = f"{table}_{date}.csv"
csv_path = os.path.join(csv_dirs, csv_name)
# create csv path
if not os.path.exists(csv_dirs):
os.makedirs(csv_dirs)
# if file doesn't exist - make it
if not os.path.isfile(csv_path):
open(csv_path, 'w').close()
# open the csv file
try:
df = pd.read_csv(csv_path)
except pd.errors.EmptyDataError:
get_todays_data(url, table, csv_path)
continue
if df.empty:
get_todays_data(url, table, csv_path)
else:
cmd = f"pycr1000 update {url} {table} {csv_path}"
subprocess.call(cmd, shell=True)
