def write_p1_json_dbx_folder( configdb=None, data=None, flog=None, sysid=const.SYSTEM_ID_DEFAULT, utc_ok_timestamp=0):
#flog.setLevel(logging.DEBUG)
_id, parameter, _label = configdb.strget(50,flog) # is Dropbox is on or off
if int(parameter) == 0: # dropbox sharing is off
return utc_ok_timestamp # do nothing
#print ("#!"+ str( abs(util.getUtcTime() - utc_ok_timestamp) ) )
if abs(util.getUtcTime() - utc_ok_timestamp) < 10:
return utc_ok_timestamp # do nothing, DBX rate limting
try:
filename = const.DIR_DBX_LOCAL + const.DBX_DIR_DATA + '/' + const.API_BASIC_JSON_PREFIX + sysid + const.API_BASIC_JSON_SUFFIX
flog.debug(inspect.stack()[0][3]+": json output =" + json.dumps( data , sort_keys=True ) + " naar bestand " + filename )
with open(filename, 'w') as outfile:
json.dump( data , outfile, sort_keys=True )
utc_ok_timestamp = util.getUtcTime()
util.setFile2user( filename,'p1mon' ) # to make sure we can process the file
except Exception as e:
flog.error(inspect.stack()[0][3]+": wegschrijven data naar Dropbox is mislukt. melding:"+str(e.args[0]))
return utc_ok_timestamp
def writeSemaphoreFile(name, flog):
try:
filename = name + '.p1mon'
if os.path.isfile(const.DIR_FILESEMAPHORE + filename):
flog.warning(inspect.stack()[0][3] + ": semafoor file " +
filename + " bestaat al, gestopt.")
return False # semaphore file already exist.
time = str(getUtcTime()).encode('utf-8')
filename = str(filename).encode('utf-8')
hashvalue = hashlib.sha1(filename + time).hexdigest()
#print ( time )
#print ( filename )
#print (hashvalue)
fo = open(const.DIR_FILESEMAPHORE + filename.decode('utf-8'), "w")
fo.write(filename.decode('utf-8') + '\n')
fo.write(time.decode('utf-8') + '\n')
fo.write(hashvalue + '\n')
fo.close()
flog.info(inspect.stack()[0][3] + " semafoor file " +
const.DIR_FILESEMAPHORE + filename.decode('utf-8') +
" gemaakt.")
return True
except Exception as e:
flog.error(inspect.stack()[0][3] + ": semafoor file error ->" + str(e))
return False
def gas_stub_instert(self, line=None, serialbuffer=None, gasmode=DUMMY_GAS_MODE_2421 ):
#voeg een dummy gas record in alleen voor ontwikkeling!!!
try:
#dev_dummy_gas_value = dev_dummy_gas_value + random.uniform(0, 0.0005) #0.0005
if util.getUtcTime() - self.timestamp_last_gas_update > DUMMY_GAS_TIME_ELAPSED:
self.dev_dummy_gas_value = self.dev_dummy_gas_value + random.uniform(0.0, 0.4) #only update after elapsed time
self.timestamp_last_gas_update = util.getUtcTime()
self.flog.debug(inspect.stack()[0][3]+": gas dummy string toevoegen.")
del serialbuffer[ len(serialbuffer)-1 ]
#if DUMMY_GAS_MODE_2421 == True:
if gasmode == DUMMY_GAS_MODE_2421:
line_1 = ''.join( filter(lambda x: x in string.printable, '0-1:24.2.1(170108160000W)('+'{0:09.3f}'.format(self.dev_dummy_gas_value)+'*m3)\r\n')).rstrip()[0:1024]
serialbuffer.append( line_1 )
#serial_buffer.append( '0-1:24.2.1(700101010000W)(00000000)' )
#flog.debug(inspect.stack()[0][3]+": dummy gas waarde = "+line_1)
self.flog.warning(inspect.stack()[0][3]+": test gas regel toegevoegd aan telegram: " + line_1 )
#if DUMMY_GAS_MODE_2423 == True: #(voor Belgie)
if gasmode == DUMMY_GAS_MODE_2423: #(voor Belgie)
line_1 = ''.join( filter(lambda x: x in string.printable, '0-1:24.2.3(190830073458S)('+'{0:09.3f}'.format(self.dev_dummy_gas_value)+'*m3)\r\n')).rstrip()[0:1024]
serialbuffer.append( line_1 )
#serial_buffer.append( '0-1:24.2.1(700101010000W)(00000000)' )
#flog.debug(inspect.stack()[0][3]+": dummy gas waarde = "+line_1)
self.flog.warning(inspect.stack()[0][3]+": test gas regel toegevoegd aan telegram: " + line_1 )
#if DUMMY_GAS_MODE_2421 == True or DUMMY_GAS_MODE_2423 == True or DUMMY_GAS_MODE_2430 == True:
#if DUMMY_GAS_MODE_2430 == True:
if gasmode == DUMMY_GAS_MODE_2430:
line_1 = ''.join( filter(lambda x: x in string.printable, '0-1:24.3.0(121030140000)(00)(60)(1)(0-1:24.2.1)(m3)\r\n')).rstrip()[0:1024]
line_2 = ''.join( filter(lambda x: x in string.printable, '('+'{0:09.3f}'.format(self.dev_dummy_gas_value)+')')).rstrip()[0:1024]
serialbuffer.append( line_1 )
serialbuffer.append( line_2 )
self.flog.debug(inspect.stack()[0][3]+": dummy gas waarde = "+line_1)
self.flog.warning(inspect.stack()[0][3]+": test gas regel 1 toegevoegd aan telegram: " + line_1 )
self.flog.warning(inspect.stack()[0][3]+": test gas regel 2 toegevoegd aan telegram: " + line_2 )
self.flog.debug(inspect.stack()[0][3]+": "+line_2)
serialbuffer.append( line )
except Exception as e:
self.flog.warning( inspect.stack()[0][3]+": test gas test insert probleem " + str( e ) )
def insert_db_serial_record(data_set=None,
status=None,
dbstatus=None,
dbserial=None,
flog=None):
try:
timestr = util.mkLocalTimeString()
#timestr_min=timestr[0:16]+":00"
if status['gas_present_in_serial_data'] == False:
#gas_verbr_m3_2421 = 0
data_set['gas_verbr_m3_2421'] = 0
sqlstr ="insert or replace into " + const.DB_SERIAL_TAB + \
" values (\
'" +timestr+"',\
'" +"0"+"',\
'" +str( data_set['verbrk_kwh_181'] )+"', \
'" +str( data_set['verbrk_kwh_182'] )+"', \
'" +str( data_set['gelvr_kwh_281'] )+"', \
'" +str( data_set['gelvr_kwh_282'] )+"', \
'" +str( data_set['tarief_code'] )+"', \
'" +str( data_set['act_verbr_kw_170'] )+"', \
'" +str( data_set['act_gelvr_kw_270'] )+"',\
'" +str( data_set['gas_verbr_m3_2421'] )+"')"
sqlstr = " ".join(sqlstr.split())
flog.debug(inspect.stack()[0][3] +
": (2)serial e-buffer insert: sql=" + sqlstr)
dbserial.insert_rec(sqlstr)
# timestamp telegram processed
dbstatus.timestamp(16, flog) # Timestring + Daylight saving.
dbstatus.strset(str(util.getUtcTime()), 87, flog) # UTC time
status['p1_record_is_ok'] = 1
dbstatus.strset(str(status['p1_record_is_ok']), 46,
flog) # P1 data is ok recieved
status['timestamp_last_insert'] = util.getUtcTime()
# if util.isMod( timestr,15 ) == True:
# backupData()
except Exception as e:
flog.error(inspect.stack()[0][3] + ": Insert gefaald. Melding=" +
str(e.args[0]))
def backupData():
global timestamp_last_backup
# set backup timestamp in status database
rt_status_db.timestamp(57,flog)
timestamp_last_backup = getUtcTime()
#setFileFlags()
flog.debug(inspect.stack()[0][3]+": Gestart")
os.system("/p1mon/scripts/P1DbCopy.py --temperature2disk --forcecopy")
#backupFile(const.FILE_DB_TEMPERATUUR_FILENAME)
#setFileFlags()
rt_status_db.timestamp(29,flog)
flog.debug(inspect.stack()[0][3]+": Gereed")
def clean_old_files():
path = const.DIR_DBX_LOCAL + const.DBX_DIR_BACKUP
# define the path
currentDirectory = pathlib.Path(path)
for currentFile in currentDirectory.iterdir():
delta_secs = abs(util.getUtcTime() -
int(os.path.getctime(currentFile)))
#print ( "Delta " + str(currentFile), " ", str( delta_secs ))
if delta_secs > 7200: #remove files that are older then 2 hours, to prevent daylight saving problems
flog.info( inspect.stack()[0][3]+": verouderde bestand " + str(currentFile) +\
" wordt verwijderd. Bestand is " + str(delta_secs ) + " seconden oud." )
os.remove(currentFile)
else:
flog.debug( inspect.stack()[0][3]+": bestand " + str(currentFile) +\
" wordt NIET verwijderd. Bestand is " + str(delta_secs ) + " seconden oud." )
def ftpCopy(ftpConnection, filename):
_head, tail = os.path.split(filename)
try:
ftpConnection.storbinary('STOR ' + tail,
open(filename, 'rb'),
blocksize=128)
except Exception as e:
rt_status_db.strset(
'fout: copy file naar server: ' + str(e.args[0]) + ' Gestopt.', 48,
flog)
flog.error(inspect.stack()[0][3] + ": copy file naar server: " +
str(e.args[0]) + " Gestopt.")
sys.exit(12)
try:
file_list = ftpConnection.nlst()
except Exception as e:
rt_status_db.strset(
'fout: directory list server : ' + str(e.args[0]) + ' Gestopt.',
48, flog)
flog.error(inspect.stack()[0][3] + ": directory list server fout: " +
str(e.args[0]) + " Gestopt.")
sys.exit(13)
flog.debug(inspect.stack()[0][3] + ": file list van FTP server: " +
str(file_list))
try:
ftpConnection.rename(tail, fileprefix + str(getUtcTime()) + '-' + tail)
except Exception as e:
rt_status_db.strset(
'fout: hernoemen van file (' + tail + ') mislukt: ' +
str(e.args[0]) + ' Gestopt.', 48, flog)
flog.error(inspect.stack()[0][3] + ": hernoemen van file (" + tail +
") mislukt: " + str(e.args[0]) + " Gestopt.")
sys.exit(14)
if int(ftp_para['maxfilecount']) > -1:
flog.debug(
inspect.stack()[0][3] +
": wissen van oude files staat aan. maximaal aantal files is " +
str(ftp_para['maxfilecount']))
ftpRemoveOldFiles(ftpConnection, file_list)
def update_json_data( jsondata=None, p1data=None ):
#convert dutch tarif code to English Dal = Low, Peak = High
if p1data['tarief_code'] == 'P':
tarief_code = 'HIGH'
else:
tarief_code = "LOW" # there are only two options (D/P)
jsondata[ apiconst.JSON_TS_LCL ] = str(util.mkLocalTimeString())
jsondata[ apiconst.JSON_TS_LCL_UTC ] = util.getUtcTime()
jsondata[ apiconst.JSON_API_API_STTS ] = 'production'
jsondata[ apiconst.JSON_API_CNSMPTN_KWH_L ] = float( p1data['verbrk_kwh_181'] )
jsondata[ apiconst.JSON_API_CNSMPTN_KWH_H ] = float( p1data['verbrk_kwh_182'] )
jsondata[ apiconst.JSON_API_PRDCTN_KWH_L ] = float( p1data['gelvr_kwh_281'] )
jsondata[ apiconst.JSON_API_PRDCTN_KWH_H ] = float( p1data['gelvr_kwh_282'] )
jsondata[ apiconst.JSON_API_TRFCD ] = str( tarief_code )
jsondata[ apiconst.JSON_API_CNSMPTN_KW ] = float( p1data['act_verbr_kw_170'] )
jsondata[ apiconst.JSON_API_PRDCTN_KW ] = float( p1data['act_gelvr_kw_270'] )
jsondata[ apiconst.JSON_API_CNSMPTN_GAS_M3 ] = float( p1data['gas_verbr_m3_2421'] )
jsondata[ apiconst.JSON_API_SYSTM_ID] = systemid.getSystemId()
def main_prod():
global ser1
global serial_buffer
global processing_speed
stub_serial_buffer = [] # for stub testing only
p1_port_shared_lib.clear_data_buffer(buffer=phase_db_record)
flog.info("Start van programma.")
# make sure as first program to run that ram is filled with the
# last data from persistent memory (disk)
p1_port_shared_lib.disk_restore_from_disk_to_ram()
# open van seriele database
try:
e_db_serial.init(const.FILE_DB_E_FILENAME, const.DB_SERIAL_TAB)
except Exception as e:
flog.critical(inspect.stack()[0][3] + " database niet te openen(1)." +
const.FILE_DB_E_FILENAME + ") melding:" + str(e.args[0]))
sys.exit(1)
flog.info(inspect.stack()[0][3] + ": database tabel: " +
const.DB_SERIAL_TAB + " succesvol geopend.")
# defrag van database
e_db_serial.defrag()
flog.info(inspect.stack()[0][3] + ": database bestand " +
const.DB_SERIAL_TAB + " gedefragmenteerd.")
# open van status database
try:
rt_status_db.init(const.FILE_DB_STATUS, const.DB_STATUS_TAB)
except Exception as e:
flog.critical(inspect.stack()[0][3] + ": database niet te openen(1)." +
const.FILE_DB_STATUS + ") melding:" + str(e.args[0]))
sys.exit(1)
flog.info(inspect.stack()[0][3] + ": database tabel: " +
const.DB_STATUS_TAB + " succesvol geopend.")
# open van config database
try:
config_db.init(const.FILE_DB_CONFIG, const.DB_CONFIG_TAB)
except Exception as e:
flog.critical(inspect.stack()[0][3] + ": database niet te openen(3)." +
const.FILE_DB_CONFIG + ") melding:" + str(e.args[0]))
sys.exit(1)
flog.info(inspect.stack()[0][3] + ": database tabel " +
const.DB_CONFIG_TAB + " succesvol geopend.")
# open van temperatuur database
try:
temperature_db.init(const.FILE_DB_TEMPERATUUR_FILENAME,
const.DB_TEMPERATUUR_TAB)
except Exception as e:
flog.critical(inspect.stack()[0][3] + ": Database niet te openen(1)." +
const.FILE_DB_TEMPERATUUR_FILENAME + ") melding:" +
str(e.args[0]))
sys.exit(1)
flog.info(inspect.stack()[0][3] + ": database tabel " +
const.DB_TEMPERATUUR_TAB + " succesvol geopend.")
# open van watermeter database
try:
watermeter_db.init(const.FILE_DB_WATERMETERV2,
const.DB_WATERMETERV2_TAB, flog)
except Exception as e:
flog.critical(inspect.stack()[0][3] + ": Database niet te openen(3)." +
const.FILE_DB_WATERMETERV2 + " melding:" +
str(e.args[0]))
sys.exit(1)
flog.info(inspect.stack()[0][3] + ": database tabel " +
const.DB_WATERMETERV2_TAB + " succesvol geopend.")
# open van fase database
try:
fase_db.init(const.FILE_DB_PHASEINFORMATION,
const.DB_FASE_REALTIME_TAB)
fase_db.defrag()
except Exception as e:
flog.critical(inspect.stack()[0][3] + " database niet te openen(1)." +
const.FILE_DB_PHASEINFORMATION + ") melding:" +
str(e.args[0]))
sys.exit(1)
flog.info(inspect.stack()[0][3] + ": database tabel: " +
const.DB_FASE_REALTIME_TAB + " succesvol geopend.")
# only make object when tests are active
if DUMMY_3PHASE_DATA == True or gas_test_mode != p1_telegram_test_lib.NO_GAS_TEST:
flog.warning(inspect.stack()[0][3] + ": test functies geactiveerd ! ")
p1_test = p1_telegram_test_lib.p1_telegram()
p1_test.init(flog, configdb=config_db)
# sets the rate limiting to 1 record per 10 secs or maximum speed processing of P1 telegrams
p1_port_shared_lib.set_p1_processing_speed(p1_processing=processing_speed,
config_db=config_db,
flog=flog)
p1_port_shared_lib.backup_data_to_disk_by_timestamp(statusdb=rt_status_db,
flog=flog)
while check_serial() == False:
flog.critical(
inspect.stack()[0][3] +
": seriele poort niet gevonden, is de kabel aangesloten?")
time.sleep(10)
# reset gas per uur waarde als we starten.
rt_status_db.strset("0", 50, flog)
rt_status_db.timestamp(5, flog)
#read serial settings from status DB
serCheckCnt = 0
check_serial_db_config_settings()
p1_port_shared_lib.get_country_day_night_mode(status=p1_status,
configdb=config_db,
flog=flog)
p1_port_shared_lib.get_gas_telgram_prefix(status=p1_status,
configdb=config_db,
flog=flog)
flog.info(inspect.stack()[0][3] + ": P1 poort instelling baudrate=" +
str(ser1.baudrate) + " bytesize=" + str(ser1.bytesize) +
" pariteit=" + str(ser1.parity) + " stopbits=" +
str(ser1.stopbits))
#serOpen(ser1)
#ser1.flushInput()
# read GAS value from status database
#if DUMMY_GAS_MODE_2421 == True or DUMMY_GAS_MODE_2423 == True or DUMMY_GAS_MODE_2430 == True:
if gas_test_mode > 0:
print("DUMMY GAS STAAT AAN IS DIT CORRECT?\r")
flog.warning(inspect.stack()[0][3] +
" #############################################")
flog.warning(inspect.stack()[0][3] +
": Dummy gas waarde staat aan met een interval van " +
str(p1_test.gas_interval()) +
" seconden. Zet uit voor productie!")
flog.warning(inspect.stack()[0][3] +
" #### DUMMY GAS STAAT AAN IS DIT CORRECT? ####")
else:
try:
rt_status_db.strset(0, 43, flog) # reset dummy gas value
except Exception as e:
flog.error(inspect.stack()[0][3] + ": Melding=" + str(e.args[0]))
# check for dummy 3Phase data
if DUMMY_3PHASE_DATA == True:
print("DUMMY 3 FASE DATA STAAT AAN IS DIT CORRECT?\r")
flog.warning(inspect.stack()[0][3] +
" #############################################")
flog.warning(
inspect.stack()[0][3] +
": Dummy 3 fase waarde staat aan. Zet uit voor productie!")
flog.warning(inspect.stack()[0][3] +
" #### DUMMY 3 FASE STAAT AAN IS DIT CORRECT? ####")
# check for dummy 3Phase data
if DUMMY_1SEC_PROCCESSING == True:
print("DUMMY 1 SECONDEN VERWERKING STAAT AAN?\r")
flog.warning(inspect.stack()[0][3] +
" #############################################")
flog.warning(
inspect.stack()[0][3] +
": 1 seconden verwerking staat aan. Zet uit voor productie!")
flog.warning(inspect.stack()[0][3] +
" #### DUMMY 1 SECONDEN STAAT AAN IS DIT CORRECT? ####")
# read crc check settings from config
p1_port_shared_lib.get_P1_crc(status=p1_status,
configdb=config_db,
flog=flog)
# set intial FQDN
p1_port_shared_lib.fqdn_from_config(verbose=True,
configdb=config_db,
data_set=json_data,
flog=flog)
day_values.init(dbstatus=rt_status_db, dbserial=e_db_serial, flog=flog)
while True:
try:
#print ('ser1.In_waiting='+str(ser1.in_waiting))
if ser1.in_waiting > 1:
try:
line = ser1.readline().decode('utf-8')
#flog.debug(inspect.stack()[0][3]+": line = " + line +" length=" + str(len(line)))
#line = filter(lambda x: x in string.printable, line_raw ).rstrip()[0:1024]
#flog.debug(inspect.stack()[0][3]+": line filtered = " + str(line_raw ) )
except:
flog.warning(inspect.stack()[0][3] +
": lezen van serieele poort mislukt.")
serial_buffer.append(line)
#flog.debug(inspect.stack()[0][3]+": [** V2.0]serial buffer bytes waiting = " + str(ser1.inWaiting()) )
#the benthouse BUG has 78 lines
if len(serial_buffer
) > 250: # normale size less then a 100 lines
flog.warning(
inspect.stack()[0][3] +
": serieele buffer te groot, gewist! Buffer lengte was "
+ str(len(serial_buffer)))
del serial_buffer[:]
#print ( serial_buffer )
#print ( "[*]len="+str(len(serial_buffer)) )
if line[:1] == '!':
#flog.debug(inspect.stack()[0][3]+": serial buffer "+str(serial_buffer))
if DUMMY_3PHASE_DATA == True:
#phase3StubInstert( line )
p1_test.phase3_stub_instert(line=line,
serialbuffer=serial_buffer)
#if DUMMY_GAS_MODE_2421 == True or DUMMY_GAS_MODE_2423 == True or DUMMY_GAS_MODE_2430 == True:
if gas_test_mode > 0:
#gasStubInstert(line)
p1_test.gas_stub_instert(line=line,
serialbuffer=serial_buffer,
gasmode=gas_test_mode)
#flog.info(inspect.stack()[0][3] + " step 0")
if processing_speed['max_processing_speed'] == False:
# rate limiting of inserts, some smart meters send more then every 10 sec. a telegram.
if abs(p1_status['timestamp_last_insert'] -
util.getUtcTime()
) < 9: # 9 or 10 sec interval is ok.
#flog.debug("Te snel een nieuw telegram ontvangen. genegeerd.")
del serial_buffer[:]
continue
#serial_buffer[-1] = "!0F9B" #DEBUG
#print ( "[*] p1 crc check = " + str(p1_crc_check_is_on) )
#flog.info(inspect.stack()[0][3] + " step 1")
# check for meters that supply a crc value.
if len(
serial_buffer[len(serial_buffer) - 1]
) > 3 and p1_status[
'p1_crc_check_is_on'] == True: # crc telegrams are 7 chars minimal (5+ cr\lf)
# check for telegrams with a CRC attached
crc_read = str(serial_buffer[len(serial_buffer) -
1][1:5])
#flog.debug("CRC in telegram gevonden -> "+crc_read)
#print ('[*] crc waarde')
#print crc_read
save_crc = serial_buffer[len(serial_buffer) - 1]
serial_buffer[
len(serial_buffer) -
1] = '!' # only process to ! not the crc itself
strvar = ''.join(serial_buffer)
#CRC16().calculate
calc_crc = str('{0:0{1}X}'.format(
CRC16().calculate(strvar), 4))
#flog.debug("Berekende CRC uit telegram -> "+calc_crc )
#flog.debug(inspect.stack()[0][3]+": serial buffer "+str(serial_buffer))
if (crc_read == calc_crc):
#flog.debug("CRC is in orde.")
#restore CRC to received data
serial_buffer[len(serial_buffer) - 1] = save_crc
else:
p1_status['crc_error_cnt'] = p1_status[
'crc_error_cnt'] + 1
#flog.debug("CRC van telegram komt niet overeen. CRC telegram ("+crc_read+"), berekende CRC ("+calc_crc+") niet verwerkt.")
del serial_buffer[:]
continue
#flog.debug(inspect.stack()[0][3]+": serial buffer "+str(serial_buffer))
#flog.setLevel(logging.DEBUG)
#print('verwerk seriele data.')
if DUMMY_1SEC_PROCCESSING == True:
stub_serial_buffer = serial_buffer.copy()
json_data[apiconst.JSON_API_RM_TMPRTR_IN] ,json_data[apiconst.JSON_API_RM_TMPRTR_OUT] =\
p1_port_shared_lib.current_room_temperature( temperaturedb=temperature_db, flog=flog )
json_data[apiconst.JSON_API_WM_CNSMPTN_LTR_M3] =\
p1_port_shared_lib.current_watermeter_count( configdb=config_db, waterdb=watermeter_db, flog=flog)
#print ( serial_buffer )
#parseSerBuffer( data_set=base_p1_data, status=p1_status, phase_db_rec=phase_db_record )
p1_port_shared_lib.write_p1_telegram_to_ram(
buffer=serial_buffer, flog=flog)
p1_port_shared_lib.parse_serial_buffer(
serialbuffer=serial_buffer,
data_set=base_p1_data,
status=p1_status,
phase_db_rec=phase_db_record,
flog=flog)
update_data_set(data_set=base_p1_data,
status=p1_status,
phase_db_record=phase_db_record,
flog=flog)
#clean the buffer
del serial_buffer[:]
p1_port_shared_lib.clear_data_buffer(buffer=base_p1_data)
else:
if processing_speed['max_processing_speed'] == True:
if DUMMY_1SEC_PROCCESSING == True:
if stub_serial_buffer != []:
# print ( "serial_buffer = " + str(stub_serial_buffer) )
serial_buffer = stub_serial_buffer.copy()
json_data[apiconst.JSON_API_RM_TMPRTR_IN] ,json_data[apiconst.JSON_API_RM_TMPRTR_OUT] =\
p1_port_shared_lib.current_room_temperature( temperaturedb=temperature_db, flog=flog )
json_data[apiconst.JSON_API_WM_CNSMPTN_LTR_M3] =\
p1_port_shared_lib.current_watermeter_count( configdb=config_db, waterdb=watermeter_db, flog=flog)
#parseSerBuffer( data_set=base_p1_data, status=p1_status, phase_db_rec=phase_db_record )
p1_port_shared_lib.write_p1_telegram_to_ram(
buffer=serial_buffer, flog=flog)
p1_port_shared_lib.parse_serial_buffer(
serialbuffer=serial_buffer,
data_set=base_p1_data,
status=p1_status,
phase_db_rec=phase_db_record,
flog=flog)
update_data_set(data_set=base_p1_data,
status=p1_status,
phase_db_record=phase_db_record,
flog=flog)
time.sleep(0.1)
serCheckCnt = serCheckCnt + 1
if serCheckCnt > 300: #check updates every 30 seconds.
# perodic check and change of fqdn.
# set intial FQDN
p1_port_shared_lib.fqdn_from_config(verbose=False,
configdb=config_db,
data_set=json_data,
flog=flog)
p1_port_shared_lib.backup_data_to_disk_by_timestamp(
statusdb=rt_status_db, flog=flog)
#print "serial interval check"
#############################################################################################
# changed in version > 1.3.1 because of the possible high frequency of updates. use less #
# often to remove stress on the database. #
#############################################################################################
p1_port_shared_lib.delete_serial_records(
p1_processing=processing_speed,
serial_db=e_db_serial,
flog=flog)
p1_port_shared_lib.delete_phase_record(
p1_processing=processing_speed,
phase_db=fase_db,
flog=flog)
p1_port_shared_lib.set_p1_processing_speed(
p1_processing=processing_speed,
config_db=config_db,
flog=flog)
check_serial_db_config_settings()
p1_port_shared_lib.get_country_day_night_mode(
status=p1_status, configdb=config_db, flog=flog)
p1_port_shared_lib.get_gas_telgram_prefix(
status=p1_status, configdb=config_db, flog=flog)
#checkCRCsettings()
p1_port_shared_lib.get_P1_crc(status=p1_status,
configdb=config_db,
flog=flog)
serCheckCnt = 0
if abs(p1_status['timestamp_last_insert'] -
util.getUtcTime()) > 51:
if p1_status['p1_record_is_ok'] == 1:
flog.warning(inspect.stack()[0][3] +
": geen P1 record te lezen.")
p1_status['p1_record_is_ok'] = 0
rt_status_db.strset(str(p1_status['p1_record_is_ok']), 46,
flog) # P1 data is not ok recieved
# print abs(last_crc_check_timestamp - getUtcTime()) , " cnt=", crc_error_cnt
# crc error messages, if any.
if p1_status['p1_crc_check_is_on'] == True:
if abs(
p1_status['last_crc_check_timestamp'] -
util.getUtcTime()
) > 900: #check every 15 minutes, to limit log entries.
p1_status[
'last_crc_check_timestamp'] = util.getUtcTime()
if p1_status['crc_error_cnt'] > 0:
flog.warning(
"aantal P1 telegram crc fouten gevonden in de afgelopen minuut = "
+ str(p1_status['crc_error_cnt']))
p1_status['crc_error_cnt'] = 0
except Exception as e:
flog.warning(inspect.stack()[0][3] +
": fout bij het wachten op seriele gegevens. Error=" +
str(e.args))
check_serial()
time.sleep(1)
ser1.bytesize = 8
ser1.parity = "N"
ser1.stopbits = 1
ser1.xonxoff = 0 # changed from version 0.9.2 onwards
ser1.rtscts = 0
ser1.timeout = 1
ser1.port = ser_devices_list[0]
base_p1_data = data_struct_lib.p1_data_base_record
phase_db_record = data_struct_lib.phase_db_record
json_data = data_struct_lib.json_basic
processing_speed = data_struct_lib.p1_processing_speed
p1_status = data_struct_lib.p1_status_record
p1_status['timestamp_last_insert'] = util.getUtcTime()
p1_status['day_night_mode'] = 0 # default NL, 1 is Belgium
day_values = day_values_lib.dayMinMaxkW()
def main_prod():
global ser1
global serial_buffer
global processing_speed
stub_serial_buffer = [] # for stub testing only
p1_port_shared_lib.clear_data_buffer(buffer=phase_db_record)
def Main(argv):
flog.info("Start van programma.")
global mqtt_para, mqtt_client
# makes link between ramdisk and www/json folder if the link not exits
# prevents errors in console.log when MQTT is not active.
make_json_topic_link()
# open van config database
try:
config_db.init(const.FILE_DB_CONFIG, const.DB_CONFIG_TAB)
except Exception as e:
flog.critical(inspect.stack()[0][3] + ": database niet te openen(2)." +
const.FILE_DB_CONFIG + ") melding:" + str(e.args[0]))
sys.exit(1)
flog.info(inspect.stack()[0][3] + ": database tabel " +
const.DB_CONFIG_TAB + " succesvol geopend.")
runCheck()
# open van status database
try:
rt_status_db.init(const.FILE_DB_STATUS, const.DB_STATUS_TAB)
except Exception as e:
flog.critical(inspect.stack()[0][3] + ": Database niet te openen(1)." +
const.FILE_DB_STATUS + ") melding:" + str(e.args[0]))
sys.exit(1)
flog.info(inspect.stack()[0][3] + ": database tabel " +
const.DB_STATUS_TAB + " succesvol geopend.")
# open van seriele database
try:
e_db_serial.init(const.FILE_DB_E_FILENAME, const.DB_SERIAL_TAB)
except Exception as e:
flog.critical(inspect.stack()[0][3] + " database niet te openen(3)." +
const.FILE_DB_E_FILENAME + ") melding:" + str(e.args[0]))
sys.exit(1)
flog.info(inspect.stack()[0][3] + ": database tabel " +
const.DB_SERIAL_TAB + " succesvol geopend.")
# open van watermeter database
try:
watermeter_db.init(const.FILE_DB_WATERMETERV2,
const.DB_WATERMETERV2_TAB, flog)
except Exception as e:
flog.critical(inspect.stack()[0][3] + ": Database niet te openen(3)." +
const.FILE_DB_WATERMETERV2 + " melding:" +
str(e.args[0]))
sys.exit(1)
flog.info(inspect.stack()[0][3] + ": database tabel " +
const.DB_WATERMETERV2_TAB + " succesvol geopend.")
# open van weer database voor huidige weer
try:
weer_db.init(const.FILE_DB_WEATHER, const.DB_WEATHER_TAB)
except Exception as e:
flog.critical(inspect.stack()[0][3] + ": database niet te openen(3)." +
const.DB_WEATHER_TAB + ") melding:" + str(e.args[0]))
sys.exit(1)
flog.debug(inspect.stack()[0][3] + ": database tabel " +
const.DB_WEATHER_TAB + " succesvol geopend.")
# open van temperatuur database
try:
temperature_db.init(const.FILE_DB_TEMPERATUUR_FILENAME,
const.DB_TEMPERATUUR_TAB)
except Exception as e:
flog.critical(inspect.stack()[0][3] + ": Database niet te openen(1)." +
const.FILE_DB_TEMPERATUUR_FILENAME + ") melding:" +
str(e.args[0]))
sys.exit(1)
flog.info(inspect.stack()[0][3] + ": database tabel " +
const.DB_TEMPERATUUR_TAB + " succesvol geopend.")
# open van power production database
try:
power_production_db.init(const.FILE_DB_POWERPRODUCTION,
const.DB_POWERPRODUCTION_TAB, flog)
except Exception as e:
flog.critical(inspect.stack()[0][3] + ": Database niet te openen(3)." +
const.FILE_DB_POWERPRODUCTION + " melding:" +
str(e.args[0]))
sys.exit(1)
flog.info(inspect.stack()[0][3] + ": database tabel " +
const.DB_POWERPRODUCTION_TAB + " succesvol geopend.")
# set proces gestart timestamp
rt_status_db.timestamp(95, flog)
checkActiveState()
setConfigFromDb()
makeTopicJsonFile()
mqtt_para[
'brokerconnectionstatustext'] = "probeer met de broker een connectie op te bouwen"
rt_status_db.strset(mqtt_para['brokerconnectionstatustext'], 97, flog)
# INIT connect to the broker
while True:
runCheck()
#flog.setLevel( logging.DEBUG )
if minimalBrokerSettingsAvailable() == False:
time.sleep(60) # wait on valid setting, quitely.
setConfigFromDb()
continue
#flog.setLevel( logging.INFO )
if initMttq() == False:
flog.info( inspect.stack()[0][3] + ": connectie met broker gefaald, wacht " + \
str(int(mqtt_para['reconnecttimeoute'])) + " seconden voor een nieuwe poging. " )
time.sleep(int(mqtt_para['reconnecttimeoute']))
setConfigFromDb()
else:
flog.info(inspect.stack()[0][3] +
": initiele connectie met broker gereed.")
break
while True:
if mqtt_para['brokerconnectionisok'] == False: # try to reconnect
#flog.setLevel( logging.DEBUG )
if minimalBrokerSettingsAvailable() == False:
time.sleep(60) # wait on valid setting, quitely.
setConfigFromDb()
continue
#flog.setLevel( logging.INFO )
if (getUtcTime() % int(mqtt_para['reconnecttimeoute'])) == 0:
if mqtt_client != None:
try:
flog.warning(inspect.stack()[0][3] +
": reconnecting()")
mqtt_client.reconnect()
except Exception as e:
flog.error( inspect.stack()[0][3] + " reconnectie probleem ->" + str( e ) + " broker host = " + \
str(mqtt_para['brokerhost']) + " broker port = " + str(mqtt_para['brokerport']) )
try:
_id, parameter, _label = config_db.strget(118, flog)
flog.debug(inspect.stack()[0][3] + ": DB configuratie flag = " +
str(parameter))
if int(parameter) == 1: # do the changes.
flog.info(inspect.stack()[0][3] +
": MQTT configuratie wordt aangepast.")
setConfigFromDb()
closeMqtt()
initMttq()
checkActiveState(
) # controleer de DB op welke publish aan of uit staat
makeTopicJsonFile()
config_db.strset('0', 118, flog) # reset config flag.
flog.info(inspect.stack()[0][3] +
": MQTT configuratie is aangepast.")
except Exception as e:
flog.warning(inspect.stack()[0][3] +
": DB configuratie flag probleem -> " +
str(e.args[0]))
if mqtt_para['brokerconnectionisok'] == True:
# smart meter processing
try:
#_id, parameter, _label = config_db.strget( 114, flog )
if mqtt_para['smartmeterpublishisactive'] == True: #is active
_id, timestamp, _label, _security = rt_status_db.strget(
16, flog)
if (mqtt_para['smartmeterprocessedtimestamp']
) != timestamp:
getPayloadFromDB(mqtt_payload_smartmeter, e_db_serial)
if len(mqtt_payload_smartmeter[0]
) > 0: # only send when we have data
mqttPublish(mqtt_client, mqtt_topics_smartmeter,
mqtt_payload_smartmeter)
mqtt_para[
'smartmeterprocessedtimestamp'] = timestamp
except Exception as e:
flog.warning(
inspect.stack()[0][3] +
": onverwachte fout bij smartmeter publish van melding:" +
str(e))
#######################################################################
# Important to determine if we have to send data the timestamp when #
# data is updated must be set in the status database by the process #
# responsible for the processing. #
# check the index in the lines like this #
# _id, timestamp, _label, _security = rt_status_db.strget( xx, flog ) #
#######################################################################
# watermeter proceessing
try:
#_id, parameter, _label = config_db.strget( 115, flog )
if mqtt_para['watermeterpublishisactive'] == True: #is active
_id, timestamp, _label, _security = rt_status_db.strget(
90, flog)
if (mqtt_para['watermeterprocessedtimestamp']
) != timestamp:
getPayloadFromDB(mqtt_payload_watermeter,
watermeter_db)
if len(mqtt_payload_watermeter[0]
) > 0: # only send when when we have data
mqttPublish(mqtt_client, mqtt_topics_watermeter,
mqtt_payload_watermeter)
mqtt_para[
'watermeterprocessedtimestamp'] = timestamp
except Exception as e:
flog.warning(
inspect.stack()[0][3] +
": onverwachte fout bij watermeter publish van melding:" +
str(e))
# weather processing
try:
#_id, parameter, _label = config_db.strget( 116, flog )
if mqtt_para['weatherpublishisactive'] == True: #is active
_id, timestamp, _label, _security = rt_status_db.strget(
45, flog)
if (mqtt_para['weatherprocessedtimestamp']) != timestamp:
getPayloadFromDB(mqtt_payload_weather, weer_db)
if len(mqtt_payload_weather[0]
) > 0: # only send when we have data
mqttPublish(mqtt_client, mqtt_topics_weather,
mqtt_payload_weather)
mqtt_para['weatherprocessedtimestamp'] = timestamp
except Exception as e:
flog.warning(
inspect.stack()[0][3] +
": onverwachte fout bij weer publish van melding:" +
str(e))
# indoor temperature processing
try:
#_id, parameter, _label = config_db.strget( 117, flog )
if mqtt_para[
'indoortemperaturepublishisactive'] == True: #is active
_id, timestamp, _label, _security = rt_status_db.strget(
58, flog)
if (mqtt_para['indoortemperatureprocessedtimestamp']
) != timestamp:
getPayloadFromDB(mqtt_payload_indoor_temperature,
temperature_db)
if len(mqtt_payload_indoor_temperature[0]
) > 0: # only send when we have data
mqttPublish(mqtt_client,
mqtt_topics_indoor_temperature,
mqtt_payload_indoor_temperature)
mqtt_para[
'indoortemperatureprocessedtimestamp'] = timestamp
except Exception as e:
flog.warning(
inspect.stack()[0][3] +
": onverwachte fout bij binnen temperatuur publish van melding:"
+ str(e))
# powerproduction processing
try:
if mqtt_para[
'powerproductionpublishisactive'] == True: #is active
_id, timestamp, _label, _security = rt_status_db.strget(
109, flog)
if (mqtt_para['powerproductionprocessedtimestamp']
) != timestamp:
getPayloadFromDB(mqtt_payload_powerproduction,
power_production_db)
if len(mqtt_payload_powerproduction[0]
) > 0: # only send when we have data
mqttPublish(mqtt_client,
mqtt_topics_powerproduction,
mqtt_payload_powerproduction)
mqtt_para[
'powerproductionprocessedtimestamp'] = timestamp
except Exception as e:
flog.warning(
inspect.stack()[0][3] +
": onverwachte fout bij opgewekte energie publish van melding:"
+ str(e))
# phase processing
try:
#_id, parameter, _label = config_db.strget( 117, flog )
if mqtt_para['phasepublishisactive'] == True: #is active
_id, timestamp, _label, _security = rt_status_db.strget(
106, flog)
if (mqtt_para['phaseprocessedtimestamp']) != timestamp:
getPhasePayloadFromDB(mqtt_payload_phase)
if len(mqtt_topics_phase[0]
) > 0: # only send when we have data
mqttPublish(mqtt_client, mqtt_topics_phase,
mqtt_payload_phase)
mqtt_para['phaseprocessedtimestamp'] = timestamp
except Exception as e:
flog.warning(
inspect.stack()[0][3] +
": onverwachte fout bij binnen temperatuur publish van melding:"
+ str(e))
flog.debug(inspect.stack()[0][3] +
": sleeping... mqtt_para['brokerconnectionisok'] = " +
str(mqtt_para['brokerconnectionisok']))
checkActiveState(
) # controleer de DB op welke publish aan of uit staat
if mqtt_para[
'anypublishisactive'] == False: # ga in langzame modes als alle publish uit staat.
runCheck()
time.sleep(30) #slow poll
else:
runCheck()
time.sleep(2)
def Main(argv):
flog.info("Start van programma.")
last_seq_id = -1
DiskRestore()
# open van config database
try:
config_db.init(const.FILE_DB_CONFIG,const.DB_CONFIG_TAB)
except Exception as e:
flog.critical(inspect.stack()[0][3]+": database niet te openen(3)."+const.FILE_DB_CONFIG+") melding:"+str(e.args[0]))
sys.exit(1)
flog.info(inspect.stack()[0][3]+": database tabel "+const.DB_CONFIG_TAB+" succesvol geopend.")
# open van status database
try:
rt_status_db.init(const.FILE_DB_STATUS,const.DB_STATUS_TAB)
except Exception as e:
flog.critical(inspect.stack()[0][3]+": Database niet te openen(1)."+const.FILE_DB_STATUS+") melding:"+str(e.args[0]))
sys.exit(1)
flog.info(inspect.stack()[0][3]+": database tabel "+const.DB_STATUS_TAB+" succesvol geopend.")
# open van temperatuur database
try:
temperature_db.init(const.FILE_DB_TEMPERATUUR_FILENAME ,const.DB_TEMPERATUUR_TAB )
temperature_db.cleanDb(flog)
temperature_db.defrag()
except Exception as e:
flog.critical(inspect.stack()[0][3]+": Database niet te openen(1)."+const.FILE_DB_TEMPERATUUR_FILENAME+") melding:"+str(e.args[0]))
sys.exit(1)
flog.info(inspect.stack()[0][3]+": database tabel "+const.DB_TEMPERATUUR_TAB +" succesvol geopend.")
# set start timestamp in status database
rt_status_db.timestamp(56,flog)
try:
# Bind the socket to the host and port
udpsocket.bind((HOST, PORT))
except Exception as e:
flog.error(inspect.stack()[0][3]+": UDP ontvanger niet te starten , gestopt -> " + str(e) )
sys.exit(1)
# one time message for log
if prgIsActive(flog) == False:
flog.info(inspect.stack()[0][3]+": programma is niet als actief geconfigureerd , wordt niet uitgevoerd.")
# fix problem with primary key.
temperature_db.change_table( flog )
temperature_db.fix_missing_month_day( flog )
# main loop blocked recieving of UDP packages / messages
while True:
# check if we are using this program (uses config db!)
while prgIsActive(flog) == False:
continue
# backup data to flash every ~15 min.
if abs( timestamp_last_backup - getUtcTime() ) > 900:
backupData()
# Receive BUFFER_SIZE bytes data
data = udpsocket.recvfrom(BUFFER_SIZE)
if data:
#print received data
flog.debug(inspect.stack()[1][3]+": udp message = "+ datetime.fromtimestamp(time()).strftime('%H:%M:%S') +' ' + str(data[0]))
# process json
try:
#print ( data )
jsondata = json.loads( data[0].decode('utf-8') )
# record id index codes
# secs = 10
# min. = 11
# hour = 12
# days = 13
# months = 14
# years = 15
timestamp = datetime.fromtimestamp(time()).strftime('%Y-%m-%d %H:%M:%S')
#timestamp = "2020-09-01 00:01:02"
# process secs values, data every 5 secs or so.
if jsondata['id'] != 'ztatz_dt':
flog.warning(inspect.stack()[0][3]+": id van json data is niet correct :"+jsondata['id'] )
else:
if last_seq_id == jsondata['seq']:
flog.debug(inspect.stack()[0][3]+": dubbele volgorde id gevonden, wordt genegeerd. seq id=:"+str(jsondata['seq']) )
continue
last_seq_id = jsondata['seq']
temperature_db.replace(timestamp,\
10, \
jsondata['t_in'], \
jsondata['t_in_avg'],\
0,\
0,\
jsondata['t_out'], \
jsondata['t_out_avg'],\
0,\
0,\
flog )
# process minutes values.
# returns avg, min, max(in), avg, min, max(out)
temp_list = temperature_db.selectAMM( timestamp[:16],10,flog )
temperature_db.replace(timestamp[:16]+":00",\
11, \
0, \
temp_list[0][0],\
temp_list[0][0],\
temp_list[0][0],\
0,\
temp_list[0][3],\
temp_list[0][3],\
temp_list[0][3],\
flog )
# process hour values.
# returns avg, min, max(in), avg, min, max(out)
temp_list = temperature_db.selectAMM( timestamp[:13],11,flog )
temperature_db.replace(timestamp[:13]+":00:00",\
12, \
0, \
temp_list[0][0],\
temp_list[0][1],\
temp_list[0][2],\
0,\
temp_list[0][3],\
temp_list[0][4],\
temp_list[0][5],\
flog )
#flog.setLevel( logging.DEBUG )
# process day values.
# returns avg, min, max(in), avg, min, max(out)
temp_list = temperature_db.selectAMM( timestamp[:10],12,flog )
temperature_db.replace(timestamp[:10]+" 00:00:00",\
13, \
0, \
temp_list[0][0],\
temp_list[0][1],\
temp_list[0][2],\
0,\
temp_list[0][3],\
temp_list[0][4],\
temp_list[0][5],\
flog )
# process month values.
# returns avg, min, max(in), avg, min, max(out)
temp_list = temperature_db.selectAMM( timestamp[:7],13,flog )
temperature_db.replace(timestamp[:7]+"-01 00:00:00",\
14, \
0, \
temp_list[0][0],\
temp_list[0][1],\
temp_list[0][2],\
0,\
temp_list[0][3],\
temp_list[0][4],\
temp_list[0][5],\
flog )
#flog.setLevel( logging.INFO )
# process year values.
# returns avg, min, max(in), avg, min, max(out)
temp_list = temperature_db.selectAMM( timestamp[:4],14,flog )
temperature_db.replace(timestamp[:4]+"-01-01 00:00:00",\
15, \
0, \
temp_list[0][0],\
temp_list[0][1],\
temp_list[0][2],\
0,\
temp_list[0][3],\
temp_list[0][4],\
temp_list[0][5],\
flog )
# set process timestamp in status database
rt_status_db.timestamp(58,flog)
#clean_database
temperature_db.cleanDb(flog)
except Exception as e:
flog.error(inspect.stack()[0][3]+": json fout melding:"+str(e.args) )
prgname = 'P1UdpDaemon'
# bind all IP
HOST = '0.0.0.0'
# Listen on Port
PORT = 30721
#Size of receive buffer
BUFFER_SIZE = 1024
# Create a UDP/IP socket
udpsocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
temperature_db = temperatureDB()
rt_status_db = rtStatusDb()
config_db = configDB()
timestamp_last_backup = getUtcTime()
def Main(argv):
flog.info("Start van programma.")
last_seq_id = -1
DiskRestore()
# open van config database
try:
config_db.init(const.FILE_DB_CONFIG,const.DB_CONFIG_TAB)
except Exception as e:
flog.critical(inspect.stack()[0][3]+": database niet te openen(3)."+const.FILE_DB_CONFIG+") melding:"+str(e.args[0]))
sys.exit(1)
flog.info(inspect.stack()[0][3]+": database tabel "+const.DB_CONFIG_TAB+" succesvol geopend.")
def Main(argv):
flog.info("Start van programma.")
parser = argparse.ArgumentParser(description="options: -fb")
parser.add_argument('-fb',
'--forcebackup',
required=False,
action="store_true")
args = parser.parse_args()
# open van status database
try:
rt_status_db.init(const.FILE_DB_STATUS, const.DB_STATUS_TAB)
except Exception as e:
flog.critical(inspect.stack()[0][3] + ": Database niet te openen(1)." +
const.FILE_DB_STATUS + ") melding:" + str(e.args[0]))
sys.exit(1)
flog.debug(inspect.stack()[0][3] + ": database tabel " +
const.DB_STATUS_TAB + " succesvol geopend.")
# open van config database
try:
config_db.init(const.FILE_DB_CONFIG, const.DB_CONFIG_TAB)
except Exception as e:
flog.critical(inspect.stack()[0][3] + ": database niet te openen(2)." +
const.FILE_DB_CONFIG + ") melding:" + str(e.args[0]))
sys.exit(2)
flog.debug(inspect.stack()[0][3] + ": database tabel " +
const.DB_CONFIG_TAB + " succesvol geopend.")
_id, do_ftp_backup, _label = config_db.strget(36, flog)
_id, do_dbx_backup, _label = config_db.strget(49, flog)
parameter = 0
# do some casting....comparing strings is a pain
do_ftp_backup = int(do_ftp_backup)
do_dbx_backup = int(do_dbx_backup)
flog.debug(inspect.stack()[0][3] + ": ftp backup=" + str(do_ftp_backup) +
" Dropbox back-up=" + str(do_dbx_backup))
# if one of the backup options is on make an export file
if do_ftp_backup == 1 or do_dbx_backup == 1:
parameter = 1
else:
flog.info(inspect.stack()[0][3] +
": Backup staat uit, geen back-up gestart")
if args.forcebackup == True: # force backup, forget preference.
flog.info(inspect.stack()[0][3] + ": geforceerde back-up gestart.")
parameter = 1
if int(parameter) == 1:
flog.debug(inspect.stack()[0][3] + ": export file wordt gemaakt.")
file_tmp_id = str(util.getUtcTime()) + "-" + systemid.getSystemId()
cmd = "/p1mon/scripts/P1SqlExport.py -e " + file_tmp_id
flog.debug(inspect.stack()[0][3] + ": export commando is ->" +
str(cmd))
retvalue = os.system("/p1mon/scripts/P1SqlExport.py -e " + file_tmp_id)
if retvalue != 0:
flog.error(inspect.stack()[0][3] +
": export van file gefaald, gestopt.")
sys.exit(3)
export_file = exportfile_base + file_tmp_id + '.zip'
if util.fileExist(export_file) == False:
flog.error(inspect.stack()[0][3] + ": export file " + export_file +
" niet gevonden, gestopt.")
sys.exit(4)
# update config database with current name of backup file.
config_db.strset(export_file, 33, flog)
if do_dbx_backup == 1:
# do dropbox back-up
flog.info(inspect.stack()[0][3] + ": Dropbox backup gestart")
try:
# try to clean old files
clean_old_files()
if len(os.listdir(const.DIR_DBX_LOCAL +
const.DBX_DIR_BACKUP)) > 10:
flog.critical(
inspect.stack()[0][3] +
": Dropbox backup bestand niet gekopierd, te veel bestanden in ram buffer."
)
else:
flog.debug(inspect.stack()[0][3] +
": copy export file naar lokale dropbox folder: " +
export_file)
shutil.copy2(export_file,
const.DIR_DBX_LOCAL + const.DBX_DIR_BACKUP)
_head, tail = os.path.split(export_file)
util.setFile2user(
const.DIR_DBX_LOCAL + const.DBX_DIR_BACKUP + '/' + tail,
'p1mon')
except Exception as e:
flog.error(inspect.stack()[0][3] + ": Dropbox back-up. melding:" +
str(e.args[0]))
if do_ftp_backup == 1:
flog.info(inspect.stack()[0][3] + ": FTP backup gestart")
retvalue = os.system("/p1mon/scripts/P1FtpCopy.py")
if retvalue > 0:
flog.error(inspect.stack()[0][3] +
": ftp backup gefaald, gestopt.")
sys.exit(5)
flog.info("programma is succesvol gestopt.")
sys.exit(0) # all is well.
def powerSwitcher():
global powerswitcher_active, powerswitcher_last_action_utc_timestamp, powerswitcher_forced_on
try:
#force powerswitcher force on (1) or (0) automatic.
#powerswitcher_forced_on_read = 0
_id, powerswitcher_forced_on_read, _label = config_db.strget( 87, flog )
if int( powerswitcher_forced_on_read ) == 1:
if powerswitcher_forced_on == 0:
flog.info( inspect.stack()[0][3] + ": powerswitcher geforceerd aangezet." )
powerswitcher_forced_on = 1
rt_status_db.strset( 0, 83 , flog )
rt_status_db.timestamp( 84, flog )
gpioPowerSwitcher.gpioOn( True )
flog.debug( inspect.stack()[0][3] + ": powerswitcher geforceerd aangezet." )
return # no other actions needed or possible.
else:
if powerswitcher_forced_on == 1:
flog.info( inspect.stack()[0][3] + ": powerswitcher geforceerd uitgezet." )
powerswitcher_forced_on = 0
gpioPowerSwitcher.gpioOn( False )
rt_status_db.timestamp( 84, flog )
rt_status_db.strset( 0, 83 , flog )
#power switcher is active.
_id, powerswitcher_is_on, _label = config_db.strget( 86, flog )
if int(powerswitcher_is_on) == 0:
flog.debug( inspect.stack()[0][3] + ": powerswitcher staat uit." )
gpioPowerSwitcher.gpioOn( False ) # make sure we switch off the load when not active.
powerswitcher_active = False
powerswitcher_last_action_utc_timestamp = 0
rt_status_db.strset( 0, 83 , flog )
return
#default values. as failsave.
on_threshold_watt = 1000
off_threshold_watt = 500
on_threshold_minutes = 2
off_threshold_minutes = 2
on_minimum_time = 0
off_minimum_time = 0
# read config information.
_id, on_threshold_watt, _label = config_db.strget( 81,flog )
_id, off_threshold_watt, _label = config_db.strget( 82,flog )
_id, on_threshold_minutes, _label = config_db.strget( 83,flog )
_id, off_threshold_minutes, _label = config_db.strget( 84,flog )
_id, on_minimum_time, _label = config_db.strget( 88,flog )
_id, off_minimum_time, _label = config_db.strget( 89,flog )
flog.debug( inspect.stack()[0][3]+": on_threshold_watt=" + str(on_threshold_watt) + ' off_threshold_watt=' + str(off_threshold_watt) + " on_threshold_minutes=" + str(on_threshold_minutes) + " off_threshold_minutes=" + str(off_threshold_minutes) + " on_minimum_time=" + str(on_minimum_time) + " off_minimum_time=" + str(off_minimum_time) )
#check for on hold time
if (powerswitcher_active == True) and ( powerswitcher_last_action_utc_timestamp + ( int(on_minimum_time) * 60 ) > util.getUtcTime() ):
flog.debug( inspect.stack()[0][3]+": aan minimum tijd is nog geldig, geen actie, output blijft aan.")
return
if (powerswitcher_active == False ) and ( powerswitcher_last_action_utc_timestamp + ( int(off_minimum_time) * 60 ) > util.getUtcTime() ):
flog.debug( inspect.stack()[0][3]+": uit minimum tijd is nog geldig, geen actie, output blijft uit.")
return
if powerswitcher_active == False:
watt_producing = powerSwitcherSql( on_threshold_minutes )
#watt_producing = 600
if watt_producing == None:
return
if watt_producing >= int( on_threshold_watt ):
powerswitcher_active = True
gpioPowerSwitcher.gpioOn( True )
powerswitcher_last_action_utc_timestamp = util.getUtcTime()
rt_status_db.strset( watt_producing, 83, flog ) # watt value uses to activate the switcher
rt_status_db.timestamp( 84, flog )
flog.info( inspect.stack()[0][3]+": Ingeschakeld op een vermogen van " + str(watt_producing) + " watt." )
else:
watt_producing = powerSwitcherSql( off_threshold_minutes )
if watt_producing == None:
return
if watt_producing <= int( off_threshold_watt ):
powerswitcher_active = False
gpioPowerSwitcher.gpioOn( False )
powerswitcher_last_action_utc_timestamp = util.getUtcTime()
rt_status_db.strset( 0, 83 , flog )
rt_status_db.timestamp( 84, flog )
flog.info( inspect.stack()[0][3]+": Uitgeschakeld op een vermogen van " + str(watt_producing) + " watt." )
except Exception as e:
flog.error( inspect.stack()[0][3]+": onverwachte fout " + str(e) )
def Main(argv):
flog.info("Start van programma.")
global ftp_para
# open van status database
try:
rt_status_db.init(const.FILE_DB_STATUS, const.DB_STATUS_TAB)
except Exception as e:
flog.critical(inspect.stack()[0][3] + ": Database niet te openen(1)." +
const.FILE_DB_STATUS + ") melding:" + str(e.args[0]))
sys.exit(1)
flog.debug(inspect.stack()[0][3] + ": database tabel " +
const.DB_STATUS_TAB + " succesvol geopend.")
# open van config database
try:
config_db.init(const.FILE_DB_CONFIG, const.DB_CONFIG_TAB)
except Exception as e:
flog.critical(inspect.stack()[0][3] + ": database niet te openen(2)." +
const.FILE_DB_CONFIG + ") melding:" + str(e.args[0]))
sys.exit(1)
flog.debug(inspect.stack()[0][3] + ": database tabel " +
const.DB_CONFIG_TAB + " succesvol geopend.")
rt_status_db.timestamp(47, flog)
# update field from database, the cli switches overwrite the DB values!
_id, ftp_para['user'], _label = config_db.strget(28, flog)
_id, ftp_para['password'], _label = config_db.strget(29, flog)
_id, ftp_para['directory'], _label = config_db.strget(30, flog)
_id, ftp_para['server'], _label = config_db.strget(31, flog)
_id, ftp_para['port'], _label = config_db.strget(32, flog)
_id, ftp_para['filename'], _label = config_db.strget(33, flog)
_id, ftp_para['maxfilecount'], _label = config_db.strget(34, flog)
_id, ftp_para['ftps'], _label = config_db.strget(35, flog) # was secure
_id, ftp_para['ftp'], _label = config_db.strget(76, flog)
_id, ftp_para['sftp'], _label = config_db.strget(77, flog)
# sftp toevoegen. ftp toevoegen
flog.debug(inspect.stack()[0][3] + ": parameters uit de DB:" +
str(ftp_para))
parser = argparse.ArgumentParser(description="ftp....")
parser.add_argument('-u', '--user', required=False)
parser.add_argument('-pw', '--password', required=False)
parser.add_argument('-dir', '--directory', required=False)
parser.add_argument('-srv', '--server', required=False)
parser.add_argument('-fname', '--filename', required=False)
parser.add_argument('-mfcnt', '--maxfilecount', required=False)
parser.add_argument('-pt', '--port', required=False)
parser.add_argument('-ftps', '--ftps', required=False,
action="store_true") # flag only
parser.add_argument('-sftp', '--sftp', required=False,
action="store_true") # flag only
parser.add_argument('-ftp', '--ftp', required=False,
action="store_true") # flag only
args = parser.parse_args()
if args.user != None:
ftp_para['user'] = args.user
if args.password != None:
ftp_para['password'] = args.password
flog.debug("password van commandline ontvangen -> " +
ftp_para['password'])
else: #decode password from database
try:
ftp_para['password'] = base64.standard_b64decode(
crypto3.p1Decrypt(ftp_para['password'],
'ftppw')).decode('utf-8')
# added by Aad
if ftp_para['password'] == '':
ftp_para['password'] = "******"
raise Exception(" wachtwoord is niet ingesteld.")
except Exception as e:
flog.error(inspect.stack()[0][3]+": password decodering gefaald. Decoded password="******" Gestopt. melding:" + str(e.args[0]) )
sys.exit(16)
flog.info("Password decryptie ok.")
flog.debug("Decoded password = "******"''"
raise Exception(
"server adres (IP adres of url) is niet ingesteld.")
except Exception as e:
flog.error(inspect.stack()[0][3] + ": Gestopt, melding: " +
str(e.args[0]))
sys.exit(17)
if args.filename != None:
ftp_para['filename'] = args.filename
if args.maxfilecount != None:
ftp_para['maxfilecount'] = int(args.maxfilecount)
if args.port != None:
ftp_para['port'] = int(args.port)
if args.ftp == True:
ftp_para['ftp'] = 1
ftp_para['ftps'] = 0
ftp_para['sftp'] = 0
if args.ftps == True:
ftp_para['ftp'] = 0
ftp_para['ftps'] = 1
ftp_para['sftp'] = 0
if args.sftp == True:
ftp_para['ftp'] = 0
ftp_para['ftps'] = 0
ftp_para['sftp'] = 1
flog.debug(inspect.stack()[0][3] + ": parameters na CLI parsing:" +
str(ftp_para))
if int(ftp_para['ftp']) == 0 and int(ftp_para['ftps']) == 0 and int(
ftp_para['sftp']) == 0:
flog.warning(inspect.stack()[0][3] +
": geen ftp, ftps of sftp opties geslecteerd!")
if os.path.isfile(ftp_para['filename']) == False:
rt_status_db.strset(
'fout: te kopieren bestand niet gevonden. Gestopt.', 48, flog)
flog.error(inspect.stack()[0][3] +
": te kopieren bestand niet gevonden. Gestopt.")
sys.exit(17)
# do normal plain text FTP
if int(ftp_para['ftp']) == 1:
flog.info(inspect.stack()[0][3] + ": probeer bestand " +
ftp_para['filename'] + " te kopieren via ftp.")
try:
ftpConnection = ftpConnect()
if len(ftp_para['directory']) > 0:
ftpChangeDirectory(ftpConnection, ftp_para['directory'])
ftpCopy(ftpConnection, ftp_para['filename'])
flog.info(inspect.stack()[0][3] + ": bestand " +
ftp_para['filename'] + " succesvol gekopierd via ftp.")
ftpConnection.quit() # be polite to ftp servers
except Exception as e:
rt_status_db.strset(
'fout: server antwoord: ' + str(e.args[0]) + ' Gestopt.', 48,
flog)
flog.error(inspect.stack()[0][3] + ": FTP server antwoord: " +
str(e.args[0]) + " Gestopt.")
sys.exit(11)
rt_status_db.strset("FTP transfer is succesvol gestopt.", 48, flog)
rt_status_db.timestamp(49, flog)
flog.info("FTP transfer is succesvol gestopt.")
sys.exit(0) # all is well.
# TODO ftps en sftp aanpassen zodat ze poort meenemen.
# do ftps FTP with SSL
if int(ftp_para['ftps']) == 1:
#################################################################
# STEP 1 copy the file #
#################################################################
try: # copying file
#flog.setLevel( logging.DEBUG )
_head, tail = os.path.split(ftp_para['filename'])
changed_filename = '//p1mon/mnt/ramdisk/' + fileprefix + str(
getUtcTime()) + '-' + tail
#server = "ftps://"+ftp_para['server']
server = "ftp://" + ftp_para['server'] #+ ":990"
if len(ftp_para['directory']) > 0:
server = server + "/" + ftp_para[
'directory'] + "/" # checked can handle // and / in path.
flog.debug(inspect.stack()[0][3] + ": server path is=" +
str(server))
shutil.copy(ftp_para['filename'], changed_filename)
flog.info(inspect.stack()[0][3] + ": probeer bestand " +
ftp_para['filename'] + " te kopieren via ftps.")
"""
cp = subprocess.run([ "curl", "--ssl-reqd", "--ftp-ssl-control", "-ssl", "--globoff", "--insecure", "-sSv", server, "--user", ftp_para['user']+":"+ftp_para['password'], "-T", changed_filename ], \
universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60 )
"""
cp = subprocess.run([ "curl", "--ftp-ssl-control", "--ftp-ssl" ,"--globoff", "--insecure", "-sSv", server, "--user", ftp_para['user']+":"+ftp_para['password'], "-T", changed_filename ], \
universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60 )
flog.debug(inspect.stack()[0][3] + ": cp.stdout=" + str(cp.stdout))
flog.debug(inspect.stack()[0][3] + ": cp.stderr=" + str(cp.stderr))
flog.debug(inspect.stack()[0][3] + ": cp.returncode=" +
str(cp.returncode))
# Added by Aad.
saveDelete(changed_filename) # remove tmp copy of file.
if cp.returncode == 0:
flog.info(inspect.stack()[0][3] + ": bestand " +
ftp_para['filename'] +
" succesvol gekopierd via ftps als " +
changed_filename)
else:
raise Exception(cp.stderr.replace("\n", ""))
except Exception as e:
rt_status_db.strset(
'fout: server antwoord: ' + str(e.args[0]) + ' Gestopt.', 48,
flog)
flog.error(inspect.stack()[0][3] + ": SFTP server antwoord: " +
str(e.args[0]) + " Gestopt.")
sys.exit(11)
#################################################################
# STEP 2 checking if we have reached the maxium number of files #
#################################################################
try: # checkin if max files are reached
_head, tail = os.path.split(ftp_para['filename'])
changed_filename = '//p1mon/mnt/ramdisk/' + fileprefix + str(
getUtcTime()) + tail
server = "ftp://" + ftp_para['server']
if len(ftp_para['directory']) > 0:
server = server + "/" + ftp_para[
'directory'] + "/" # checked can handle // and / in path.
flog.debug(inspect.stack()[0][3] + ": server path is=" +
str(server))
cp = subprocess.run([ "curl", "--ftp-ssl-control", "--ftp-ssl", "--globoff", "--insecure", "-sS", server, "--user", ftp_para['user']+":"+ftp_para['password'], "--list-only" ], \
universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60 )
#flog.debug( inspect.stack()[0][3]+": cp.stdout=" + str(cp.stdout) )
#flog.debug( inspect.stack()[0][3]+": cp.stderr=" + str(cp.stderr) )
#flog.debug( inspect.stack()[0][3]+": cp.returncode=" + str(cp.returncode ) )
if cp.returncode == 0: # process output, proces files based on epoch value in filename
filtered_file_list = grep("\AP1BU-\d+", cp.stdout.split('\n'))
if len(filtered_file_list) < int(ftp_para['maxfilecount']) - 1:
flog.info(inspect.stack()[0][3] +
": maximale aantal van files " +
str(ftp_para['maxfilecount']) +
" niet gehaald (" +
str(len(filtered_file_list)) + ")")
else:
flog.info(inspect.stack()[0][3] +
": maximale aantal van files overschreden " +
str(ftp_para['maxfilecount']))
ftpsRemoveOldFiles(filtered_file_list)
else:
raise Exception(cp.stderr.replace("\n", ""))
except Exception as e:
rt_status_db.strset(
'fout: directory list server : ' + str(e.args[0]) +
' Gestopt.', 48, flog)
flog.error(inspect.stack()[0][3] +
": SFTP directory list server fout: " + str(e.args[0]) +
" Gestopt.")
sys.exit(13)
rt_status_db.strset("FTPS transfer is succesvol gestopt.", 48, flog)
rt_status_db.timestamp(49, flog)
flog.info("FTPS transfer is succesvol gestopt.")
sys.exit(0) # all is well.
# do sftp FTP with SSH
if int(ftp_para['sftp']) == 1:
#################################################################
# STEP 1 copy the file #
#################################################################
try: # copying file
_head, tail = os.path.split(ftp_para['filename'])
tmp_path = '/var/log/p1monitor/'
changed_filename = tmp_path + fileprefix + str(
getUtcTime()) + '-' + tail
server = "sftp://" + ftp_para['server'] + "/"
if len(ftp_para['directory']) > 0:
server = server + ftp_para[
'directory'] + "/" # checked can handle // and / in path.
else:
server = server + "home/"
flog.debug(inspect.stack()[0][3] + ": server path is=" +
str(server))
#print ( "changed_filename " + changed_filename + " filename " + ftp_para['filename'] )
shutil.copy(ftp_para['filename'], changed_filename)
flog.info(inspect.stack()[0][3] + ": probeer bestand " +
ftp_para['filename'] + " te kopieren via sftp.")
cp = subprocess.run([ "curl","--globoff", "--insecure", "-sSv", server, "--user", ftp_para['user']+":"+ftp_para['password'], "-T", changed_filename ], \
universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60 )
flog.debug(inspect.stack()[0][3] + ": cp.stdout=" + str(cp.stdout))
flog.debug(inspect.stack()[0][3] + ": cp.stderr=" + str(cp.stderr))
flog.debug(inspect.stack()[0][3] + ": cp.returncode=" +
str(cp.returncode))
#Added by Aad.
saveDelete(changed_filename) # remove tmp copy of file.
if cp.returncode == 0:
flog.info(inspect.stack()[0][3] + ": bestand " +
ftp_para['filename'] +
" succesvol gekopierd via ftps als " +
changed_filename)
else:
raise Exception(cp.stderr.replace("\n", ""))
except Exception as e:
rt_status_db.strset(
'fout: server antwoord: ' + str(e.args[0]) + ' Gestopt.', 48,
flog)
flog.error(inspect.stack()[0][3] + ": SFTP server antwoord: " +
str(e.args[0]) + " Gestopt.")
sys.exit(11)
#################################################################
# STEP 2 checking if we have reached the maxium number of files #
#################################################################
try: # checkin if max files are reached
#_head,tail = os.path.split( ftp_para['filename'] )
#changed_filename = '//p1mon/mnt/ramdisk/' + fileprefix + str(getUtcTime()) + '-' + tail
server = "sftp://" + ftp_para['server'] + "/"
if len(ftp_para['directory']) > 0:
server = server + ftp_para[
'directory'] + "/" # checked can handle // and / in path.
else:
server = server + "home/"
flog.debug(inspect.stack()[0][3] + ": server path is=" +
str(server))
# Modified by Aad: for clarity, conform curl params: --list-only parameter added.
cp = subprocess.run([ "curl", "--globoff", "--insecure", "-sSv", server, "--user", ftp_para['user']+":"+ftp_para['password'], "--list-only" ], \
universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60 )
flog.debug(inspect.stack()[0][3] + ": cp.stdout=" + str(cp.stdout))
flog.debug(inspect.stack()[0][3] + ": cp.stderr=" + str(cp.stderr))
flog.debug(inspect.stack()[0][3] + ": cp.returncode=" +
str(cp.returncode))
# pre-process output that could have a unspecified layout.
filtered_file_list = []
matched_lines = [
line for line in cp.stdout.split('\n') if fileprefix in line
]
for line in matched_lines:
#filtered_file_list.append ( line[ line.find('P1BU-'): ] )
#modified by Aad: absolute find filter replaced by variable.
filtered_file_list.append(line[line.find(fileprefix):])
if cp.returncode == 0: # process output, proces files based on epoch value in filename
if len(filtered_file_list) < int(ftp_para['maxfilecount']) - 1:
flog.info(inspect.stack()[0][3] +
": maximale aantal van files " +
str(ftp_para['maxfilecount']) +
" niet gehaald (" +
str(len(filtered_file_list)) + ")")
else:
flog.info(inspect.stack()[0][3] +
": maximale aantal van files overschreden " +
str(ftp_para['maxfilecount']))
# sftpRemoveOldFiles( filtered_file_list ) # orignal
if sftpRemoveOldFiles(filtered_file_list) == False:
flog.info(inspect.stack()[0][3] +
" Een of meerdere bestanden >" +
str(ftp_para['maxfilecount']) +
" zijn niet gewist.")
else:
raise Exception(cp.stderr.replace("\n", ""))
except Exception as e:
rt_status_db.strset(
'fout: directory list server : ' + str(e.args[0]) +
' Gestopt.', 48, flog)
flog.error(inspect.stack()[0][3] +
": SFTP directory list server fout: " + str(e.args[0]) +
" Gestopt.")
sys.exit(13)
rt_status_db.strset("SFTP transfer is succesvol gestopt.", 48, flog)
rt_status_db.timestamp(49, flog)
flog.info("SFTP transfer is succesvol gestopt.")
sys.exit(0) # all is well.
