def write_p1_json_dbx_folder( configdb=None, data=None, flog=None, sysid=const.SYSTEM_ID_DEFAULT, utc_ok_timestamp=0): #flog.setLevel(logging.DEBUG) _id, parameter, _label = configdb.strget(50,flog) # is Dropbox is on or off if int(parameter) == 0: # dropbox sharing is off return utc_ok_timestamp # do nothing #print ("#!"+ str( abs(util.getUtcTime() - utc_ok_timestamp) ) ) if abs(util.getUtcTime() - utc_ok_timestamp) < 10: return utc_ok_timestamp # do nothing, DBX rate limting try: filename = const.DIR_DBX_LOCAL + const.DBX_DIR_DATA + '/' + const.API_BASIC_JSON_PREFIX + sysid + const.API_BASIC_JSON_SUFFIX flog.debug(inspect.stack()[0][3]+": json output =" + json.dumps( data , sort_keys=True ) + " naar bestand " + filename ) with open(filename, 'w') as outfile: json.dump( data , outfile, sort_keys=True ) utc_ok_timestamp = util.getUtcTime() util.setFile2user( filename,'p1mon' ) # to make sure we can process the file except Exception as e: flog.error(inspect.stack()[0][3]+": wegschrijven data naar Dropbox is mislukt. melding:"+str(e.args[0])) return utc_ok_timestamp
def writeSemaphoreFile(name, flog): try: filename = name + '.p1mon' if os.path.isfile(const.DIR_FILESEMAPHORE + filename): flog.warning(inspect.stack()[0][3] + ": semafoor file " + filename + " bestaat al, gestopt.") return False # semaphore file already exist. time = str(getUtcTime()).encode('utf-8') filename = str(filename).encode('utf-8') hashvalue = hashlib.sha1(filename + time).hexdigest() #print ( time ) #print ( filename ) #print (hashvalue) fo = open(const.DIR_FILESEMAPHORE + filename.decode('utf-8'), "w") fo.write(filename.decode('utf-8') + '\n') fo.write(time.decode('utf-8') + '\n') fo.write(hashvalue + '\n') fo.close() flog.info(inspect.stack()[0][3] + " semafoor file " + const.DIR_FILESEMAPHORE + filename.decode('utf-8') + " gemaakt.") return True except Exception as e: flog.error(inspect.stack()[0][3] + ": semafoor file error ->" + str(e)) return False
def gas_stub_instert(self, line=None, serialbuffer=None, gasmode=DUMMY_GAS_MODE_2421 ): #voeg een dummy gas record in alleen voor ontwikkeling!!! try: #dev_dummy_gas_value = dev_dummy_gas_value + random.uniform(0, 0.0005) #0.0005 if util.getUtcTime() - self.timestamp_last_gas_update > DUMMY_GAS_TIME_ELAPSED: self.dev_dummy_gas_value = self.dev_dummy_gas_value + random.uniform(0.0, 0.4) #only update after elapsed time self.timestamp_last_gas_update = util.getUtcTime() self.flog.debug(inspect.stack()[0][3]+": gas dummy string toevoegen.") del serialbuffer[ len(serialbuffer)-1 ] #if DUMMY_GAS_MODE_2421 == True: if gasmode == DUMMY_GAS_MODE_2421: line_1 = ''.join( filter(lambda x: x in string.printable, '0-1:24.2.1(170108160000W)('+'{0:09.3f}'.format(self.dev_dummy_gas_value)+'*m3)\r\n')).rstrip()[0:1024] serialbuffer.append( line_1 ) #serial_buffer.append( '0-1:24.2.1(700101010000W)(00000000)' ) #flog.debug(inspect.stack()[0][3]+": dummy gas waarde = "+line_1) self.flog.warning(inspect.stack()[0][3]+": test gas regel toegevoegd aan telegram: " + line_1 ) #if DUMMY_GAS_MODE_2423 == True: #(voor Belgie) if gasmode == DUMMY_GAS_MODE_2423: #(voor Belgie) line_1 = ''.join( filter(lambda x: x in string.printable, '0-1:24.2.3(190830073458S)('+'{0:09.3f}'.format(self.dev_dummy_gas_value)+'*m3)\r\n')).rstrip()[0:1024] serialbuffer.append( line_1 ) #serial_buffer.append( '0-1:24.2.1(700101010000W)(00000000)' ) #flog.debug(inspect.stack()[0][3]+": dummy gas waarde = "+line_1) self.flog.warning(inspect.stack()[0][3]+": test gas regel toegevoegd aan telegram: " + line_1 ) #if DUMMY_GAS_MODE_2421 == True or DUMMY_GAS_MODE_2423 == True or DUMMY_GAS_MODE_2430 == True: #if DUMMY_GAS_MODE_2430 == True: if gasmode == DUMMY_GAS_MODE_2430: line_1 = ''.join( filter(lambda x: x in string.printable, '0-1:24.3.0(121030140000)(00)(60)(1)(0-1:24.2.1)(m3)\r\n')).rstrip()[0:1024] line_2 = ''.join( filter(lambda x: x in string.printable, '('+'{0:09.3f}'.format(self.dev_dummy_gas_value)+')')).rstrip()[0:1024] serialbuffer.append( line_1 ) serialbuffer.append( line_2 ) self.flog.debug(inspect.stack()[0][3]+": dummy gas waarde = "+line_1) self.flog.warning(inspect.stack()[0][3]+": test gas regel 1 toegevoegd aan telegram: " + line_1 ) self.flog.warning(inspect.stack()[0][3]+": test gas regel 2 toegevoegd aan telegram: " + line_2 ) self.flog.debug(inspect.stack()[0][3]+": "+line_2) serialbuffer.append( line ) except Exception as e: self.flog.warning( inspect.stack()[0][3]+": test gas test insert probleem " + str( e ) )
def insert_db_serial_record(data_set=None, status=None, dbstatus=None, dbserial=None, flog=None): try: timestr = util.mkLocalTimeString() #timestr_min=timestr[0:16]+":00" if status['gas_present_in_serial_data'] == False: #gas_verbr_m3_2421 = 0 data_set['gas_verbr_m3_2421'] = 0 sqlstr ="insert or replace into " + const.DB_SERIAL_TAB + \ " values (\ '" +timestr+"',\ '" +"0"+"',\ '" +str( data_set['verbrk_kwh_181'] )+"', \ '" +str( data_set['verbrk_kwh_182'] )+"', \ '" +str( data_set['gelvr_kwh_281'] )+"', \ '" +str( data_set['gelvr_kwh_282'] )+"', \ '" +str( data_set['tarief_code'] )+"', \ '" +str( data_set['act_verbr_kw_170'] )+"', \ '" +str( data_set['act_gelvr_kw_270'] )+"',\ '" +str( data_set['gas_verbr_m3_2421'] )+"')" sqlstr = " ".join(sqlstr.split()) flog.debug(inspect.stack()[0][3] + ": (2)serial e-buffer insert: sql=" + sqlstr) dbserial.insert_rec(sqlstr) # timestamp telegram processed dbstatus.timestamp(16, flog) # Timestring + Daylight saving. dbstatus.strset(str(util.getUtcTime()), 87, flog) # UTC time status['p1_record_is_ok'] = 1 dbstatus.strset(str(status['p1_record_is_ok']), 46, flog) # P1 data is ok recieved status['timestamp_last_insert'] = util.getUtcTime() # if util.isMod( timestr,15 ) == True: # backupData() except Exception as e: flog.error(inspect.stack()[0][3] + ": Insert gefaald. Melding=" + str(e.args[0]))
def backupData(): global timestamp_last_backup # set backup timestamp in status database rt_status_db.timestamp(57,flog) timestamp_last_backup = getUtcTime() #setFileFlags() flog.debug(inspect.stack()[0][3]+": Gestart") os.system("/p1mon/scripts/P1DbCopy.py --temperature2disk --forcecopy") #backupFile(const.FILE_DB_TEMPERATUUR_FILENAME) #setFileFlags() rt_status_db.timestamp(29,flog) flog.debug(inspect.stack()[0][3]+": Gereed")
def clean_old_files(): path = const.DIR_DBX_LOCAL + const.DBX_DIR_BACKUP # define the path currentDirectory = pathlib.Path(path) for currentFile in currentDirectory.iterdir(): delta_secs = abs(util.getUtcTime() - int(os.path.getctime(currentFile))) #print ( "Delta " + str(currentFile), " ", str( delta_secs )) if delta_secs > 7200: #remove files that are older then 2 hours, to prevent daylight saving problems flog.info( inspect.stack()[0][3]+": verouderde bestand " + str(currentFile) +\ " wordt verwijderd. Bestand is " + str(delta_secs ) + " seconden oud." ) os.remove(currentFile) else: flog.debug( inspect.stack()[0][3]+": bestand " + str(currentFile) +\ " wordt NIET verwijderd. Bestand is " + str(delta_secs ) + " seconden oud." )
def ftpCopy(ftpConnection, filename): _head, tail = os.path.split(filename) try: ftpConnection.storbinary('STOR ' + tail, open(filename, 'rb'), blocksize=128) except Exception as e: rt_status_db.strset( 'fout: copy file naar server: ' + str(e.args[0]) + ' Gestopt.', 48, flog) flog.error(inspect.stack()[0][3] + ": copy file naar server: " + str(e.args[0]) + " Gestopt.") sys.exit(12) try: file_list = ftpConnection.nlst() except Exception as e: rt_status_db.strset( 'fout: directory list server : ' + str(e.args[0]) + ' Gestopt.', 48, flog) flog.error(inspect.stack()[0][3] + ": directory list server fout: " + str(e.args[0]) + " Gestopt.") sys.exit(13) flog.debug(inspect.stack()[0][3] + ": file list van FTP server: " + str(file_list)) try: ftpConnection.rename(tail, fileprefix + str(getUtcTime()) + '-' + tail) except Exception as e: rt_status_db.strset( 'fout: hernoemen van file (' + tail + ') mislukt: ' + str(e.args[0]) + ' Gestopt.', 48, flog) flog.error(inspect.stack()[0][3] + ": hernoemen van file (" + tail + ") mislukt: " + str(e.args[0]) + " Gestopt.") sys.exit(14) if int(ftp_para['maxfilecount']) > -1: flog.debug( inspect.stack()[0][3] + ": wissen van oude files staat aan. maximaal aantal files is " + str(ftp_para['maxfilecount'])) ftpRemoveOldFiles(ftpConnection, file_list)
def update_json_data( jsondata=None, p1data=None ): #convert dutch tarif code to English Dal = Low, Peak = High if p1data['tarief_code'] == 'P': tarief_code = 'HIGH' else: tarief_code = "LOW" # there are only two options (D/P) jsondata[ apiconst.JSON_TS_LCL ] = str(util.mkLocalTimeString()) jsondata[ apiconst.JSON_TS_LCL_UTC ] = util.getUtcTime() jsondata[ apiconst.JSON_API_API_STTS ] = 'production' jsondata[ apiconst.JSON_API_CNSMPTN_KWH_L ] = float( p1data['verbrk_kwh_181'] ) jsondata[ apiconst.JSON_API_CNSMPTN_KWH_H ] = float( p1data['verbrk_kwh_182'] ) jsondata[ apiconst.JSON_API_PRDCTN_KWH_L ] = float( p1data['gelvr_kwh_281'] ) jsondata[ apiconst.JSON_API_PRDCTN_KWH_H ] = float( p1data['gelvr_kwh_282'] ) jsondata[ apiconst.JSON_API_TRFCD ] = str( tarief_code ) jsondata[ apiconst.JSON_API_CNSMPTN_KW ] = float( p1data['act_verbr_kw_170'] ) jsondata[ apiconst.JSON_API_PRDCTN_KW ] = float( p1data['act_gelvr_kw_270'] ) jsondata[ apiconst.JSON_API_CNSMPTN_GAS_M3 ] = float( p1data['gas_verbr_m3_2421'] ) jsondata[ apiconst.JSON_API_SYSTM_ID] = systemid.getSystemId()
def main_prod(): global ser1 global serial_buffer global processing_speed stub_serial_buffer = [] # for stub testing only p1_port_shared_lib.clear_data_buffer(buffer=phase_db_record) flog.info("Start van programma.") # make sure as first program to run that ram is filled with the # last data from persistent memory (disk) p1_port_shared_lib.disk_restore_from_disk_to_ram() # open van seriele database try: e_db_serial.init(const.FILE_DB_E_FILENAME, const.DB_SERIAL_TAB) except Exception as e: flog.critical(inspect.stack()[0][3] + " database niet te openen(1)." + const.FILE_DB_E_FILENAME + ") melding:" + str(e.args[0])) sys.exit(1) flog.info(inspect.stack()[0][3] + ": database tabel: " + const.DB_SERIAL_TAB + " succesvol geopend.") # defrag van database e_db_serial.defrag() flog.info(inspect.stack()[0][3] + ": database bestand " + const.DB_SERIAL_TAB + " gedefragmenteerd.") # open van status database try: rt_status_db.init(const.FILE_DB_STATUS, const.DB_STATUS_TAB) except Exception as e: flog.critical(inspect.stack()[0][3] + ": database niet te openen(1)." + const.FILE_DB_STATUS + ") melding:" + str(e.args[0])) sys.exit(1) flog.info(inspect.stack()[0][3] + ": database tabel: " + const.DB_STATUS_TAB + " succesvol geopend.") # open van config database try: config_db.init(const.FILE_DB_CONFIG, const.DB_CONFIG_TAB) except Exception as e: flog.critical(inspect.stack()[0][3] + ": database niet te openen(3)." + const.FILE_DB_CONFIG + ") melding:" + str(e.args[0])) sys.exit(1) flog.info(inspect.stack()[0][3] + ": database tabel " + const.DB_CONFIG_TAB + " succesvol geopend.") # open van temperatuur database try: temperature_db.init(const.FILE_DB_TEMPERATUUR_FILENAME, const.DB_TEMPERATUUR_TAB) except Exception as e: flog.critical(inspect.stack()[0][3] + ": Database niet te openen(1)." + const.FILE_DB_TEMPERATUUR_FILENAME + ") melding:" + str(e.args[0])) sys.exit(1) flog.info(inspect.stack()[0][3] + ": database tabel " + const.DB_TEMPERATUUR_TAB + " succesvol geopend.") # open van watermeter database try: watermeter_db.init(const.FILE_DB_WATERMETERV2, const.DB_WATERMETERV2_TAB, flog) except Exception as e: flog.critical(inspect.stack()[0][3] + ": Database niet te openen(3)." + const.FILE_DB_WATERMETERV2 + " melding:" + str(e.args[0])) sys.exit(1) flog.info(inspect.stack()[0][3] + ": database tabel " + const.DB_WATERMETERV2_TAB + " succesvol geopend.") # open van fase database try: fase_db.init(const.FILE_DB_PHASEINFORMATION, const.DB_FASE_REALTIME_TAB) fase_db.defrag() except Exception as e: flog.critical(inspect.stack()[0][3] + " database niet te openen(1)." + const.FILE_DB_PHASEINFORMATION + ") melding:" + str(e.args[0])) sys.exit(1) flog.info(inspect.stack()[0][3] + ": database tabel: " + const.DB_FASE_REALTIME_TAB + " succesvol geopend.") # only make object when tests are active if DUMMY_3PHASE_DATA == True or gas_test_mode != p1_telegram_test_lib.NO_GAS_TEST: flog.warning(inspect.stack()[0][3] + ": test functies geactiveerd ! ") p1_test = p1_telegram_test_lib.p1_telegram() p1_test.init(flog, configdb=config_db) # sets the rate limiting to 1 record per 10 secs or maximum speed processing of P1 telegrams p1_port_shared_lib.set_p1_processing_speed(p1_processing=processing_speed, config_db=config_db, flog=flog) p1_port_shared_lib.backup_data_to_disk_by_timestamp(statusdb=rt_status_db, flog=flog) while check_serial() == False: flog.critical( inspect.stack()[0][3] + ": seriele poort niet gevonden, is de kabel aangesloten?") time.sleep(10) # reset gas per uur waarde als we starten. rt_status_db.strset("0", 50, flog) rt_status_db.timestamp(5, flog) #read serial settings from status DB serCheckCnt = 0 check_serial_db_config_settings() p1_port_shared_lib.get_country_day_night_mode(status=p1_status, configdb=config_db, flog=flog) p1_port_shared_lib.get_gas_telgram_prefix(status=p1_status, configdb=config_db, flog=flog) flog.info(inspect.stack()[0][3] + ": P1 poort instelling baudrate=" + str(ser1.baudrate) + " bytesize=" + str(ser1.bytesize) + " pariteit=" + str(ser1.parity) + " stopbits=" + str(ser1.stopbits)) #serOpen(ser1) #ser1.flushInput() # read GAS value from status database #if DUMMY_GAS_MODE_2421 == True or DUMMY_GAS_MODE_2423 == True or DUMMY_GAS_MODE_2430 == True: if gas_test_mode > 0: print("DUMMY GAS STAAT AAN IS DIT CORRECT?\r") flog.warning(inspect.stack()[0][3] + " #############################################") flog.warning(inspect.stack()[0][3] + ": Dummy gas waarde staat aan met een interval van " + str(p1_test.gas_interval()) + " seconden. Zet uit voor productie!") flog.warning(inspect.stack()[0][3] + " #### DUMMY GAS STAAT AAN IS DIT CORRECT? ####") else: try: rt_status_db.strset(0, 43, flog) # reset dummy gas value except Exception as e: flog.error(inspect.stack()[0][3] + ": Melding=" + str(e.args[0])) # check for dummy 3Phase data if DUMMY_3PHASE_DATA == True: print("DUMMY 3 FASE DATA STAAT AAN IS DIT CORRECT?\r") flog.warning(inspect.stack()[0][3] + " #############################################") flog.warning( inspect.stack()[0][3] + ": Dummy 3 fase waarde staat aan. Zet uit voor productie!") flog.warning(inspect.stack()[0][3] + " #### DUMMY 3 FASE STAAT AAN IS DIT CORRECT? ####") # check for dummy 3Phase data if DUMMY_1SEC_PROCCESSING == True: print("DUMMY 1 SECONDEN VERWERKING STAAT AAN?\r") flog.warning(inspect.stack()[0][3] + " #############################################") flog.warning( inspect.stack()[0][3] + ": 1 seconden verwerking staat aan. Zet uit voor productie!") flog.warning(inspect.stack()[0][3] + " #### DUMMY 1 SECONDEN STAAT AAN IS DIT CORRECT? ####") # read crc check settings from config p1_port_shared_lib.get_P1_crc(status=p1_status, configdb=config_db, flog=flog) # set intial FQDN p1_port_shared_lib.fqdn_from_config(verbose=True, configdb=config_db, data_set=json_data, flog=flog) day_values.init(dbstatus=rt_status_db, dbserial=e_db_serial, flog=flog) while True: try: #print ('ser1.In_waiting='+str(ser1.in_waiting)) if ser1.in_waiting > 1: try: line = ser1.readline().decode('utf-8') #flog.debug(inspect.stack()[0][3]+": line = " + line +" length=" + str(len(line))) #line = filter(lambda x: x in string.printable, line_raw ).rstrip()[0:1024] #flog.debug(inspect.stack()[0][3]+": line filtered = " + str(line_raw ) ) except: flog.warning(inspect.stack()[0][3] + ": lezen van serieele poort mislukt.") serial_buffer.append(line) #flog.debug(inspect.stack()[0][3]+": [** V2.0]serial buffer bytes waiting = " + str(ser1.inWaiting()) ) #the benthouse BUG has 78 lines if len(serial_buffer ) > 250: # normale size less then a 100 lines flog.warning( inspect.stack()[0][3] + ": serieele buffer te groot, gewist! Buffer lengte was " + str(len(serial_buffer))) del serial_buffer[:] #print ( serial_buffer ) #print ( "[*]len="+str(len(serial_buffer)) ) if line[:1] == '!': #flog.debug(inspect.stack()[0][3]+": serial buffer "+str(serial_buffer)) if DUMMY_3PHASE_DATA == True: #phase3StubInstert( line ) p1_test.phase3_stub_instert(line=line, serialbuffer=serial_buffer) #if DUMMY_GAS_MODE_2421 == True or DUMMY_GAS_MODE_2423 == True or DUMMY_GAS_MODE_2430 == True: if gas_test_mode > 0: #gasStubInstert(line) p1_test.gas_stub_instert(line=line, serialbuffer=serial_buffer, gasmode=gas_test_mode) #flog.info(inspect.stack()[0][3] + " step 0") if processing_speed['max_processing_speed'] == False: # rate limiting of inserts, some smart meters send more then every 10 sec. a telegram. if abs(p1_status['timestamp_last_insert'] - util.getUtcTime() ) < 9: # 9 or 10 sec interval is ok. #flog.debug("Te snel een nieuw telegram ontvangen. genegeerd.") del serial_buffer[:] continue #serial_buffer[-1] = "!0F9B" #DEBUG #print ( "[*] p1 crc check = " + str(p1_crc_check_is_on) ) #flog.info(inspect.stack()[0][3] + " step 1") # check for meters that supply a crc value. if len( serial_buffer[len(serial_buffer) - 1] ) > 3 and p1_status[ 'p1_crc_check_is_on'] == True: # crc telegrams are 7 chars minimal (5+ cr\lf) # check for telegrams with a CRC attached crc_read = str(serial_buffer[len(serial_buffer) - 1][1:5]) #flog.debug("CRC in telegram gevonden -> "+crc_read) #print ('[*] crc waarde') #print crc_read save_crc = serial_buffer[len(serial_buffer) - 1] serial_buffer[ len(serial_buffer) - 1] = '!' # only process to ! not the crc itself strvar = ''.join(serial_buffer) #CRC16().calculate calc_crc = str('{0:0{1}X}'.format( CRC16().calculate(strvar), 4)) #flog.debug("Berekende CRC uit telegram -> "+calc_crc ) #flog.debug(inspect.stack()[0][3]+": serial buffer "+str(serial_buffer)) if (crc_read == calc_crc): #flog.debug("CRC is in orde.") #restore CRC to received data serial_buffer[len(serial_buffer) - 1] = save_crc else: p1_status['crc_error_cnt'] = p1_status[ 'crc_error_cnt'] + 1 #flog.debug("CRC van telegram komt niet overeen. CRC telegram ("+crc_read+"), berekende CRC ("+calc_crc+") niet verwerkt.") del serial_buffer[:] continue #flog.debug(inspect.stack()[0][3]+": serial buffer "+str(serial_buffer)) #flog.setLevel(logging.DEBUG) #print('verwerk seriele data.') if DUMMY_1SEC_PROCCESSING == True: stub_serial_buffer = serial_buffer.copy() json_data[apiconst.JSON_API_RM_TMPRTR_IN] ,json_data[apiconst.JSON_API_RM_TMPRTR_OUT] =\ p1_port_shared_lib.current_room_temperature( temperaturedb=temperature_db, flog=flog ) json_data[apiconst.JSON_API_WM_CNSMPTN_LTR_M3] =\ p1_port_shared_lib.current_watermeter_count( configdb=config_db, waterdb=watermeter_db, flog=flog) #print ( serial_buffer ) #parseSerBuffer( data_set=base_p1_data, status=p1_status, phase_db_rec=phase_db_record ) p1_port_shared_lib.write_p1_telegram_to_ram( buffer=serial_buffer, flog=flog) p1_port_shared_lib.parse_serial_buffer( serialbuffer=serial_buffer, data_set=base_p1_data, status=p1_status, phase_db_rec=phase_db_record, flog=flog) update_data_set(data_set=base_p1_data, status=p1_status, phase_db_record=phase_db_record, flog=flog) #clean the buffer del serial_buffer[:] p1_port_shared_lib.clear_data_buffer(buffer=base_p1_data) else: if processing_speed['max_processing_speed'] == True: if DUMMY_1SEC_PROCCESSING == True: if stub_serial_buffer != []: # print ( "serial_buffer = " + str(stub_serial_buffer) ) serial_buffer = stub_serial_buffer.copy() json_data[apiconst.JSON_API_RM_TMPRTR_IN] ,json_data[apiconst.JSON_API_RM_TMPRTR_OUT] =\ p1_port_shared_lib.current_room_temperature( temperaturedb=temperature_db, flog=flog ) json_data[apiconst.JSON_API_WM_CNSMPTN_LTR_M3] =\ p1_port_shared_lib.current_watermeter_count( configdb=config_db, waterdb=watermeter_db, flog=flog) #parseSerBuffer( data_set=base_p1_data, status=p1_status, phase_db_rec=phase_db_record ) p1_port_shared_lib.write_p1_telegram_to_ram( buffer=serial_buffer, flog=flog) p1_port_shared_lib.parse_serial_buffer( serialbuffer=serial_buffer, data_set=base_p1_data, status=p1_status, phase_db_rec=phase_db_record, flog=flog) update_data_set(data_set=base_p1_data, status=p1_status, phase_db_record=phase_db_record, flog=flog) time.sleep(0.1) serCheckCnt = serCheckCnt + 1 if serCheckCnt > 300: #check updates every 30 seconds. # perodic check and change of fqdn. # set intial FQDN p1_port_shared_lib.fqdn_from_config(verbose=False, configdb=config_db, data_set=json_data, flog=flog) p1_port_shared_lib.backup_data_to_disk_by_timestamp( statusdb=rt_status_db, flog=flog) #print "serial interval check" ############################################################################################# # changed in version > 1.3.1 because of the possible high frequency of updates. use less # # often to remove stress on the database. # ############################################################################################# p1_port_shared_lib.delete_serial_records( p1_processing=processing_speed, serial_db=e_db_serial, flog=flog) p1_port_shared_lib.delete_phase_record( p1_processing=processing_speed, phase_db=fase_db, flog=flog) p1_port_shared_lib.set_p1_processing_speed( p1_processing=processing_speed, config_db=config_db, flog=flog) check_serial_db_config_settings() p1_port_shared_lib.get_country_day_night_mode( status=p1_status, configdb=config_db, flog=flog) p1_port_shared_lib.get_gas_telgram_prefix( status=p1_status, configdb=config_db, flog=flog) #checkCRCsettings() p1_port_shared_lib.get_P1_crc(status=p1_status, configdb=config_db, flog=flog) serCheckCnt = 0 if abs(p1_status['timestamp_last_insert'] - util.getUtcTime()) > 51: if p1_status['p1_record_is_ok'] == 1: flog.warning(inspect.stack()[0][3] + ": geen P1 record te lezen.") p1_status['p1_record_is_ok'] = 0 rt_status_db.strset(str(p1_status['p1_record_is_ok']), 46, flog) # P1 data is not ok recieved # print abs(last_crc_check_timestamp - getUtcTime()) , " cnt=", crc_error_cnt # crc error messages, if any. if p1_status['p1_crc_check_is_on'] == True: if abs( p1_status['last_crc_check_timestamp'] - util.getUtcTime() ) > 900: #check every 15 minutes, to limit log entries. p1_status[ 'last_crc_check_timestamp'] = util.getUtcTime() if p1_status['crc_error_cnt'] > 0: flog.warning( "aantal P1 telegram crc fouten gevonden in de afgelopen minuut = " + str(p1_status['crc_error_cnt'])) p1_status['crc_error_cnt'] = 0 except Exception as e: flog.warning(inspect.stack()[0][3] + ": fout bij het wachten op seriele gegevens. Error=" + str(e.args)) check_serial() time.sleep(1)
ser1.bytesize = 8 ser1.parity = "N" ser1.stopbits = 1 ser1.xonxoff = 0 # changed from version 0.9.2 onwards ser1.rtscts = 0 ser1.timeout = 1 ser1.port = ser_devices_list[0] base_p1_data = data_struct_lib.p1_data_base_record phase_db_record = data_struct_lib.phase_db_record json_data = data_struct_lib.json_basic processing_speed = data_struct_lib.p1_processing_speed p1_status = data_struct_lib.p1_status_record p1_status['timestamp_last_insert'] = util.getUtcTime() p1_status['day_night_mode'] = 0 # default NL, 1 is Belgium day_values = day_values_lib.dayMinMaxkW() def main_prod(): global ser1 global serial_buffer global processing_speed stub_serial_buffer = [] # for stub testing only p1_port_shared_lib.clear_data_buffer(buffer=phase_db_record)
def Main(argv): flog.info("Start van programma.") global mqtt_para, mqtt_client # makes link between ramdisk and www/json folder if the link not exits # prevents errors in console.log when MQTT is not active. make_json_topic_link() # open van config database try: config_db.init(const.FILE_DB_CONFIG, const.DB_CONFIG_TAB) except Exception as e: flog.critical(inspect.stack()[0][3] + ": database niet te openen(2)." + const.FILE_DB_CONFIG + ") melding:" + str(e.args[0])) sys.exit(1) flog.info(inspect.stack()[0][3] + ": database tabel " + const.DB_CONFIG_TAB + " succesvol geopend.") runCheck() # open van status database try: rt_status_db.init(const.FILE_DB_STATUS, const.DB_STATUS_TAB) except Exception as e: flog.critical(inspect.stack()[0][3] + ": Database niet te openen(1)." + const.FILE_DB_STATUS + ") melding:" + str(e.args[0])) sys.exit(1) flog.info(inspect.stack()[0][3] + ": database tabel " + const.DB_STATUS_TAB + " succesvol geopend.") # open van seriele database try: e_db_serial.init(const.FILE_DB_E_FILENAME, const.DB_SERIAL_TAB) except Exception as e: flog.critical(inspect.stack()[0][3] + " database niet te openen(3)." + const.FILE_DB_E_FILENAME + ") melding:" + str(e.args[0])) sys.exit(1) flog.info(inspect.stack()[0][3] + ": database tabel " + const.DB_SERIAL_TAB + " succesvol geopend.") # open van watermeter database try: watermeter_db.init(const.FILE_DB_WATERMETERV2, const.DB_WATERMETERV2_TAB, flog) except Exception as e: flog.critical(inspect.stack()[0][3] + ": Database niet te openen(3)." + const.FILE_DB_WATERMETERV2 + " melding:" + str(e.args[0])) sys.exit(1) flog.info(inspect.stack()[0][3] + ": database tabel " + const.DB_WATERMETERV2_TAB + " succesvol geopend.") # open van weer database voor huidige weer try: weer_db.init(const.FILE_DB_WEATHER, const.DB_WEATHER_TAB) except Exception as e: flog.critical(inspect.stack()[0][3] + ": database niet te openen(3)." + const.DB_WEATHER_TAB + ") melding:" + str(e.args[0])) sys.exit(1) flog.debug(inspect.stack()[0][3] + ": database tabel " + const.DB_WEATHER_TAB + " succesvol geopend.") # open van temperatuur database try: temperature_db.init(const.FILE_DB_TEMPERATUUR_FILENAME, const.DB_TEMPERATUUR_TAB) except Exception as e: flog.critical(inspect.stack()[0][3] + ": Database niet te openen(1)." + const.FILE_DB_TEMPERATUUR_FILENAME + ") melding:" + str(e.args[0])) sys.exit(1) flog.info(inspect.stack()[0][3] + ": database tabel " + const.DB_TEMPERATUUR_TAB + " succesvol geopend.") # open van power production database try: power_production_db.init(const.FILE_DB_POWERPRODUCTION, const.DB_POWERPRODUCTION_TAB, flog) except Exception as e: flog.critical(inspect.stack()[0][3] + ": Database niet te openen(3)." + const.FILE_DB_POWERPRODUCTION + " melding:" + str(e.args[0])) sys.exit(1) flog.info(inspect.stack()[0][3] + ": database tabel " + const.DB_POWERPRODUCTION_TAB + " succesvol geopend.") # set proces gestart timestamp rt_status_db.timestamp(95, flog) checkActiveState() setConfigFromDb() makeTopicJsonFile() mqtt_para[ 'brokerconnectionstatustext'] = "probeer met de broker een connectie op te bouwen" rt_status_db.strset(mqtt_para['brokerconnectionstatustext'], 97, flog) # INIT connect to the broker while True: runCheck() #flog.setLevel( logging.DEBUG ) if minimalBrokerSettingsAvailable() == False: time.sleep(60) # wait on valid setting, quitely. setConfigFromDb() continue #flog.setLevel( logging.INFO ) if initMttq() == False: flog.info( inspect.stack()[0][3] + ": connectie met broker gefaald, wacht " + \ str(int(mqtt_para['reconnecttimeoute'])) + " seconden voor een nieuwe poging. " ) time.sleep(int(mqtt_para['reconnecttimeoute'])) setConfigFromDb() else: flog.info(inspect.stack()[0][3] + ": initiele connectie met broker gereed.") break while True: if mqtt_para['brokerconnectionisok'] == False: # try to reconnect #flog.setLevel( logging.DEBUG ) if minimalBrokerSettingsAvailable() == False: time.sleep(60) # wait on valid setting, quitely. setConfigFromDb() continue #flog.setLevel( logging.INFO ) if (getUtcTime() % int(mqtt_para['reconnecttimeoute'])) == 0: if mqtt_client != None: try: flog.warning(inspect.stack()[0][3] + ": reconnecting()") mqtt_client.reconnect() except Exception as e: flog.error( inspect.stack()[0][3] + " reconnectie probleem ->" + str( e ) + " broker host = " + \ str(mqtt_para['brokerhost']) + " broker port = " + str(mqtt_para['brokerport']) ) try: _id, parameter, _label = config_db.strget(118, flog) flog.debug(inspect.stack()[0][3] + ": DB configuratie flag = " + str(parameter)) if int(parameter) == 1: # do the changes. flog.info(inspect.stack()[0][3] + ": MQTT configuratie wordt aangepast.") setConfigFromDb() closeMqtt() initMttq() checkActiveState( ) # controleer de DB op welke publish aan of uit staat makeTopicJsonFile() config_db.strset('0', 118, flog) # reset config flag. flog.info(inspect.stack()[0][3] + ": MQTT configuratie is aangepast.") except Exception as e: flog.warning(inspect.stack()[0][3] + ": DB configuratie flag probleem -> " + str(e.args[0])) if mqtt_para['brokerconnectionisok'] == True: # smart meter processing try: #_id, parameter, _label = config_db.strget( 114, flog ) if mqtt_para['smartmeterpublishisactive'] == True: #is active _id, timestamp, _label, _security = rt_status_db.strget( 16, flog) if (mqtt_para['smartmeterprocessedtimestamp'] ) != timestamp: getPayloadFromDB(mqtt_payload_smartmeter, e_db_serial) if len(mqtt_payload_smartmeter[0] ) > 0: # only send when we have data mqttPublish(mqtt_client, mqtt_topics_smartmeter, mqtt_payload_smartmeter) mqtt_para[ 'smartmeterprocessedtimestamp'] = timestamp except Exception as e: flog.warning( inspect.stack()[0][3] + ": onverwachte fout bij smartmeter publish van melding:" + str(e)) ####################################################################### # Important to determine if we have to send data the timestamp when # # data is updated must be set in the status database by the process # # responsible for the processing. # # check the index in the lines like this # # _id, timestamp, _label, _security = rt_status_db.strget( xx, flog ) # ####################################################################### # watermeter proceessing try: #_id, parameter, _label = config_db.strget( 115, flog ) if mqtt_para['watermeterpublishisactive'] == True: #is active _id, timestamp, _label, _security = rt_status_db.strget( 90, flog) if (mqtt_para['watermeterprocessedtimestamp'] ) != timestamp: getPayloadFromDB(mqtt_payload_watermeter, watermeter_db) if len(mqtt_payload_watermeter[0] ) > 0: # only send when when we have data mqttPublish(mqtt_client, mqtt_topics_watermeter, mqtt_payload_watermeter) mqtt_para[ 'watermeterprocessedtimestamp'] = timestamp except Exception as e: flog.warning( inspect.stack()[0][3] + ": onverwachte fout bij watermeter publish van melding:" + str(e)) # weather processing try: #_id, parameter, _label = config_db.strget( 116, flog ) if mqtt_para['weatherpublishisactive'] == True: #is active _id, timestamp, _label, _security = rt_status_db.strget( 45, flog) if (mqtt_para['weatherprocessedtimestamp']) != timestamp: getPayloadFromDB(mqtt_payload_weather, weer_db) if len(mqtt_payload_weather[0] ) > 0: # only send when we have data mqttPublish(mqtt_client, mqtt_topics_weather, mqtt_payload_weather) mqtt_para['weatherprocessedtimestamp'] = timestamp except Exception as e: flog.warning( inspect.stack()[0][3] + ": onverwachte fout bij weer publish van melding:" + str(e)) # indoor temperature processing try: #_id, parameter, _label = config_db.strget( 117, flog ) if mqtt_para[ 'indoortemperaturepublishisactive'] == True: #is active _id, timestamp, _label, _security = rt_status_db.strget( 58, flog) if (mqtt_para['indoortemperatureprocessedtimestamp'] ) != timestamp: getPayloadFromDB(mqtt_payload_indoor_temperature, temperature_db) if len(mqtt_payload_indoor_temperature[0] ) > 0: # only send when we have data mqttPublish(mqtt_client, mqtt_topics_indoor_temperature, mqtt_payload_indoor_temperature) mqtt_para[ 'indoortemperatureprocessedtimestamp'] = timestamp except Exception as e: flog.warning( inspect.stack()[0][3] + ": onverwachte fout bij binnen temperatuur publish van melding:" + str(e)) # powerproduction processing try: if mqtt_para[ 'powerproductionpublishisactive'] == True: #is active _id, timestamp, _label, _security = rt_status_db.strget( 109, flog) if (mqtt_para['powerproductionprocessedtimestamp'] ) != timestamp: getPayloadFromDB(mqtt_payload_powerproduction, power_production_db) if len(mqtt_payload_powerproduction[0] ) > 0: # only send when we have data mqttPublish(mqtt_client, mqtt_topics_powerproduction, mqtt_payload_powerproduction) mqtt_para[ 'powerproductionprocessedtimestamp'] = timestamp except Exception as e: flog.warning( inspect.stack()[0][3] + ": onverwachte fout bij opgewekte energie publish van melding:" + str(e)) # phase processing try: #_id, parameter, _label = config_db.strget( 117, flog ) if mqtt_para['phasepublishisactive'] == True: #is active _id, timestamp, _label, _security = rt_status_db.strget( 106, flog) if (mqtt_para['phaseprocessedtimestamp']) != timestamp: getPhasePayloadFromDB(mqtt_payload_phase) if len(mqtt_topics_phase[0] ) > 0: # only send when we have data mqttPublish(mqtt_client, mqtt_topics_phase, mqtt_payload_phase) mqtt_para['phaseprocessedtimestamp'] = timestamp except Exception as e: flog.warning( inspect.stack()[0][3] + ": onverwachte fout bij binnen temperatuur publish van melding:" + str(e)) flog.debug(inspect.stack()[0][3] + ": sleeping... mqtt_para['brokerconnectionisok'] = " + str(mqtt_para['brokerconnectionisok'])) checkActiveState( ) # controleer de DB op welke publish aan of uit staat if mqtt_para[ 'anypublishisactive'] == False: # ga in langzame modes als alle publish uit staat. runCheck() time.sleep(30) #slow poll else: runCheck() time.sleep(2)
def Main(argv): flog.info("Start van programma.") last_seq_id = -1 DiskRestore() # open van config database try: config_db.init(const.FILE_DB_CONFIG,const.DB_CONFIG_TAB) except Exception as e: flog.critical(inspect.stack()[0][3]+": database niet te openen(3)."+const.FILE_DB_CONFIG+") melding:"+str(e.args[0])) sys.exit(1) flog.info(inspect.stack()[0][3]+": database tabel "+const.DB_CONFIG_TAB+" succesvol geopend.") # open van status database try: rt_status_db.init(const.FILE_DB_STATUS,const.DB_STATUS_TAB) except Exception as e: flog.critical(inspect.stack()[0][3]+": Database niet te openen(1)."+const.FILE_DB_STATUS+") melding:"+str(e.args[0])) sys.exit(1) flog.info(inspect.stack()[0][3]+": database tabel "+const.DB_STATUS_TAB+" succesvol geopend.") # open van temperatuur database try: temperature_db.init(const.FILE_DB_TEMPERATUUR_FILENAME ,const.DB_TEMPERATUUR_TAB ) temperature_db.cleanDb(flog) temperature_db.defrag() except Exception as e: flog.critical(inspect.stack()[0][3]+": Database niet te openen(1)."+const.FILE_DB_TEMPERATUUR_FILENAME+") melding:"+str(e.args[0])) sys.exit(1) flog.info(inspect.stack()[0][3]+": database tabel "+const.DB_TEMPERATUUR_TAB +" succesvol geopend.") # set start timestamp in status database rt_status_db.timestamp(56,flog) try: # Bind the socket to the host and port udpsocket.bind((HOST, PORT)) except Exception as e: flog.error(inspect.stack()[0][3]+": UDP ontvanger niet te starten , gestopt -> " + str(e) ) sys.exit(1) # one time message for log if prgIsActive(flog) == False: flog.info(inspect.stack()[0][3]+": programma is niet als actief geconfigureerd , wordt niet uitgevoerd.") # fix problem with primary key. temperature_db.change_table( flog ) temperature_db.fix_missing_month_day( flog ) # main loop blocked recieving of UDP packages / messages while True: # check if we are using this program (uses config db!) while prgIsActive(flog) == False: continue # backup data to flash every ~15 min. if abs( timestamp_last_backup - getUtcTime() ) > 900: backupData() # Receive BUFFER_SIZE bytes data data = udpsocket.recvfrom(BUFFER_SIZE) if data: #print received data flog.debug(inspect.stack()[1][3]+": udp message = "+ datetime.fromtimestamp(time()).strftime('%H:%M:%S') +' ' + str(data[0])) # process json try: #print ( data ) jsondata = json.loads( data[0].decode('utf-8') ) # record id index codes # secs = 10 # min. = 11 # hour = 12 # days = 13 # months = 14 # years = 15 timestamp = datetime.fromtimestamp(time()).strftime('%Y-%m-%d %H:%M:%S') #timestamp = "2020-09-01 00:01:02" # process secs values, data every 5 secs or so. if jsondata['id'] != 'ztatz_dt': flog.warning(inspect.stack()[0][3]+": id van json data is niet correct :"+jsondata['id'] ) else: if last_seq_id == jsondata['seq']: flog.debug(inspect.stack()[0][3]+": dubbele volgorde id gevonden, wordt genegeerd. seq id=:"+str(jsondata['seq']) ) continue last_seq_id = jsondata['seq'] temperature_db.replace(timestamp,\ 10, \ jsondata['t_in'], \ jsondata['t_in_avg'],\ 0,\ 0,\ jsondata['t_out'], \ jsondata['t_out_avg'],\ 0,\ 0,\ flog ) # process minutes values. # returns avg, min, max(in), avg, min, max(out) temp_list = temperature_db.selectAMM( timestamp[:16],10,flog ) temperature_db.replace(timestamp[:16]+":00",\ 11, \ 0, \ temp_list[0][0],\ temp_list[0][0],\ temp_list[0][0],\ 0,\ temp_list[0][3],\ temp_list[0][3],\ temp_list[0][3],\ flog ) # process hour values. # returns avg, min, max(in), avg, min, max(out) temp_list = temperature_db.selectAMM( timestamp[:13],11,flog ) temperature_db.replace(timestamp[:13]+":00:00",\ 12, \ 0, \ temp_list[0][0],\ temp_list[0][1],\ temp_list[0][2],\ 0,\ temp_list[0][3],\ temp_list[0][4],\ temp_list[0][5],\ flog ) #flog.setLevel( logging.DEBUG ) # process day values. # returns avg, min, max(in), avg, min, max(out) temp_list = temperature_db.selectAMM( timestamp[:10],12,flog ) temperature_db.replace(timestamp[:10]+" 00:00:00",\ 13, \ 0, \ temp_list[0][0],\ temp_list[0][1],\ temp_list[0][2],\ 0,\ temp_list[0][3],\ temp_list[0][4],\ temp_list[0][5],\ flog ) # process month values. # returns avg, min, max(in), avg, min, max(out) temp_list = temperature_db.selectAMM( timestamp[:7],13,flog ) temperature_db.replace(timestamp[:7]+"-01 00:00:00",\ 14, \ 0, \ temp_list[0][0],\ temp_list[0][1],\ temp_list[0][2],\ 0,\ temp_list[0][3],\ temp_list[0][4],\ temp_list[0][5],\ flog ) #flog.setLevel( logging.INFO ) # process year values. # returns avg, min, max(in), avg, min, max(out) temp_list = temperature_db.selectAMM( timestamp[:4],14,flog ) temperature_db.replace(timestamp[:4]+"-01-01 00:00:00",\ 15, \ 0, \ temp_list[0][0],\ temp_list[0][1],\ temp_list[0][2],\ 0,\ temp_list[0][3],\ temp_list[0][4],\ temp_list[0][5],\ flog ) # set process timestamp in status database rt_status_db.timestamp(58,flog) #clean_database temperature_db.cleanDb(flog) except Exception as e: flog.error(inspect.stack()[0][3]+": json fout melding:"+str(e.args) )
prgname = 'P1UdpDaemon' # bind all IP HOST = '0.0.0.0' # Listen on Port PORT = 30721 #Size of receive buffer BUFFER_SIZE = 1024 # Create a UDP/IP socket udpsocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) temperature_db = temperatureDB() rt_status_db = rtStatusDb() config_db = configDB() timestamp_last_backup = getUtcTime() def Main(argv): flog.info("Start van programma.") last_seq_id = -1 DiskRestore() # open van config database try: config_db.init(const.FILE_DB_CONFIG,const.DB_CONFIG_TAB) except Exception as e: flog.critical(inspect.stack()[0][3]+": database niet te openen(3)."+const.FILE_DB_CONFIG+") melding:"+str(e.args[0])) sys.exit(1) flog.info(inspect.stack()[0][3]+": database tabel "+const.DB_CONFIG_TAB+" succesvol geopend.")
def Main(argv): flog.info("Start van programma.") parser = argparse.ArgumentParser(description="options: -fb") parser.add_argument('-fb', '--forcebackup', required=False, action="store_true") args = parser.parse_args() # open van status database try: rt_status_db.init(const.FILE_DB_STATUS, const.DB_STATUS_TAB) except Exception as e: flog.critical(inspect.stack()[0][3] + ": Database niet te openen(1)." + const.FILE_DB_STATUS + ") melding:" + str(e.args[0])) sys.exit(1) flog.debug(inspect.stack()[0][3] + ": database tabel " + const.DB_STATUS_TAB + " succesvol geopend.") # open van config database try: config_db.init(const.FILE_DB_CONFIG, const.DB_CONFIG_TAB) except Exception as e: flog.critical(inspect.stack()[0][3] + ": database niet te openen(2)." + const.FILE_DB_CONFIG + ") melding:" + str(e.args[0])) sys.exit(2) flog.debug(inspect.stack()[0][3] + ": database tabel " + const.DB_CONFIG_TAB + " succesvol geopend.") _id, do_ftp_backup, _label = config_db.strget(36, flog) _id, do_dbx_backup, _label = config_db.strget(49, flog) parameter = 0 # do some casting....comparing strings is a pain do_ftp_backup = int(do_ftp_backup) do_dbx_backup = int(do_dbx_backup) flog.debug(inspect.stack()[0][3] + ": ftp backup=" + str(do_ftp_backup) + " Dropbox back-up=" + str(do_dbx_backup)) # if one of the backup options is on make an export file if do_ftp_backup == 1 or do_dbx_backup == 1: parameter = 1 else: flog.info(inspect.stack()[0][3] + ": Backup staat uit, geen back-up gestart") if args.forcebackup == True: # force backup, forget preference. flog.info(inspect.stack()[0][3] + ": geforceerde back-up gestart.") parameter = 1 if int(parameter) == 1: flog.debug(inspect.stack()[0][3] + ": export file wordt gemaakt.") file_tmp_id = str(util.getUtcTime()) + "-" + systemid.getSystemId() cmd = "/p1mon/scripts/P1SqlExport.py -e " + file_tmp_id flog.debug(inspect.stack()[0][3] + ": export commando is ->" + str(cmd)) retvalue = os.system("/p1mon/scripts/P1SqlExport.py -e " + file_tmp_id) if retvalue != 0: flog.error(inspect.stack()[0][3] + ": export van file gefaald, gestopt.") sys.exit(3) export_file = exportfile_base + file_tmp_id + '.zip' if util.fileExist(export_file) == False: flog.error(inspect.stack()[0][3] + ": export file " + export_file + " niet gevonden, gestopt.") sys.exit(4) # update config database with current name of backup file. config_db.strset(export_file, 33, flog) if do_dbx_backup == 1: # do dropbox back-up flog.info(inspect.stack()[0][3] + ": Dropbox backup gestart") try: # try to clean old files clean_old_files() if len(os.listdir(const.DIR_DBX_LOCAL + const.DBX_DIR_BACKUP)) > 10: flog.critical( inspect.stack()[0][3] + ": Dropbox backup bestand niet gekopierd, te veel bestanden in ram buffer." ) else: flog.debug(inspect.stack()[0][3] + ": copy export file naar lokale dropbox folder: " + export_file) shutil.copy2(export_file, const.DIR_DBX_LOCAL + const.DBX_DIR_BACKUP) _head, tail = os.path.split(export_file) util.setFile2user( const.DIR_DBX_LOCAL + const.DBX_DIR_BACKUP + '/' + tail, 'p1mon') except Exception as e: flog.error(inspect.stack()[0][3] + ": Dropbox back-up. melding:" + str(e.args[0])) if do_ftp_backup == 1: flog.info(inspect.stack()[0][3] + ": FTP backup gestart") retvalue = os.system("/p1mon/scripts/P1FtpCopy.py") if retvalue > 0: flog.error(inspect.stack()[0][3] + ": ftp backup gefaald, gestopt.") sys.exit(5) flog.info("programma is succesvol gestopt.") sys.exit(0) # all is well.
def powerSwitcher(): global powerswitcher_active, powerswitcher_last_action_utc_timestamp, powerswitcher_forced_on try: #force powerswitcher force on (1) or (0) automatic. #powerswitcher_forced_on_read = 0 _id, powerswitcher_forced_on_read, _label = config_db.strget( 87, flog ) if int( powerswitcher_forced_on_read ) == 1: if powerswitcher_forced_on == 0: flog.info( inspect.stack()[0][3] + ": powerswitcher geforceerd aangezet." ) powerswitcher_forced_on = 1 rt_status_db.strset( 0, 83 , flog ) rt_status_db.timestamp( 84, flog ) gpioPowerSwitcher.gpioOn( True ) flog.debug( inspect.stack()[0][3] + ": powerswitcher geforceerd aangezet." ) return # no other actions needed or possible. else: if powerswitcher_forced_on == 1: flog.info( inspect.stack()[0][3] + ": powerswitcher geforceerd uitgezet." ) powerswitcher_forced_on = 0 gpioPowerSwitcher.gpioOn( False ) rt_status_db.timestamp( 84, flog ) rt_status_db.strset( 0, 83 , flog ) #power switcher is active. _id, powerswitcher_is_on, _label = config_db.strget( 86, flog ) if int(powerswitcher_is_on) == 0: flog.debug( inspect.stack()[0][3] + ": powerswitcher staat uit." ) gpioPowerSwitcher.gpioOn( False ) # make sure we switch off the load when not active. powerswitcher_active = False powerswitcher_last_action_utc_timestamp = 0 rt_status_db.strset( 0, 83 , flog ) return #default values. as failsave. on_threshold_watt = 1000 off_threshold_watt = 500 on_threshold_minutes = 2 off_threshold_minutes = 2 on_minimum_time = 0 off_minimum_time = 0 # read config information. _id, on_threshold_watt, _label = config_db.strget( 81,flog ) _id, off_threshold_watt, _label = config_db.strget( 82,flog ) _id, on_threshold_minutes, _label = config_db.strget( 83,flog ) _id, off_threshold_minutes, _label = config_db.strget( 84,flog ) _id, on_minimum_time, _label = config_db.strget( 88,flog ) _id, off_minimum_time, _label = config_db.strget( 89,flog ) flog.debug( inspect.stack()[0][3]+": on_threshold_watt=" + str(on_threshold_watt) + ' off_threshold_watt=' + str(off_threshold_watt) + " on_threshold_minutes=" + str(on_threshold_minutes) + " off_threshold_minutes=" + str(off_threshold_minutes) + " on_minimum_time=" + str(on_minimum_time) + " off_minimum_time=" + str(off_minimum_time) ) #check for on hold time if (powerswitcher_active == True) and ( powerswitcher_last_action_utc_timestamp + ( int(on_minimum_time) * 60 ) > util.getUtcTime() ): flog.debug( inspect.stack()[0][3]+": aan minimum tijd is nog geldig, geen actie, output blijft aan.") return if (powerswitcher_active == False ) and ( powerswitcher_last_action_utc_timestamp + ( int(off_minimum_time) * 60 ) > util.getUtcTime() ): flog.debug( inspect.stack()[0][3]+": uit minimum tijd is nog geldig, geen actie, output blijft uit.") return if powerswitcher_active == False: watt_producing = powerSwitcherSql( on_threshold_minutes ) #watt_producing = 600 if watt_producing == None: return if watt_producing >= int( on_threshold_watt ): powerswitcher_active = True gpioPowerSwitcher.gpioOn( True ) powerswitcher_last_action_utc_timestamp = util.getUtcTime() rt_status_db.strset( watt_producing, 83, flog ) # watt value uses to activate the switcher rt_status_db.timestamp( 84, flog ) flog.info( inspect.stack()[0][3]+": Ingeschakeld op een vermogen van " + str(watt_producing) + " watt." ) else: watt_producing = powerSwitcherSql( off_threshold_minutes ) if watt_producing == None: return if watt_producing <= int( off_threshold_watt ): powerswitcher_active = False gpioPowerSwitcher.gpioOn( False ) powerswitcher_last_action_utc_timestamp = util.getUtcTime() rt_status_db.strset( 0, 83 , flog ) rt_status_db.timestamp( 84, flog ) flog.info( inspect.stack()[0][3]+": Uitgeschakeld op een vermogen van " + str(watt_producing) + " watt." ) except Exception as e: flog.error( inspect.stack()[0][3]+": onverwachte fout " + str(e) )
def Main(argv): flog.info("Start van programma.") global ftp_para # open van status database try: rt_status_db.init(const.FILE_DB_STATUS, const.DB_STATUS_TAB) except Exception as e: flog.critical(inspect.stack()[0][3] + ": Database niet te openen(1)." + const.FILE_DB_STATUS + ") melding:" + str(e.args[0])) sys.exit(1) flog.debug(inspect.stack()[0][3] + ": database tabel " + const.DB_STATUS_TAB + " succesvol geopend.") # open van config database try: config_db.init(const.FILE_DB_CONFIG, const.DB_CONFIG_TAB) except Exception as e: flog.critical(inspect.stack()[0][3] + ": database niet te openen(2)." + const.FILE_DB_CONFIG + ") melding:" + str(e.args[0])) sys.exit(1) flog.debug(inspect.stack()[0][3] + ": database tabel " + const.DB_CONFIG_TAB + " succesvol geopend.") rt_status_db.timestamp(47, flog) # update field from database, the cli switches overwrite the DB values! _id, ftp_para['user'], _label = config_db.strget(28, flog) _id, ftp_para['password'], _label = config_db.strget(29, flog) _id, ftp_para['directory'], _label = config_db.strget(30, flog) _id, ftp_para['server'], _label = config_db.strget(31, flog) _id, ftp_para['port'], _label = config_db.strget(32, flog) _id, ftp_para['filename'], _label = config_db.strget(33, flog) _id, ftp_para['maxfilecount'], _label = config_db.strget(34, flog) _id, ftp_para['ftps'], _label = config_db.strget(35, flog) # was secure _id, ftp_para['ftp'], _label = config_db.strget(76, flog) _id, ftp_para['sftp'], _label = config_db.strget(77, flog) # sftp toevoegen. ftp toevoegen flog.debug(inspect.stack()[0][3] + ": parameters uit de DB:" + str(ftp_para)) parser = argparse.ArgumentParser(description="ftp....") parser.add_argument('-u', '--user', required=False) parser.add_argument('-pw', '--password', required=False) parser.add_argument('-dir', '--directory', required=False) parser.add_argument('-srv', '--server', required=False) parser.add_argument('-fname', '--filename', required=False) parser.add_argument('-mfcnt', '--maxfilecount', required=False) parser.add_argument('-pt', '--port', required=False) parser.add_argument('-ftps', '--ftps', required=False, action="store_true") # flag only parser.add_argument('-sftp', '--sftp', required=False, action="store_true") # flag only parser.add_argument('-ftp', '--ftp', required=False, action="store_true") # flag only args = parser.parse_args() if args.user != None: ftp_para['user'] = args.user if args.password != None: ftp_para['password'] = args.password flog.debug("password van commandline ontvangen -> " + ftp_para['password']) else: #decode password from database try: ftp_para['password'] = base64.standard_b64decode( crypto3.p1Decrypt(ftp_para['password'], 'ftppw')).decode('utf-8') # added by Aad if ftp_para['password'] == '': ftp_para['password'] = "******" raise Exception(" wachtwoord is niet ingesteld.") except Exception as e: flog.error(inspect.stack()[0][3]+": password decodering gefaald. Decoded password="******" Gestopt. melding:" + str(e.args[0]) ) sys.exit(16) flog.info("Password decryptie ok.") flog.debug("Decoded password = "******"''" raise Exception( "server adres (IP adres of url) is niet ingesteld.") except Exception as e: flog.error(inspect.stack()[0][3] + ": Gestopt, melding: " + str(e.args[0])) sys.exit(17) if args.filename != None: ftp_para['filename'] = args.filename if args.maxfilecount != None: ftp_para['maxfilecount'] = int(args.maxfilecount) if args.port != None: ftp_para['port'] = int(args.port) if args.ftp == True: ftp_para['ftp'] = 1 ftp_para['ftps'] = 0 ftp_para['sftp'] = 0 if args.ftps == True: ftp_para['ftp'] = 0 ftp_para['ftps'] = 1 ftp_para['sftp'] = 0 if args.sftp == True: ftp_para['ftp'] = 0 ftp_para['ftps'] = 0 ftp_para['sftp'] = 1 flog.debug(inspect.stack()[0][3] + ": parameters na CLI parsing:" + str(ftp_para)) if int(ftp_para['ftp']) == 0 and int(ftp_para['ftps']) == 0 and int( ftp_para['sftp']) == 0: flog.warning(inspect.stack()[0][3] + ": geen ftp, ftps of sftp opties geslecteerd!") if os.path.isfile(ftp_para['filename']) == False: rt_status_db.strset( 'fout: te kopieren bestand niet gevonden. Gestopt.', 48, flog) flog.error(inspect.stack()[0][3] + ": te kopieren bestand niet gevonden. Gestopt.") sys.exit(17) # do normal plain text FTP if int(ftp_para['ftp']) == 1: flog.info(inspect.stack()[0][3] + ": probeer bestand " + ftp_para['filename'] + " te kopieren via ftp.") try: ftpConnection = ftpConnect() if len(ftp_para['directory']) > 0: ftpChangeDirectory(ftpConnection, ftp_para['directory']) ftpCopy(ftpConnection, ftp_para['filename']) flog.info(inspect.stack()[0][3] + ": bestand " + ftp_para['filename'] + " succesvol gekopierd via ftp.") ftpConnection.quit() # be polite to ftp servers except Exception as e: rt_status_db.strset( 'fout: server antwoord: ' + str(e.args[0]) + ' Gestopt.', 48, flog) flog.error(inspect.stack()[0][3] + ": FTP server antwoord: " + str(e.args[0]) + " Gestopt.") sys.exit(11) rt_status_db.strset("FTP transfer is succesvol gestopt.", 48, flog) rt_status_db.timestamp(49, flog) flog.info("FTP transfer is succesvol gestopt.") sys.exit(0) # all is well. # TODO ftps en sftp aanpassen zodat ze poort meenemen. # do ftps FTP with SSL if int(ftp_para['ftps']) == 1: ################################################################# # STEP 1 copy the file # ################################################################# try: # copying file #flog.setLevel( logging.DEBUG ) _head, tail = os.path.split(ftp_para['filename']) changed_filename = '//p1mon/mnt/ramdisk/' + fileprefix + str( getUtcTime()) + '-' + tail #server = "ftps://"+ftp_para['server'] server = "ftp://" + ftp_para['server'] #+ ":990" if len(ftp_para['directory']) > 0: server = server + "/" + ftp_para[ 'directory'] + "/" # checked can handle // and / in path. flog.debug(inspect.stack()[0][3] + ": server path is=" + str(server)) shutil.copy(ftp_para['filename'], changed_filename) flog.info(inspect.stack()[0][3] + ": probeer bestand " + ftp_para['filename'] + " te kopieren via ftps.") """ cp = subprocess.run([ "curl", "--ssl-reqd", "--ftp-ssl-control", "-ssl", "--globoff", "--insecure", "-sSv", server, "--user", ftp_para['user']+":"+ftp_para['password'], "-T", changed_filename ], \ universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60 ) """ cp = subprocess.run([ "curl", "--ftp-ssl-control", "--ftp-ssl" ,"--globoff", "--insecure", "-sSv", server, "--user", ftp_para['user']+":"+ftp_para['password'], "-T", changed_filename ], \ universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60 ) flog.debug(inspect.stack()[0][3] + ": cp.stdout=" + str(cp.stdout)) flog.debug(inspect.stack()[0][3] + ": cp.stderr=" + str(cp.stderr)) flog.debug(inspect.stack()[0][3] + ": cp.returncode=" + str(cp.returncode)) # Added by Aad. saveDelete(changed_filename) # remove tmp copy of file. if cp.returncode == 0: flog.info(inspect.stack()[0][3] + ": bestand " + ftp_para['filename'] + " succesvol gekopierd via ftps als " + changed_filename) else: raise Exception(cp.stderr.replace("\n", "")) except Exception as e: rt_status_db.strset( 'fout: server antwoord: ' + str(e.args[0]) + ' Gestopt.', 48, flog) flog.error(inspect.stack()[0][3] + ": SFTP server antwoord: " + str(e.args[0]) + " Gestopt.") sys.exit(11) ################################################################# # STEP 2 checking if we have reached the maxium number of files # ################################################################# try: # checkin if max files are reached _head, tail = os.path.split(ftp_para['filename']) changed_filename = '//p1mon/mnt/ramdisk/' + fileprefix + str( getUtcTime()) + tail server = "ftp://" + ftp_para['server'] if len(ftp_para['directory']) > 0: server = server + "/" + ftp_para[ 'directory'] + "/" # checked can handle // and / in path. flog.debug(inspect.stack()[0][3] + ": server path is=" + str(server)) cp = subprocess.run([ "curl", "--ftp-ssl-control", "--ftp-ssl", "--globoff", "--insecure", "-sS", server, "--user", ftp_para['user']+":"+ftp_para['password'], "--list-only" ], \ universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60 ) #flog.debug( inspect.stack()[0][3]+": cp.stdout=" + str(cp.stdout) ) #flog.debug( inspect.stack()[0][3]+": cp.stderr=" + str(cp.stderr) ) #flog.debug( inspect.stack()[0][3]+": cp.returncode=" + str(cp.returncode ) ) if cp.returncode == 0: # process output, proces files based on epoch value in filename filtered_file_list = grep("\AP1BU-\d+", cp.stdout.split('\n')) if len(filtered_file_list) < int(ftp_para['maxfilecount']) - 1: flog.info(inspect.stack()[0][3] + ": maximale aantal van files " + str(ftp_para['maxfilecount']) + " niet gehaald (" + str(len(filtered_file_list)) + ")") else: flog.info(inspect.stack()[0][3] + ": maximale aantal van files overschreden " + str(ftp_para['maxfilecount'])) ftpsRemoveOldFiles(filtered_file_list) else: raise Exception(cp.stderr.replace("\n", "")) except Exception as e: rt_status_db.strset( 'fout: directory list server : ' + str(e.args[0]) + ' Gestopt.', 48, flog) flog.error(inspect.stack()[0][3] + ": SFTP directory list server fout: " + str(e.args[0]) + " Gestopt.") sys.exit(13) rt_status_db.strset("FTPS transfer is succesvol gestopt.", 48, flog) rt_status_db.timestamp(49, flog) flog.info("FTPS transfer is succesvol gestopt.") sys.exit(0) # all is well. # do sftp FTP with SSH if int(ftp_para['sftp']) == 1: ################################################################# # STEP 1 copy the file # ################################################################# try: # copying file _head, tail = os.path.split(ftp_para['filename']) tmp_path = '/var/log/p1monitor/' changed_filename = tmp_path + fileprefix + str( getUtcTime()) + '-' + tail server = "sftp://" + ftp_para['server'] + "/" if len(ftp_para['directory']) > 0: server = server + ftp_para[ 'directory'] + "/" # checked can handle // and / in path. else: server = server + "home/" flog.debug(inspect.stack()[0][3] + ": server path is=" + str(server)) #print ( "changed_filename " + changed_filename + " filename " + ftp_para['filename'] ) shutil.copy(ftp_para['filename'], changed_filename) flog.info(inspect.stack()[0][3] + ": probeer bestand " + ftp_para['filename'] + " te kopieren via sftp.") cp = subprocess.run([ "curl","--globoff", "--insecure", "-sSv", server, "--user", ftp_para['user']+":"+ftp_para['password'], "-T", changed_filename ], \ universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60 ) flog.debug(inspect.stack()[0][3] + ": cp.stdout=" + str(cp.stdout)) flog.debug(inspect.stack()[0][3] + ": cp.stderr=" + str(cp.stderr)) flog.debug(inspect.stack()[0][3] + ": cp.returncode=" + str(cp.returncode)) #Added by Aad. saveDelete(changed_filename) # remove tmp copy of file. if cp.returncode == 0: flog.info(inspect.stack()[0][3] + ": bestand " + ftp_para['filename'] + " succesvol gekopierd via ftps als " + changed_filename) else: raise Exception(cp.stderr.replace("\n", "")) except Exception as e: rt_status_db.strset( 'fout: server antwoord: ' + str(e.args[0]) + ' Gestopt.', 48, flog) flog.error(inspect.stack()[0][3] + ": SFTP server antwoord: " + str(e.args[0]) + " Gestopt.") sys.exit(11) ################################################################# # STEP 2 checking if we have reached the maxium number of files # ################################################################# try: # checkin if max files are reached #_head,tail = os.path.split( ftp_para['filename'] ) #changed_filename = '//p1mon/mnt/ramdisk/' + fileprefix + str(getUtcTime()) + '-' + tail server = "sftp://" + ftp_para['server'] + "/" if len(ftp_para['directory']) > 0: server = server + ftp_para[ 'directory'] + "/" # checked can handle // and / in path. else: server = server + "home/" flog.debug(inspect.stack()[0][3] + ": server path is=" + str(server)) # Modified by Aad: for clarity, conform curl params: --list-only parameter added. cp = subprocess.run([ "curl", "--globoff", "--insecure", "-sSv", server, "--user", ftp_para['user']+":"+ftp_para['password'], "--list-only" ], \ universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60 ) flog.debug(inspect.stack()[0][3] + ": cp.stdout=" + str(cp.stdout)) flog.debug(inspect.stack()[0][3] + ": cp.stderr=" + str(cp.stderr)) flog.debug(inspect.stack()[0][3] + ": cp.returncode=" + str(cp.returncode)) # pre-process output that could have a unspecified layout. filtered_file_list = [] matched_lines = [ line for line in cp.stdout.split('\n') if fileprefix in line ] for line in matched_lines: #filtered_file_list.append ( line[ line.find('P1BU-'): ] ) #modified by Aad: absolute find filter replaced by variable. filtered_file_list.append(line[line.find(fileprefix):]) if cp.returncode == 0: # process output, proces files based on epoch value in filename if len(filtered_file_list) < int(ftp_para['maxfilecount']) - 1: flog.info(inspect.stack()[0][3] + ": maximale aantal van files " + str(ftp_para['maxfilecount']) + " niet gehaald (" + str(len(filtered_file_list)) + ")") else: flog.info(inspect.stack()[0][3] + ": maximale aantal van files overschreden " + str(ftp_para['maxfilecount'])) # sftpRemoveOldFiles( filtered_file_list ) # orignal if sftpRemoveOldFiles(filtered_file_list) == False: flog.info(inspect.stack()[0][3] + " Een of meerdere bestanden >" + str(ftp_para['maxfilecount']) + " zijn niet gewist.") else: raise Exception(cp.stderr.replace("\n", "")) except Exception as e: rt_status_db.strset( 'fout: directory list server : ' + str(e.args[0]) + ' Gestopt.', 48, flog) flog.error(inspect.stack()[0][3] + ": SFTP directory list server fout: " + str(e.args[0]) + " Gestopt.") sys.exit(13) rt_status_db.strset("SFTP transfer is succesvol gestopt.", 48, flog) rt_status_db.timestamp(49, flog) flog.info("SFTP transfer is succesvol gestopt.") sys.exit(0) # all is well.