def update_json_data( jsondata=None, p1data=None ): #convert dutch tarif code to English Dal = Low, Peak = High if p1data['tarief_code'] == 'P': tarief_code = 'HIGH' else: tarief_code = "LOW" # there are only two options (D/P) jsondata[ apiconst.JSON_TS_LCL ] = str(util.mkLocalTimeString()) jsondata[ apiconst.JSON_TS_LCL_UTC ] = util.getUtcTime() jsondata[ apiconst.JSON_API_API_STTS ] = 'production' jsondata[ apiconst.JSON_API_CNSMPTN_KWH_L ] = float( p1data['verbrk_kwh_181'] ) jsondata[ apiconst.JSON_API_CNSMPTN_KWH_H ] = float( p1data['verbrk_kwh_182'] ) jsondata[ apiconst.JSON_API_PRDCTN_KWH_L ] = float( p1data['gelvr_kwh_281'] ) jsondata[ apiconst.JSON_API_PRDCTN_KWH_H ] = float( p1data['gelvr_kwh_282'] ) jsondata[ apiconst.JSON_API_TRFCD ] = str( tarief_code ) jsondata[ apiconst.JSON_API_CNSMPTN_KW ] = float( p1data['act_verbr_kw_170'] ) jsondata[ apiconst.JSON_API_PRDCTN_KW ] = float( p1data['act_gelvr_kw_270'] ) jsondata[ apiconst.JSON_API_CNSMPTN_GAS_M3 ] = float( p1data['gas_verbr_m3_2421'] ) jsondata[ apiconst.JSON_API_SYSTM_ID] = systemid.getSystemId()
import p1_telegram_test_lib import time import util temperature_db = sqldb.temperatureDB() e_db_serial = sqldb.SqlDb1() rt_status_db = sqldb.rtStatusDb() config_db = sqldb.configDB() watermeter_db = sqldb.WatermeterDBV2() fase_db = sqldb.PhaseDB() prgname = 'P1SerReader' serial_buffer = [] dev_dummy_gas_value = 0 timestamp_last_gas_update = 0 system_id = systemid.getSystemId() # list of serial devices tried to use ser_devices_list = ["/dev/ttyUSB0", "/dev/ttyUSB1"] ################################################################################### # let op deze optie geef veel foutmelding en in de log deze kunnen geen kwaad # ################################################################################### DUMMY_1SEC_PROCCESSING = False ######### DEZE OP FALSE ZETTEN BIJ PRODUCTIE CODE!!!! DUMMY_3PHASE_DATA = False ######### DEZE OP FALSE ZETTEN BIJ PRODUCTIE CODE!!!! # zet deze op p1_telegram_test_lib.NO_GAS_TEST om de test uit te zetten. #gas_test_mode=p1_telegram_test_lib.DUMMY_GAS_MODE_2421 gas_test_mode = p1_telegram_test_lib.NO_GAS_TEST #Set COM port config
def Main(argv): flog.info("Start van programma.") parser = argparse.ArgumentParser(description="options: -fb") parser.add_argument('-fb', '--forcebackup', required=False, action="store_true") args = parser.parse_args() # open van status database try: rt_status_db.init(const.FILE_DB_STATUS, const.DB_STATUS_TAB) except Exception as e: flog.critical(inspect.stack()[0][3] + ": Database niet te openen(1)." + const.FILE_DB_STATUS + ") melding:" + str(e.args[0])) sys.exit(1) flog.debug(inspect.stack()[0][3] + ": database tabel " + const.DB_STATUS_TAB + " succesvol geopend.") # open van config database try: config_db.init(const.FILE_DB_CONFIG, const.DB_CONFIG_TAB) except Exception as e: flog.critical(inspect.stack()[0][3] + ": database niet te openen(2)." + const.FILE_DB_CONFIG + ") melding:" + str(e.args[0])) sys.exit(2) flog.debug(inspect.stack()[0][3] + ": database tabel " + const.DB_CONFIG_TAB + " succesvol geopend.") _id, do_ftp_backup, _label = config_db.strget(36, flog) _id, do_dbx_backup, _label = config_db.strget(49, flog) parameter = 0 # do some casting....comparing strings is a pain do_ftp_backup = int(do_ftp_backup) do_dbx_backup = int(do_dbx_backup) flog.debug(inspect.stack()[0][3] + ": ftp backup=" + str(do_ftp_backup) + " Dropbox back-up=" + str(do_dbx_backup)) # if one of the backup options is on make an export file if do_ftp_backup == 1 or do_dbx_backup == 1: parameter = 1 else: flog.info(inspect.stack()[0][3] + ": Backup staat uit, geen back-up gestart") if args.forcebackup == True: # force backup, forget preference. flog.info(inspect.stack()[0][3] + ": geforceerde back-up gestart.") parameter = 1 if int(parameter) == 1: flog.debug(inspect.stack()[0][3] + ": export file wordt gemaakt.") file_tmp_id = str(util.getUtcTime()) + "-" + systemid.getSystemId() cmd = "/p1mon/scripts/P1SqlExport.py -e " + file_tmp_id flog.debug(inspect.stack()[0][3] + ": export commando is ->" + str(cmd)) retvalue = os.system("/p1mon/scripts/P1SqlExport.py -e " + file_tmp_id) if retvalue != 0: flog.error(inspect.stack()[0][3] + ": export van file gefaald, gestopt.") sys.exit(3) export_file = exportfile_base + file_tmp_id + '.zip' if util.fileExist(export_file) == False: flog.error(inspect.stack()[0][3] + ": export file " + export_file + " niet gevonden, gestopt.") sys.exit(4) # update config database with current name of backup file. config_db.strset(export_file, 33, flog) if do_dbx_backup == 1: # do dropbox back-up flog.info(inspect.stack()[0][3] + ": Dropbox backup gestart") try: # try to clean old files clean_old_files() if len(os.listdir(const.DIR_DBX_LOCAL + const.DBX_DIR_BACKUP)) > 10: flog.critical( inspect.stack()[0][3] + ": Dropbox backup bestand niet gekopierd, te veel bestanden in ram buffer." ) else: flog.debug(inspect.stack()[0][3] + ": copy export file naar lokale dropbox folder: " + export_file) shutil.copy2(export_file, const.DIR_DBX_LOCAL + const.DBX_DIR_BACKUP) _head, tail = os.path.split(export_file) util.setFile2user( const.DIR_DBX_LOCAL + const.DBX_DIR_BACKUP + '/' + tail, 'p1mon') except Exception as e: flog.error(inspect.stack()[0][3] + ": Dropbox back-up. melding:" + str(e.args[0])) if do_ftp_backup == 1: flog.info(inspect.stack()[0][3] + ": FTP backup gestart") retvalue = os.system("/p1mon/scripts/P1FtpCopy.py") if retvalue > 0: flog.error(inspect.stack()[0][3] + ": ftp backup gefaald, gestopt.") sys.exit(5) flog.info("programma is succesvol gestopt.") sys.exit(0) # all is well.
def Main(argv): global no_status_messages my_pid = os.getpid() flog.info("Start van programma met process id " + str(my_pid) ) pid_list, _process_list = listOfPidByName( prgname ) #print ( pid_list ) pid_list.remove( my_pid ) # remove own pid from the count flog.debug( inspect.stack()[0][3] + ": pid list clean (andere lopende proces id's) " + str(pid_list ) ) if len( pid_list ) > 0: # more then 1 because the script is started from os.system() msg_str = "Gestopt een andere versie van het programma is actief." flog.info( inspect.stack()[0][3] + ": " + msg_str ) sys.exit(1) timestart = time.time() # used te calculate the processing time. flog.info( inspect.stack()[0][3] + ": Wordt uitgevoerd als user -> " + pwd.getpwuid( os.getuid() ).pw_name ) parser = argparse.ArgumentParser( description = prgname ) parser.add_argument( '-i' , '--importfile', help="Naam van het export bestand om te importeren.", required=False ) parser.add_argument( '-rm', '--rmstatus', help="Maak geen status bestand aan", action='store_true', required=False ) args = parser.parse_args() no_status_messages = args.rmstatus # default False when set True initStatusFile() # open van config database try: config_db.init( const.FILE_DB_CONFIG,const.DB_CONFIG_TAB ) except Exception as e: msg = ": database niet te openen ("+ const.FILE_DB_CONFIG + ") melding: " + str(e.args[0] ) writeLineToStatusFile( msg ) stop( 2 ) msg = "database " + const.DB_CONFIG + " succesvol geopend." writeLineToStatusFile( msg ) try: # update field from database, the cli switches overwrite the DB values! _id, importfile, _label = config_db.strget( 138,flog ) except Exception as e: msg = "fout met lezen van import bestand uit database -> " + str(e) writeLineToStatusFile( msg ) flog.error(inspect.stack()[0][3]+": " + msg ) stop( 2 ) # check must be done after DB read of import filename if args.importfile != None: importfile = args.importfile if importfile == '': msg = "gestopt importfile ontbreekt." writeLineToStatusFile( msg ) flog.error( inspect.stack()[0][3]+": " + msg ) stop( 2 ) try: extension = os.path.splitext( importfile )[1] print ( extension ) msg = "ZIP file " + importfile + " gevonden." writeLineToStatusFile( msg ) flog.info(inspect.stack()[0][3]+": " + msg ) if extension != '.zip': msg = "Geen passend ZIP file gevonden, gestopt." writeLineToStatusFile( msg ) flog.warning(inspect.stack()[0][3]+": " + msg) stop( 3 ) except Exception as e: msg = "Geen passend ZIP file gevonden, gestopt -> " + str(e) flog.error(inspect.stack()[0][3]+": " + msg ) stop( 3 ) openDatabases() try: zf = zipfile.ZipFile( importfile ) _head,tail = os.path.split( importfile ) msgToInfoLogAndStatusFile( "ZIP file " + tail + " succesvol geopend." ) except Exception as e: msg = "ZIP file " + importfile + " probleem =>" + str(e) writeLineToStatusFile( msg ) flog.critical(inspect.stack()[0][3]+": " + msg ) stop( 30 ) # set file rights setFile2user( const.FILE_DB_E_FILENAME, 'p1mon' ) setFile2user( const.FILE_DB_E_HISTORIE, 'p1mon' ) setFile2user( const.FILE_DB_CONFIG, 'p1mon' ) setFile2user( const.FILE_DB_STATUS, 'p1mon' ) setFile2user( const.FILE_DB_FINANCIEEL, 'p1mon' ) setFile2user( const.FILE_DB_WEATHER, 'p1mon' ) setFile2user( const.FILE_DB_WEATHER_HISTORIE, 'p1mon' ) setFile2user( const.FILE_DB_TEMPERATUUR_FILENAME, 'p1mon' ) setFile2user( const.FILE_DB_WATERMETER, 'p1mon' ) setFile2user( const.FILE_DB_WATERMETERV2, 'p1mon' ) setFile2user( const.FILE_DB_POWERPRODUCTION,'p1mon') setFile2user( const.FILE_DB_PHASEINFORMATION, 'p1mon') msgToInfoLogAndStatusFile( "file rechten van database bestanden correct gezet." ) dbIntegrityCheck( config_db, const.FILE_DB_CONFIG ) dbIntegrityCheck( e_db_history_min, const.FILE_DB_E_HISTORIE ) dbIntegrityCheck( e_db_financieel_dag, const.FILE_DB_FINANCIEEL ) dbIntegrityCheck( weer_db, const.FILE_DB_WEATHER ) dbIntegrityCheck( weer_history_db_uur, const.FILE_DB_WEATHER_HISTORIE ) dbIntegrityCheck( temperature_db, const.FILE_DB_TEMPERATUUR_FILENAME ) dbIntegrityCheck( watermeter_db_uur, const.FILE_DB_WATERMETER ) dbIntegrityCheck( watermeter_db, const.FILE_DB_WATERMETERV2 ) dbIntegrityCheck( fase_db, const.FILE_DB_PHASEINFORMATION ) dbIntegrityCheck( power_production_db, const.FILE_DB_POWERPRODUCTION ) try: for fname in zf.namelist(): #filter out the manifest file first if fname == const.FILE_EXPORT_MANIFEST[1:]: data = zf.read(const.FILE_EXPORT_MANIFEST[1:]).decode('utf-8') json_data = json.loads(data) statusdata['records_total'] = json_data['record_count'] #statusdata['export_timestamp'] = str(json_data['timestamp']) msgToInfoLogAndStatusFile( "manifest is correct verwerkt uit ZIP file." ) msgToInfoLogAndStatusFile( "aantal te importeren records " + str(statusdata['records_total']) + "." ) msgToInfoLogAndStatusFile( "import bestand creatie datum " + str(json_data['timestamp']) + "." ) break except Exception as e: msg = "manifest bestand is niet correct in ZIP file. -> " + str(e) writeLineToStatusFile( msg ) flog.warning(inspect.stack()[0][3] + ": " + msg ) # custom www folder try: #raise Exception("dummy exception voor test.") for fname in zf.namelist(): if fname[:len(const.FILE_PREFIX_CUSTOM_UI)-1] == const.FILE_PREFIX_CUSTOM_UI[1:]: zf.extract(fname,'/') # komt dan terecht in /p1mon/var/tmp exportcode = fname[len(const.FILE_PREFIX_CUSTOM_UI)-1:-3] cmd = 'sudo tar -zxf ' + const.FILE_PREFIX_CUSTOM_UI + exportcode + '.gz -C ' + const.DIR_WWW_CUSTOM + " 2>/dev/null" if os.system( cmd ) > 0: flog.error(inspect.stack()[0][3]+" custom www import gefaald.") msg = "custom www bestanden konden niet worden verwerkt of waren niet aanwezig." writeLineToStatusFile( msg ) flog.warning( inspect.stack()[0][3] + ": " + msg ) else: msgToInfoLogAndStatusFile( "custom www bestanden succesvol verwerkt." ) except Exception as e: msg = "ZIP file verwerking probleem tijdens custom www verwerking -> " + str(e) writeLineToStatusFile( msg ) flog.error(inspect.stack()[0][3]+": ZIP file verwerking ging mis tijdens custom www verwerking "+str(e)) try: for fname in zf.namelist(): _head,tail = os.path.split(fname) #print ( "fname=", fname ) #print ( "tail=", tail ) #print ( "configuratie=" +str( tail.startswith( 'configuratie' ) ) ) #print ( "financieel=" +str( tail.startswith( 'finacieel' ) ) ) #print ( "historie=" +str( tail.startswith( 'historie' ) ) ) #print ( "01_weer_historie=" +str( tail.startswith( '01_weer_historie' ) ) ) #print ( "weer=" +str( tail.startswith( 'weer' ) ) ) #print ( "manifest.json=" +str( tail.startswith( 'manifest.json' ) ) ) #print ( "02_temperatuur=" +str( tail.startswith( '02_temperatuur' ) ) ) #print ( "03_watermeter=" +str( tail.startswith( '03_watermeter' ) ) ) ############################################# if tail.startswith( const.DB_WATERMETERV2 ): processImportDataSet( const.DB_WATERMETERV2 , watermeter_db, zf, fname, 'replace into watermeter*' ) ########################################### elif tail.startswith( const.DB_WATERMETER ): processImportDataSet( const.DB_WATERMETER , watermeter_db_uur, zf, fname, 'replace into watermeter*' ) ############################################ elif tail.startswith( const.DB_TEMPERATURE ): processImportDataSet( const.DB_TEMPERATURE , temperature_db, zf, fname, 'replace into temperatuur*' ) temperature_db.fix_missing_month_day( flog ) ############################################ elif tail.startswith( const.DB_E_HISTORIE ): processImportDataSet( const.DB_E_HISTORIE, e_db_history_min, zf, fname, 'replace into e_history*' ) ############################################ # this is fix from version 0.9.19 > to fix the typo in "finacieel" text elif tail.startswith( const.DB_FINANCIEEL ) or tail.startswith("finacieel"): processImportDataSet( const.DB_FINANCIEEL, e_db_financieel_dag, zf, fname, 'replace into e_financieel*' ) ############################################ elif tail.startswith( const.DB_CONFIG ): processImportDataSet( const.DB_CONFIG, config_db, zf, fname, 'update ' + const.DB_CONFIG_TAB + ' set PARAMETER=*' ) ############################################ elif tail.startswith( const.DB_WEER_HISTORY ): processImportDataSet( const.DB_WEER_HISTORY, weer_history_db_uur, zf, fname, 'replace into weer_history*' ) ############################################ elif tail.startswith( const.DB_WEER ): processImportDataSet( const.DB_WEER, weer_db, zf, fname, 'replace into weer*' ) ############################################ elif tail.startswith( const.DB_PHASEINFORMATION ): processImportDataSet( const.DB_PHASEINFORMATION, fase_db, zf, fname, 'replace into ' + const.DB_FASE_REALTIME_TAB + '*' ) ############################################ elif tail.startswith( const.DB_POWERPRODUCTION ): processImportDataSet( const.DB_POWERPRODUCTION, power_production_db, zf, fname, 'replace into ' + const.DB_POWERPRODUCTION_TAB + '*' ) except Exception as e: msg = "ZIP file verwerking globale fout -> " + str(e) writeLineToStatusFile( msg ) flog.error( inspect.stack()[0][3] + ": " + msg ) zf.close msgToInfoLogAndStatusFile( "alle data uit het ZIP bestand verwerkt." ) # waterbase V1 to V2 database conversion # set de normale verwerking van de water datbase uit. # doe de conversie msgToInfoLogAndStatusFile( "watermeting conversie van oude naar nieuwe database gestart." ) os.system("/p1mon/scripts/P1WatermeterDbV1toV2.py") msgToInfoLogAndStatusFile( "watermeting conversie van oude naar nieuwe database gereed." ) # lees systeem ID uit en zet deze in de config database. # versleuteld om dat deze data in een back-up bestand terecht kan komen. try: msgToInfoLogAndStatusFile( 'System ID zetten in configuratie database: ' + str( systemid.getSystemId() ) ) #flog.info(inspect.stack()[0][3]+': System ID zetten in configuratie database: ' + str( systemid.getSystemId() ) ) sysid_encrypted = crypto3.p1Encrypt( systemid.getSystemId(),"sysid" ).encode('utf-8').decode('utf-8') config_db.strset( sysid_encrypted ,58, flog ) except Exception as e: msg = " System ID zetten mislukt -> " + str(e.args[0]) writeLineToStatusFile( msg ) flog.warning( inspect.stack()[0][3] + ": " + msg ) setSoftwareVersionInformation() msgToInfoLogAndStatusFile( 'WiFi wordt aangepast.' ) if os.system('sudo /p1mon/scripts/P1SetWifi.py') > 0: msg = "Wifi aanpassen gefaald." writeLineToStatusFile( msg ) flog.error( inspect.stack()[0][3] + ": " + msg ) else: msgToInfoLogAndStatusFile( 'WiFi aanpassingen gereed.' ) msgToInfoLogAndStatusFile( 'CRON wordt aangepast.' ) if os.system('sudo -u p1mon /p1mon/scripts/P1Scheduler.py' ) > 0: msg = "CRON update gefaald." writeLineToStatusFile( msg ) flog.error( inspect.stack()[0][3] + ": " + msg ) else: msgToInfoLogAndStatusFile( 'CRON aanpassingen gereed.' ) msgToInfoLogAndStatusFile( 'Internet API wordt aangepast.' ) _id, api_is_active, _label = config_db.strget( 163, flog ) if int( api_is_active ) == 1: # the Internet API is active process the changes, make https config and get certificates. config_db.strset( 1, 162, flog ) # set the flag so the watchdog processes the changes msgToInfoLogAndStatusFile( 'Internet API aanpassingen worden doorgevoerd, duur maximaal 60 sec.' ) time.sleep(10) #make sure that all is copied to disk msgToInfoLogAndStatusFile( "Databases worden van RAM naar het SDHC kaartje geschreven." ) os.system( "/p1mon/scripts/P1DbCopy.py --allcopy2disk --forcecopy" ) msgToInfoLogAndStatusFile( "Databases kopieren gereed." ) msgToInfoLogAndStatusFile( "ZIP bestand wordt verwijderd." ) try: time.sleep(1) # allow file to be unlocked. os.remove( importfile ) # remove uploaded file. except Exception as e: flog.error(inspect.stack()[0][3]+": "+str(e)) msg = "verwijderen van import bestand " + importfile + " gefaald." writeLineToStatusFile( msg ) flog.error( inspect.stack()[0][3] + ": " + msg ) msgToInfoLogAndStatusFile( 'Vaste IP adressen worden aangepast als deze actief zijn.' ) config_db.strset( 3, 168, flog ) # set the flag so the watchdog processes the changes timestop = time.time() m, s = divmod( timestop - timestart , 60) # make minutes and seconds from total secs count msgToInfoLogAndStatusFile( "Gereed verwerkings tijd is " + '{:02d}:{:02d}.'.format( int(m), int(s) ) ) stop( 0 )