def update_json_data( jsondata=None, p1data=None ):
#convert dutch tarif code to English Dal = Low, Peak = High
if p1data['tarief_code'] == 'P':
tarief_code = 'HIGH'
else:
tarief_code = "LOW" # there are only two options (D/P)
jsondata[ apiconst.JSON_TS_LCL ] = str(util.mkLocalTimeString())
jsondata[ apiconst.JSON_TS_LCL_UTC ] = util.getUtcTime()
jsondata[ apiconst.JSON_API_API_STTS ] = 'production'
jsondata[ apiconst.JSON_API_CNSMPTN_KWH_L ] = float( p1data['verbrk_kwh_181'] )
jsondata[ apiconst.JSON_API_CNSMPTN_KWH_H ] = float( p1data['verbrk_kwh_182'] )
jsondata[ apiconst.JSON_API_PRDCTN_KWH_L ] = float( p1data['gelvr_kwh_281'] )
jsondata[ apiconst.JSON_API_PRDCTN_KWH_H ] = float( p1data['gelvr_kwh_282'] )
jsondata[ apiconst.JSON_API_TRFCD ] = str( tarief_code )
jsondata[ apiconst.JSON_API_CNSMPTN_KW ] = float( p1data['act_verbr_kw_170'] )
jsondata[ apiconst.JSON_API_PRDCTN_KW ] = float( p1data['act_gelvr_kw_270'] )
jsondata[ apiconst.JSON_API_CNSMPTN_GAS_M3 ] = float( p1data['gas_verbr_m3_2421'] )
jsondata[ apiconst.JSON_API_SYSTM_ID] = systemid.getSystemId()
import p1_telegram_test_lib import time import util temperature_db = sqldb.temperatureDB() e_db_serial = sqldb.SqlDb1() rt_status_db = sqldb.rtStatusDb() config_db = sqldb.configDB() watermeter_db = sqldb.WatermeterDBV2() fase_db = sqldb.PhaseDB() prgname = 'P1SerReader' serial_buffer = [] dev_dummy_gas_value = 0 timestamp_last_gas_update = 0 system_id = systemid.getSystemId() # list of serial devices tried to use ser_devices_list = ["/dev/ttyUSB0", "/dev/ttyUSB1"] ################################################################################### # let op deze optie geef veel foutmelding en in de log deze kunnen geen kwaad # ################################################################################### DUMMY_1SEC_PROCCESSING = False ######### DEZE OP FALSE ZETTEN BIJ PRODUCTIE CODE!!!! DUMMY_3PHASE_DATA = False ######### DEZE OP FALSE ZETTEN BIJ PRODUCTIE CODE!!!! # zet deze op p1_telegram_test_lib.NO_GAS_TEST om de test uit te zetten. #gas_test_mode=p1_telegram_test_lib.DUMMY_GAS_MODE_2421 gas_test_mode = p1_telegram_test_lib.NO_GAS_TEST #Set COM port config
def Main(argv):
flog.info("Start van programma.")
parser = argparse.ArgumentParser(description="options: -fb")
parser.add_argument('-fb',
'--forcebackup',
required=False,
action="store_true")
args = parser.parse_args()
# open van status database
try:
rt_status_db.init(const.FILE_DB_STATUS, const.DB_STATUS_TAB)
except Exception as e:
flog.critical(inspect.stack()[0][3] + ": Database niet te openen(1)." +
const.FILE_DB_STATUS + ") melding:" + str(e.args[0]))
sys.exit(1)
flog.debug(inspect.stack()[0][3] + ": database tabel " +
const.DB_STATUS_TAB + " succesvol geopend.")
# open van config database
try:
config_db.init(const.FILE_DB_CONFIG, const.DB_CONFIG_TAB)
except Exception as e:
flog.critical(inspect.stack()[0][3] + ": database niet te openen(2)." +
const.FILE_DB_CONFIG + ") melding:" + str(e.args[0]))
sys.exit(2)
flog.debug(inspect.stack()[0][3] + ": database tabel " +
const.DB_CONFIG_TAB + " succesvol geopend.")
_id, do_ftp_backup, _label = config_db.strget(36, flog)
_id, do_dbx_backup, _label = config_db.strget(49, flog)
parameter = 0
# do some casting....comparing strings is a pain
do_ftp_backup = int(do_ftp_backup)
do_dbx_backup = int(do_dbx_backup)
flog.debug(inspect.stack()[0][3] + ": ftp backup=" + str(do_ftp_backup) +
" Dropbox back-up=" + str(do_dbx_backup))
# if one of the backup options is on make an export file
if do_ftp_backup == 1 or do_dbx_backup == 1:
parameter = 1
else:
flog.info(inspect.stack()[0][3] +
": Backup staat uit, geen back-up gestart")
if args.forcebackup == True: # force backup, forget preference.
flog.info(inspect.stack()[0][3] + ": geforceerde back-up gestart.")
parameter = 1
if int(parameter) == 1:
flog.debug(inspect.stack()[0][3] + ": export file wordt gemaakt.")
file_tmp_id = str(util.getUtcTime()) + "-" + systemid.getSystemId()
cmd = "/p1mon/scripts/P1SqlExport.py -e " + file_tmp_id
flog.debug(inspect.stack()[0][3] + ": export commando is ->" +
str(cmd))
retvalue = os.system("/p1mon/scripts/P1SqlExport.py -e " + file_tmp_id)
if retvalue != 0:
flog.error(inspect.stack()[0][3] +
": export van file gefaald, gestopt.")
sys.exit(3)
export_file = exportfile_base + file_tmp_id + '.zip'
if util.fileExist(export_file) == False:
flog.error(inspect.stack()[0][3] + ": export file " + export_file +
" niet gevonden, gestopt.")
sys.exit(4)
# update config database with current name of backup file.
config_db.strset(export_file, 33, flog)
if do_dbx_backup == 1:
# do dropbox back-up
flog.info(inspect.stack()[0][3] + ": Dropbox backup gestart")
try:
# try to clean old files
clean_old_files()
if len(os.listdir(const.DIR_DBX_LOCAL +
const.DBX_DIR_BACKUP)) > 10:
flog.critical(
inspect.stack()[0][3] +
": Dropbox backup bestand niet gekopierd, te veel bestanden in ram buffer."
)
else:
flog.debug(inspect.stack()[0][3] +
": copy export file naar lokale dropbox folder: " +
export_file)
shutil.copy2(export_file,
const.DIR_DBX_LOCAL + const.DBX_DIR_BACKUP)
_head, tail = os.path.split(export_file)
util.setFile2user(
const.DIR_DBX_LOCAL + const.DBX_DIR_BACKUP + '/' + tail,
'p1mon')
except Exception as e:
flog.error(inspect.stack()[0][3] + ": Dropbox back-up. melding:" +
str(e.args[0]))
if do_ftp_backup == 1:
flog.info(inspect.stack()[0][3] + ": FTP backup gestart")
retvalue = os.system("/p1mon/scripts/P1FtpCopy.py")
if retvalue > 0:
flog.error(inspect.stack()[0][3] +
": ftp backup gefaald, gestopt.")
sys.exit(5)
flog.info("programma is succesvol gestopt.")
sys.exit(0) # all is well.
def Main(argv):
global no_status_messages
my_pid = os.getpid()
flog.info("Start van programma met process id " + str(my_pid) )
pid_list, _process_list = listOfPidByName( prgname )
#print ( pid_list )
pid_list.remove( my_pid ) # remove own pid from the count
flog.debug( inspect.stack()[0][3] + ": pid list clean (andere lopende proces id's) " + str(pid_list ) )
if len( pid_list ) > 0: # more then 1 because the script is started from os.system()
msg_str = "Gestopt een andere versie van het programma is actief."
flog.info( inspect.stack()[0][3] + ": " + msg_str )
sys.exit(1)
timestart = time.time() # used te calculate the processing time.
flog.info( inspect.stack()[0][3] + ": Wordt uitgevoerd als user -> " + pwd.getpwuid( os.getuid() ).pw_name )
parser = argparse.ArgumentParser( description = prgname )
parser.add_argument( '-i' , '--importfile', help="Naam van het export bestand om te importeren.", required=False )
parser.add_argument( '-rm', '--rmstatus', help="Maak geen status bestand aan", action='store_true', required=False )
args = parser.parse_args()
no_status_messages = args.rmstatus # default False when set True
initStatusFile()
# open van config database
try:
config_db.init( const.FILE_DB_CONFIG,const.DB_CONFIG_TAB )
except Exception as e:
msg = ": database niet te openen ("+ const.FILE_DB_CONFIG + ") melding: " + str(e.args[0] )
writeLineToStatusFile( msg )
stop( 2 )
msg = "database " + const.DB_CONFIG + " succesvol geopend."
writeLineToStatusFile( msg )
try:
# update field from database, the cli switches overwrite the DB values!
_id, importfile, _label = config_db.strget( 138,flog )
except Exception as e:
msg = "fout met lezen van import bestand uit database -> " + str(e)
writeLineToStatusFile( msg )
flog.error(inspect.stack()[0][3]+": " + msg )
stop( 2 )
# check must be done after DB read of import filename
if args.importfile != None:
importfile = args.importfile
if importfile == '':
msg = "gestopt importfile ontbreekt."
writeLineToStatusFile( msg )
flog.error( inspect.stack()[0][3]+": " + msg )
stop( 2 )
try:
extension = os.path.splitext( importfile )[1]
print ( extension )
msg = "ZIP file " + importfile + " gevonden."
writeLineToStatusFile( msg )
flog.info(inspect.stack()[0][3]+": " + msg )
if extension != '.zip':
msg = "Geen passend ZIP file gevonden, gestopt."
writeLineToStatusFile( msg )
flog.warning(inspect.stack()[0][3]+": " + msg)
stop( 3 )
except Exception as e:
msg = "Geen passend ZIP file gevonden, gestopt -> " + str(e)
flog.error(inspect.stack()[0][3]+": " + msg )
stop( 3 )
openDatabases()
try:
zf = zipfile.ZipFile( importfile )
_head,tail = os.path.split( importfile )
msgToInfoLogAndStatusFile( "ZIP file " + tail + " succesvol geopend." )
except Exception as e:
msg = "ZIP file " + importfile + " probleem =>" + str(e)
writeLineToStatusFile( msg )
flog.critical(inspect.stack()[0][3]+": " + msg )
stop( 30 )
# set file rights
setFile2user( const.FILE_DB_E_FILENAME, 'p1mon' )
setFile2user( const.FILE_DB_E_HISTORIE, 'p1mon' )
setFile2user( const.FILE_DB_CONFIG, 'p1mon' )
setFile2user( const.FILE_DB_STATUS, 'p1mon' )
setFile2user( const.FILE_DB_FINANCIEEL, 'p1mon' )
setFile2user( const.FILE_DB_WEATHER, 'p1mon' )
setFile2user( const.FILE_DB_WEATHER_HISTORIE, 'p1mon' )
setFile2user( const.FILE_DB_TEMPERATUUR_FILENAME, 'p1mon' )
setFile2user( const.FILE_DB_WATERMETER, 'p1mon' )
setFile2user( const.FILE_DB_WATERMETERV2, 'p1mon' )
setFile2user( const.FILE_DB_POWERPRODUCTION,'p1mon')
setFile2user( const.FILE_DB_PHASEINFORMATION, 'p1mon')
msgToInfoLogAndStatusFile( "file rechten van database bestanden correct gezet." )
dbIntegrityCheck( config_db, const.FILE_DB_CONFIG )
dbIntegrityCheck( e_db_history_min, const.FILE_DB_E_HISTORIE )
dbIntegrityCheck( e_db_financieel_dag, const.FILE_DB_FINANCIEEL )
dbIntegrityCheck( weer_db, const.FILE_DB_WEATHER )
dbIntegrityCheck( weer_history_db_uur, const.FILE_DB_WEATHER_HISTORIE )
dbIntegrityCheck( temperature_db, const.FILE_DB_TEMPERATUUR_FILENAME )
dbIntegrityCheck( watermeter_db_uur, const.FILE_DB_WATERMETER )
dbIntegrityCheck( watermeter_db, const.FILE_DB_WATERMETERV2 )
dbIntegrityCheck( fase_db, const.FILE_DB_PHASEINFORMATION )
dbIntegrityCheck( power_production_db, const.FILE_DB_POWERPRODUCTION )
try:
for fname in zf.namelist(): #filter out the manifest file first
if fname == const.FILE_EXPORT_MANIFEST[1:]:
data = zf.read(const.FILE_EXPORT_MANIFEST[1:]).decode('utf-8')
json_data = json.loads(data)
statusdata['records_total'] = json_data['record_count']
#statusdata['export_timestamp'] = str(json_data['timestamp'])
msgToInfoLogAndStatusFile( "manifest is correct verwerkt uit ZIP file." )
msgToInfoLogAndStatusFile( "aantal te importeren records " + str(statusdata['records_total']) + "." )
msgToInfoLogAndStatusFile( "import bestand creatie datum " + str(json_data['timestamp']) + "." )
break
except Exception as e:
msg = "manifest bestand is niet correct in ZIP file. -> " + str(e)
writeLineToStatusFile( msg )
flog.warning(inspect.stack()[0][3] + ": " + msg )
# custom www folder
try:
#raise Exception("dummy exception voor test.")
for fname in zf.namelist():
if fname[:len(const.FILE_PREFIX_CUSTOM_UI)-1] == const.FILE_PREFIX_CUSTOM_UI[1:]:
zf.extract(fname,'/') # komt dan terecht in /p1mon/var/tmp
exportcode = fname[len(const.FILE_PREFIX_CUSTOM_UI)-1:-3]
cmd = 'sudo tar -zxf ' + const.FILE_PREFIX_CUSTOM_UI + exportcode + '.gz -C ' + const.DIR_WWW_CUSTOM + " 2>/dev/null"
if os.system( cmd ) > 0:
flog.error(inspect.stack()[0][3]+" custom www import gefaald.")
msg = "custom www bestanden konden niet worden verwerkt of waren niet aanwezig."
writeLineToStatusFile( msg )
flog.warning( inspect.stack()[0][3] + ": " + msg )
else:
msgToInfoLogAndStatusFile( "custom www bestanden succesvol verwerkt." )
except Exception as e:
msg = "ZIP file verwerking probleem tijdens custom www verwerking -> " + str(e)
writeLineToStatusFile( msg )
flog.error(inspect.stack()[0][3]+": ZIP file verwerking ging mis tijdens custom www verwerking "+str(e))
try:
for fname in zf.namelist():
_head,tail = os.path.split(fname)
#print ( "fname=", fname )
#print ( "tail=", tail )
#print ( "configuratie=" +str( tail.startswith( 'configuratie' ) ) )
#print ( "financieel=" +str( tail.startswith( 'finacieel' ) ) )
#print ( "historie=" +str( tail.startswith( 'historie' ) ) )
#print ( "01_weer_historie=" +str( tail.startswith( '01_weer_historie' ) ) )
#print ( "weer=" +str( tail.startswith( 'weer' ) ) )
#print ( "manifest.json=" +str( tail.startswith( 'manifest.json' ) ) )
#print ( "02_temperatuur=" +str( tail.startswith( '02_temperatuur' ) ) )
#print ( "03_watermeter=" +str( tail.startswith( '03_watermeter' ) ) )
#############################################
if tail.startswith( const.DB_WATERMETERV2 ):
processImportDataSet( const.DB_WATERMETERV2 , watermeter_db, zf, fname, 'replace into watermeter*' )
###########################################
elif tail.startswith( const.DB_WATERMETER ):
processImportDataSet( const.DB_WATERMETER , watermeter_db_uur, zf, fname, 'replace into watermeter*' )
############################################
elif tail.startswith( const.DB_TEMPERATURE ):
processImportDataSet( const.DB_TEMPERATURE , temperature_db, zf, fname, 'replace into temperatuur*' )
temperature_db.fix_missing_month_day( flog )
############################################
elif tail.startswith( const.DB_E_HISTORIE ):
processImportDataSet( const.DB_E_HISTORIE, e_db_history_min, zf, fname, 'replace into e_history*' )
############################################
# this is fix from version 0.9.19 > to fix the typo in "finacieel" text
elif tail.startswith( const.DB_FINANCIEEL ) or tail.startswith("finacieel"):
processImportDataSet( const.DB_FINANCIEEL, e_db_financieel_dag, zf, fname, 'replace into e_financieel*' )
############################################
elif tail.startswith( const.DB_CONFIG ):
processImportDataSet( const.DB_CONFIG, config_db, zf, fname, 'update ' + const.DB_CONFIG_TAB + ' set PARAMETER=*' )
############################################
elif tail.startswith( const.DB_WEER_HISTORY ):
processImportDataSet( const.DB_WEER_HISTORY, weer_history_db_uur, zf, fname, 'replace into weer_history*' )
############################################
elif tail.startswith( const.DB_WEER ):
processImportDataSet( const.DB_WEER, weer_db, zf, fname, 'replace into weer*' )
############################################
elif tail.startswith( const.DB_PHASEINFORMATION ):
processImportDataSet( const.DB_PHASEINFORMATION, fase_db, zf, fname, 'replace into ' + const.DB_FASE_REALTIME_TAB + '*' )
############################################
elif tail.startswith( const.DB_POWERPRODUCTION ):
processImportDataSet( const.DB_POWERPRODUCTION, power_production_db, zf, fname, 'replace into ' + const.DB_POWERPRODUCTION_TAB + '*' )
except Exception as e:
msg = "ZIP file verwerking globale fout -> " + str(e)
writeLineToStatusFile( msg )
flog.error( inspect.stack()[0][3] + ": " + msg )
zf.close
msgToInfoLogAndStatusFile( "alle data uit het ZIP bestand verwerkt." )
# waterbase V1 to V2 database conversion
# set de normale verwerking van de water datbase uit.
# doe de conversie
msgToInfoLogAndStatusFile( "watermeting conversie van oude naar nieuwe database gestart." )
os.system("/p1mon/scripts/P1WatermeterDbV1toV2.py")
msgToInfoLogAndStatusFile( "watermeting conversie van oude naar nieuwe database gereed." )
# lees systeem ID uit en zet deze in de config database.
# versleuteld om dat deze data in een back-up bestand terecht kan komen.
try:
msgToInfoLogAndStatusFile( 'System ID zetten in configuratie database: ' + str( systemid.getSystemId() ) )
#flog.info(inspect.stack()[0][3]+': System ID zetten in configuratie database: ' + str( systemid.getSystemId() ) )
sysid_encrypted = crypto3.p1Encrypt( systemid.getSystemId(),"sysid" ).encode('utf-8').decode('utf-8')
config_db.strset( sysid_encrypted ,58, flog )
except Exception as e:
msg = " System ID zetten mislukt -> " + str(e.args[0])
writeLineToStatusFile( msg )
flog.warning( inspect.stack()[0][3] + ": " + msg )
setSoftwareVersionInformation()
msgToInfoLogAndStatusFile( 'WiFi wordt aangepast.' )
if os.system('sudo /p1mon/scripts/P1SetWifi.py') > 0:
msg = "Wifi aanpassen gefaald."
writeLineToStatusFile( msg )
flog.error( inspect.stack()[0][3] + ": " + msg )
else:
msgToInfoLogAndStatusFile( 'WiFi aanpassingen gereed.' )
msgToInfoLogAndStatusFile( 'CRON wordt aangepast.' )
if os.system('sudo -u p1mon /p1mon/scripts/P1Scheduler.py' ) > 0:
msg = "CRON update gefaald."
writeLineToStatusFile( msg )
flog.error( inspect.stack()[0][3] + ": " + msg )
else:
msgToInfoLogAndStatusFile( 'CRON aanpassingen gereed.' )
msgToInfoLogAndStatusFile( 'Internet API wordt aangepast.' )
_id, api_is_active, _label = config_db.strget( 163, flog )
if int( api_is_active ) == 1: # the Internet API is active process the changes, make https config and get certificates.
config_db.strset( 1, 162, flog ) # set the flag so the watchdog processes the changes
msgToInfoLogAndStatusFile( 'Internet API aanpassingen worden doorgevoerd, duur maximaal 60 sec.' )
time.sleep(10)
#make sure that all is copied to disk
msgToInfoLogAndStatusFile( "Databases worden van RAM naar het SDHC kaartje geschreven." )
os.system( "/p1mon/scripts/P1DbCopy.py --allcopy2disk --forcecopy" )
msgToInfoLogAndStatusFile( "Databases kopieren gereed." )
msgToInfoLogAndStatusFile( "ZIP bestand wordt verwijderd." )
try:
time.sleep(1) # allow file to be unlocked.
os.remove( importfile ) # remove uploaded file.
except Exception as e:
flog.error(inspect.stack()[0][3]+": "+str(e))
msg = "verwijderen van import bestand " + importfile + " gefaald."
writeLineToStatusFile( msg )
flog.error( inspect.stack()[0][3] + ": " + msg )
msgToInfoLogAndStatusFile( 'Vaste IP adressen worden aangepast als deze actief zijn.' )
config_db.strset( 3, 168, flog ) # set the flag so the watchdog processes the changes
timestop = time.time()
m, s = divmod( timestop - timestart , 60) # make minutes and seconds from total secs count
msgToInfoLogAndStatusFile( "Gereed verwerkings tijd is " + '{:02d}:{:02d}.'.format( int(m), int(s) ) )
stop( 0 )