Exemplo n.º 1
0
def update_json_data( jsondata=None, p1data=None ):

    #convert dutch tarif code to English Dal = Low, Peak = High 
    if p1data['tarief_code'] == 'P': 
        tarief_code = 'HIGH'
    else: 
        tarief_code = "LOW" # there are only two options (D/P)

    jsondata[ apiconst.JSON_TS_LCL ]                   = str(util.mkLocalTimeString())
    jsondata[ apiconst.JSON_TS_LCL_UTC ]               = util.getUtcTime()
    jsondata[ apiconst.JSON_API_API_STTS ]             = 'production'
    jsondata[ apiconst.JSON_API_CNSMPTN_KWH_L ]        = float( p1data['verbrk_kwh_181'] )
    jsondata[ apiconst.JSON_API_CNSMPTN_KWH_H ]        = float( p1data['verbrk_kwh_182'] )
    jsondata[ apiconst.JSON_API_PRDCTN_KWH_L ]         = float( p1data['gelvr_kwh_281']  )
    jsondata[ apiconst.JSON_API_PRDCTN_KWH_H ]         = float( p1data['gelvr_kwh_282'] )
    jsondata[ apiconst.JSON_API_TRFCD ]                = str( tarief_code )
    jsondata[ apiconst.JSON_API_CNSMPTN_KW ]           = float( p1data['act_verbr_kw_170'] )
    jsondata[ apiconst.JSON_API_PRDCTN_KW ]            = float( p1data['act_gelvr_kw_270'] )
    jsondata[ apiconst.JSON_API_CNSMPTN_GAS_M3 ]       = float( p1data['gas_verbr_m3_2421'] )
    jsondata[ apiconst.JSON_API_SYSTM_ID]              = systemid.getSystemId()
Exemplo n.º 2
0
import p1_telegram_test_lib
import time
import util

temperature_db = sqldb.temperatureDB()
e_db_serial = sqldb.SqlDb1()
rt_status_db = sqldb.rtStatusDb()
config_db = sqldb.configDB()
watermeter_db = sqldb.WatermeterDBV2()
fase_db = sqldb.PhaseDB()

prgname = 'P1SerReader'
serial_buffer = []
dev_dummy_gas_value = 0
timestamp_last_gas_update = 0
system_id = systemid.getSystemId()

# list of serial devices tried to use
ser_devices_list = ["/dev/ttyUSB0", "/dev/ttyUSB1"]

###################################################################################
# let op deze optie geef veel foutmelding en in de log deze kunnen geen kwaad     #
###################################################################################
DUMMY_1SEC_PROCCESSING = False  ######### DEZE OP FALSE ZETTEN BIJ PRODUCTIE CODE!!!!
DUMMY_3PHASE_DATA = False  ######### DEZE OP FALSE ZETTEN BIJ PRODUCTIE CODE!!!!

# zet deze op p1_telegram_test_lib.NO_GAS_TEST om de test uit te zetten.
#gas_test_mode=p1_telegram_test_lib.DUMMY_GAS_MODE_2421
gas_test_mode = p1_telegram_test_lib.NO_GAS_TEST

#Set COM port config
Exemplo n.º 3
0
def Main(argv):
    flog.info("Start van programma.")

    parser = argparse.ArgumentParser(description="options: -fb")
    parser.add_argument('-fb',
                        '--forcebackup',
                        required=False,
                        action="store_true")
    args = parser.parse_args()

    # open van status database
    try:
        rt_status_db.init(const.FILE_DB_STATUS, const.DB_STATUS_TAB)
    except Exception as e:
        flog.critical(inspect.stack()[0][3] + ": Database niet te openen(1)." +
                      const.FILE_DB_STATUS + ") melding:" + str(e.args[0]))
        sys.exit(1)
    flog.debug(inspect.stack()[0][3] + ": database tabel " +
               const.DB_STATUS_TAB + " succesvol geopend.")

    # open van config database
    try:
        config_db.init(const.FILE_DB_CONFIG, const.DB_CONFIG_TAB)
    except Exception as e:
        flog.critical(inspect.stack()[0][3] + ": database niet te openen(2)." +
                      const.FILE_DB_CONFIG + ") melding:" + str(e.args[0]))
        sys.exit(2)
    flog.debug(inspect.stack()[0][3] + ": database tabel " +
               const.DB_CONFIG_TAB + " succesvol geopend.")

    _id, do_ftp_backup, _label = config_db.strget(36, flog)
    _id, do_dbx_backup, _label = config_db.strget(49, flog)
    parameter = 0
    # do some casting....comparing strings is a pain
    do_ftp_backup = int(do_ftp_backup)
    do_dbx_backup = int(do_dbx_backup)

    flog.debug(inspect.stack()[0][3] + ": ftp backup=" + str(do_ftp_backup) +
               " Dropbox back-up=" + str(do_dbx_backup))

    # if one of the backup options is on make an export file
    if do_ftp_backup == 1 or do_dbx_backup == 1:
        parameter = 1
    else:
        flog.info(inspect.stack()[0][3] +
                  ": Backup staat uit, geen back-up gestart")

    if args.forcebackup == True:  # force backup, forget preference.
        flog.info(inspect.stack()[0][3] + ": geforceerde back-up gestart.")
        parameter = 1

    if int(parameter) == 1:
        flog.debug(inspect.stack()[0][3] + ": export file wordt gemaakt.")
        file_tmp_id = str(util.getUtcTime()) + "-" + systemid.getSystemId()
        cmd = "/p1mon/scripts/P1SqlExport.py -e " + file_tmp_id
        flog.debug(inspect.stack()[0][3] + ": export commando is ->" +
                   str(cmd))
        retvalue = os.system("/p1mon/scripts/P1SqlExport.py -e " + file_tmp_id)
        if retvalue != 0:
            flog.error(inspect.stack()[0][3] +
                       ": export van file gefaald, gestopt.")
            sys.exit(3)
        export_file = exportfile_base + file_tmp_id + '.zip'
        if util.fileExist(export_file) == False:
            flog.error(inspect.stack()[0][3] + ": export file " + export_file +
                       " niet gevonden, gestopt.")
            sys.exit(4)
        # update config database with current name of backup file.
        config_db.strset(export_file, 33, flog)

    if do_dbx_backup == 1:
        # do dropbox back-up
        flog.info(inspect.stack()[0][3] + ": Dropbox backup gestart")
        try:
            # try to clean old files
            clean_old_files()

            if len(os.listdir(const.DIR_DBX_LOCAL +
                              const.DBX_DIR_BACKUP)) > 10:
                flog.critical(
                    inspect.stack()[0][3] +
                    ": Dropbox backup bestand niet gekopierd, te veel bestanden in ram buffer."
                )
            else:
                flog.debug(inspect.stack()[0][3] +
                           ": copy export file naar lokale dropbox folder: " +
                           export_file)
                shutil.copy2(export_file,
                             const.DIR_DBX_LOCAL + const.DBX_DIR_BACKUP)
                _head, tail = os.path.split(export_file)
                util.setFile2user(
                    const.DIR_DBX_LOCAL + const.DBX_DIR_BACKUP + '/' + tail,
                    'p1mon')
        except Exception as e:
            flog.error(inspect.stack()[0][3] + ": Dropbox back-up. melding:" +
                       str(e.args[0]))

    if do_ftp_backup == 1:
        flog.info(inspect.stack()[0][3] + ": FTP backup gestart")
        retvalue = os.system("/p1mon/scripts/P1FtpCopy.py")
        if retvalue > 0:
            flog.error(inspect.stack()[0][3] +
                       ": ftp backup gefaald, gestopt.")
            sys.exit(5)

    flog.info("programma is succesvol gestopt.")
    sys.exit(0)  # all is well.
Exemplo n.º 4
0
def Main(argv):

    global no_status_messages
    
    my_pid = os.getpid()
    flog.info("Start van programma met process id " + str(my_pid) )
    pid_list, _process_list = listOfPidByName( prgname )
    #print ( pid_list )
    pid_list.remove( my_pid ) # remove own pid from the count
    flog.debug( inspect.stack()[0][3] + ": pid list clean (andere lopende proces id's) " + str(pid_list ) )
    if len( pid_list ) > 0: # more then 1 because the script is started from os.system()
        msg_str = "Gestopt een andere versie van het programma is actief."
        flog.info( inspect.stack()[0][3] + ": " + msg_str )
        sys.exit(1)

    timestart = time.time() # used te calculate the processing time.

    flog.info( inspect.stack()[0][3] +  ": Wordt uitgevoerd als user -> " + pwd.getpwuid( os.getuid() ).pw_name  )
    
    parser = argparse.ArgumentParser( description = prgname )
    parser.add_argument( '-i' , '--importfile', help="Naam van het export bestand om te importeren.",     required=False  ) 
    parser.add_argument( '-rm', '--rmstatus',   help="Maak geen status bestand aan", action='store_true', required=False ) 

    args = parser.parse_args()
    no_status_messages = args.rmstatus # default False when set True


    initStatusFile()

    # open van config database
    try: 
        config_db.init( const.FILE_DB_CONFIG,const.DB_CONFIG_TAB )
    except Exception as e:
        msg = ": database niet te openen ("+ const.FILE_DB_CONFIG + ") melding: " + str(e.args[0] )
        writeLineToStatusFile( msg )
        stop( 2 ) 
    
    msg =  "database " + const.DB_CONFIG + " succesvol geopend."
    writeLineToStatusFile( msg )

    try:
        # update field from database, the cli switches overwrite the DB values!
        _id, importfile, _label = config_db.strget( 138,flog )
    except Exception as e:
        msg = "fout met lezen van import bestand uit database -> " + str(e) 
        writeLineToStatusFile( msg )
        flog.error(inspect.stack()[0][3]+": " + msg )
        stop( 2 ) 
        
    # check must be done after DB read of import filename
    if args.importfile != None:
        importfile = args.importfile

    if importfile == '':
        msg = "gestopt importfile ontbreekt."
        writeLineToStatusFile( msg )
        flog.error( inspect.stack()[0][3]+": " + msg )
        stop( 2 ) 

    try:
        extension = os.path.splitext( importfile )[1]
        print ( extension )
        msg = "ZIP file " + importfile + " gevonden."
        writeLineToStatusFile( msg )
        flog.info(inspect.stack()[0][3]+": " + msg )
        if extension != '.zip':
            msg = "Geen passend ZIP file gevonden, gestopt."
            writeLineToStatusFile( msg )
            flog.warning(inspect.stack()[0][3]+": " + msg)  
            stop( 3 ) 
    except Exception as e:
        msg = "Geen passend ZIP file gevonden, gestopt -> " + str(e)
        flog.error(inspect.stack()[0][3]+": " + msg )
        stop( 3 ) 

    openDatabases()
 
    try:
        zf = zipfile.ZipFile( importfile )
        _head,tail = os.path.split( importfile ) 
        msgToInfoLogAndStatusFile( "ZIP file " + tail + " succesvol geopend." )
    except Exception as e:
        msg = "ZIP file " + importfile + " probleem =>" + str(e)
        writeLineToStatusFile( msg )
        flog.critical(inspect.stack()[0][3]+": " + msg )  
        stop( 30 ) 

    # set file rights
    setFile2user( const.FILE_DB_E_FILENAME, 'p1mon' )
    setFile2user( const.FILE_DB_E_HISTORIE, 'p1mon' )
    setFile2user( const.FILE_DB_CONFIG, 'p1mon' )
    setFile2user( const.FILE_DB_STATUS, 'p1mon' )
    setFile2user( const.FILE_DB_FINANCIEEL, 'p1mon' ) 
    setFile2user( const.FILE_DB_WEATHER, 'p1mon' )
    setFile2user( const.FILE_DB_WEATHER_HISTORIE, 'p1mon' )
    setFile2user( const.FILE_DB_TEMPERATUUR_FILENAME, 'p1mon' )
    setFile2user( const.FILE_DB_WATERMETER, 'p1mon' )
    setFile2user( const.FILE_DB_WATERMETERV2, 'p1mon' )
    setFile2user( const.FILE_DB_POWERPRODUCTION,'p1mon')
    setFile2user( const.FILE_DB_PHASEINFORMATION, 'p1mon')
    msgToInfoLogAndStatusFile( "file rechten van database bestanden correct gezet." )

    dbIntegrityCheck( config_db,           const.FILE_DB_CONFIG )
    dbIntegrityCheck( e_db_history_min,    const.FILE_DB_E_HISTORIE )
    dbIntegrityCheck( e_db_financieel_dag, const.FILE_DB_FINANCIEEL )
    dbIntegrityCheck( weer_db,             const.FILE_DB_WEATHER )
    dbIntegrityCheck( weer_history_db_uur, const.FILE_DB_WEATHER_HISTORIE )
    dbIntegrityCheck( temperature_db,      const.FILE_DB_TEMPERATUUR_FILENAME )
    dbIntegrityCheck( watermeter_db_uur,   const.FILE_DB_WATERMETER )
    dbIntegrityCheck( watermeter_db,       const.FILE_DB_WATERMETERV2 )
    dbIntegrityCheck( fase_db,             const.FILE_DB_PHASEINFORMATION )
    dbIntegrityCheck( power_production_db, const.FILE_DB_POWERPRODUCTION )

    try:
       
        for fname in zf.namelist():  #filter out the manifest file first
            if fname == const.FILE_EXPORT_MANIFEST[1:]:
                data = zf.read(const.FILE_EXPORT_MANIFEST[1:]).decode('utf-8')
                json_data = json.loads(data)
                statusdata['records_total']     = json_data['record_count']
                #statusdata['export_timestamp']  = str(json_data['timestamp'])
                msgToInfoLogAndStatusFile( "manifest is correct verwerkt uit ZIP file." )
                msgToInfoLogAndStatusFile( "aantal te importeren records " + str(statusdata['records_total']) + "." )
                msgToInfoLogAndStatusFile( "import bestand creatie datum " + str(json_data['timestamp']) + "." )
                break

    except Exception as e:
        msg = "manifest bestand is niet correct in ZIP file. -> " + str(e)
        writeLineToStatusFile( msg )
        flog.warning(inspect.stack()[0][3] + ": " + msg )

    # custom www folder
    try:
        #raise Exception("dummy exception voor test.")
        for fname in zf.namelist():
            if fname[:len(const.FILE_PREFIX_CUSTOM_UI)-1] == const.FILE_PREFIX_CUSTOM_UI[1:]:
                
                zf.extract(fname,'/') # komt dan terecht in /p1mon/var/tmp
                
                exportcode = fname[len(const.FILE_PREFIX_CUSTOM_UI)-1:-3]

                cmd = 'sudo tar -zxf ' + const.FILE_PREFIX_CUSTOM_UI + exportcode + '.gz -C ' + const.DIR_WWW_CUSTOM + " 2>/dev/null"
                if os.system( cmd ) > 0:
                    flog.error(inspect.stack()[0][3]+" custom www import gefaald.")
                    msg = "custom www bestanden konden niet worden verwerkt of waren niet aanwezig."
                    writeLineToStatusFile( msg )
                    flog.warning( inspect.stack()[0][3] + ": " + msg )
                else:
                    msgToInfoLogAndStatusFile( "custom www bestanden succesvol verwerkt." )

    except Exception as e:
        msg = "ZIP file verwerking probleem tijdens custom www verwerking -> " + str(e)
        writeLineToStatusFile( msg )
        flog.error(inspect.stack()[0][3]+": ZIP file verwerking ging mis tijdens custom www verwerking "+str(e))    

    try:

        for fname in zf.namelist():
            
            _head,tail = os.path.split(fname)
           
            #print ( "fname=", fname )
            #print ( "tail=", tail ) 
            #print ( "configuratie="   +str( tail.startswith( 'configuratie' ) ) )
            #print ( "financieel="      +str( tail.startswith( 'finacieel' ) ) )
            #print ( "historie="       +str( tail.startswith( 'historie' ) ) )
            #print ( "01_weer_historie="  +str( tail.startswith( '01_weer_historie' ) ) )
            #print ( "weer="           +str( tail.startswith( 'weer' ) ) )
            #print ( "manifest.json="  +str( tail.startswith( 'manifest.json' ) ) )
            #print ( "02_temperatuur="  +str( tail.startswith( '02_temperatuur' ) ) )
            #print ( "03_watermeter="  +str( tail.startswith( '03_watermeter' ) ) )

            #############################################
            if tail.startswith( const.DB_WATERMETERV2  ):
                processImportDataSet( const.DB_WATERMETERV2 , watermeter_db, zf, fname, 'replace into watermeter*' )
    
            ###########################################
            elif tail.startswith( const.DB_WATERMETER ):
                processImportDataSet(  const.DB_WATERMETER  , watermeter_db_uur, zf, fname, 'replace into watermeter*' )

            ############################################
            elif tail.startswith( const.DB_TEMPERATURE ):
                processImportDataSet( const.DB_TEMPERATURE , temperature_db, zf, fname, 'replace into temperatuur*' )
                temperature_db.fix_missing_month_day( flog )

            ############################################
            elif tail.startswith( const.DB_E_HISTORIE ):
                processImportDataSet( const.DB_E_HISTORIE, e_db_history_min, zf, fname, 'replace into e_history*' )

            ############################################
            # this is fix from version 0.9.19 > to fix the typo in "finacieel" text
            elif tail.startswith( const.DB_FINANCIEEL ) or tail.startswith("finacieel"):
                processImportDataSet( const.DB_FINANCIEEL, e_db_financieel_dag, zf, fname, 'replace into e_financieel*' )

            ############################################
            elif tail.startswith( const.DB_CONFIG ):
                processImportDataSet( const.DB_CONFIG, config_db, zf, fname, 'update ' + const.DB_CONFIG_TAB + ' set PARAMETER=*' )

            ############################################
            elif tail.startswith( const.DB_WEER_HISTORY ):
                processImportDataSet( const.DB_WEER_HISTORY, weer_history_db_uur, zf, fname, 'replace into weer_history*' )

            ############################################
            elif tail.startswith( const.DB_WEER ):
                processImportDataSet( const.DB_WEER, weer_db, zf, fname, 'replace into weer*' )

            ############################################
            elif tail.startswith( const.DB_PHASEINFORMATION ):
                processImportDataSet( const.DB_PHASEINFORMATION, fase_db, zf, fname, 'replace into ' + const.DB_FASE_REALTIME_TAB + '*' )
                
            ############################################
            elif tail.startswith( const.DB_POWERPRODUCTION ):
                processImportDataSet( const.DB_POWERPRODUCTION, power_production_db, zf, fname, 'replace into ' + const.DB_POWERPRODUCTION_TAB + '*' )

    except Exception as e:
        msg = "ZIP file verwerking globale fout -> " + str(e)
        writeLineToStatusFile( msg )
        flog.error( inspect.stack()[0][3] + ": " + msg )

    zf.close 
    msgToInfoLogAndStatusFile( "alle data uit het ZIP bestand verwerkt." )
    
    # waterbase V1 to V2 database conversion
    # set de normale verwerking van de water datbase uit.
    # doe de conversie
    msgToInfoLogAndStatusFile( "watermeting conversie van oude naar nieuwe database gestart." )
    os.system("/p1mon/scripts/P1WatermeterDbV1toV2.py")
    msgToInfoLogAndStatusFile( "watermeting conversie van oude naar nieuwe database gereed." )

    # lees systeem ID uit en zet deze in de config database. 
    # versleuteld om dat deze data in een back-up bestand terecht kan komen.
    try: 
        msgToInfoLogAndStatusFile( 'System ID zetten in configuratie database: ' + str( systemid.getSystemId() ) )
        #flog.info(inspect.stack()[0][3]+': System ID zetten in configuratie database: ' + str( systemid.getSystemId() ) )
        sysid_encrypted  = crypto3.p1Encrypt( systemid.getSystemId(),"sysid" ).encode('utf-8').decode('utf-8')
        config_db.strset( sysid_encrypted ,58, flog ) 
    except Exception as e:
        msg = " System ID zetten mislukt -> " + str(e.args[0])
        writeLineToStatusFile( msg )
        flog.warning( inspect.stack()[0][3] + ": " + msg )

    setSoftwareVersionInformation()

    msgToInfoLogAndStatusFile( 'WiFi wordt aangepast.' )
    if os.system('sudo /p1mon/scripts/P1SetWifi.py') > 0:
        msg = "Wifi aanpassen gefaald."
        writeLineToStatusFile( msg )
        flog.error( inspect.stack()[0][3] + ": " + msg )
    else:
        msgToInfoLogAndStatusFile( 'WiFi aanpassingen gereed.' )

    msgToInfoLogAndStatusFile( 'CRON wordt aangepast.' )
    if os.system('sudo -u p1mon /p1mon/scripts/P1Scheduler.py' ) > 0:
        msg = "CRON update gefaald."
        writeLineToStatusFile( msg )
        flog.error( inspect.stack()[0][3] + ": " + msg )
    else:
        msgToInfoLogAndStatusFile( 'CRON aanpassingen gereed.' )

    msgToInfoLogAndStatusFile( 'Internet API wordt aangepast.' )
    _id, api_is_active, _label = config_db.strget( 163, flog ) 
    if int( api_is_active ) == 1:        # the Internet API is active process the changes, make https config and get certificates.
        config_db.strset( 1, 162, flog ) # set the flag so the watchdog processes the changes
        msgToInfoLogAndStatusFile( 'Internet API aanpassingen worden doorgevoerd, duur maximaal 60 sec.' )
        time.sleep(10)

    #make sure that all is copied to disk
    msgToInfoLogAndStatusFile( "Databases worden van RAM naar het SDHC kaartje geschreven." )
    os.system( "/p1mon/scripts/P1DbCopy.py --allcopy2disk --forcecopy" )
    msgToInfoLogAndStatusFile( "Databases kopieren gereed." )

    msgToInfoLogAndStatusFile( "ZIP bestand wordt verwijderd." )
    try:
        time.sleep(1) # allow file to be unlocked.
        os.remove( importfile ) # remove uploaded file.
    except Exception as e:
        flog.error(inspect.stack()[0][3]+": "+str(e))
        msg = "verwijderen van import bestand " + importfile + " gefaald."
        writeLineToStatusFile( msg )
        flog.error( inspect.stack()[0][3] + ": " + msg )

    msgToInfoLogAndStatusFile( 'Vaste IP adressen worden aangepast als deze actief zijn.' )
    config_db.strset( 3, 168, flog ) # set the flag so the watchdog processes the changes

    timestop = time.time()
    m, s = divmod( timestop - timestart , 60) # make minutes and seconds from total secs count
    msgToInfoLogAndStatusFile( "Gereed verwerkings tijd is " + '{:02d}:{:02d}.'.format( int(m), int(s) ) )

    stop( 0 )