Example #1
0
def makeTopicJsonFile():
    list_of_topics = []
    if mqtt_para['smartmeterpublishisactive'] == True:
        topicToJson(mqtt_topics_smartmeter, list_of_topics)
    if mqtt_para['watermeterpublishisactive'] == True:
        topicToJson(mqtt_topics_watermeter, list_of_topics)
    if mqtt_para['weatherpublishisactive'] == True:
        topicToJson(mqtt_topics_weather, list_of_topics)
    if mqtt_para['indoortemperaturepublishisactive'] == True:
        topicToJson(mqtt_topics_indoor_temperature, list_of_topics)
    if mqtt_para['phasepublishisactive'] == True:
        topicToJson(mqtt_topics_phase, list_of_topics)
    if mqtt_para['powerproductionpublishisactive'] == True:
        topicToJson(mqtt_topics_powerproduction, list_of_topics)

    try:
        filename = const.FILE_MQTT_TOPICS
        flog.debug(inspect.stack()[0][3] + ": topics json output =" +
                   json.dumps(list_of_topics, sort_keys=True) +
                   " naar bestand " + filename)
        with open(filename, 'w') as outfile:
            json.dump(list_of_topics, outfile, sort_keys=True)
        setFile2user(filename, 'p1mon')  # to make sure we can process the file
    except Exception as e:
        flog.error(inspect.stack()[0][3] +
                   ": wegschrijven data naar ramdisk is mislukt. melding:" +
                   str(e.args[0]))
def write_p1_json_dbx_folder( configdb=None, data=None, flog=None, sysid=const.SYSTEM_ID_DEFAULT, utc_ok_timestamp=0):
    
    #flog.setLevel(logging.DEBUG)
    _id, parameter, _label = configdb.strget(50,flog) # is Dropbox is on or off 
    if int(parameter) == 0: # dropbox sharing is off
        return utc_ok_timestamp # do nothing

    #print ("#!"+ str( abs(util.getUtcTime() - utc_ok_timestamp) ) )

    if abs(util.getUtcTime() - utc_ok_timestamp) < 10:
         return utc_ok_timestamp # do nothing, DBX rate limting

    try:
        filename = const.DIR_DBX_LOCAL + const.DBX_DIR_DATA + '/' + const.API_BASIC_JSON_PREFIX + sysid + const.API_BASIC_JSON_SUFFIX
        flog.debug(inspect.stack()[0][3]+": json output =" + json.dumps( data , sort_keys=True ) + " naar bestand " + filename )

        with open(filename, 'w') as outfile:
            json.dump( data , outfile, sort_keys=True )

        utc_ok_timestamp = util.getUtcTime() 
        util.setFile2user( filename,'p1mon' ) # to make sure we can process the file

    except Exception as e:
        flog.error(inspect.stack()[0][3]+": wegschrijven data naar Dropbox is mislukt. melding:"+str(e.args[0]))

    return utc_ok_timestamp
Example #3
0
def writeManifestFile():
    flog.debug(inspect.stack()[0][3]+": Manifest file " + const.FILE_EXPORT_MANIFEST + " maken.")
    try:    
        manifestdata = {
            'timestamp': util.mkLocalTimeString(),
            'record_count': statusdata['record_count']
        }
        fo = open(const.FILE_EXPORT_MANIFEST, "w")
        fo.write(json.dumps(manifestdata))
        fo.close() 
        util.setFile2user(const.FILE_EXPORT_MANIFEST,'p1mon')  
    except Exception as e:
        flog.error(inspect.stack()[0][3]+": manifest bestand kon niet worden weggeschreven -> "+str(e))
def write_p1_json_to_ram( data=None, flog=None, sysid=const.SYSTEM_ID_DEFAULT ):
    #flog.setLevel(logging.DEBUG)
   
    try:
        filename     = const.DIR_RAMDISK  + const.API_BASIC_JSON_PREFIX + sysid + const.API_BASIC_JSON_SUFFIX
        filename_tmp = const.DIR_RAMDISK  + const.API_BASIC_JSON_PREFIX + sysid + const.API_BASIC_JSON_SUFFIX + ".tmp"
        flog.debug(inspect.stack()[0][3]+": json output =" + json.dumps( data , sort_keys=True ) + " naar bestand " + filename)
        with open( filename_tmp, 'w') as outfile:
            json.dump( data , outfile, sort_keys=True )
        util.setFile2user( filename_tmp,'p1mon' ) # to make sure we can process the file
        if os.path.exists( filename_tmp ):
            if os.path.getsize( filename_tmp ) > 250: # only write the file if it has some content
                shutil.move( filename_tmp, filename ) # beter then os.rename/os.move
    except Exception as e:
        flog.error(inspect.stack()[0][3]+": wegschrijven data naar ramdisk is mislukt. melding:"+str(e.args[0]))
Example #5
0

def saveExit(signum, frame):
    stop()


def stop():
    global mqtt_client
    if mqtt_client != None:
        closeMqtt()
    signal.signal(signal.SIGINT, original_sigint)
    flog.info(inspect.stack()[0][3] + " SIGINT ontvangen, gestopt.")
    sys.exit(0)


#-------------------------------
if __name__ == "__main__":
    try:
        logfile = const.DIR_FILELOG + prgname + ".log"
        setFile2user(logfile, 'p1mon')
        flog = fileLogger(logfile, prgname)
        #### aanpassen bij productie
        flog.setLevel(logging.INFO)
        flog.consoleOutputOn(True)
    except Exception as e:
        print("critical geen logging mogelijke, gestopt.:" + str(e.args[0]))
        sys.exit(10)  #  error: no logging check file rights

    original_sigint = signal.getsignal(signal.SIGINT)
    signal.signal(signal.SIGINT, saveExit)
    Main(sys.argv[1:])
Example #6
0
def setFileFlags():
    setFile2user(const.FILE_DB_POWERPRODUCTION, 'p1mon')
    _head, tail = os.path.split(const.FILE_DB_POWERPRODUCTION)
    setFile2user(const.DIR_FILEDISK + tail, 'p1mon')
Example #7
0
def setFileFlags():
    setFile2user(const.FILE_DB_WATERMETERV2, 'p1mon')
    _head, tail = os.path.split(const.FILE_DB_WATERMETERV2)
    setFile2user(const.DIR_FILEDISK + tail, 'p1mon')
Example #8
0
        t = time.localtime()
        fp.write(
            "%04d-%02d-%02d %02d:%02d:%02d" %
            (t.tm_year, t.tm_mon, t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec) +
            " " + msg + '\n')
        fp.close()
    except Exception as e:
        flog.error("status file schrijf fout: " + str(e))


#-------------------------------
if __name__ == "__main__":
    try:
        os.umask(0o002)
        logfile = const.DIR_FILELOG + prgname + ".log"
        util.setFile2user(logfile, 'p1mon')
        flog = logger.fileLogger(logfile, prgname)
        flog.setLevel(logger.logging.INFO)
        flog.consoleOutputOn(True)

        status_fp = open(const.FILE_UPGRADE_AIDE_STATUS, "w")
        subprocess.run(
            ['sudo', 'chmod', '0666', const.FILE_UPGRADE_AIDE_STATUS],
            stdout=subprocess.DEVNULL,
            stderr=subprocess.DEVNULL)
        status_fp.close()

    except Exception as e:
        print("critical geen logging mogelijke, gestopt.:" + str(e.args[0]))
        sys.exit(1)  #  error: no logging check file rights
Example #9
0
def setFileFlags():
	setFile2user(const.FILE_DB_TEMPERATUUR_FILENAME,'p1mon')
	dummy,tail = os.path.split(const.FILE_DB_TEMPERATUUR_FILENAME)   
	setFile2user(const.DIR_FILEDISK+tail,'p1mon')
Example #10
0
def Main(argv):
    flog.info("Start van programma.")

    parser = argparse.ArgumentParser(description="options: -fb")
    parser.add_argument('-fb',
                        '--forcebackup',
                        required=False,
                        action="store_true")
    args = parser.parse_args()

    # open van status database
    try:
        rt_status_db.init(const.FILE_DB_STATUS, const.DB_STATUS_TAB)
    except Exception as e:
        flog.critical(inspect.stack()[0][3] + ": Database niet te openen(1)." +
                      const.FILE_DB_STATUS + ") melding:" + str(e.args[0]))
        sys.exit(1)
    flog.debug(inspect.stack()[0][3] + ": database tabel " +
               const.DB_STATUS_TAB + " succesvol geopend.")

    # open van config database
    try:
        config_db.init(const.FILE_DB_CONFIG, const.DB_CONFIG_TAB)
    except Exception as e:
        flog.critical(inspect.stack()[0][3] + ": database niet te openen(2)." +
                      const.FILE_DB_CONFIG + ") melding:" + str(e.args[0]))
        sys.exit(2)
    flog.debug(inspect.stack()[0][3] + ": database tabel " +
               const.DB_CONFIG_TAB + " succesvol geopend.")

    _id, do_ftp_backup, _label = config_db.strget(36, flog)
    _id, do_dbx_backup, _label = config_db.strget(49, flog)
    parameter = 0
    # do some casting....comparing strings is a pain
    do_ftp_backup = int(do_ftp_backup)
    do_dbx_backup = int(do_dbx_backup)

    flog.debug(inspect.stack()[0][3] + ": ftp backup=" + str(do_ftp_backup) +
               " Dropbox back-up=" + str(do_dbx_backup))

    # if one of the backup options is on make an export file
    if do_ftp_backup == 1 or do_dbx_backup == 1:
        parameter = 1
    else:
        flog.info(inspect.stack()[0][3] +
                  ": Backup staat uit, geen back-up gestart")

    if args.forcebackup == True:  # force backup, forget preference.
        flog.info(inspect.stack()[0][3] + ": geforceerde back-up gestart.")
        parameter = 1

    if int(parameter) == 1:
        flog.debug(inspect.stack()[0][3] + ": export file wordt gemaakt.")
        file_tmp_id = str(util.getUtcTime()) + "-" + systemid.getSystemId()
        cmd = "/p1mon/scripts/P1SqlExport.py -e " + file_tmp_id
        flog.debug(inspect.stack()[0][3] + ": export commando is ->" +
                   str(cmd))
        retvalue = os.system("/p1mon/scripts/P1SqlExport.py -e " + file_tmp_id)
        if retvalue != 0:
            flog.error(inspect.stack()[0][3] +
                       ": export van file gefaald, gestopt.")
            sys.exit(3)
        export_file = exportfile_base + file_tmp_id + '.zip'
        if util.fileExist(export_file) == False:
            flog.error(inspect.stack()[0][3] + ": export file " + export_file +
                       " niet gevonden, gestopt.")
            sys.exit(4)
        # update config database with current name of backup file.
        config_db.strset(export_file, 33, flog)

    if do_dbx_backup == 1:
        # do dropbox back-up
        flog.info(inspect.stack()[0][3] + ": Dropbox backup gestart")
        try:
            # try to clean old files
            clean_old_files()

            if len(os.listdir(const.DIR_DBX_LOCAL +
                              const.DBX_DIR_BACKUP)) > 10:
                flog.critical(
                    inspect.stack()[0][3] +
                    ": Dropbox backup bestand niet gekopierd, te veel bestanden in ram buffer."
                )
            else:
                flog.debug(inspect.stack()[0][3] +
                           ": copy export file naar lokale dropbox folder: " +
                           export_file)
                shutil.copy2(export_file,
                             const.DIR_DBX_LOCAL + const.DBX_DIR_BACKUP)
                _head, tail = os.path.split(export_file)
                util.setFile2user(
                    const.DIR_DBX_LOCAL + const.DBX_DIR_BACKUP + '/' + tail,
                    'p1mon')
        except Exception as e:
            flog.error(inspect.stack()[0][3] + ": Dropbox back-up. melding:" +
                       str(e.args[0]))

    if do_ftp_backup == 1:
        flog.info(inspect.stack()[0][3] + ": FTP backup gestart")
        retvalue = os.system("/p1mon/scripts/P1FtpCopy.py")
        if retvalue > 0:
            flog.error(inspect.stack()[0][3] +
                       ": ftp backup gefaald, gestopt.")
            sys.exit(5)

    flog.info("programma is succesvol gestopt.")
    sys.exit(0)  # all is well.
Example #11
0
def setFileFlags():
    setFile2user(const.FILE_DB_E_FILENAME,'p1mon')
    setFile2user(const.FILE_DB_E_HISTORIE,'p1mon')
    setFile2user(const.FILE_DB_CONFIG,'p1mon')
    setFile2user(const.FILE_DB_STATUS,'p1mon')
    setFile2user(const.FILE_DB_FINANCIEEL,'p1mon')
    setFile2user(const.FILE_DB_WEATHER,'p1mon')
    setFile2user(const.FILE_DB_WEATHER_HISTORIE,'p1mon')
    setFile2user(const.FILE_DB_TEMPERATUUR_FILENAME,'p1mon')
    setFile2user(const.FILE_DB_WATERMETER,'p1mon')

    dummy,tail = os.path.split(const.FILE_DB_E_FILENAME)
    setFile2user(const.DIR_FILEDISK+tail,'p1mon')
    dummy,tail = os.path.split(const.FILE_DB_E_HISTORIE)
    setFile2user(const.DIR_FILEDISK+tail,'p1mon')
    dummy,tail = os.path.split(const.FILE_DB_CONFIG)
    setFile2user(const.DIR_FILEDISK+tail,'p1mon')
    dummy,tail = os.path.split(const.FILE_DB_STATUS)
    setFile2user(const.DIR_FILEDISK+tail,'p1mon')
    dummy,tail = os.path.split(const.FILE_DB_FINANCIEEL)
    setFile2user(const.DIR_FILEDISK+tail,'p1mon')
    dummy,tail = os.path.split(const.FILE_DB_WEATHER)
    setFile2user(const.DIR_FILEDISK+tail,'p1mon')
    dummy,tail = os.path.split(const.FILE_DB_WEATHER_HISTORIE)
    setFile2user(const.DIR_FILEDISK+tail,'p1mon') 
    dummy,tail = os.path.split(const.FILE_DB_TEMPERATUUR_FILENAME)
    setFile2user(const.DIR_FILEDISK+tail,'p1mon')
    dummy,tail = os.path.split( const.FILE_DB_WATERMETERV2 )
    setFile2user(const.DIR_FILEDISK+tail,'p1mon')
Example #12
0
def Main(argv):

    global no_status_messages
    
    my_pid = os.getpid()
    flog.info("Start van programma met process id " + str(my_pid) )
    pid_list, _process_list = listOfPidByName( prgname )
    #print ( pid_list )
    pid_list.remove( my_pid ) # remove own pid from the count
    flog.debug( inspect.stack()[0][3] + ": pid list clean (andere lopende proces id's) " + str(pid_list ) )
    if len( pid_list ) > 0: # more then 1 because the script is started from os.system()
        msg_str = "Gestopt een andere versie van het programma is actief."
        flog.info( inspect.stack()[0][3] + ": " + msg_str )
        sys.exit(1)

    timestart = time.time() # used te calculate the processing time.

    flog.info( inspect.stack()[0][3] +  ": Wordt uitgevoerd als user -> " + pwd.getpwuid( os.getuid() ).pw_name  )
    
    parser = argparse.ArgumentParser( description = prgname )
    parser.add_argument( '-i' , '--importfile', help="Naam van het export bestand om te importeren.",     required=False  ) 
    parser.add_argument( '-rm', '--rmstatus',   help="Maak geen status bestand aan", action='store_true', required=False ) 

    args = parser.parse_args()
    no_status_messages = args.rmstatus # default False when set True


    initStatusFile()

    # open van config database
    try: 
        config_db.init( const.FILE_DB_CONFIG,const.DB_CONFIG_TAB )
    except Exception as e:
        msg = ": database niet te openen ("+ const.FILE_DB_CONFIG + ") melding: " + str(e.args[0] )
        writeLineToStatusFile( msg )
        stop( 2 ) 
    
    msg =  "database " + const.DB_CONFIG + " succesvol geopend."
    writeLineToStatusFile( msg )

    try:
        # update field from database, the cli switches overwrite the DB values!
        _id, importfile, _label = config_db.strget( 138,flog )
    except Exception as e:
        msg = "fout met lezen van import bestand uit database -> " + str(e) 
        writeLineToStatusFile( msg )
        flog.error(inspect.stack()[0][3]+": " + msg )
        stop( 2 ) 
        
    # check must be done after DB read of import filename
    if args.importfile != None:
        importfile = args.importfile

    if importfile == '':
        msg = "gestopt importfile ontbreekt."
        writeLineToStatusFile( msg )
        flog.error( inspect.stack()[0][3]+": " + msg )
        stop( 2 ) 

    try:
        extension = os.path.splitext( importfile )[1]
        print ( extension )
        msg = "ZIP file " + importfile + " gevonden."
        writeLineToStatusFile( msg )
        flog.info(inspect.stack()[0][3]+": " + msg )
        if extension != '.zip':
            msg = "Geen passend ZIP file gevonden, gestopt."
            writeLineToStatusFile( msg )
            flog.warning(inspect.stack()[0][3]+": " + msg)  
            stop( 3 ) 
    except Exception as e:
        msg = "Geen passend ZIP file gevonden, gestopt -> " + str(e)
        flog.error(inspect.stack()[0][3]+": " + msg )
        stop( 3 ) 

    openDatabases()
 
    try:
        zf = zipfile.ZipFile( importfile )
        _head,tail = os.path.split( importfile ) 
        msgToInfoLogAndStatusFile( "ZIP file " + tail + " succesvol geopend." )
    except Exception as e:
        msg = "ZIP file " + importfile + " probleem =>" + str(e)
        writeLineToStatusFile( msg )
        flog.critical(inspect.stack()[0][3]+": " + msg )  
        stop( 30 ) 

    # set file rights
    setFile2user( const.FILE_DB_E_FILENAME, 'p1mon' )
    setFile2user( const.FILE_DB_E_HISTORIE, 'p1mon' )
    setFile2user( const.FILE_DB_CONFIG, 'p1mon' )
    setFile2user( const.FILE_DB_STATUS, 'p1mon' )
    setFile2user( const.FILE_DB_FINANCIEEL, 'p1mon' ) 
    setFile2user( const.FILE_DB_WEATHER, 'p1mon' )
    setFile2user( const.FILE_DB_WEATHER_HISTORIE, 'p1mon' )
    setFile2user( const.FILE_DB_TEMPERATUUR_FILENAME, 'p1mon' )
    setFile2user( const.FILE_DB_WATERMETER, 'p1mon' )
    setFile2user( const.FILE_DB_WATERMETERV2, 'p1mon' )
    setFile2user( const.FILE_DB_POWERPRODUCTION,'p1mon')
    setFile2user( const.FILE_DB_PHASEINFORMATION, 'p1mon')
    msgToInfoLogAndStatusFile( "file rechten van database bestanden correct gezet." )

    dbIntegrityCheck( config_db,           const.FILE_DB_CONFIG )
    dbIntegrityCheck( e_db_history_min,    const.FILE_DB_E_HISTORIE )
    dbIntegrityCheck( e_db_financieel_dag, const.FILE_DB_FINANCIEEL )
    dbIntegrityCheck( weer_db,             const.FILE_DB_WEATHER )
    dbIntegrityCheck( weer_history_db_uur, const.FILE_DB_WEATHER_HISTORIE )
    dbIntegrityCheck( temperature_db,      const.FILE_DB_TEMPERATUUR_FILENAME )
    dbIntegrityCheck( watermeter_db_uur,   const.FILE_DB_WATERMETER )
    dbIntegrityCheck( watermeter_db,       const.FILE_DB_WATERMETERV2 )
    dbIntegrityCheck( fase_db,             const.FILE_DB_PHASEINFORMATION )
    dbIntegrityCheck( power_production_db, const.FILE_DB_POWERPRODUCTION )

    try:
       
        for fname in zf.namelist():  #filter out the manifest file first
            if fname == const.FILE_EXPORT_MANIFEST[1:]:
                data = zf.read(const.FILE_EXPORT_MANIFEST[1:]).decode('utf-8')
                json_data = json.loads(data)
                statusdata['records_total']     = json_data['record_count']
                #statusdata['export_timestamp']  = str(json_data['timestamp'])
                msgToInfoLogAndStatusFile( "manifest is correct verwerkt uit ZIP file." )
                msgToInfoLogAndStatusFile( "aantal te importeren records " + str(statusdata['records_total']) + "." )
                msgToInfoLogAndStatusFile( "import bestand creatie datum " + str(json_data['timestamp']) + "." )
                break

    except Exception as e:
        msg = "manifest bestand is niet correct in ZIP file. -> " + str(e)
        writeLineToStatusFile( msg )
        flog.warning(inspect.stack()[0][3] + ": " + msg )

    # custom www folder
    try:
        #raise Exception("dummy exception voor test.")
        for fname in zf.namelist():
            if fname[:len(const.FILE_PREFIX_CUSTOM_UI)-1] == const.FILE_PREFIX_CUSTOM_UI[1:]:
                
                zf.extract(fname,'/') # komt dan terecht in /p1mon/var/tmp
                
                exportcode = fname[len(const.FILE_PREFIX_CUSTOM_UI)-1:-3]

                cmd = 'sudo tar -zxf ' + const.FILE_PREFIX_CUSTOM_UI + exportcode + '.gz -C ' + const.DIR_WWW_CUSTOM + " 2>/dev/null"
                if os.system( cmd ) > 0:
                    flog.error(inspect.stack()[0][3]+" custom www import gefaald.")
                    msg = "custom www bestanden konden niet worden verwerkt of waren niet aanwezig."
                    writeLineToStatusFile( msg )
                    flog.warning( inspect.stack()[0][3] + ": " + msg )
                else:
                    msgToInfoLogAndStatusFile( "custom www bestanden succesvol verwerkt." )

    except Exception as e:
        msg = "ZIP file verwerking probleem tijdens custom www verwerking -> " + str(e)
        writeLineToStatusFile( msg )
        flog.error(inspect.stack()[0][3]+": ZIP file verwerking ging mis tijdens custom www verwerking "+str(e))    

    try:

        for fname in zf.namelist():
            
            _head,tail = os.path.split(fname)
           
            #print ( "fname=", fname )
            #print ( "tail=", tail ) 
            #print ( "configuratie="   +str( tail.startswith( 'configuratie' ) ) )
            #print ( "financieel="      +str( tail.startswith( 'finacieel' ) ) )
            #print ( "historie="       +str( tail.startswith( 'historie' ) ) )
            #print ( "01_weer_historie="  +str( tail.startswith( '01_weer_historie' ) ) )
            #print ( "weer="           +str( tail.startswith( 'weer' ) ) )
            #print ( "manifest.json="  +str( tail.startswith( 'manifest.json' ) ) )
            #print ( "02_temperatuur="  +str( tail.startswith( '02_temperatuur' ) ) )
            #print ( "03_watermeter="  +str( tail.startswith( '03_watermeter' ) ) )

            #############################################
            if tail.startswith( const.DB_WATERMETERV2  ):
                processImportDataSet( const.DB_WATERMETERV2 , watermeter_db, zf, fname, 'replace into watermeter*' )
    
            ###########################################
            elif tail.startswith( const.DB_WATERMETER ):
                processImportDataSet(  const.DB_WATERMETER  , watermeter_db_uur, zf, fname, 'replace into watermeter*' )

            ############################################
            elif tail.startswith( const.DB_TEMPERATURE ):
                processImportDataSet( const.DB_TEMPERATURE , temperature_db, zf, fname, 'replace into temperatuur*' )
                temperature_db.fix_missing_month_day( flog )

            ############################################
            elif tail.startswith( const.DB_E_HISTORIE ):
                processImportDataSet( const.DB_E_HISTORIE, e_db_history_min, zf, fname, 'replace into e_history*' )

            ############################################
            # this is fix from version 0.9.19 > to fix the typo in "finacieel" text
            elif tail.startswith( const.DB_FINANCIEEL ) or tail.startswith("finacieel"):
                processImportDataSet( const.DB_FINANCIEEL, e_db_financieel_dag, zf, fname, 'replace into e_financieel*' )

            ############################################
            elif tail.startswith( const.DB_CONFIG ):
                processImportDataSet( const.DB_CONFIG, config_db, zf, fname, 'update ' + const.DB_CONFIG_TAB + ' set PARAMETER=*' )

            ############################################
            elif tail.startswith( const.DB_WEER_HISTORY ):
                processImportDataSet( const.DB_WEER_HISTORY, weer_history_db_uur, zf, fname, 'replace into weer_history*' )

            ############################################
            elif tail.startswith( const.DB_WEER ):
                processImportDataSet( const.DB_WEER, weer_db, zf, fname, 'replace into weer*' )

            ############################################
            elif tail.startswith( const.DB_PHASEINFORMATION ):
                processImportDataSet( const.DB_PHASEINFORMATION, fase_db, zf, fname, 'replace into ' + const.DB_FASE_REALTIME_TAB + '*' )
                
            ############################################
            elif tail.startswith( const.DB_POWERPRODUCTION ):
                processImportDataSet( const.DB_POWERPRODUCTION, power_production_db, zf, fname, 'replace into ' + const.DB_POWERPRODUCTION_TAB + '*' )

    except Exception as e:
        msg = "ZIP file verwerking globale fout -> " + str(e)
        writeLineToStatusFile( msg )
        flog.error( inspect.stack()[0][3] + ": " + msg )

    zf.close 
    msgToInfoLogAndStatusFile( "alle data uit het ZIP bestand verwerkt." )
    
    # waterbase V1 to V2 database conversion
    # set de normale verwerking van de water datbase uit.
    # doe de conversie
    msgToInfoLogAndStatusFile( "watermeting conversie van oude naar nieuwe database gestart." )
    os.system("/p1mon/scripts/P1WatermeterDbV1toV2.py")
    msgToInfoLogAndStatusFile( "watermeting conversie van oude naar nieuwe database gereed." )

    # lees systeem ID uit en zet deze in de config database. 
    # versleuteld om dat deze data in een back-up bestand terecht kan komen.
    try: 
        msgToInfoLogAndStatusFile( 'System ID zetten in configuratie database: ' + str( systemid.getSystemId() ) )
        #flog.info(inspect.stack()[0][3]+': System ID zetten in configuratie database: ' + str( systemid.getSystemId() ) )
        sysid_encrypted  = crypto3.p1Encrypt( systemid.getSystemId(),"sysid" ).encode('utf-8').decode('utf-8')
        config_db.strset( sysid_encrypted ,58, flog ) 
    except Exception as e:
        msg = " System ID zetten mislukt -> " + str(e.args[0])
        writeLineToStatusFile( msg )
        flog.warning( inspect.stack()[0][3] + ": " + msg )

    setSoftwareVersionInformation()

    msgToInfoLogAndStatusFile( 'WiFi wordt aangepast.' )
    if os.system('sudo /p1mon/scripts/P1SetWifi.py') > 0:
        msg = "Wifi aanpassen gefaald."
        writeLineToStatusFile( msg )
        flog.error( inspect.stack()[0][3] + ": " + msg )
    else:
        msgToInfoLogAndStatusFile( 'WiFi aanpassingen gereed.' )

    msgToInfoLogAndStatusFile( 'CRON wordt aangepast.' )
    if os.system('sudo -u p1mon /p1mon/scripts/P1Scheduler.py' ) > 0:
        msg = "CRON update gefaald."
        writeLineToStatusFile( msg )
        flog.error( inspect.stack()[0][3] + ": " + msg )
    else:
        msgToInfoLogAndStatusFile( 'CRON aanpassingen gereed.' )

    msgToInfoLogAndStatusFile( 'Internet API wordt aangepast.' )
    _id, api_is_active, _label = config_db.strget( 163, flog ) 
    if int( api_is_active ) == 1:        # the Internet API is active process the changes, make https config and get certificates.
        config_db.strset( 1, 162, flog ) # set the flag so the watchdog processes the changes
        msgToInfoLogAndStatusFile( 'Internet API aanpassingen worden doorgevoerd, duur maximaal 60 sec.' )
        time.sleep(10)

    #make sure that all is copied to disk
    msgToInfoLogAndStatusFile( "Databases worden van RAM naar het SDHC kaartje geschreven." )
    os.system( "/p1mon/scripts/P1DbCopy.py --allcopy2disk --forcecopy" )
    msgToInfoLogAndStatusFile( "Databases kopieren gereed." )

    msgToInfoLogAndStatusFile( "ZIP bestand wordt verwijderd." )
    try:
        time.sleep(1) # allow file to be unlocked.
        os.remove( importfile ) # remove uploaded file.
    except Exception as e:
        flog.error(inspect.stack()[0][3]+": "+str(e))
        msg = "verwijderen van import bestand " + importfile + " gefaald."
        writeLineToStatusFile( msg )
        flog.error( inspect.stack()[0][3] + ": " + msg )

    msgToInfoLogAndStatusFile( 'Vaste IP adressen worden aangepast als deze actief zijn.' )
    config_db.strset( 3, 168, flog ) # set the flag so the watchdog processes the changes

    timestop = time.time()
    m, s = divmod( timestop - timestart , 60) # make minutes and seconds from total secs count
    msgToInfoLogAndStatusFile( "Gereed verwerkings tijd is " + '{:02d}:{:02d}.'.format( int(m), int(s) ) )

    stop( 0 )