Example #1
0
def backupData(configFile):
  
  dbFile = configSettings.getEntry('//environment/database/db/name')
  db = dhecDB(dbFile, "dhec_testing_logger")
  backupDb = configSettings.getEntry('//environment/database/db/backup/filePath')
  dbSchema = configSettings.getEntry('//environment/database/db/backup/sqlSchemaFile')
  db.backupData(backupDb, dbSchema)
def checkForPlatformAndSensor( rainGauge, xeniaDB, configSettings ):
  platformHandle = "dhec.%s.raingauge" %(rainGauge)
  id =  xeniaDB.platformExists( platformHandle )
  #Error occured.
  if( id == None ):
    print( "DB Error: %s Code: %s\n" %(xeniaDB.self.lastErrorMsg, xeniaDB.lastErrorCode) )
  #Platform doesn't exist, let's add it.
  elif( id == -1 ):
    info={}
    #Check to see if the organization exists.
    orgID = xeniaDB.organizationExists('dhec');
    if( orgID == -1 ):
      info['short_name'] = 'dhec'
      #info['active'] = 1
      info['description'] = 'South Carolina Department of Heath and Environmental Control'
      orgID = xeniaDB.addOrganization(info)
      if( orgID != None ):
        print( "Successfully added organization: %s to database\n" %(info['short_name']) )
      else:
        print( "Error adding organization: %s to database.\nError: %s" %(info['short_name'],xeniaDB.lastErrorMsg) )
        sys.exit(-1)
    dhecDatabase = dhecDB(configSettings.dbSettings['dbName'], None )
    sql = "SELECT * FROM platforms WHERE name = '%s';" %( rainGauge )
    dbCursor = dhecDatabase.executeQuery(sql)
    if( dbCursor != None ):        
      row = dbCursor.fetchone()
      info['fixed_latitude'] = row['latitude']
      info['fixed_longitude'] = row['longitude']
      info['description'] = row['description']
      info['active'] = 't' #int(row['active'])
      info['platform_handle'] = platformHandle
      info['short_name'] = rainGauge
      info['organization_id'] = orgID
      id = xeniaDB.addPlatform( info )
      if( id != None ):
        print( "Successfully added platform: %s to database\n" %(platformHandle) )
      else:
        print( "Error adding platform: %s to database.\nError: %s" %(platformHandle,xeniaDB.lastErrorMsg) )

  obsID = xeniaDB.sensorExists( 'precipitation', 'millimeter', platformHandle, 1 )
  if(obsID == -1):
    obsID = xeniaDB.addSensor('precipitation','millimeter',platformHandle, 't', 0, 1, False)
    if( obsID != None ):
      print( "Successfully added obs: %s on platform: %s to database\n" %('precipitation', platformHandle) )
    else:
      print( "Error adding obs: %s on platform: %s to database\nError: %s" %('precipitation', platformHandle,xeniaDB.lastErrorMsg) )
Example #3
0
def createXMRGSummaryFiles(configFile):
  
  try:
    
    xmrgData = dhecXMRGProcessing(configFile)
    db = dhecDB(xmrgData.configSettings.dbSettings['dbName'], xmrgData.configSettings.loggerName)
    if(db.logger != None):
      db.logger.debug("Beginning createXMRGSummaryFiles");
    else:
      print("Beginning createXMRGSummaryFiles")      
    outputDirectory = xmrgData.configSettings.getEntry('//xmrgData/processingSettings/summaryDirectory')
    if(outputDirectory == None or len(outputDirectory) == 0):
      if(db.logger != None):
        db.logger.error("No output directory given in config file for //xmrgData/processingSettings/summaryDirectory. Cannot continue.")
      else:
        print("No output directory given in config file for //xmrgData/processingSettings/summaryDirectory. Cannot continue.")      
          
    #Starting date is yesterday at 12am. We take the current date and subtract a day, then set time to 12am.
    dateTime = (time.mktime(time.localtime()) - (24 * 3600))
    dateTime = time.localtime(dateTime)
    #now set the hour,min,second to 12am(00:00:00)
    dateTime = time.struct_time((dateTime[0],dateTime[1],dateTime[2],0,0,0,0,0,0))
    #Now finally let's get the UTC time.
    localTime = time.strftime("%Y-%m-%d_%H-%M-%S", dateTime)
    dateTime = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(time.mktime(dateTime)))
    rgCursor = db.getRainGauges();
    for row in rgCursor:    
      platformHandle = row['platform_handle']
      shortName = row['short_name']
      if(db.logger != None):
        db.logger.debug("Processing %s" %(platformHandle));
      else:
        print("Processing %s" %(platformHandle))      

      #Open the output file.
      fileName = "%s/%s-%s.csv" %(outputDirectory, shortName,localTime)
      if(db.logger != None):
        db.logger.debug("Opening output file %s" %(fileName));
      else:
        print("Opening output file %s" %(fileName))      
      outFile = open(fileName, "w")

      xmrgData = db.createXMRGStats(dateTime, shortName)
      #Loop through and convert values to strings.
      for key in xmrgData:
        xmrgData[key] = ("" if xmrgData[key] == None else str(xmrgData[key])) 
      outbuf = ''
      header = '';
      if(shortName == 'nmb1'):
        header = ''
        outbuf = ''
      
      elif(shortName == 'nmb2'):
        header = 'Sum past 6 days rainfall'
        outbuf = xmrgData['radarSum144']
      
      elif(shortName == 'nmb3'):
        header = "Yesterday's rainfall,Intensity of yesterday's rain,Sum past 2 days rainfall,Sum past 6 days rainfall"
        outbuf = "%s,%s,%s,%s" %\
        (xmrgData['radarSum24'],
         xmrgData['radarIntensity'],
         xmrgData['radarSum48'],
         xmrgData['radarSum144'])
      
      elif(shortName == 'mb1'):
        header = "Yesterday's rainfall,Intensity of yesterday's rain,# Previous dry days,Two day delay rain,Sum past 2 days rainfall"
        outbuf = "%s,%s,%s,%s,%s"\
           % (xmrgData['radarSum24'],
              xmrgData['radarIntensity'],
              xmrgData['radarDryCnt'],
              xmrgData['radarsum2daydelay'],
              xmrgData['radarSum48'])
      
      elif(shortName == 'mb2'):
        header = "Yesterday's rainfall,Intensity of yesterday's rain,# Previous dry days,One day delay rain,Sum past 6 days rainfall"
        outbuf = "%s,%s,%s,%s,%s"\
        % (xmrgData['radarSum24'],
           xmrgData['radarIntensity'],
           xmrgData['radarDryCnt'],
           xmrgData['radarsum1daydelay'],
           xmrgData['radarSum144'])
      
      elif(shortName == 'mb3'):
        header = "Yesterday's rainfall,Intensity of yesterday's rain,One day delay rain,Sum past 3 days rainfall,Sum past 6 days rainfall,Sum past 7 days rainfall"
        outbuf = "%s,%s,%s,%s,%s,%s"\
        % (xmrgData['radarSum24'],
           xmrgData['radarIntensity'],
           xmrgData['radarsum1daydelay'],
           xmrgData['radarSum72'],
           xmrgData['radarSum144'],
           xmrgData['radarSum168'])
  
      elif(shortName == 'mb4'):
        header = "Yesterday's rainfall,Intensity of yesterday's rain,# Previous dry days,Two day delay rain,Three day delay rain,Sum past 2 days rainfall,Sum past 3 days rainfall,Sum past 7 days rainfall"
        outbuf = "%s,%s,%s,%s,%s,%s,%s,%s" \
        % (xmrgData['radarSum24'],
           xmrgData['radarIntensity'],
           xmrgData['radarDryCnt'],
           xmrgData['radarsum2daydelay'],
           xmrgData['radarsum3daydelay'],
           xmrgData['radarSum48'],
           xmrgData['radarSum72'],
           xmrgData['radarSum168'])
  
      elif(shortName == 'surfside'):
        header = "Yesterday's rainfall,Intensity of yesterday's rain,# Previous dry days,Two day delay rain,Sum past 2 days rainfall,Sum past 3 days rainfall,Sum past 4 days rainfall,Sum past 5 days rainfall,Sum past 6 days rainfall"
        outbuf = "%s,%s,%s,%s,%s,%s,%s,%s,%s"\
        %( xmrgData['radarSum24'],
           xmrgData['radarIntensity'],
           xmrgData['radarDryCnt'],
           xmrgData['radarsum2daydelay'],
           xmrgData['radarSum48'],
           xmrgData['radarSum72'],
           xmrgData['radarSum96'],
           xmrgData['radarSum120'],
           xmrgData['radarSum144'])
        
      elif(shortName == 'gardcty'):
        header = "Yesterday's rainfall,Intensity of yesterday's rain,Sum past 5 days rainfall"
        outbuf = "%s,%s,%s"\
        %(xmrgData['radarSum24'],
          xmrgData['radarIntensity'],
          xmrgData['radarSum120'])
        
      outFile.write(header + "\n")
      outFile.write(outbuf + "\n")
      if(db.logger != None):
        db.logger.debug("Closing output file %s" %(fileName));
      else:
        print("Closing output file %s" %(fileName))      
      
      outFile.close()
  except Exception, e:
    if(db.logger != None):
      db.logger.critical(str(e) + ' Terminating script.', exc_info=1)
    #else:
    #  print("ErrMsg: %s SQL: '%s'" % (e.args[0], sql))      
    sys.exit(-1)
   dbRowsNotInserted = 0        
   for file in fileList:
     #Make sure we are trying to process a file and not a directory.
     fullPath = configSettings.rainGaugeFileDir + file
     if( os.path.isfile(fullPath) != True ):
       print( "%s is not a file, skipping" % (fullPath) )
       continue
       
     print( "Begin processing file: %s" % fullPath )
     try:
       rainGaugeFile = readRainGaugeData()
       rainGaugeFile.openFile( fullPath )
       #Get the row id and the summary id.
       rainGaugeId = file.split('.')
       platformHandle = "dhec.%s.raingauge" %(rainGaugeId[0])
       dhecDatabase = dhecDB(configSettings.dbSettings['dbName'], None )
       sql = "SELECT * FROM platforms WHERE name = '%s';" %( rainGaugeId[0] )
       dbCursor = dhecDatabase.executeQuery(sql)
       if( dbCursor == None ):
         print( "Unable to get rain gauge info from dhec database.\n" )
         sys.exit(-1)                  
       rainGaugeNfo = dbCursor.fetchone()
       dbCursor.close()
       
       #Verify the platform and sensor exists in the xenia database.
       if( options.checkPlatformAndSensor ):
         checkForPlatformAndSensor(rainGaugeId[0], xeniaDB, configSettings)
 
       dataRow = rainGaugeFile.processLine()
       while( dataRow != None ):
         if( dataRow.ID > 0 ):