def __init__(self, testing=False): """ Constructor. :param testing: True if testing mode is being used. """ self.logger = SEKLogger(__name__, 'info') self.dbUtil = MSGDBUtil() self.dupeChecker = MSGWeatherDataDupeChecker()
def __init__(self, testing = False): """ Constructor. :param testing: True if testing mode is being used. """ self.logger = SEKLogger(__name__, 'info') self.dbUtil = MSGDBUtil() self.dupeChecker = MSGWeatherDataDupeChecker()
class MSGNOAAWeatherDataInserter(object): """ Performs weather data insertion to a database. """ def __init__(self, testing=False): """ Constructor. :param testing: True if testing mode is being used. """ self.logger = SEKLogger(__name__, 'info') self.dbUtil = MSGDBUtil() self.dupeChecker = MSGWeatherDataDupeChecker() def insertDataDict(self, conn, tableName, listOfDataDicts, commit=False): """ Given a table name and a dictionary of column names and values, insert them to the db. :param conn: A database connection. :param tableName: Name of the DB table to be inserted to. :param columnsAndValues: Dictionary of columns and values to be inserted to the DB. :param (optional) commit: A flag indicated that DB transactions will be committed. :returns: Set of datetimes processed. """ cur = conn.cursor() processedDateTimes = set() for row in listOfDataDicts: # Add a creation timestamp using the SQL function. row['created'] = 'NOW()' cols = [] vals = [] for col in row.keys(): # Prepare the columns and values for insertion via SQL. cols.append(col) if (row[col] != 'NULL'): # Surround each value with single quotes... vals.append("'%s'" % row[col]) else: # Except for NULL values. vals.append("%s" % row[col]) sql = """INSERT INTO "%s" (%s) VALUES (%s)""" % ( tableName, ','.join(cols), ','.join(vals)) if self.dupeChecker.duplicateExists(cur, row['wban'], row['datetime'], row['record_type']): self.logger.log("Dupe found, dropping dupe.", 'info') else: processedDateTimes.add( dt.datetime.strptime(row['datetime'], "%Y-%m-%d %H:%M")) if self.dbUtil.executeSQL(cur, sql, exitOnFail=False) is False: # An error occurred. for col in sorted(row.keys()): print "%s: %s" % (col, row[col]) sys.exit(-1) if commit: try: conn.commit() except: self.logger.log("ERROR: Commit failed.", 'debug') return processedDateTimes
class MSGNOAAWeatherDataInserter(object): """ Performs weather data insertion to a database. """ def __init__(self, testing = False): """ Constructor. :param testing: True if testing mode is being used. """ self.logger = SEKLogger(__name__, 'info') self.dbUtil = MSGDBUtil() self.dupeChecker = MSGWeatherDataDupeChecker() def insertDataDict(self, conn, tableName, listOfDataDicts, commit = False): """ Given a table name and a dictionary of column names and values, insert them to the db. :param conn: A database connection. :param tableName: Name of the DB table to be inserted to. :param columnsAndValues: Dictionary of columns and values to be inserted to the DB. :param (optional) commit: A flag indicated that DB transactions will be committed. :returns: Set of datetimes processed. """ cur = conn.cursor() processedDateTimes = set() for row in listOfDataDicts: # Add a creation timestamp using the SQL function. row['created'] = 'NOW()' cols = [] vals = [] for col in row.keys(): # Prepare the columns and values for insertion via SQL. cols.append(col) if (row[col] != 'NULL'): # Surround each value with single quotes... vals.append("'%s'" % row[col]) else: # Except for NULL values. vals.append("%s" % row[col]) sql = """INSERT INTO "%s" (%s) VALUES (%s)""" % ( tableName, ','.join(cols), ','.join(vals)) if self.dupeChecker.duplicateExists(cur, row['wban'], row['datetime'], row['record_type']): self.logger.log("Dupe found, dropping dupe.", 'info') else: processedDateTimes.add( dt.datetime.strptime(row['datetime'], "%Y-%m-%d %H:%M")) if self.dbUtil.executeSQL(cur, sql, exitOnFail = False) is False: # An error occurred. for col in sorted(row.keys()): print "%s: %s" % (col, row[col]) sys.exit(-1) if commit: try: conn.commit() except: self.logger.log("ERROR: Commit failed.", 'debug') return processedDateTimes