def __init__(self):
self.configer = MSGConfiger()
self.pool = None
self.fileList = []
self.dateList = []
self.weatherUtil = MSGWeatherDataUtil()
global weatherDataPath
weatherDataPath = self.configer.configOptionValue(
WEATHER_DATA, 'weather_data_path')
global weatherDataURL
weatherDataURL = self.configer.configOptionValue(
WEATHER_DATA, 'weather_data_url')
global weatherDataPattern
weatherDataPattern = self.configer.configOptionValue(
WEATHER_DATA, 'weather_data_pattern')
def setUp(self):
self.weatherUtil = MSGWeatherDataUtil()
self.logger = SEKLogger(__name__, 'DEBUG')
self.dbConnector = MSGDBConnector()
self.cursor = self.dbConnector.conn.cursor()
self.configer = MSGConfiger()
msgBody += "Processing %s.\n" % msg
fileObject = gzip.open(fullPath, "rb")
weatherDays = inserter.insertDataDict(conn,
'WeatherNOAA',
dataParser.parseWeatherData(
fileObject,
[KAHULUI_AIRPORT]),
commit=True)
allDays += weatherDays
fileObject.close()
if TESTING:
break
else: # Only process the latest data from the last loaded date.
weatherUtil = MSGWeatherDataUtil()
keepList = weatherUtil.getKeepList(weatherUtil.fileList,
connector.conn.cursor())
print "keep list = %s" % keepList
keepDates = [weatherUtil.datePart(filename=k) for k in keepList]
hourlyNames = [k + 'hourly.txt.gz' for k in keepDates]
for n in hourlyNames:
fullPath = os.path.join(root, n)
msg = fullPath
print msg
msgBody += "Processing %s.\n" % msg
fileObject = gzip.open(fullPath, "rb")
weatherDays = inserter.insertDataDict(conn,
