class WeatherDataLoadingTester(unittest.TestCase):
def setUp(self):
self.weatherUtil = MSGWeatherDataUtil()
self.logger = SEKLogger(__name__, 'DEBUG')
self.dbConnector = MSGDBConnector()
self.cursor = self.dbConnector.conn.cursor()
self.configer = MSGConfiger()
def testLoadDataSinceLastLoaded(self):
"""
Data should be loaded since the last data present in the database.
"""
pass
def testRetrieveDataSinceLastLoaded(self):
"""
Data since the last loaded date is retrieved.
"""
pass
def testGetLastLoadedDate(self):
myDate = self.weatherUtil.getLastDateLoaded(self.cursor).strftime(
"%Y-%m-%d %H:%M:%S")
pattern = '^(\d+-\d+-\d+\s\d+:\d+:\d+)$'
match = re.match(pattern, myDate)
assert match and (match.group(1) == myDate), "Date format is valid."
def testWeatherDataPattern(self):
myPattern = self.configer.configOptionValue('Weather Data',
'weather_data_pattern')
testString = """<A HREF="someURL">QCLCD201208.zip</A>"""
match = re.match(myPattern, testString)
self.logger.log("pattern = %s" % myPattern, 'info')
if match:
self.logger.log("match = %s" % match)
self.logger.log("match group = %s" % match.group(1))
else:
self.logger.log("match not found")
assert match and match.group(
1) == 'QCLCD201208.zip', "Download filename was matched."
def testWeatherDataURL(self):
myURL = self.configer.configOptionValue('Weather Data',
'weather_data_url')
pass
def __init__(self):
self.configer = MSGConfiger()
self.pool = None
self.fileList = []
self.dateList = []
self.weatherUtil = MSGWeatherDataUtil()
global weatherDataPath
weatherDataPath = self.configer.configOptionValue(
WEATHER_DATA, 'weather_data_path')
global weatherDataURL
weatherDataURL = self.configer.configOptionValue(
WEATHER_DATA, 'weather_data_url')
global weatherDataPattern
weatherDataPattern = self.configer.configOptionValue(
WEATHER_DATA, 'weather_data_pattern')
def setUp(self):
self.weatherUtil = MSGWeatherDataUtil()
self.logger = SEKLogger(__name__, 'DEBUG')
self.dbConnector = MSGDBConnector()
self.cursor = self.dbConnector.conn.cursor()
self.configer = MSGConfiger()
"""
Save retrieval results stored in a global string.
"""
global MSG_BODY
global WEATHER_DATA_PATH
fp = open("%s/retrieval-results.txt" % WEATHER_DATA_PATH, "wb")
fp.write(MSG_BODY)
fp.close()
if __name__ == "__main__":
dbConnector = MSGDBConnector()
cursor = dbConnector.conn.cursor()
weatherUtil = MSGWeatherDataUtil()
timeUtil = MSGTimeUtil()
msg = "Downloading NOAA weather data (%s)." % timeUtil.conciseNow()
print msg
MSG_BODY = "%s\n" % msg
msg = "Last loaded date is %s." % weatherUtil.datePart(datetime=weatherUtil.getLastDateLoaded(cursor))
print msg
MSG_BODY += "%s\n" % msg
retriever = MSGWeatherDataRetriever()
configer = MSGConfiger()
WEATHER_DATA_PATH = configer.configOptionValue("Weather Data", "weather_data_path")
msg = "Using URL %s." % weatherDataURL
print msg
msgBody += "Processing %s.\n" % msg
fileObject = gzip.open(fullPath, "rb")
weatherDays = inserter.insertDataDict(conn, 'WeatherNOAA',
dataParser.parseWeatherData(
fileObject,
[KAHULUI_AIRPORT]),
commit = True)
allDays += weatherDays
fileObject.close()
if TESTING:
break
else: # Only process the latest data from the last loaded date.
weatherUtil = MSGWeatherDataUtil()
keepList = weatherUtil.getKeepList(weatherUtil.fileList,
connector.conn.cursor())
print "keep list = %s" % keepList
keepDates = [weatherUtil.datePart(filename = k) for k in keepList]
hourlyNames = [k + 'hourly.txt.gz' for k in keepDates]
for n in hourlyNames:
fullPath = os.path.join(root, n)
msg = fullPath
print msg
msgBody += "Processing %s.\n" % msg
fileObject = gzip.open(fullPath, "rb")
weatherDays = inserter.insertDataDict(conn, 'WeatherNOAA',
msgBody += "Processing %s.\n" % msg
fileObject = gzip.open(fullPath, "rb")
weatherDays = inserter.insertDataDict(conn,
'WeatherNOAA',
dataParser.parseWeatherData(
fileObject,
[KAHULUI_AIRPORT]),
commit=True)
allDays += weatherDays
fileObject.close()
if TESTING:
break
else: # Only process the latest data from the last loaded date.
weatherUtil = MSGWeatherDataUtil()
keepList = weatherUtil.getKeepList(weatherUtil.fileList,
connector.conn.cursor())
print "keep list = %s" % keepList
keepDates = [weatherUtil.datePart(filename=k) for k in keepList]
hourlyNames = [k + 'hourly.txt.gz' for k in keepDates]
for n in hourlyNames:
fullPath = os.path.join(root, n)
msg = fullPath
print msg
msgBody += "Processing %s.\n" % msg
fileObject = gzip.open(fullPath, "rb")
weatherDays = inserter.insertDataDict(conn,
"""
Save retrieval results stored in a global string.
"""
global MSG_BODY
global WEATHER_DATA_PATH
fp = open('%s/retrieval-results.txt' % WEATHER_DATA_PATH, 'wb')
fp.write(MSG_BODY)
fp.close()
if __name__ == '__main__':
dbConnector = MSGDBConnector()
cursor = dbConnector.conn.cursor()
weatherUtil = MSGWeatherDataUtil()
timeUtil = MSGTimeUtil()
msg = "Downloading NOAA weather data (%s)." % timeUtil.conciseNow()
print msg
MSG_BODY = '%s\n' % msg
msg = "Last loaded date is %s." % weatherUtil.datePart(
datetime=weatherUtil.getLastDateLoaded(cursor))
print msg
MSG_BODY += '%s\n' % msg
retriever = MSGWeatherDataRetriever()
configer = MSGConfiger()
WEATHER_DATA_PATH = configer.configOptionValue('Weather Data',
'weather_data_path')
