retriever.pool = multiprocessing.Pool(int(multiprocessingLimit))
results = retriever.pool.map(performDownloading, retriever.fileList)
retriever.pool.close()
retriever.pool.join()
if False in results:
msg = "An error occurred during primary retrieval."
print msg
MSG_BODY += "%s\n" % msg
# Force download on intermediate dates between last loaded date and now.
# If a date is less than the last loaded date, remove it from the list,
# effectively.
# Get the keep list.
keepList = weatherUtil.getKeepList(retriever.fileList, cursor)
if keepList:
msg = "Performing secondary retrieval."
print msg
MSG_BODY += "%s\n" % msg
retriever.pool = multiprocessing.Pool(int(multiprocessingLimit))
results = retriever.pool.map(performDownloadingWithForcedDownload, keepList)
retriever.pool.close()
retriever.pool.join()
if False in results:
msg = "An error occurred during secondary retrieval."
print msg
MSG_BODY += "%s\n" % msg
fileObject = gzip.open(fullPath, "rb")
weatherDays = inserter.insertDataDict(conn,
'WeatherNOAA',
dataParser.parseWeatherData(
fileObject,
[KAHULUI_AIRPORT]),
commit=True)
allDays += weatherDays
fileObject.close()
if TESTING:
break
else: # Only process the latest data from the last loaded date.
weatherUtil = MSGWeatherDataUtil()
keepList = weatherUtil.getKeepList(weatherUtil.fileList,
connector.conn.cursor())
print "keep list = %s" % keepList
keepDates = [weatherUtil.datePart(filename=k) for k in keepList]
hourlyNames = [k + 'hourly.txt.gz' for k in keepDates]
for n in hourlyNames:
fullPath = os.path.join(root, n)
msg = fullPath
print msg
msgBody += "Processing %s.\n" % msg
fileObject = gzip.open(fullPath, "rb")
weatherDays = inserter.insertDataDict(conn,
'WeatherNOAA',
dataParser.parseWeatherData(
msgBody += "Processing %s.\n" % msg
fileObject = gzip.open(fullPath, "rb")
weatherDays = inserter.insertDataDict(conn, 'WeatherNOAA',
dataParser.parseWeatherData(
fileObject,
[KAHULUI_AIRPORT]),
commit = True)
allDays += weatherDays
fileObject.close()
if TESTING:
break
else: # Only process the latest data from the last loaded date.
weatherUtil = MSGWeatherDataUtil()
keepList = weatherUtil.getKeepList(weatherUtil.fileList,
connector.conn.cursor())
print "keep list = %s" % keepList
keepDates = [weatherUtil.datePart(filename = k) for k in keepList]
hourlyNames = [k + 'hourly.txt.gz' for k in keepDates]
for n in hourlyNames:
fullPath = os.path.join(root, n)
msg = fullPath
print msg
msgBody += "Processing %s.\n" % msg
fileObject = gzip.open(fullPath, "rb")
weatherDays = inserter.insertDataDict(conn, 'WeatherNOAA',
dataParser.parseWeatherData(
fileObject,
retriever.pool = multiprocessing.Pool(int(multiprocessingLimit))
results = retriever.pool.map(performDownloading, retriever.fileList)
retriever.pool.close()
retriever.pool.join()
if False in results:
msg = "An error occurred during primary retrieval."
print msg
MSG_BODY += '%s\n' % msg
# Force download on intermediate dates between last loaded date and now.
# If a date is less than the last loaded date, remove it from the list,
# effectively.
# Get the keep list.
keepList = weatherUtil.getKeepList(retriever.fileList, cursor)
if keepList:
msg = "Performing secondary retrieval."
print msg
MSG_BODY += '%s\n' % msg
retriever.pool = multiprocessing.Pool(int(multiprocessingLimit))
results = retriever.pool.map(performDownloadingWithForcedDownload,
keepList)
retriever.pool.close()
retriever.pool.join()
if False in results:
msg = "An error occurred during secondary retrieval."
print msg
MSG_BODY += '%s\n' % msg
