def main():
timewarp = ThreadPoolExecutor(2)
for fiscal_year in settings.FISCAL_YEARS:
results = timewarp.map(lambda combs: apply(download_and_analyze, combs),
usaspending.file_param_combs(fiscal_year))
for result in results:
success = result[0]
if success:
analyses = result[1]
if analyses:
for dt1, field_analyses in analyses.items():
for field_name, analysis in field_analyses.items():
print "Analysis completed for {fy}, {m}/{y}, {a}, {st}, {fld}".format(
fy=analysis['fiscal_year'],
m=analysis['month'],
y=analysis['year'],
a=analysis['agency'],
st=analysis['spending_type'],
fld=analysis['field_name'])
else:
error = result[1]
if isinstance(error, DownloadFileFailure):
print >>sys.stderr, "Failed to download %s because %s" % (error.filename, error.cause)
else:
print >>sys.stderr, str(error)
class Network(object):
def __init__(self, concurrency, timeout):
self._executor = ThreadPoolExecutor(concurrency)
self._timeout = timeout
def _request(self, request):
try:
session = requests.Session()
prepared = session.prepare_request(request)
response = session.send(prepared, timeout=self._timeout)
except Exception as exc:
logger.warning('Exception {}: {}'.format(type(exc), exc))
callback = request.kwargs['hooks']['response']
response = FakeResponse(400, 'No Response')
callback(response)
def perform_requests(self, requests):
return self._executor.map(self._request, requests)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self._executor.shutdown(wait=False)
def get_current_prices(self, instance_type='c4.large'):
thread_pool = TPE(4)
filename = 'spotprices_{0}.pkl'.format(instance_type)
if self._price_data_is_old(filename):
workers = thread_pool.map(lambda x, instance_type=instance_type: self.get_price_for_region(region=x, instance_type=instance_type), self._get_regions())
results = list(zip(self._get_regions(),[result for result in workers]))
result_file = open(filename,'wb')
pickle.dump(results,result_file)
result_file.close()
else:
result_file = open(filename,'rb')
results = pickle.load(result_file)
return results
def find_all_instances(self):
thread_pool = TPE(4)
workers = thread_pool.map(lambda x: self.find_instances(region=x), self._get_regions())
results = list(zip(self._get_regions(),[result for result in workers]))
return results