Ejemplo n.º 1
0
def main():
    timewarp = ThreadPoolExecutor(2)
    for fiscal_year in settings.FISCAL_YEARS:
        results = timewarp.map(lambda combs: apply(download_and_analyze, combs),
                               usaspending.file_param_combs(fiscal_year))
        for result in results:
            success = result[0]
            if success:
                analyses = result[1]
                if analyses:
                    for dt1, field_analyses in analyses.items():
                        for field_name, analysis in field_analyses.items():
                            print "Analysis completed for {fy}, {m}/{y}, {a}, {st}, {fld}".format(
                                fy=analysis['fiscal_year'],
                                m=analysis['month'],
                                y=analysis['year'],
                                a=analysis['agency'],
                                st=analysis['spending_type'],
                                fld=analysis['field_name'])
            else:
                error = result[1]
                if isinstance(error, DownloadFileFailure):
                    print >>sys.stderr, "Failed to download %s because %s" % (error.filename, error.cause)
                else:
                    print >>sys.stderr, str(error)
Ejemplo n.º 2
0
class Network(object):

    def __init__(self, concurrency, timeout):
        self._executor = ThreadPoolExecutor(concurrency)
        self._timeout = timeout

    def _request(self, request):
        try:
            session = requests.Session()
            prepared = session.prepare_request(request)
            response = session.send(prepared, timeout=self._timeout)
        except Exception as exc:
            logger.warning('Exception {}: {}'.format(type(exc), exc))
            callback = request.kwargs['hooks']['response']
            response = FakeResponse(400, 'No Response')
            callback(response)

    def perform_requests(self, requests):
        return self._executor.map(self._request, requests)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self._executor.shutdown(wait=False)
Ejemplo n.º 3
0
    def get_current_prices(self, instance_type='c4.large'):
        thread_pool = TPE(4)
        filename = 'spotprices_{0}.pkl'.format(instance_type)
        if self._price_data_is_old(filename):
            workers = thread_pool.map(lambda x, instance_type=instance_type: self.get_price_for_region(region=x, instance_type=instance_type), self._get_regions())
            results = list(zip(self._get_regions(),[result for result in workers]))
            result_file = open(filename,'wb')
            pickle.dump(results,result_file)
            result_file.close()
        else:
            result_file = open(filename,'rb')
            results = pickle.load(result_file)

        return results
Ejemplo n.º 4
0
 def find_all_instances(self):
     thread_pool = TPE(4)
     workers = thread_pool.map(lambda x: self.find_instances(region=x), self._get_regions())
     results = list(zip(self._get_regions(),[result for result in workers]))
     return results