def main(): test=httpscan(ADDRESS,TIMES) #start = time.time() try: with futures.ThreadPoolExecutor(max_workers=THREADS) as executor: tasks =dict((executor.submit(test.scan,port),port) for port in map(str,xrange(START,STOP))) futures.as_completed(tasks) except KeyboardInterrupt: print "Scan done\n" finally: print "Have a nice day"
def main(): test = httpscan(ADDRESS, TIMES) #start = time.time() try: with futures.ThreadPoolExecutor(max_workers=THREADS) as executor: tasks = dict((executor.submit(test.scan, port), port) for port in map(str, xrange(START, STOP))) futures.as_completed(tasks) except KeyboardInterrupt: print "Scan done\n" finally: print "Have a nice day"
def Audit(services): if services.has_key('url'): url = services['url'] #print "!!!!!" target_url=getname(url) test(target_url) # 这里执行 得到 DIR if url[-1]!="/": url += "/" print url bttest=btscan(url) with futures.ThreadPoolExecutor(max_workers=10) as executor: #默认10线程 tasks=dict((executor.submit(bttest.dirscan,dir),dir) for dir in DIR) futures.as_completed(tasks)
def fetch_all(git_repo_url, search_projects, auth=None, stream=sys.stdout): storage, stash = _init_fetch(git_repo_url, auth) start_time = time.time() exclude = set(['etk']) repos = [] for project in search_projects: repos += ['{}/{}'.format(project, slug) for slug in stash.iter_repos(project) if slug not in exclude] with futures.ThreadPoolExecutor(max_workers=8) as executor: # submit fetch jobs for each repo, creating a mapping future => (repo_name, sub_stream) future_to_repo = {} for repo_name in repos: sub_stream = StringIO() future = executor.submit(fetch, repo_name, storage, stash, sub_stream) future_to_repo[future] = (repo_name, sub_stream) # as fetches get gone, print its status along with sub-stream contents for index, future in enumerate(futures.as_completed(future_to_repo)): percent = int(((index + 1.0) / len(repos)) * 100.0) repo_name, sub_stream = future_to_repo[future] print >> stream, '=== Fetched %s (%d of %d: %d%%) ===' % (repo_name, index + 1, len(repos), percent) print >> stream, sub_stream.getvalue() yield print >> stream total_seconds = time.time()-start_time print >> stream, 'Total Time:', total_seconds storage.set_last_fetch_all_status(datetime.datetime.today(), datetime.timedelta(seconds=total_seconds))
def process(self): LOGGER.warn('process, start ') start_time = time.time() with futures.ThreadPoolExecutor(len(self.collectors)) as tp_executor: results = { tp_executor.submit( collector.collect): collector for collector in self.collectors} futures.as_completed(results) duration = time.time() - start_time self.next_wake_interval = self._wake_interval - duration if self.next_wake_interval < 1: LOGGER.warn('process, poll interval took greater than %is', duration) self.next_wake_interval = int(self._wake_interval) LOGGER.warn('process, end in %.2fs, next poll will begin at %is from now', duration, self.next_wake_interval)
def cli(): """Preprocess the samples for BDT optimization. """ # Load the configuration module. config = load_config() # Create the output directory. safe_makedirs('sample') # Preprocess the samples in parallel. To guard against deadlock, the number # of workers is chosen to be the smaller of the number of available cores # or the number of samples to preprocess. samples = config.SIGNAL + config.BACKGROUND max_workers = min(multiprocessing.cpu_count(), len(samples)) tasks = [] with futures.ProcessPoolExecutor(max_workers) as executor: for sample in samples: # The configuration module cannot be pickled, so pass the options directly. tasks.append( executor.submit(worker, sample, config.DIRECTORY, config.SELECTION, config.BRANCHES, config.TARGET_LUMI)) with click.progressbar(label='Preprocessing Samples', length=len(samples), show_pos=True, show_percent=False) as bar: for task in futures.as_completed(tasks): bar.update(1)
def upload_multiple(file_list, max_workers=5): """Upload a bunch of CSV files to Sailthru. Using Python's concurrent.futures library, asynchronously uploads the files to Sailthru, a few at a time. Arguments: file_list: List of filenames to be uploaded. max_workers: The numbers of asynchronous uploads to allow at once. """ with futures.ThreadPoolExecutor(max_workers=max_workers) as executor: future_to_file = {executor.submit(upload, filename): filename for filename in file_list} count = 0 total = len(file_list) for future in futures.as_completed(future_to_file): file_name = future_to_file[future] if future.exception() is not None: print '%r generated an exception: %s' % (file_name, future.exception()) else: # Successful upload count += 1 print ('%s/%s files have been successfully uploaded' % (count, total))
def upload_multiple(file_list, max_workers=5): """Upload a bunch of CSV files to Sailthru. Using Python's concurrent.futures library, asynchronously uploads the files to Sailthru, a few at a time. Arguments: file_list: List of filenames to be uploaded. max_workers: The numbers of asynchronous uploads to allow at once. """ with futures.ThreadPoolExecutor(max_workers=max_workers) as executor: future_to_file = { executor.submit(upload, filename): filename for filename in file_list } count = 0 total = len(file_list) for future in futures.as_completed(future_to_file): file_name = future_to_file[future] if future.exception() is not None: print '%r generated an exception: %s' % (file_name, future.exception()) else: # Successful upload count += 1 print('%s/%s files have been successfully uploaded' % (count, total))
def go(): print '-------------------------' with futures.ThreadPoolExecutor(max_workers=10) as executor: quote_futures = [executor.submit(pull_quote, q) for q in PULL] print [f.result() for f in futures.as_completed(quote_futures)] threading.Timer(PULL_INTERVAL, go).start()
def process(self): LOGGER.warn('process, start ') start_time = time.time() with futures.ThreadPoolExecutor(len(self.collectors)) as tp_executor: results = { tp_executor.submit(collector.collect): collector for collector in self.collectors } futures.as_completed(results) duration = time.time() - start_time self.next_wake_interval = self._wake_interval - duration if self.next_wake_interval < 1: LOGGER.warn('process, poll interval took greater than %is', duration) self.next_wake_interval = int(self._wake_interval) LOGGER.warn( 'process, end in %.2fs, next poll will begin at %is from now', duration, self.next_wake_interval)
def multiScan(self): with futures.ThreadPoolExecutor(max_workers=self.threads) as executor: future_to_url = dict((executor.submit(self.scan, target), target) for target in self.targets) for future in futures.as_completed(future_to_url): target = future_to_url[future] try: ret = future.result() except Exception as exc: print('%r generated an exception: %s' % (target, exc)) # logger('%r generated an exception: %s' % (url, exc)) else: print('%r returns: %s' % (target, str(ret))) self.results.append((target[0],target[1],ret))
def search_results(request): query = request.params.get('q', '') with futures.ThreadPoolExecutor(max_workers=3) as executor: workers = [ executor.submit(dataset.search, query, request), executor.submit(organization.search, query), executor.submit(wiki.search, query), ] results = dict(worker.result() for worker in futures.as_completed(workers)) return templates.render_site('search.html', request, search_query=query, has_ckan=False, **results)
def multiScan(self): with futures.ThreadPoolExecutor(max_workers=self.threads) as executor: future_to_url = dict((executor.submit(self.scan, target), target) for target in self.targets) for future in futures.as_completed(future_to_url): target = future_to_url[future] try: ret = future.result() except Exception as exc: print('%r generated an exception: %s' % (target, exc)) # logger('%r generated an exception: %s' % (url, exc)) else: print('%r returns: %s' % (target, str(ret))) self.results.append((target[0], target[1], ret))
def json2agreementmatrix(jsonflist,start=2,maxlen=0,task_type='all'): """ Multi process function to convert 2 json file annotation combination to agreement values (alpha,kappa,Avg Observed agreement) Args: jsonflist (list): list of json filenames. start (int): combination group size to begin with. maxlen(int): maximum count starting from :data:'start' Kwargs: state (bool): Current state to be in. Returns: A dict mapping annotator combination to agreement values then pickled, yamled and csved. Raises: Future.Exception """ future_list=[] detaildata={} flen=len(jsonflist) assert start+maxlen-2<=flen with futures.ProcessPoolExecutor() as executor: for cnt in range(start,start+maxlen+1): for tpl in list(itertools.combinations(jsonflist,cnt)): future_list.append(executor.submit(getagreement,tpl,os.path.dirname(jsonflist[0]),task_type)) for future in futures.as_completed(future_list): if future.exception() is not None: print('%r generated an exception: %s' % (future, future.exception())) else: detaildata.update( future.result()) yaml.dump(detaildata,open(os.path.dirname(jsonflist[0])+'\\'+str(start)+'-'+str(start+maxlen)+'out.yaml','w')) csvdump(detaildata,open(os.path.dirname(jsonflist[0])+'\\'+str(start)+'-'+str(start+maxlen)+'out.csv','w')) print "Dumped output" return detaildata
def find_all_devices(self, device_cfg): devices_mp = dict() discovery_tp_size = self.config.get('discovery_tp_size', 5) devices = utils.get_device_list(device_cfg) with futures.ThreadPoolExecutor(discovery_tp_size) as tp_executor: results = { tp_executor.submit(self.find_device, device): device for device in devices } devices = [fut.result() for fut in futures.as_completed(results)] for device in devices: if device is not None and device.get( 'ip_address') not in devices_mp: devices_mp[device.get('ip_address')] = device LOGGER.info('get_all_devices, device_count [%d]', len(devices_mp.values())) self.active_devices = devices_mp.values() return self.active_devices
def find_all_devices(self, device_cfg): devices_mp = dict() discovery_tp_size = self.config.get('discovery_tp_size', 5) devices = utils.get_device_list(device_cfg) with futures.ThreadPoolExecutor(discovery_tp_size) as tp_executor: results = { tp_executor.submit( self.find_device, device): device for device in devices} devices = [fut.result() for fut in futures.as_completed(results)] for device in devices: if device is not None and device.get( 'ip_address') not in devices_mp: devices_mp[device.get('ip_address')] = device LOGGER.info( 'get_all_devices, device_count [%d]', len( devices_mp.values())) self.active_devices = devices_mp.values() return self.active_devices
def main(): url = 'https://s3.amazonaws.com/pydemo/test.zip' async_downloads = [] session = FuturesSession(executor=ThreadPoolExecutor(max_workers=4)) for cnt in range(4): print "getting %d..." % cnt def cb(sess, resp, n=cnt): fn = '/tmp/%d-test-futures.zip' % n print "writing %s" % fn write_part(fn, resp) return "done with %s" % fn s = session.get(url, stream=True, timeout=10, #! background_callback=cb) async_downloads.append(s) for f in futures.as_completed(async_downloads): exp = f.exception() if exp is not None: raise exp print f.result()
def slurp_puppet_facts(source=None, source_url=None, auth=None, facts=None, ssl_verify=None): """ Spawn N threads for every fact being fetched. Tell N ForemanFactSlurp instances to fetch their data, process that data, and then insert the data into the ExteriorData table. """ future_facts = get_future_facts( source, source_url, auth, facts, ssl_verify ) # Clear everything we saw last time ExternalData.objects.filter(source=source).delete() with futures.ThreadPoolExecutor(max_workers=len(future_facts)) as executor: facts = dict( (executor.submit(ff.slurp), ff) for ff in future_facts ) for fact in futures.as_completed(facts): # This will can an exception and rollback our delete() fact.result() facts[fact].process()
def slurp_puppet_facts(source=None, source_url=None, auth=None, facts=None, ssl_verify=None): """ Spawn N threads for every fact being fetched. Tell N ForemanFactSlurp instances to fetch their data, process that data, and then insert the data into the ExteriorData table. """ future_facts = get_future_facts(source, source_url, auth, facts, ssl_verify) # Clear everything we saw last time ExternalData.objects.filter(source=source).delete() with futures.ThreadPoolExecutor(max_workers=len(future_facts)) as executor: facts = dict((executor.submit(ff.slurp), ff) for ff in future_facts) for fact in futures.as_completed(facts): # This will can an exception and rollback our delete() fact.result() facts[fact].process()
def fetch_all(git_repo_url, search_projects, auth=None, stream=sys.stdout): storage, stash = _init_fetch(git_repo_url, auth) start_time = time.time() exclude = set(['etk']) repos = [] for project in search_projects: repos += [ '{}/{}'.format(project, slug) for slug in stash.iter_repos(project) if slug not in exclude ] with futures.ThreadPoolExecutor(max_workers=8) as executor: # submit fetch jobs for each repo, creating a mapping future => (repo_name, sub_stream) future_to_repo = {} for repo_name in repos: sub_stream = StringIO() future = executor.submit(fetch, repo_name, storage, stash, sub_stream) future_to_repo[future] = (repo_name, sub_stream) # as fetches get gone, print its status along with sub-stream contents for index, future in enumerate(futures.as_completed(future_to_repo)): percent = int(((index + 1.0) / len(repos)) * 100.0) repo_name, sub_stream = future_to_repo[future] print >> stream, '=== Fetched %s (%d of %d: %d%%) ===' % ( repo_name, index + 1, len(repos), percent) print >> stream, sub_stream.getvalue() yield print >> stream total_seconds = time.time() - start_time print >> stream, 'Total Time:', total_seconds storage.set_last_fetch_all_status( datetime.datetime.today(), datetime.timedelta(seconds=total_seconds))
def Audit(services): global ret, http404page retinfo = {} output = 'plugin run' + os.linesep # first get http404 code page http404page = get404Page(services['url']+'/404.html') # print 'http404page=',http404page urls = generateUrls(services['url']) # pprint(urls) # threads lock = threading.Lock() threads = [] maxthreads = 20 # for url in urls: # th = threading.Thread(target=httpcrack,args=(url,lock)) # threads.append(th) # i = 0 # while i<len(threads): # if i+maxthreads >len(threads): # numthreads = len(threads) - i # else: # numthreads = maxthreads # print 'threads:',i,' - ', i + numthreads # # start threads # for j in range(numthreads): # threads[i+j].start() # # wait for threads # for j in range(numthreads): # threads[i+j].join() # i += maxthreads # 改用futures模块 # with futures.ThreadPoolExecutor(max_workers=maxthreads) as executor: #默认10线程 # future_to_url = dict((executor.submit(httpcrack, url), url) # for url in urls) with futures.ThreadPoolExecutor(max_workers=maxthreads) as executor: # Start the load operations and mark each future with its URL future_to_url = dict((executor.submit(httpcrack, url), url) for url in urls) # try: for future in futures.as_completed(future_to_url): url = future_to_url[future] try: ret = future.result() except Exception as exc: # print('%r generated an exception: %s' % (url, exc)) logger('%r generated an exception: %s' % (url, exc)) else: # print('%r returns: %s' % (url, str(ret))) logger('%r returns: %s' % (url, str(ret))) # except (KeyboardInterrupt, SystemExit): # print "Exiting..." # return (retinfo,output) # ctrl + c KeyboardInterrupt # try: # executor = futures.ThreadPoolExecutor(max_workers=maxthreads) # future_to_url = dict((executor.submit(httpcrack, url, lock), url) for url in urls) # for future in futures.as_completed(future_to_url): # url = future_to_url[future] # data = future.result() # except IndexError,e: # pass # except KeyboardInterrupt,e: # print 'KeyboardInterrupt found' # executor.shutdown(wait=False) if ret != '': retinfo = {'level':'low','content':ret} # security_warning(str(ret)) return (retinfo,output)
def cosn_expval2(self, nmax=None, rbins=500, rmax=None, truncate=5.0, nthreads=None, epsabs=1.0e-7, epsrel=1.0e-7): '''Calculates the expectation values of cos^n(theta) for the fit as a function r up to rmax and for n from 0 to nmax. rbins specifies the number of data points to calculate. rmax specifies the maximum radius to consider and is specified in dimensions of the original image that was fitted. nthreads specifies the number of threads to be used. If None, then the number of CPU cores is used as the number of threads. truncate specifies the number of basis function sigmas we consider either side of each point when calculating the intensity at each point. For example if truncate is 5.0, at each point we'll consider all basis functions whose centre lies within 5.0 * sigma of that point. 5.0 is the default. epsabs and epsrel specify the absolute and relative error desired when performing the numerical integration over theta when calculating the expectatino values. The default values should suffice. ''' if self.coef is None: logger.error('no fit done') raise AttributeError if rmax is None: rmax = self.rmax elif rmax > self.rmax: logger.error('rmax exceeds that of original data') raise ValueError if nmax is None: nmax = self.lmax elif nmax > self.lmax: logger.error('nmax exceeds lmax of the fit') raise ValueError if self.oddl: oddl = 1 else: oddl = 0 # Calculate r values. Set enpoint=False here, since the r # values are the lowest value of r in each bin. r = numpy.linspace(0.0, rmax, rbins, endpoint=False) expval = numpy.zeros((nmax + 1, rbins)) def __worker(rbin): rr = r[rbin] expval[:, rbin] = cosn_expval_point( rr, nmax, self.coef, self.kmax, self.rkstep, self.sigma, self.lmax, oddl, truncate, epsabs, epsrel) # import time # time.sleep(1) if nthreads is None: nthreads = multiprocessing.cpu_count() with futures.ThreadPoolExecutor(max_workers=nthreads) as executor: jobs = dict((executor.submit(__worker, rbin), rbin) for rbin in xrange(rbins)) jobs_done = futures.as_completed(jobs) while True: try: job = next(jobs_done) if job.exception() is not None: logger.error(job.exception()) for j in jobs: if not j.done(): # and not j.running(): j.cancel() raise job.exception() except StopIteration: break except KeyboardInterrupt: logger.info('Ctrl-c received, exiting.') for j in jobs: if not j.done(): # and not j.running(): j.cancel() raise return r, expval
def Audit(services): global ret, http404page retinfo = {} output = 'plugin run' + os.linesep # first get http404 code page http404page = get404Page(services['url'] + '/404.html') urls = generateUrls(services['url']) # pprint(urls) # threads lock = threading.Lock() threads = [] maxthreads = 20 # for url in urls: # th = threading.Thread(target=httpcrack,args=(url,lock)) # threads.append(th) # i = 0 # while i<len(threads): # if i+maxthreads >len(threads): # numthreads = len(threads) - i # else: # numthreads = maxthreads # print 'threads:',i,' - ', i + numthreads # # start threads # for j in range(numthreads): # threads[i+j].start() # # wait for threads # for j in range(numthreads): # threads[i+j].join() # i += maxthreads # 改用futures模块 # with futures.ThreadPoolExecutor(max_workers=maxthreads) as executor: #默认10线程 # future_to_url = dict((executor.submit(httpcrack, url), url) for url in urls) with futures.ThreadPoolExecutor(max_workers=maxthreads) as executor: # Start the load operations and mark each future with its URL future_to_url = dict( (executor.submit(httpcrack, url), url) for url in urls) # try: for future in futures.as_completed(future_to_url): url = future_to_url[future] try: ret = future.result() except Exception as exc: # print('%r generated an exception: %s' % (url, exc)) logger('%r generated an exception: %s' % (url, exc)) else: # print('%r returns: %s' % (url, str(ret))) logger('%r returns: %s' % (url, str(ret))) # except (KeyboardInterrupt, SystemExit): # print "Exiting..." # return (retinfo,output) if ret != '': retinfo = {'level': 'medium', 'content': ret} return (retinfo, output)
class portscan(): def __init__(self, IP): self.ip = IP #self.timeout=TIMEOUT def scan(self, port): s = socket.socket() s.settimeout(0.1) #timeout 手动修改 > < try: s.connect((self.ip, port)) except Exception, e: #print e pass else: print '%s open' % port s.close() if __name__ == "__main__": start = time.time() IP = url2ip(ADDRESS) test = portscan(IP) #test.scan(port) print "ip ------------>" + str(IP) with futures.ThreadPoolExecutor(max_workers=THREADS) as executor: tasks = dict((executor.submit(test.scan, port), port) for port in range(START, STOP)) futures.as_completed(tasks)
def overlap_factor(self, rbins=500, rmax=None, truncate=5.0, nthreads=None): if self.coef is None: logger.error('no fit done') raise AttributeError if rmax is None: rmax = self.rmax elif rmax > self.rmax: logger.error('rmax exceeds that of original data') raise ValueError if self.oddl: oddl = 1 else: oddl = 0 df = self.detectionfn if df.oddl: df_oddl = 1 else: df_oddl = 0 # Calculate r values. Set enpoint=False here, since the r # values are the lowest value of r in each bin. r = numpy.linspace(0.0, rmax, rbins, endpoint=False) lmax = min(self.lmax, df.lmax) wt = numpy.fromfunction(lambda l: 1 / (2 * l + 1), (lmax + 1,)) # Construct array holding the legendre polynomials evaluated at self.beta plbeta = legendre.legval(math.cos(self.beta), numpy.diag(numpy.ones(lmax + 1))) overlap = numpy.zeros(rbins) def __worker(rbin): rr = r[rbin] # Axis distribution beta parameters beta_axis = beta_coeffs_point( rr, self.coef, self.kmax, self.rkstep, self.sigma, self.lmax, oddl, truncate) # Detection function beta parameters in the detection frame beta_df = beta_coeffs_point( rr, df.coef, df.kmax, df.rkstep, df.sigma, df.lmax, df_oddl, truncate) # Detection function beta parameters in the lab frame beta_df = beta_df[0:lmax + 1] * plbeta # The following is too slow # res = scipy.optimize.basinhopping( # lambda x: -legendre.legval(x, beta_df), # [0.0], stepsize = 1.0/lmax, T=1.0, # minimizer_kwargs=dict(method='TNC', bounds=((-1.0,1.0),)) # ) # Find approximate position of maxmimum. Note that there # are lmax maxima/minima between -1..1, so this value of # Ns should suffice. res = scipy.optimize.brute( lambda x, *args: -legendre.legval(x, beta_df), ((-1.0, 1.0),), Ns=5.0 * lmax, ) # Refine position of maximum - note that the 'finish' # option of optimize.brute doesn't work with fmin_tnc etc respolish = scipy.optimize.minimize(lambda x: -legendre.legval(x, beta_df), [res[0]], bounds=((-1.0,1.0),), method='TNC') maxval = -respolish.fun beta = beta_axis[0:lmax + 1] * beta_df[0:lmax + 1] * wt overlap[rbin] = beta.sum() / maxval if nthreads is None: nthreads = multiprocessing.cpu_count() with futures.ThreadPoolExecutor(max_workers=nthreads) as executor: jobs = dict((executor.submit(__worker, rbin), rbin) for rbin in xrange(rbins)) jobs_done = futures.as_completed(jobs) while True: try: job = next(jobs_done) if job.exception() is not None: logger.error(job.exception()) for j in jobs: if not j.done(): # and not j.running(): j.cancel() raise job.exception() except StopIteration: break except KeyboardInterrupt: logger.info('Ctrl-c received, exiting.') for j in jobs: if not j.done(): # and not j.running(): j.cancel() raise return r, overlap
values = OrderedDict(result) r = json.dumps(values, sort_keys=False, indent=2, separators=(',', ': ')) now1 = datetime.datetime.now() - now0 #print "runtime in seconds: " #print now1.seconds #print r fout.write('runtime in seconds: ' + str(now1.seconds) + '\n' + r + '\n') fout.close() print str(fname) + " complete****************************\n" return r f = open("uniqueURI", 'r') theurls = f.readlines() count = 1 q = Queue.Queue() with futures.ThreadPoolExecutor(max_workers=NUMTHREADS) as executor: for u in theurls: urifutures = executor.submit(cd, u, count) count = count + 1 for future in futures.as_completed(urifutures): try: data = future.result() except Exception as exc: print "{} generated an exception: {}".format(u, exc)
ip=socket.gethostbyname(ADDRESS) return ip except: print "sorry,something wrong" class portscan(): def __init__(self,IP): self.ip=IP #self.timeout=TIMEOUT def scan(self,port): s=socket.socket() s.settimeout(0.1) #timeout 手动修改 > < try: s.connect((self.ip,port)) except Exception,e: #print e pass else: print '%s open'% port s.close() if __name__=="__main__": start = time.time() IP=url2ip(ADDRESS) test=portscan(IP) #test.scan(port) print "ip ------------>"+str(IP) with futures.ThreadPoolExecutor(max_workers=THREADS) as executor: tasks=dict((executor.submit(test.scan,port),port) for port in range(START,STOP)) futures.as_completed(tasks)
def calc_matrix_threaded(self, Rbins, Thetabins, kmax, lmax, detectionfn, alpha=0.0, beta=0.0, sigma=None, oddl=True, method='cquad', epsabs=0.0, epsrel=1.0e-7, wkspsize=100000, nthreads=None): """Calculates an inversion matrix using multiple threads. kmax determines the number of radial basis functions (from k=0..kmax). lmax determines the maximum value of l for the legendre polynomials (l=0..lmax). Rbins specifies the number of radial bins in the image to be inverted. Thetabins specifies the number of angular bins in the image to be inverted. sigma specifes the width of the Gaussian radial basis functions. This is defined according to the normal convention for Gaussian functions i.e. FWHM=2*sigma*sqrt(2*ln2), and NOT as defined in the Garcia, Lahon, Powis paper. If sigma is not specified it is set automatically such that the half-maximum of the Gaussian occurs midway between each radial function. Note that sigma is specified indimension of bins of the image, NOT in the dimensions of the detection function. method specifies the integration method to be used. Currently supported values are 'qaws', 'qags' and 'cquad' corresponding to the different GSL integration functions of the same name. epsabs and epsrel specify the desired integration tolerance when calculating the basis functions. The defaults should suffice. wkspsize specifies the maximum number of subintervals used for the numerical integration of the basis functions. detectionfn specifies a detection function. At present this can be None, or an instance of PbasexFit from a previous fit. alpha specifies the azimuthal angle between the frame that the detection function is specified in and the lab frame. This is in radians. If None, we assume 0.0 for this angle. beta specifies the polar angle between the frame that the detection function is specified in and the lab frame. This is in radians. If None, we assume 0.0 for this angle. nthreads specifies the number of threads to use. If this has the default value of None, the number of threads used will be equal to the number of CPU cores. """ if not isinstance(detectionfn, pbasex.PbasexFit): raise TypeError('detectionfn is not an instance of PbasexFit') # Spacing of radial basis function centres. The most obvious # choice here is rkspacing = Rbins / (kmax + 1.0), but we'd # actually like to have the last basis function centered on # the largest value of R, so instead we choose: rkspacing = Rbins / float(kmax) if sigma is None: # If sigma is not specified, we calculate a reasonable # value based on rkspacing. In the original Powis et al # paper they had rkspacing=2, and set their sigma (=2 # sigma^2) = 2 pixels. We do similar here, and then make # it 20% bigger to avoid oscillations. sigma = m.sqrt(rkspacing / 2.0) * 1.2 # We need to rescale the detection function parameters to # express them in terms of bins for the actual matrix # calculation. The detection function rmax specifies the # maximum radial value we can sensibly consider. The stored # values of rkstep and sigma in the detection function are # currently scaled according to that rmax (i.e. are in # dimensions of the image used to generate the detection # function fit). So, since our matrix calculation is actually # done in terms of bins, rather than absolute scale, we need # to rescale the detection function parameters accordingly. df_rscale = detectionfn.rmax / Rbins df_rkstep = detectionfn.rkstep / df_rscale df_sigma = detectionfn.sigma / df_rscale # It seems like a good idea to normalize the detection # coefficients to a maximum value of one, however, this seems # to lead to instability, so we don't do it. But we leave this # here as a reminder not to do it in the future. #detectionfn.coef /= detectionfn.coef.max() if detectionfn.oddl is True: df_oddl = 1 else: df_oddl = 0 if oddl is False: linc = 2 mtx = numpy.empty([kmax + 1, lmax / 2 + 1, Rbins, Thetabins]) else: linc = 1 mtx = numpy.empty([kmax + 1, lmax + 1, Rbins, Thetabins]) def __worker(k, l): rk = rkspacing * k logger.info( 'Calculating basis function for k={0}, l={1}'.format(k, l)) bf = basisfn_detfn1( k, l, Rbins, Thetabins, sigma, rk, epsabs, epsrel, wkspsize, detectionfn.coef, detectionfn.kmax, df_sigma, df_rkstep, detectionfn.lmax, df_oddl, alpha, beta, method) if oddl is True: mtx[k, l] = bf else: mtx[k, l / 2] = bf logger.info( 'Finished calculating basis function for k={0}, l={1}'.format( k, l) ) if nthreads is None: nthreads = multiprocessing.cpu_count() with futures.ThreadPoolExecutor(max_workers=nthreads) as executor: jobs = [] for k in numpy.arange(kmax + 1): for l in numpy.arange(0, lmax + 1, linc): j = executor.submit(__worker, k, l) jobs.append(j) jobs_done = futures.as_completed(jobs) while True: try: job = next(jobs_done) if job.exception() is not None: logger.error(job.exception()) for j in jobs: if not j.done(): # and not j.running(): j.cancel() raise job.exception() except StopIteration: break except KeyboardInterrupt: logger.info('Ctrl-c received, exiting.') for j in jobs: if not j.done(): # and not j.running(): j.cancel() raise self.matrix = mtx self.kmax = kmax self.sigma = sigma self.lmax = lmax self.oddl = oddl self.Rbins = Rbins self.Thetabins = Thetabins self.epsabs = epsabs self.epsrel = epsrel self.method = method # It's important we save this as part of the matrix # object, as subsequent fits with this matrix are only # valid if they have the same binning and scaling. Note # that we can always calculate the relevant rscale from # (self.Rbins / self.rmax) if needs be, so we don't save # that. self.rmax = detectionfn.rmax # Also save the detectionfn details for future reference self.detectionfn = detectionfn self.alpha = alpha self.beta = beta
def writeMeABASHScript(self, var, username, password, cmd): mutex = Lock() sys.stdout.write("#!/bin/bash\n") sys.stdout.write("#. threads: %d\n" % self._threads) sys.stdout.write("#. timeout: %0.1f\n" % self._timeout) sys.stdout.write("\n") with futures.ThreadPoolExecutor(max_workers=self._threads) as executor: #. See if stdin from the shell has any data to offer... #stdindata = [] #if select([sys.stdin,],[],[],0.0)[0]: # stdindata = [_.strip('\n') for _ in sys.stdin.readlines()] stdindata = [] if password: cmd = """sudo -S %s""" % cmd stdindata.append(password) queue = { executor.submit(self.connect, username, qdn, cmd, stdindata): qdn for qdn in self._clients.keys() } sys.stdout.write("local -A %s\n" % var) #. stdout sys.stdout.write("local -A %s_o\n" % var) #. stdout sys.stdout.write("local -A %s_e\n" % var) #. stderr sys.stdout.write("local -A %s_w\n" % var) #. warnings i = 0 total = len(self._clients.keys()) #for future in futures.as_completed(queue, timeout=self._timeout): for future in futures.as_completed(queue): i += 1 qdn = queue[future] data = None try: #data = future.result(timeout=self._timeout) data = future.result() except futures.TimeoutError: sys.stdout.write("#. %s is taking too long. Oh well.\n" % qdn) else: mutex.acquire() try: sys.stdout.write("#. %s -={\n" % qdn) _qdn, stdout, stderr, e = data qdn_hash = sha(qdn).hexdigest() sys.stdout.write( "#. Query with host %s (%d of %d) : %s\n" % ( qdn, i, total, future._state ) ) sys.stdout.write("%s[%s]=%d\n" % (var, qdn_hash, e)) if len(stdout) > 0: sys.stdout.write("read -r -d '' %s_o[%s] <<-!\n" % (var, qdn_hash)) sys.stdout.write(''.join(stdout).strip()) sys.stdout.write("\n!\n") if len(stderr) > 0: sys.stdout.write("read -r -d '' %s_e[%s] <<-!\n" % (var, qdn_hash)) sys.stdout.write(''.join(stderr).strip()) sys.stdout.write("\n!\n") sys.stdout.write("#. }=- %s\n\n" % qdn) finally: mutex.release() sys.stdout.write("#. All done.\n")
def writeMeABASHScript(self, var, username, password, cmd): mutex = Lock() sys.stdout.write("#!/bin/bash\n") sys.stdout.write("#. threads: %d\n" % self._threads) sys.stdout.write("#. timeout: %0.1f\n" % self._timeout) sys.stdout.write("\n") with futures.ThreadPoolExecutor(max_workers=self._threads) as executor: #. See if stdin from the shell has any data to offer... #stdindata = [] #if select([sys.stdin,],[],[],0.0)[0]: # stdindata = [_.strip('\n') for _ in sys.stdin.readlines()] stdindata = [] if password: cmd = """sudo -S %s""" % cmd stdindata.append(password) queue = { executor.submit(self.connect, username, qdn, cmd, stdindata): qdn for qdn in self._clients.keys() } sys.stdout.write("local -A %s\n" % var) #. stdout sys.stdout.write("local -A %s_o\n" % var) #. stdout sys.stdout.write("local -A %s_e\n" % var) #. stderr sys.stdout.write("local -A %s_w\n" % var) #. warnings i = 0 total = len(self._clients.keys()) #for future in futures.as_completed(queue, timeout=self._timeout): for future in futures.as_completed(queue): i += 1 qdn = queue[future] data = None try: #data = future.result(timeout=self._timeout) data = future.result() except futures.TimeoutError: sys.stdout.write("#. %s is taking too long. Oh well.\n" % qdn) else: mutex.acquire() try: sys.stdout.write("#. %s -={\n" % qdn) _qdn, stdout, stderr, e = data qdn_hash = sha(qdn).hexdigest() sys.stdout.write( "#. Query with host %s (%d of %d) : %s\n" % (qdn, i, total, future._state)) sys.stdout.write("%s[%s]=%d\n" % (var, qdn_hash, e)) if len(stdout) > 0: sys.stdout.write("read -r -d '' %s_o[%s] <<-!\n" % (var, qdn_hash)) sys.stdout.write(''.join(stdout).strip()) sys.stdout.write("\n!\n") if len(stderr) > 0: sys.stdout.write("read -r -d '' %s_e[%s] <<-!\n" % (var, qdn_hash)) sys.stdout.write(''.join(stderr).strip()) sys.stdout.write("\n!\n") sys.stdout.write("#. }=- %s\n\n" % qdn) finally: mutex.release() sys.stdout.write("#. All done.\n")
if uri_json[ECD]: print 'Found creation date: {}'.format(uri_json[ECD]) return (uri, uri_json[ECD]) else: print 'Found no ECD' return None if __name__ == '__main__': # Remove already completed links with open('site_mementos') as infile: input_uris = [line.split(' ')[0] for line in infile if line.rstrip('\n').split(' ')[1] != '0'] with open('site_ecd_all') as prevfile: prev = [line.split(' ')[0] for line in prevfile] uris = [uri for uri in input_uris if uri not in prev] print 'Starting on uri #{}'.format(len(uris)) # Work on the rest with open('site_ecd_all','a') as outfile: with futures.ThreadPoolExecutor(max_workers=8) as executor: urifutures = [executor.submit(getdate, uri) for uri in uris] for future in futures.as_completed(urifutures): try: data = future.result() except Exception as exc: print '{} generated an exception: {}'.format(uri, exc) if len(data) == 2: print 'Writing data: {}'.format(data) outfile.write('{} {}\n'.format(data[0], data[1])) else: print 'Found no data'
def Audit(services): global ret, http404page retinfo = {} output = 'plugin run' + os.linesep # first check no exists file noexistscode = getNoExists(services['url']+'/noexists_1231232.html') if not noexistscode: logger('no exists file return code 200, stop plugin') return # first get http404 code page http404page = get404Page(services['url']+'/404.html') urls = generateUrls(services['url']) # pprint(urls) # threads lock = threading.Lock() threads = [] maxthreads = 20 # for url in urls: # th = threading.Thread(target=httpcrack,args=(url,lock)) # threads.append(th) # i = 0 # while i<len(threads): # if i+maxthreads >len(threads): # numthreads = len(threads) - i # else: # numthreads = maxthreads # print 'threads:',i,' - ', i + numthreads # # start threads # for j in range(numthreads): # threads[i+j].start() # # wait for threads # for j in range(numthreads): # threads[i+j].join() # i += maxthreads # 改用futures模块 # with futures.ThreadPoolExecutor(max_workers=maxthreads) as executor: #默认10线程 # future_to_url = dict((executor.submit(httpcrack, url), url) for url in urls) with futures.ThreadPoolExecutor(max_workers=maxthreads) as executor: # Start the load operations and mark each future with its URL future_to_url = dict((executor.submit(httpcrack, url), url) for url in urls) # try: for future in futures.as_completed(future_to_url): url = future_to_url[future] try: ret = future.result() except Exception as exc: # print('%r generated an exception: %s' % (url, exc)) logger('%r generated an exception: %s' % (url, exc)) else: # print('%r returns: %s' % (url, str(ret))) logger('%r returns: %s' % (url, str(ret))) # except (KeyboardInterrupt, SystemExit): # print "Exiting..." # return (retinfo,output) if ret != '': retinfo = {'level':'medium','content':ret} return (retinfo,output)
#!/usr/bin/env python #### https://pypi.python.org/pypi/futures/#downloads #### Backport of the concurrent.futures package from Python 3.2 import futures import urllib2 URLS = ['http://www.foxnews.com/', 'http://www.cnn.com/', 'http://europe.wsj.com/', 'http://www.bbc.co.uk/', 'http://google.com/', 'http://yahoo.com/'] def load_url(url, timeout): return urllib2.urlopen(url, timeout=timeout).read() with futures.ThreadPoolExecutor(max_workers=5) as executor: future_to_url = dict((executor.submit(load_url, url, 60), url) for url in URLS) for future in futures.as_completed(future_to_url): url = future_to_url[future] if future.exception() is not None: print '%r generated an exception: %s' % (url, future.exception()) else: print '%r page is %d bytes' % (url, len(future.result()))