def handler(event, contest): logger.info("Start!") executor = ThreadPoolExecutor(max_workers=1000) main_loop = asyncio.new_event_loop() main_loop.set_default_executor(executor) asyncio.set_event_loop(main_loop) poll = Poll(main_loop) cal = poll.cal update_and_delete = UpdateAndDelete(main_loop, executor) table = event['table'] queue_url = event['queueUrl'] message_count = event['messageCount'] poll.messages(sqs, queue_url, message_count) logger.info("Receive API count: {}".format(poll.fetch_count)) logger.info("Fetched messages: {}".format(poll.message_count)) update_and_delete.execute(sqs_client, db, queue_url, table, cal.stats) logger.info("Update API count: {}".format(update_and_delete.update_count)) logger.info("Delete API count: {}".format(update_and_delete.delete_count)) logger.info("Delete Message count: {}".format( update_and_delete.deleted_message_count)) main_loop.close() executor.shutdown() return "Lambda job finished successfully."
class TaskPool(BasePool): """Thread Task Pool.""" body_can_be_buffer = True signal_safe = False def __init__(self, *args, **kwargs): super(TaskPool, self).__init__(*args, **kwargs) # from 3.5, it is calculated from number of CPUs if (3, 0) <= sys.version_info < (3, 5) and self.limit is None: self.limit = 5 self.executor = ThreadPoolExecutor(max_workers=self.limit) def on_stop(self): self.executor.shutdown() super(TaskPool, self).on_stop() def on_apply(self, target, args=None, kwargs=None, callback=None, accept_callback=None, **_): f = self.executor.submit(apply_target, target, args, kwargs, callback, accept_callback) return ApplyResult(f) def _get_info(self): return { 'max-concurrency': self.limit, 'threads': len(self.executor._threads) # TODO use a public api to retrieve the current number of threads # in the executor when available. (Currently not available). }
class ModulePool: def __init__(self, workers=1): self._jobs = {} self._pool = ThreadPoolExecutor(workers) def _launch(self, func, hook, dependencies): if dependencies: for dependency in dependencies: while True: if dependency in self._jobs.keys(): self._jobs[dependency].result() break else: sleep(0.1) func(hook) def schedule(self, func, hook): innerfunc, args, module, hookname, dependencies = hook job = self._pool.submit(self._launch, func, hook, dependencies) self._jobs[module] = job def shutdown(self): for job in as_completed([self._jobs[j] for j in self._jobs]): job.result() self._pool.shutdown()
def main(): """Main method executed when run""" if len(argv) != 2: print("Usage: flymirror.py [rules_file]") return if not file_exists(argv[1]): print("Error: rules file", argv[1], "does not exist.") return config = read_config(argv[1]) URLS.put(config.start) # Start the loops in another thread # include 2 extra threads for the loopers pool = ThreadPoolExecutor(int(config.workers) + 2) perfprint("[START]") pool.submit(download_loop, pool) pool.submit(handle_response_loop, pool, config) # Join on both the queues at once (Yeah, this is hacky -- may break in later versions) while URLS.unfinished_tasks or RESPONSES.unfinished_tasks: sleep(0.3) # Shut everything down (may take 1 second) DONE.put(True) pool.shutdown() perfprint("[END]")
def start(self): left_lng = 103.9213455517 top_lat = 30.7828453209 right_lng = 104.2178123382 bottom_lat = 30.4781772402 offset = 0.002 if os.path.isfile(self.db_name): os.remove(self.db_name) try: with sqlite3.connect(self.db_name) as c: c.execute('''CREATE TABLE mobike (Time DATETIME, bikeIds VARCHAR(12), bikeType TINYINT,distId INTEGER,distNum TINYINT, type TINYINT, x DOUBLE, y DOUBLE)''') except Exception as ex: pass executor = ThreadPoolExecutor(max_workers=250) print("Start") self.total = 0 lat_range = np.arange(top_lat, bottom_lat, -offset) for lat in lat_range: lng_range = np.arange(left_lng, right_lng, offset) for lon in lng_range: self.total += 1 executor.submit(self.get_nearby_bikes, (lat, lon)) executor.shutdown() self.group_data()
def handler(event, contest): logger.info("Start!") executor = ThreadPoolExecutor(max_workers=100) cal = Sum() queue_url = event['queueUrl'] message_count = event['messageCount'] queue = sqs.Queue(queue_url) num_of_calls = message_count // batch_count queues = [] for i in range(num_of_calls): queues.append(queue) message_count = 0 responses = executor.map(one_request, queues) for response in responses: message_count += len(response) for msg in response: cal.add(msg) logger.info("Receive API count: {}".format(num_of_calls)) logger.info("Fetched messages: {}".format(message_count)) executor.shutdown()
class Session(object): def __init__(self, hostnames=None, port=8080, max_workers=6, catalog="hive", schema="default", user='******', result_mode='dict'): self.catalog = catalog self.schema = schema self.hostnames = hostnames self.port = port self.user = user self.result_mode = result_mode self.executor = ThreadPoolExecutor(max_workers=max_workers) self.default_timeout = None self.wait_step = 0.2 self.wait_max_step = 10 def __enter__(self): return self def __exit__(self, type, value, traceback): self.close() def close(self): self.executor.shutdown() def query_async(self, q, args=None): q = Query(self, q, args) return self.executor.submit(q.execute_wait) def query(self, q, args=None): q = Query(self, q, args) q.execute() return q
class ScheduledIOPool(Thread): """Schedule events to an IO worker pool. """ def __init__(self, workers, delay): super(ScheduledIOPool, self).__init__() self.scheduler = sched.scheduler(time.time, time.sleep) self.thread_pool = ThreadPoolExecutor(max_workers=workers) self.delay = delay self.shutdown = False self.daemon = True def shutdown(self): self.shutdown = True # TODO shutdown def run(self): while not self.shutdown: try: self.scheduler.run() time.sleep(.1) # TODO: no wait/notify when queue is empty except: # TODO log print "Unexpected error scheduling IO:" traceback.print_exc() time.sleep(.1) self.thread_pool.shutdown() def cancel(self, event): return self.scheduler.cancel(event) def schedule(self, *args): return self.scheduler.enter(self.delay, 1, self.thread_pool.submit, args)
class ThreadPool: def __init__(self, workers=1): self._jobs = [] self._pool = ThreadPoolExecutor(workers) def _finish(self, job): try: self._jobs.remove(job) except ValueError: pass try: e = job.exception() if e: raise e except Exception as e: error(str(e), exc_info=True) def schedule(self, func): job = self._pool.submit(func) job.add_done_callback(self._finish) self._jobs.append(job) def shutdown(self): for job in as_completed(self._jobs): job.result() self._pool.shutdown()
class ActiveVoice: def __init__(self): # See https://msdn.microsoft.com/en-us/library/ms723602%28v=vs.85%29.aspx for doc self._voice = win32com.client.Dispatch("SAPI.SpVoice") self._executor = ThreadPoolExecutor(max_workers=1) def shutdown(self): self._executor.shutdown(False) def speak(self, text, volume=DEFAULT_VOLUME, rate=DEFAULT_RATE): volume = max(MIN_VOLUME, min(MAX_VOLUME, volume)) rate = max(MIN_RATE, min(MAX_RATE, rate)) def do_speak(): self._voice.Volume = volume self._voice.Rate = rate self._voice.Speak(text) self._executor.submit(do_speak) def wait(self, time_in_sec): self._executor.submit(time.sleep, time_in_sec) def list_avatars(self): # pointless, only one on Windows 10! avatars = [] for index, token in enumerate(self._voice.GetVoices("", "")): avatars.append( '%d: %s' % (index+1, token.GetDescription()) ) return avatars
def main(cmd, args): logging.basicConfig( level=logging.INFO, format=("%(relativeCreated)04d %(process)05d %(threadName)-10s " "%(levelname)-5s %(msg)s")) pool = Pool(max_workers=1) arg1 = tmp_fname arg2 = 'arg2' stage_file_name = cmd if (glob_stage == 1): f = pool.submit(check_output, ["ec-perl", "-w", stage_file_name, args, arg2], shell=True) print "stage 1 started. Initiate call to" + stage_file_name print "Stage is " + str(glob_stage) else: f = pool.submit(check_output, ["ec-perl", stage_file_name, " ", args, arg2], shell=True) print "stage 2 started. Initiate call to" + stage_file_name + " " + args + " " + arg2 + "'" print "Stage is " + str(glob_stage) f.add_done_callback(callback) pool.shutdown(wait=False)
class BaseDataLayer(Layer): def setup(self, bottom, top): param = eval(self.param_str_) self.batch_size_ = param['batch_size'] self.data_setup(bottom, top) top[0].reshape(*self.data_.shape) self.executor_ = ThreadPoolExecutor(max_workers=1) self.thread_ = self.executor_.submit(self.internal_thread_entry) def reshape(self, bottom, top): pass def forward(self, bottom, top): self.thread_.result() top[0].reshape(*self.data_.shape) top[0].data[...] = self.data_ self.thread_ = self.executor_.submit(self.internal_thread_entry) def data_setup(self, bottom, top): raise NotImplementedError() def internal_thread_entry(self): raise NotImplementedError() def __del__(self): self.thread_.result() self.executor_.shutdown() super(self.__class__, self).__del__()
def upload_file(upload_file_name, temp_file_name='encoded.csv', split_file_format="{orig_file}_{id}.{orig_ext}", parent_folder_id='0B46HJMu9Db4xTUxhQ0x4WHpfVmM'): file_name = os.path.basename(upload_file_name) # Encode file. base64.encode(open(upload_file_name), open(temp_file_name, 'w+')) # Split file. num_split_files, file_names = splitfile(temp_file_name, SPLIT_SIZE, split_file_format) # Start upload threads. start = time.time() file_id = uuid.uuid1() thread_pool = ThreadPoolExecutor(max_workers=MAX_DOWNLOADS) for i in range(num_split_files): current_file_name = file_names[i] up_t = upload_worker.UploadWorker(index=i + 1, file_id=file_id, filename=file_name, parent_folder_id=parent_folder_id, total_file_num=num_split_files, upload_file_name=current_file_name) future = thread_pool.submit(up_t.run) # Wait for completion. thread_pool.shutdown() end = time.time() m, s = divmod(end - start, 60) print "Overall time taken: ", m, "m ", s, "s" return file_id
class HttpThreadpool(object): def __init__(self, max_workers=10, queue_size=200): self.executor = ThreadPoolExecutor(max_workers, queue_size) @retry(max_tries=3) def _download(self, url): req = urllib2.Request(url) req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.93 Safari/537.36') rsp = urllib2.urlopen(req, timeout=30) return rsp.read() def download_and_process(self, url, body_process): return self.executor.submit(self._download_and_process, url, body_process) def _download_and_process(self, url, body_process): body_func, body_args, body_kw = body_process body = self._download(url) try: body_func(body, *body_args, **body_kw) except Exception as e: print url, traceback.format_exec() def shutdown(self): self.executor.shutdown()
class LoaferRunner: def __init__(self, loop=None, max_workers=None, on_stop_callback=None): self._on_stop_callback = on_stop_callback self.loop = loop or asyncio.get_event_loop() # XXX: See https://github.com/python/asyncio/issues/258 # The minimum value depends on the number of cores in the machine # See https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor self._executor = ThreadPoolExecutor(max_workers) self.loop.set_default_executor(self._executor) def start(self, future=None, run_forever=True): start = 'starting Loafer, pid={}, run_forever={}' logger.info(start.format(os.getpid(), run_forever)) self.loop.add_signal_handler(signal.SIGINT, self.stop) self.loop.add_signal_handler(signal.SIGTERM, self.stop) try: if run_forever: self.loop.run_forever() else: self.loop.run_until_complete(future) self.stop() except CancelledError: self.loop.close() def stop(self, *args, **kwargs): logger.info('stopping Loafer ...') if callable(self._on_stop_callback): self._on_stop_callback() self._executor.shutdown(wait=True) if self.loop.is_running(): self.loop.stop()
def test(no_workers, pipeline, input, state, run_function = run_pipeline): executor = ThreadPoolExecutor(max_workers = no_workers) try: result = run_function(executor, pipeline, input, state) finally: executor.shutdown(True) return result
class ExecutorEventEmitter(BaseEventEmitter): """An event emitter class which runs handlers in a ``concurrent.futures`` executor. If using python 2, this will fall back to trying to use the ``futures`` backported library (caveats there apply). By default, this class creates a default ``ThreadPoolExecutor``, but a custom executor may also be passed in explicitly to, for instance, use a ``ProcessPoolExecutor`` instead. This class runs all emitted events on the configured executor. Errors captured by the resulting Future are automatically emitted on the ``error`` event. This is unlike the BaseEventEmitter, which have no error handling. The underlying executor may be shut down by calling the ``shutdown`` method. Alternately you can treat the event emitter as a context manager:: with ExecutorEventEmitter() as ee: # Underlying executor open @ee.on('data') def handler(data): print(data) ee.emit('event') # Underlying executor closed Since the function call is scheduled on an executor, emit is always non-blocking. No effort is made to ensure thread safety, beyond using an executor. """ def __init__(self, executor=None): super(ExecutorEventEmitter, self).__init__() if executor: self._executor = executor else: self._executor = ThreadPoolExecutor() def _emit_run(self, f, args, kwargs): future = self._executor.submit(f, *args, **kwargs) @future.add_done_callback def _callback(f): exc = f.exception() if exc: self.emit('error', exc) def shutdown(self, wait=True): """Call ``shutdown`` on the internal executor.""" self._executor.shutdown(wait=wait) def __enter__(self): return self def __exit__(self, type, value, traceback): self.shutdown()
def poem_from_template(template, db, sound_cache=None): executor = ThreadPoolExecutor(4) letter_sound_map = map_letters_to_sounds(db, template, sound_cache) process_tmpl_line = threaded(partial(extract_ruleset, db, letter_sound_map), partial(ruleset_to_line, db)) poem_lines = executor.map(process_tmpl_line, template) executor.shutdown() return list(poem_lines)
def fetch(self, tiles): """ Execute all tile requests. :param tiles: List of tile requests. """ pool = ThreadPoolExecutor(max_workers=32) pool.map(self.fetch_tile, tiles, timeout=5) pool.shutdown()
class LoaferRunner: def __init__(self, loop=None, max_workers=None, on_stop_callback=None): self._on_stop_callback = on_stop_callback self.loop = loop or asyncio.get_event_loop() # XXX: See https://github.com/python/asyncio/issues/258 # The minimum value depends on the number of cores in the machine # See https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor self._executor = ThreadPoolExecutor(max_workers) self.loop.set_default_executor(self._executor) def start(self, future=None, run_forever=None, debug=False): if debug: self.loop.set_debug(enabled=debug) if future: logger.warning( 'runner `future` argument is deprecated and will be removed in the next major version' ) if run_forever: logger.warning( 'runner `run_forever` argument is deprecated and will be removed in the next major version' ) self.loop.add_signal_handler(signal.SIGINT, self.prepare_stop) self.loop.add_signal_handler(signal.SIGTERM, self.prepare_stop) try: self.loop.run_forever() finally: self.stop() self.loop.close() logger.debug('loop.is_running={}'.format(self.loop.is_running())) logger.debug('loop.is_closed={}'.format(self.loop.is_closed())) def prepare_stop(self, *args): if self.loop.is_running(): # signals loop.run_forever to exit in the next iteration self.loop.stop() def stop(self, *args, **kwargs): logger.info('stopping Loafer ...') if callable(self._on_stop_callback): self._on_stop_callback() logger.info('cancel schedulled operations ...') for task in asyncio.Task.all_tasks(self.loop): task.cancel() if task.cancelled() or task.done(): continue with suppress(CancelledError): self.loop.run_until_complete(task) self._executor.shutdown(wait=True)
def search(self, query): n_days = (self.until - self.since).days tp = ThreadPoolExecutor(max_workers=self.n_threads) for i in range(0, n_days): since_query = self.since + datetime.timedelta(days=i) until_query = self.since + datetime.timedelta(days=(i + 1)) day_query = "%s since:%s until:%s" % (query, since_query.strftime("%Y-%m-%d"), until_query.strftime("%Y-%m-%d")) tp.submit(self.perform_search, day_query) tp.shutdown(wait=True)
class ThreadedResolverTest(AsyncTestCase, _ResolverTestMixin): def setUp(self): super(ThreadedResolverTest, self).setUp() from concurrent.futures import ThreadPoolExecutor self.executor = ThreadPoolExecutor(2) self.resolver = Resolver(self.io_loop, self.executor) def tearDown(self): self.executor.shutdown() super(ThreadedResolverTest, self).tearDown()
def launch(self, *args, **kwargs): """ Launch a new pool with one thread for that one process. Then we call the callback. """ pool = Pool(max_workers=1) f = pool.submit(Executor, *args, **kwargs) f.add_done_callback(self.callback) pool.shutdown(wait=True) return f
class Foo(object): def __init__(self): self._thread_pool = ThreadPoolExecutor(8) @run_on_executor(executor="_thread_pool") def foo(self, i): logging.debug("{0} {1}".format(current_thread(), i)) def shutdown(self): self._thread_pool.shutdown()
class Executor(): def __init__(self, io_loop=None): self.io_loop = io_loop or tornado.ioloop.IOLoop.instance() self.executor = ThreadPoolExecutor(2) @tornado.concurrent.run_on_executor def wait_and_return_a_value(self): return 20 def destroy(self): self.executor.shutdown(1)
def main(): executor=ThreadPoolExecutor(max_workers=3) futures=[] for i in range(6): print("Threads: {}".format(len(executor._threads))) futures.append(executor.submit(spam,i)) print("main thread exit") for future in concurrent.futures.as_completed(futures): print(future.result()) executor.shutdown()
def stop(): logging.info("Stopping tasks on server...") executor = ThreadPoolExecutor(MAX_SUBMISSION_THREADS) global _unfinished_run_ids stopTasks = set() for runId in _unfinished_run_ids: stopTasks.add(executor.submit(_stop_run, runId)) for task in stopTasks: task.result() executor.shutdown(wait=True) logging.info("Stopped all tasks.")
def console_main(): """ This function handles all the console action. """ setproctitle('image-scraper') scraper = ImageScraper() scraper.get_arguments() print("\nImageScraper\n============\nRequesting page....\n") try: scraper.get_html() except PageLoadError as err: if err.status_code is None: print("ImageScraper is unable to acces the internet.") else: print("Page failed to load. Status code: {0}".format(err.status_code)) sys.exit() scraper.get_img_list() if len(scraper.images) == 0: sys.exit("Sorry, no images found.") if scraper.no_to_download is None: scraper.no_to_download = len(scraper.images) print("Found {0} images: ".format(len(scraper.images))) try: scraper.process_download_path() except DirectoryAccessError: print("Sorry, the directory can't be accessed.") sys.exit() except DirectoryCreateError: print("Sorry, the directory can't be created.") sys.exit() if scraper.dump_urls: for img_url in scraper.images: print(img_url) status_flags = {'count': 0, 'percent': 0.0, 'failed': 0, 'under_min_or_over_max_filesize': 0} widgets = ['Progress: ', Percentage(), ' ', Bar(marker=RotatingMarker()), ' ', ETA(), ' ', FileTransferSpeed()] pbar = ProgressBar(widgets=widgets, maxval=100).start() pool = ThreadPoolExecutor(max_workers=scraper.nthreads) status_lock = threading.Lock() for img_url in scraper.images: if status_flags['count'] == scraper.no_to_download: break pool.submit(download_worker_fn, scraper, img_url, pbar, status_flags, status_lock) status_flags['count'] += 1 pool.shutdown(wait=True) pbar.finish() print("\nDone!\nDownloaded {0} images\nFailed: {1}\n".format( status_flags['count']-status_flags['failed']-status_flags['under_min_or_over_max_filesize'], status_flags['failed'])) return
def poem_from_template(template, db: Database, corpus_id, sound_cache=None): engine = get_engine(db) conn = engine.connect() executor = ThreadPoolExecutor(4) letter_sound_map = map_letters_to_sounds(conn, corpus_id, template, sound_cache) process_tmpl_line = threaded(partial(extract_ruleset, conn, corpus_id, letter_sound_map), partial(ruleset_to_line, conn, corpus_id)) poem_lines = executor.map(process_tmpl_line, template) executor.shutdown() return list(poem_lines)
def test_propagation_time(nb_worlds, nb_changes): executor = ThreadPoolExecutor(max_workers=nb_worlds) pool = Pool(nb_worlds) pipes = [] res = [] for i in range(nb_worlds-1): print("Setting up passthrough between world %d and world %d" % (i, i+1)) #f = executor.submit(passthrough, "world%d" % i, "world%d" % (i+1)) conn1, conn2 = Pipe() pipes.append(conn1) res.append(pool.apply_async(passthrough, ["world%d" % i, "world%d" % (i+1), conn2])) time.sleep(0.5) ctx = underworlds.Context("test_client") entry_world = ctx.worlds["world0"] exit_world = ctx.worlds["world%d" % (nb_worlds-1)] future = executor.submit(wait_for_changes, exit_world, nb_changes) print("\n\n\nPropagating %d change(s) from world %s..." % (nb_changes, entry_world)) profileonce("start test with %d worlds" % nb_worlds) for i in range(nb_changes): n = Node() n.name = "node_%d" % i entry_world.scene.append_and_propagate(n) time.sleep(0.01) seen, duration = future.result() profileonce("end test") if seen is None: logger.error("The changes have not been seen!") duration = 0 else: print("It took %s to be notified of the %d change(s) in world %s" % (ms(duration), nb_changes, exit_world)) executor.shutdown(wait=True) for p in pipes: p.send(True) pool.close() pool.join() ctx.close() return duration
class AWSBenchmark(Benchmark): """AWSBenchmark an extension of Benchmark class, to run benchmarks on AWS """ @classmethod def fetch_results(cls, instances_file, filter=None, force_update=False): bench = cls(None, None) bench._load_instances(normalize_path(instances_file)) inst = next(inst for inst in bench.instances.values()) bench.sid = inst.session bucket_name = re.match(r's3://([\w\-.]+)/.*', inst.s3_dir).group(1) bench.s3 = boto3.resource('s3', region_name=bench.region) bench.bucket = bench._create_s3_bucket(bucket_name, auto_create=False) filter = ( lambda items: [k for k, v in items]) if filter is None else filter for iid in filter(bench.instances.items()): if force_update: bench.instances[iid].success = False bench._download_results(iid) def __init__(self, framework_name, benchmark_name, constraint_name, region=None): """ :param framework_name: :param benchmark_name: :param constraint_name: :param region: """ super().__init__(framework_name, benchmark_name, constraint_name) self.suid = datetime_iso( micros=True, no_sep=True ) # short sid for AWS entities whose name length is limited self.region = region if region \ else rconfig().aws.region if rconfig().aws['region'] \ else boto3.session.Session().region_name self.ami = rconfig().aws.ec2.regions[self.region].ami self.cloudwatch = None self.ec2 = None self.iam = None self.s3 = None self.bucket = None self.uploaded_resources = None self.instance_profile = None self.instances = {} self.jobs = [] self.exec = None self.monitoring = None self._validate2() def _validate(self): if rconfig().aws.ec2.terminate_instances not in [ 'always', 'success', 'never', True, False ]: raise ValueError( "`terminate_instances` setting should be one among ['always', 'success', 'never']" ) if self.parallel_jobs == 0 or self.parallel_jobs > rconfig( ).max_parallel_jobs: log.warning("Forcing parallelization to its upper limit: %s.", rconfig().max_parallel_jobs) self.parallel_jobs = rconfig().max_parallel_jobs def _validate2(self): if self.ami is None: raise ValueError("Region {} not supported by AMI yet.".format( self.region)) def setup(self, mode): if mode == Benchmark.SetupMode.skip: log.warning( "AWS setup mode set to unsupported {mode}, ignoring.".format( mode=mode)) # S3 setup to exchange files between local and ec2 instances self.s3 = boto3.resource('s3', region_name=self.region) self.bucket = self._create_s3_bucket() self.uploaded_resources = self._upload_resources() # IAM setup to secure exchanges between s3 and ec2 instances self.iam = boto3.resource('iam', region_name=self.region) if mode == Benchmark.SetupMode.force: log.warning("Cleaning up previously created IAM entities if any.") self._delete_iam_entities() self.instance_profile = self._create_instance_profile() # EC2 setup to prepare creation of ec2 instances self.ec2 = boto3.resource('ec2', region_name=self.region) self.cloudwatch = boto3.resource('cloudwatch', region_name=self.region) def cleanup(self): self._stop_all_instances() self._monitoring_stop() self._exec_stop() if rconfig().aws.s3.delete_resources is True: self._delete_resources() if rconfig().aws.iam.temporary is True: self._delete_iam_entities() if rconfig().aws.s3.temporary is True: self._delete_s3_bucket() def run(self, task_name=None, fold=None): task_defs = self._get_task_defs(task_name) # validates tasks self._exec_start() self._monitoring_start() if self.parallel_jobs > 1: if rconfig().aws.minimize_instances: # use one instance per task: all folds executed on same instance try: jobs = flatten([ self._make_aws_job([task_def.name], fold) for task_def in task_defs ]) results = self._run_jobs(jobs) return self._process_results(results, task_name=task_name) finally: self.cleanup() else: # use one instance per fold per task return super().run(task_name, fold) else: # use one instance for all try: task_names = None if task_name is None else [ task_def.name for task_def in task_defs ] job = self._make_aws_job(task_names, fold) results = self._run_jobs([job]) return self._process_results(results, task_name=task_name) finally: self.cleanup() def _make_job(self, task_def, fold=int): return self._make_aws_job([task_def.name], [fold]) def _exec_start(self): if self.exec is not None: return self.exec = ThreadPoolExecutor(max_workers=1, thread_name_prefix="aws_exec_") def _exec_stop(self): if self.exec is None: return try: self.exec.shutdown(wait=True) except: pass finally: self.exec = None def _exec_send(self, fn, *args, **kwargs): if self.exec is not None: self.exec.submit(fn, *args, **kwargs) else: log.warning( "Sending exec function while executor is not started: executing the function in the calling thread." ) try: fn(*args, **kwargs) except: pass def _make_aws_job(self, task_names=None, folds=None): task_names = [] if task_names is None else task_names folds = [] if folds is None else [str(f) for f in folds] task_def = self._get_task_def(task_names[0]) if len(task_names) >= 1 \ else self._get_task_def('__defaults__', include_disabled=True, fail_on_missing=False) instance_def = ns( type=task_def.ec2_instance_type, volume_type=task_def.ec2_volume_type, ) if task_def else ns( type='.'.join([ rconfig().aws.ec2.instance_type.series, rconfig().aws.ec2.instance_type.map.default ]), volume_type=rconfig().aws.ec2.volume_type, ) if task_def and task_def.min_vol_size_mb > 0: instance_def.volume_size = math.ceil( (task_def.min_vol_size_mb + rconfig().benchmarks.os_vol_size_mb) / 1024.) else: instance_def.volume_size = None timeout_secs = task_def.max_runtime_seconds if task_def \ else sum([task.max_runtime_seconds for task in self.benchmark_def]) timeout_secs += rconfig().aws.overhead_time_seconds job = Job('_'.join([ 'aws', self.benchmark_name, self.constraint_name, '.'.join(task_names) if len(task_names) > 0 else 'all', '.'.join(folds), self.framework_name ])) job.instance_id = None def _run(job_self): resources_root = "/custom" if rconfig( ).aws.use_docker else "/s3bucket/user" job_self.instance_id = self._start_instance( instance_def, script_params= "{framework} {benchmark} {constraint} {task_param} {folds_param} -Xseed={seed}" .format( framework=self.framework_name, benchmark=self.benchmark_name if self.benchmark_path.startswith(rconfig().root_dir) else "{}/{}.yaml".format(resources_root, self.benchmark_name), constraint=self.constraint_name, task_param='' if len(task_names) == 0 else ' '.join(['-t'] + task_names), folds_param='' if len(folds) == 0 else ' '.join(['-f'] + folds), seed=rget().seed(int(folds[0])) if len(folds) == 1 else rconfig().seed, ), # instance_key='_'.join([job.name, datetime_iso(micros=True, time_sep='.')]), instance_key=job.name, timeout_secs=timeout_secs) try: return self._wait_for_results(job_self) except Exception as e: fold = int(folds[0]) if len(folds) > 0 else -1 results = TaskResult(task_def=task_def, fold=fold) return results.compute_scores(self.framework_name, [], result=ErrorResult(e)) def _on_done(job_self): terminate = self._download_results(job_self.instance_id) if not terminate and rconfig( ).aws.ec2.terminate_instances == 'success': log.warning( "[WARNING]: EC2 Instance %s won't be terminated as we couldn't download the results: " "please terminate it manually or restart it (after clearing its UserData) if you want to inspect the instance.", job_self.instance_id) self._stop_instance(job_self.instance_id, terminate=terminate) job._run = _run.__get__(job) job._on_done = _on_done.__get__(job) return job def _wait_for_results(self, job): instance = self.instances[job.instance_id].instance last_console_line = -1 def log_console(): nonlocal last_console_line try: output = instance.console_output(Latest=True) if 'Output' in output: output = output[ 'Output'] # note that console_output only returns the last 64kB of console new_log, last_line = tail(output, from_line=last_console_line, include_line=False) if last_line is not None: last_console_line = last_line['line'] if new_log: log.info(new_log) except Exception as e: log.exception(e) interrupt = threading.Event() while not interrupt.is_set(): if job.instance_id in self.instances: inst_desc = self.instances[job.instance_id] if inst_desc['abort']: self._update_instance(job.instance_id, status='aborted') raise Exception("Aborting instance {} for job {}.".format( job.instance_id, job.name)) try: state = instance.state['Name'] log.info("[%s] checking job %s on instance %s: %s.", datetime_iso(), job.name, job.instance_id, state) log_console() self._update_instance(job.instance_id, status=state) if instance.state['Code'] > 16: # ended instance log.info("EC2 instance %s is %s: %s", job.instance_id, state, instance.state_reason['Message']) interrupt.set() except Exception as e: log.exception(e) finally: interrupt.wait(rconfig().aws.query_frequency_seconds) def _get_cpu_activity(self, iid, delta_minutes=60, period_minutes=5): now = dt.datetime.utcnow() resp = self.cloudwatch.meta.client.get_metric_statistics( Namespace='AWS/EC2', MetricName='CPUUtilization', Dimensions=[dict(Name='InstanceId', Value=iid)], StartTime=now - dt.timedelta(minutes=delta_minutes), EndTime=now, Period=60 * period_minutes, Statistics=['Average'], Unit='Percent') return [ activity['Average'] for activity in sorted(resp['Datapoints'], key=op.itemgetter('Timestamp'), reverse=True) ] def _is_hanging(self, iid): cpu_config = rconfig().aws.ec2.monitoring.cpu activity = self._get_cpu_activity( iid, delta_minutes=cpu_config.delta_minutes, period_minutes=cpu_config.period_minutes) threshold = cpu_config.threshold min_activity_len = int(cpu_config.delta_minutes / cpu_config.period_minutes) return len(activity) >= min_activity_len and all( [a < threshold for a in activity]) def _monitoring_start(self): if self.monitoring is not None: return interrupt = threading.Event() def cpu_monitor(): cpu_config = rconfig().aws.ec2.monitoring.cpu if cpu_config.query_frequency_seconds <= 0: return while not interrupt.is_set(): try: hanging_instances = list( filter(self._is_hanging, self.instances.keys())) for inst in hanging_instances: if inst in self.instances: inst_desc = self.instances[inst] log.warning( "WARN: Instance %s (%s) has no CPU activity in the last %s minutes.", inst, inst_desc.key, cpu_config.delta_minutes) if cpu_config.abort_inactive_instances: inst_desc.abort = True except Exception as e: log.exception(e) finally: interrupt.wait(cpu_config.query_frequency_seconds) self.monitoring = ns(executor=ThreadPoolExecutor( max_workers=1, thread_name_prefix="aws_monitoring_"), interrupt=interrupt) self.monitoring.executor.submit(cpu_monitor) def _monitoring_stop(self): if self.monitoring is None: return try: self.monitoring.interrupt.set() self.monitoring.executor.shutdown(wait=False) except: pass finally: self.monitoring = None def _start_instance(self, instance_def, script_params="", instance_key=None, timeout_secs=-1): log.info("Starting new EC2 instance with params: %s.", script_params) inst_key = instance_key.lower() if instance_key \ else "{}_p{}_i{}".format(self.sid, re.sub(r"[\s-]", '', script_params), datetime_iso(micros=True, time_sep='.')).lower() # TODO: don't know if it would be considerably faster to reuse previously stopped instances sometimes # instead of always creating a new one: # would still need to set a new UserData though before restarting the instance. try: ebs = dict(VolumeType=instance_def.volume_type) if instance_def.volume_size: ebs['VolumeSize'] = instance_def.volume_size instance = self.ec2.create_instances( BlockDeviceMappings=[ dict(DeviceName=rconfig().aws.ec2.root_device_name, Ebs=ebs) ], IamInstanceProfile=dict(Name=self.instance_profile.name), ImageId=self.ami, InstanceType=instance_def.type, MinCount=1, MaxCount=1, SubnetId=rconfig().aws.ec2.subnet_id, UserData=self._ec2_startup_script( inst_key, script_params=script_params, timeout_secs=timeout_secs))[0] log.info("Started EC2 instance %s.", instance.id) self.instances[instance.id] = ns(instance=instance, key=inst_key, status='started', success='', start_time=datetime_iso(), stop_time='') except Exception as e: fake_iid = "no_instance_{}".format(len(self.instances) + 1) self.instances[fake_iid] = ns(instance=None, key=inst_key, status='failed', success=False, start_time=datetime_iso(), stop_time=datetime_iso()) raise e finally: self._exec_send(self._save_instances) return instance.id def _stop_instance(self, instance_id, terminate=None): instance = self.instances[instance_id].instance self.instances[instance_id].instance = None if instance is None: return terminate_config = rconfig().aws.ec2.terminate_instances if terminate_config in ['always', True]: terminate = True elif terminate_config in ['never', False]: terminate = False else: terminate = False if terminate is None else terminate try: log.info("%s EC2 instances %s.", "Terminating" if terminate else "Stopping", instance_id) if terminate: response = instance.terminate() else: response = instance.stop() log.info("%s EC2 instances %s with response %s.", "Terminated" if terminate else "Stopped", instance_id, response) except Exception as e: log.error( "ERROR: EC2 instance %s could not be %s!\n" "Even if the instance should stop by itself after a certain timeout, " "you may want to stop/terminate it manually:\n%s", instance_id, "terminated" if terminate else "stopped", str(e)) finally: try: state = response['TerminatingInstances'][0]['CurrentState'][ 'Name'] log.info("Instance %s state: %s.", instance_id, state) self._update_instance(instance_id, status=state, stop_time=datetime_iso()) except: pass def _update_instance(self, instance_id, **kwargs): do_save = False if len(kwargs): do_save = True inst = self.instances[instance_id] for k, v in kwargs.items(): if k in inst and inst[k] != v: inst[k] = v do_save = True if do_save: self._exec_send(lambda: self._save_instances()) def _stop_all_instances(self): for iid in self.instances.keys(): self._stop_instance(iid) def _save_instances(self): write_csv( [(iid, self.instances[iid].status, self.instances[iid].success, self.instances[iid].start_time, self.instances[iid].stop_time, self.sid, self.instances[iid].key, self._s3_key(self.sid, instance_key_or_id=iid, absolute=True)) for iid in self.instances.keys()], columns=[ 'ec2', 'status', 'success', 'start_time', 'stop_time', 'session', 'instance_key', 's3 dir' ], path=os.path.join(self.output_dirs.session, 'instances.csv')) def _load_instances(self, instances_file): df = read_csv(instances_file) self.instances = { row['ec2']: ns( status=row['status'], success=row['success'], session=row['session'], key=row['instance_key'], s3_dir=row['s3 dir'], ) for idx, row in df.iterrows() } def _s3_key(self, main_dir, *subdirs, instance_key_or_id=None, absolute=False, encode=False): root_key = str_def(rconfig().aws.s3.root_key) if instance_key_or_id is None: ikey = '' elif instance_key_or_id in self.instances.keys(): ikey = self.instances[instance_key_or_id].key else: ikey = instance_key_or_id tokens = [main_dir, ikey, *subdirs] if encode: tokens = map(uenc, tokens) rel_key = url_join(root_key, *tokens) return url_join('s3://', self.bucket.name, rel_key) if absolute else rel_key def _s3_session(self, *subdirs, **kwargs): return self._s3_key(self.sid, *subdirs, **kwargs) def _s3_user(self, *subdirs, **kwargs): return self._s3_key(self.sid, 'user', *subdirs, **kwargs) def _s3_input(self, *subdirs, **kwargs): return self._s3_key(self.sid, 'input', *subdirs, **kwargs) def _s3_output(self, instance_key_or_id, *subdirs, **kwargs): return self._s3_key(self.sid, 'output', *subdirs, instance_key_or_id=instance_key_or_id, **kwargs) def _create_s3_bucket(self, bucket_name=None, auto_create=True): # cf. s3 restrictions: https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html if bucket_name is None: bucket_name = rconfig().aws.s3.bucket if rconfig().aws.s3.temporary: bucket_name += ('-' + self.suid) try: self.s3.meta.client.head_bucket(Bucket=bucket_name) bucket = self.s3.Bucket(bucket_name) except botocore.exceptions.ClientError as e: error_code = int(e.response['Error']['Code']) if error_code == 404 and auto_create: log.info("%s bucket doesn't exist, creating it in region %s.", bucket_name, self.region) bucket = self.s3.create_bucket( Bucket=bucket_name, CreateBucketConfiguration=dict( LocationConstraint=self.region)) log.info("S3 bucket %s was successfully created.", bucket_name) else: if error_code == 403: log.error( "You don't have access rights to S3 bucket %s.\n" "Please ensure that you specified a unique `aws.s3.bucket` in your config file" " or verify that your AWS account is correctly configured" " (cf. docs/README.md for more details).", bucket_name) elif error_code == 404: log.error( "S3 bucket %s does not exist and auto-creation is disabled.", bucket_name) raise e return bucket def _delete_s3_bucket(self): if self.bucket: # we can only delete 1000 objects at a time using this API, # but this is intended only for temporary buckets, so no need for pagination to_delete = [dict(Key=o.key) for o in self.bucket.objects.all()] if len(to_delete) > 0: log.info("Deleting objects from S3 bucket %s: %s", self.bucket.name, to_delete) self.bucket.delete_objects( Delete=dict(Objects=to_delete, Quiet=True)) log.info("Deleting s3 bucket %s.", self.bucket.name) self.bucket.delete() log.info("S3 bucket %s was successfully deleted.", self.bucket.name) def _upload_resources(self): def dest_path(res_path): in_app_dir = res_path.startswith(rconfig().root_dir) if in_app_dir: return None in_input_dir = res_path.startswith(rconfig().input_dir) in_user_dir = res_path.startswith(rconfig().user_dir) name = (os.path.relpath(res_path, start=rconfig().input_dir) if in_input_dir else os.path.relpath(res_path, start=rconfig().user_dir) if in_user_dir else os.path.basename(res_path)) return self._s3_input(name) if in_input_dir else self._s3_user( name) upload_paths = [self.benchmark_path] + rconfig().aws.resource_files upload_files = list_all_files(upload_paths, exclude=rconfig().aws.resource_ignore) log.debug("Uploading files to S3: %s", upload_files) uploaded_resources = [] for res in upload_files: upload_path = dest_path(res) if upload_path is None: log.debug("Skipping upload of `%s` to s3 bucket.", res) continue log.info("Uploading `%s` to `%s` on s3 bucket %s.", res, upload_path, self.bucket.name) self.bucket.upload_file(res, upload_path) uploaded_resources.append(upload_path) return uploaded_resources def _delete_resources(self): if self.uploaded_resources is None: return log.info("Deleting uploaded resources `%s` from s3 bucket %s.", self.uploaded_resources, self.bucket.name) self.bucket.delete_objects(Delete=dict( Objects=[dict(Key=res) for res in self.uploaded_resources])) def _download_results(self, instance_id): """ :param instance_id: :return: True iff the main result/scoring file has been successfully downloaded. Other failures are only logged. """ def download_file(obj, dest, dest_display_path=None): dest_display_path = dest if dest_display_path is None else dest_display_path try: log.info("Downloading `%s` from s3 bucket %s to `%s`.", obj.key, self.bucket.name, dest_display_path) if isinstance(dest, str): touch(dest) obj.download_file(dest) else: obj.download_fileobj(dest) except Exception as e: log.error("Failed downloading `%s` from s3 bucket %s: %s", obj.key, self.bucket.name, str(e)) log.exception(e) success = self.instances[instance_id].success is True try: instance_output_key = self._s3_output(instance_id, encode=True) objs = [ o.Object() for o in self.bucket.objects.filter(Prefix=instance_output_key) ] session_key = self._s3_session(encode=True) # result_key = self._s3_output(instance_id, Scoreboard.results_file, encode=True) for obj in objs: rel_path = url_relpath(obj.key, start=session_key) dest_path = os.path.join(self.output_dirs.session, rel_path) download_file(obj, dest_path) # if obj.key == result_key: if not success and os.path.basename( obj.key) == Scoreboard.results_file: if rconfig().results.save: self._exec_send( lambda path: self._append(Scoreboard.load_df(path) ), dest_path) success = True except Exception as e: log.error( "Failed downloading benchmark results from s3 bucket %s: %s", self.bucket.name, str(e)) log.exception(e) log.info("Instance `%s` success=%s", instance_id, success) self._update_instance(instance_id, success=success) return success def _create_instance_profile(self): """ see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html for steps defined here. for restrictions, cf. https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-limits.html :return: """ s3c = rconfig().aws.s3 iamc = rconfig().aws.iam bucket_prefix = (s3c.bucket + '-') if (s3c.temporary and not iamc.temporary) else self.bucket.name role_name = iamc.role_name profile_name = iamc.instance_profile_name if iamc.temporary: role_name += ('-' + self.suid) profile_name += ('-' + self.suid) irole = None try: self.iam.meta.client.get_role(RoleName=role_name) irole = self.iam.Role(role_name) except botocore.exceptions.ClientError as e: log.info("Role %s doesn't exist, creating it: [%s].", role_name, str(e)) if not irole: ec2_role_trust_policy_json = json.dumps({ # trust role 'Version': '2012-10-17', # version of the policy language, cf. https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_version.html 'Statement': [{ 'Effect': 'Allow', 'Principal': { 'Service': 'ec2.amazonaws.com' }, 'Action': 'sts:AssumeRole' }] }) irole = self.iam.create_role( RoleName=role_name, AssumeRolePolicyDocument=ec2_role_trust_policy_json, MaxSessionDuration=iamc.max_role_session_duration_secs) log.info("Role %s successfully created.", role_name) if iamc.s3_policy_name not in [p.name for p in irole.policies.all()]: resource_prefix = "arn:aws:s3:::{bucket}*/{root_key}".format( bucket=bucket_prefix, root_key=str_def(s3c.root_key) ) # ARN format for s3, cf. https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-arn-format.html s3_policy_json = json.dumps({ 'Version': '2012-10-17', 'Statement': [ { 'Effect': 'Allow', 'Action': 's3:List*', 'Resource': 'arn:aws:s3:::{}*'.format(bucket_prefix) }, { 'Effect': 'Allow', 'Action': 's3:GetObject', # S3 actions, cf. https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html 'Resource': resource_prefix + '*' }, { 'Effect': 'Allow', 'Action': 's3:PutObject', 'Resource': resource_prefix + '*' # technically, we could grant write access for each instance only to its own 'directory', but this is not necessary } ] }) self.iam.meta.client.put_role_policy( RoleName=irole.name, PolicyName=iamc.s3_policy_name, PolicyDocument=s3_policy_json) iprofile = None try: self.iam.meta.client.get_instance_profile( InstanceProfileName=profile_name) iprofile = self.iam.InstanceProfile(profile_name) except botocore.exceptions.ClientError as e: log.info("Instance profile %s doesn't exist, creating it: [%s].", profile_name, str(e)) if not iprofile: iprofile = self.iam.create_instance_profile( InstanceProfileName=profile_name) log.info("Instance profile %s successfully created.", profile_name) waiting_time = iamc.credentials_propagation_waiting_time_secs steps = math.ceil(waiting_time / 10) for i in range(steps): log.info( "Waiting for new credentials propagation, time left = %ss.", round(waiting_time * (1 - i / steps))) time.sleep(waiting_time / steps) if irole.name not in [r.name for r in iprofile.roles]: iprofile.add_role(RoleName=irole.name) return iprofile def _delete_iam_entities(self): iamc = rconfig().aws.iam iprofile = self.instance_profile if iprofile is None: profile_name = iamc.instance_profile_name if iamc.temporary: profile_name += ('-' + self.suid) try: self.iam.meta.client.get_instance_profile( InstanceProfileName=profile_name) iprofile = self.iam.InstanceProfile(profile_name) except botocore.exceptions.ClientError as e: log.info( "Instance profile %s doesn't exist, nothing to delete: [%s]", profile_name, str(e)) if iprofile is not None: for role in iprofile.roles: log.info("Removing role %s from instance profile %s.", role.name, iprofile.name) iprofile.remove_role(RoleName=role.name) self._delete_iam_entities_from_role(role.name) log.info("Deleting instance profile %s.", iprofile.name) iprofile.delete() log.info("Instance profile %s was successfully deleted.", iprofile.name) else: role_name = iamc.role_name if iamc.temporary: role_name += ('-' + self.suid) self._delete_iam_entities_from_role(role_name) def _delete_iam_entities_from_role(self, role_name): iamc = rconfig().aws.iam try: self.iam.meta.client.get_role(RoleName=role_name) irole = self.iam.Role(role_name) for policy in irole.policies.all(): log.info("Deleting role policy %s from role %s.", policy.name, policy.role_name) policy.delete() log.info("Policy %s was successfully deleted.", policy.name) for profile in irole.instance_profiles.all(): log.info("Removing instance profile %s from role %s.", profile.name, irole.name) profile.remove_role(RoleName=irole.name) log.info("Deleting instance profile %s.", profile.name) profile.delete() log.info("Instance profile %s was successfully deleted.", profile.name) log.info("Deleting role %s.", irole.name) irole.delete() log.info("Role %s was successfully deleted.", irole.name) except botocore.exceptions.ClientError as e: log.info("Role %s doesn't exist, skipping its deletion: [%s]", iamc.role_name, str(e)) def _ec2_startup_script(self, instance_key, script_params="", timeout_secs=-1): """ Generates the UserData is cloud-config format for the EC2 instance: this script is automatically executed by the instance at the end of its boot process. This cloud-config version is currently preferred as the runcmd are always executed sequentially, regardless of the previous one raising an error. Especially, the power_state directive is always executed. Notes about cloud-config syntax: - runcmd: all command are executed sequentially. If one raises an error, the next one is still executed afterwards. - power_state: * delay (in mn) passed to the shutdown command is executed, * timeout (in sec) waiting for cloud-init to complete before triggering shutdown. :param instance_key: the unique local identifier for the instance. This is different from EC2 instance id as we don't know it yet. Mainly used to put output files to dedicated key on s3. :param script_params: the custom params passed to the benchmark script, usually only task, fold params :return: the UserData for the new ec2 instance """ script_extra_params = "--session=" cloud_config = """ #cloud-config package_update: true package_upgrade: false packages: - python3 - python3-pip - docker.io runcmd: - apt-get -y remove unattended-upgrades - systemctl stop apt-daily.timer - systemctl disable apt-daily.timer - systemctl disable apt-daily.service - systemctl daemon-reload - mkdir -p /s3bucket/input - mkdir -p /s3bucket/output - mkdir -p /s3bucket/user - pip3 install -U awscli - aws s3 cp '{s3_input}' /s3bucket/input --recursive - aws s3 cp '{s3_user}' /s3bucket/user --recursive - docker run {docker_options} -v /s3bucket/input:/input -v /s3bucket/output:/output -v /s3bucket/user:/custom --rm {image} {params} -i /input -o /output -u /custom -s skip -Xrun_mode=aws.docker {extra_params} - aws s3 cp /s3bucket/output '{s3_output}' --recursive - rm -f /var/lib/cloud/instances/*/sem/config_scripts_user final_message: "AutoML benchmark (docker) {ikey} completed after $UPTIME s" power_state: delay: "+1" mode: poweroff message: "I'm losing power" timeout: {timeout} condition: True """ if rconfig().aws.use_docker else """ #cloud-config package_update: true package_upgrade: false packages: - curl - wget - unzip - git - python3 - python3-pip - python3-venv runcmd: - apt-get -y remove unattended-upgrades - systemctl stop apt-daily.timer - systemctl disable apt-daily.timer - systemctl disable apt-daily.service - systemctl daemon-reload - pip3 install -U awscli - python3 -m venv /venvs/bench - alias PIP='/venvs/bench/bin/pip3' - alias PY='/venvs/bench/bin/python3 -W ignore' - alias PIP_REQ='xargs -L 1 /venvs/bench/bin/pip3 install --no-cache-dir' - mkdir -p /s3bucket/input - mkdir -p /s3bucket/output - mkdir -p /s3bucket/user - mkdir /repo - cd /repo - git clone --depth 1 --single-branch --branch {branch} {repo} . - PIP install -U pip=={pip_version} - PIP_REQ < requirements.txt # - until aws s3 ls '{s3_base_url}'; do echo "waiting for credentials"; sleep 10; done - aws s3 cp '{s3_input}' /s3bucket/input --recursive - aws s3 cp '{s3_user}' /s3bucket/user --recursive - PY {script} {params} -i /s3bucket/input -o /s3bucket/output -u /s3bucket/user -s only --session= - PY {script} {params} -i /s3bucket/input -o /s3bucket/output -u /s3bucket/user -Xrun_mode=aws -Xproject_repository={repo}#{branch} {extra_params} - aws s3 cp /s3bucket/output '{s3_output}' --recursive - rm -f /var/lib/cloud/instances/*/sem/config_scripts_user final_message: "AutoML benchmark {ikey} completed after $UPTIME s" power_state: delay: "+1" mode: poweroff message: "I'm losing power" timeout: {timeout} condition: True """ return cloud_config.format( repo=rget().project_info.repo, branch=rget().project_info.branch, image=rconfig().docker.image or DockerBenchmark.docker_image_name(self.framework_def), pip_version=rconfig().versions.pip, s3_base_url=self._s3_session(absolute=True, encode=True), s3_user=self._s3_user(absolute=True, encode=True), s3_input=self._s3_input(absolute=True, encode=True), s3_output=self._s3_output(instance_key, absolute=True, encode=True), script=rconfig().script, ikey=instance_key, params=script_params, extra_params=script_extra_params, docker_options=rconfig().docker.run_extra_options, timeout=timeout_secs if timeout_secs > 0 else rconfig().aws.max_timeout_seconds, ) def _ec2_startup_script_bash(self, instance_key, script_params="", timeout_secs=-1): """ Backup UserData version if the cloud-config version doesn't work as expected. Generates the UserData is bash format for the EC2 instance: this script is automatically executed by the instance at the end of its boot process. TODO: current version doesn't handle errors at all, that's why the cloud-config version is currently preferred. :param instance_key: the unique local identifier for the instance. This is different from EC2 instance id as we don't know it yet. Mainly used to put output files to dedicated key on s3. :param script_params: the custom params passed to the benchmark script, usually only task, fold params :return: the UserData for the new ec2 instance """ script_extra_params = "--session=" return """#!/bin/bash apt-get update #apt-get -y upgrade apt-get -y install curl wget unzip git apt-get -y install python3 python3-pip python3-venv #apt-get -y install docker.io pip3 install -U awscli python3 -m venv /venvs/bench alias PIP='/venvs/bench/bin/pip3' alias PY='/venvs/bench/bin/python3 -W ignore' mkdir -p /s3bucket/input mkdir -p /s3bucket/output mkdir -p /s3bucket/user mkdir ~/repo cd ~/repo git clone --depth 1 --single-branch --branch {branch} {repo} . PIP install -U pip=={pip_version} xargs -L 1 PIP install --no-cache-dir < requirements.txt PIP install -U awscli aws s3 cp '{s3_input}' /s3bucket/input --recursive aws s3 cp '{s3_user}' /s3bucket/user --recursive PY {script} {params} -i /s3bucket/input -o /s3bucket/output -u /s3bucket/user -s only --session= PY {script} {params} -i /s3bucket/input -o /s3bucket/output -u /s3bucket/user -Xrun_mode=aws -Xproject_repository={repo}#{branch} {extra_params} aws s3 cp /s3bucket/output '{s3_output}' --recursive rm -f /var/lib/cloud/instances/*/sem/config_scripts_user shutdown -P +1 "I'm losing power" """.format( repo=rget().project_info.repo, branch=rget().project_info.branch, pip_version=rconfig().versions.pip, s3_base_url=self._s3_session(absolute=True, encode=True), s3_user=self._s3_user(absolute=True, encode=True), s3_input=self._s3_input(absolute=True, encode=True), s3_output=self._s3_output(instance_key, absolute=True, encode=True), script=rconfig().script, ikey=instance_key, params=script_params, extra_params=script_extra_params, )
def do_work(named_proc, resolver, rndc_cmd, kill_method, n_workers, n_queries): """Creates a number of A queries to run in parallel in order simulate a slightly more realistic test scenario. The main idea of this function is to create and send a bunch of A queries to a target named instance and during this process a request for shutting down named will be issued. In the process of shutting down named, a couple control connections are created (by launching rndc) to ensure that the crash was fixed. if kill_method=="rndc" named will be asked to shutdown by means of rndc stop. if kill_method=="sigterm" named will be killed by SIGTERM on POSIX systems or by TerminateProcess() on Windows systems. :param named_proc: named process instance :type named_proc: subprocess.Popen :param resolver: target resolver :type resolver: dns.resolver.Resolver :param rndc_cmd: rndc command with default arguments :type rndc_cmd: list of strings, e.g. ["rndc", "-p", "23750"] :kill_method: "rndc" or "sigterm" :type kill_method: str :param n_workers: Number of worker threads to create :type n_workers: int :param n_queries: Total number of queries to send :type n_queries: int """ # pylint: disable-msg=too-many-arguments # pylint: disable-msg=too-many-locals # helper function, args must be a list or tuple with arguments to rndc. def launch_rndc(args): return subprocess.call(rndc_cmd + args, timeout=10) # We're going to execute queries in parallel by means of a thread pool. # dnspython functions block, so we need to circunvent that. executor = ThreadPoolExecutor(n_workers + 1) # Helper dict, where keys=Future objects and values are tags used # to process results later. futures = {} # 50% of work will be A queries. # 1 work will be rndc stop. # Remaining work will be rndc status (so we test parallel control # connections that were crashing named). shutdown = True for i in range(n_queries): if i < (n_queries // 2): # Half work will be standard A queries. # Among those we split 50% queries relname='www', # 50% queries relname=random characters if random.randrange(2) == 1: tag = "good" relname = "www" else: tag = "bad" length = random.randint(4, 10) relname = "".join(letters[random.randrange(len(letters))] for i in range(length)) qname = relname + ".test" futures[executor.submit(resolver.query, qname, 'A')] = tag elif shutdown: # We attempt to stop named in the middle shutdown = False if kill_method == "rndc": futures[executor.submit(launch_rndc, ['stop'])] = 'stop' else: futures[executor.submit(named_proc.terminate)] = 'kill' else: # We attempt to send couple rndc commands while named is # being shutdown futures[executor.submit(launch_rndc, ['status'])] = 'status' ret_code = -1 for future in as_completed(futures): try: result = future.result() # If tag is "stop", result is an instance of # subprocess.CompletedProcess, then we check returncode # attribute to know if rncd stop command finished successfully. # # if tag is "kill" then the main function will check if # named process exited gracefully after SIGTERM signal. if futures[future] == "stop": ret_code = result except (dns.resolver.NXDOMAIN, dns.exception.Timeout): pass if kill_method == "rndc": assert ret_code == 0 executor.shutdown()
class KinesisProducer: """Basic Kinesis Producer. Parameters ---------- stream_name : string Name of the stream to send the records. batch_size : int Numbers of records to batch before flushing the queue. batch_time : int Maximum of seconds to wait before flushing the queue. max_retries: int Maximum number of times to retry the put operation. kinesis_client: boto3.client Kinesis client. Attributes ---------- records : array Queue of formated records. pool: concurrent.futures.ThreadPoolExecutor Pool of threads handling client I/O. """ def __init__(self, stream_name, batch_size=500, batch_time=5, max_retries=5, threads=10, kinesis_client=boto3.client('kinesis')): self.stream_name = stream_name self.queue = Queue() self.batch_size = batch_size self.batch_time = batch_time self.max_retries = max_retries self.kinesis_client = kinesis_client self.pool = ThreadPoolExecutor(threads) self.last_flush = time.time() self.monitor_running = threading.Event() self.monitor_running.set() self.pool.submit(self.monitor) def monitor(self): """Flushes the queue periodically.""" while self.monitor_running.is_set(): if time.time() - self.last_flush > self.batch_time: if not self.queue.empty(): logger.info( "Flushing the queue (time without flush exceeded)") self.flush_queue() time.sleep(self.batch_time) def put_record(self, data, partition_key=None): """Add data to the record queue in the proper format. Parameters ---------- data : str Data to send. partition_key: str Hash that determines which shard a given data record belongs to. """ # Byte encode the data data = encode_data(data) # Create a random partition key if not provided if not partition_key: partition_key = uuid.uuid4().hex # Build the record record = {'Data': data, 'PartitionKey': partition_key} # Flush the queue if it reaches the batch size if self.queue.qsize() >= self.batch_size: logger.info("Flushing the queue (batch size reached)") self.pool.submit(self.flush_queue) # Append the record logger.info('Putting record "{}"'.format(record['Data'][:100])) self.queue.put(record) def close(self): """Flushes the queue and waits for the executor to finish.""" self.flush_queue() self.monitor_running.clear() self.pool.shutdown() def flush_queue(self): """Grab all the current records in the queue and send them.""" records = [] while not self.queue.empty() and len(records) < self.batch_size: records.append(self.queue.get()) if records: self.send_records(records) self.last_flush = time.time() def send_records(self, records, attempt=0): """Send records to the Kinesis stream. Falied records are sent again with an exponential backoff decay. Parameters ---------- records : array Array of formated records to send. attempt: int Number of times the records have been sent without success. """ # If we already tried more times than we wanted, save to a file if attempt > self.max_retries: logger.warning('Writing {} records to file'.format(len(records))) with open('failed_records.dlq', 'ab') as f: for r in records: f.write(r.get('Data')) return # Sleep before retrying if attempt: time.sleep(2**attempt * .1) response = self.kinesis_client.put_records(StreamName=self.stream_name, Records=records) failed_record_count = response['FailedRecordCount'] # Grab failed records if failed_record_count: logger.warning('Retrying failed records') failed_records = [] for i, record in enumerate(response['Records']): if record.get('ErrorCode'): failed_records.append(records[i]) # Recursive call attempt += 1 self.send_records(failed_records, attempt=attempt)
# name = name*2 # time.sleep(2) # print("当前线程是%s,结果是%s" % (threading.current_thread(), name)) # print('thread %s is running...' % threading.current_thread()) # for i in range(1,5): # t = threading.Thread(target=work,args=(i,)) # t.start() # # print('ok') # t.join() # # print("=====================") # # # t1 = threading.Thread(target=work,args=(11,)) # t2 = threading.Thread(target=work,args=(22,)) # t3 = threading.Thread(target=work,args=(22,)) # t1.start() # t2.start() # t3.start() # t1.join() # t2.join() # t3.join() print("++++++++++++++++++++++++") th = ThreadPoolExecutor(max_workers=3) for s in range(1, 10): th.submit(work, s) th.shutdown() print('````````````````````````````')
from get_mmf_url import mmf_url_list from get_fund_info import get_info import csv from concurrent.futures import ThreadPoolExecutor if __name__ == '__main__': executor = ThreadPoolExecutor(max_workers=250) with open('data.csv', 'w') as f: writer = csv.writer(f) writer.writerow([ 'id', 'size', 'annualized', 'prop_dt', 'prop_large', 'prop_retail', 'prop_insider', 'share' ]) for num, url in enumerate(mmf_url_list[2800:]): print(num, url) row = executor.submit(get_info, url) if row.result() != None: writer.writerow(row.result()) executor.shutdown()
class SseResultDownloader: def __init__(self, web_interface, result_poll_interval): logging.debug("Server-Send Events are used to get state of runs.") self._web_interface = web_interface self._run_finished_url = web_interface._web_interface_url + "runs/finished" self._result_poll_interval = result_poll_interval self._sse_client = None self._shutdown = False self._new_runs = False self._state_receive_executor = ThreadPoolExecutor(max_workers=1) def _log_future_exception_and_fallback(self, result): e = result.exception() if e is not None: if (self._shutdown and isinstance(e, AttributeError) and str(e) == "'NoneType' object has no attribute 'read'"): # This is harmless, it occurs because SSEClient reads on closed connection. logging.debug('Error during result processing:', exc_info=True) else: logging.warning('Error during result processing:', exc_info=True) if not self._shutdown: self._fall_back() def _should_reconnect(self, error): if self._new_runs: return False elif type(error) == HTTPError and error.response is not None \ and error.response.status >= 400 and error.response.status < 500 : logging.debug("Exception in SSE connection: %s", error) return False else: return True def _start_sse_connection(self): while (self._new_runs): run_ids = set(self._web_interface._unfinished_runs.keys()) self._new_runs = False # nothing to do if len(run_ids) == 0: return params = [] for run_id in run_ids: params.append(("run", run_id)) headers = {"Accept-Encoding": "UTF-8"} logging.debug("Creating Server-Send Event connection.") try: self._sse_client = ShouldReconnectSeeClient( self._run_finished_url, self._should_reconnect, session=self._web_interface._connection, headers=headers, data=params) except Exception as e: logging.warning("Creating SSE connection failed: %s", e) self._fall_back() return for message in self._sse_client: data = message.data tokens = data.split(" ") if len(tokens) == 2: run_id = tokens[0] state = tokens[1] if state == "FINISHED": if run_id in run_ids: logging.debug('Run %s finished.', run_id) self._web_interface._download_result_async( run_id) elif state == "UNKNOWN": logging.debug( 'Run %s is not known by the webclient, trying to get the result.', run_id) self._web_interface._download_async(run_id) elif state == "ERROR": self._web_interface._run_failed(run_id) else: logging.warning( 'Received unknown run state %s for run %s.', state, run_id) run_ids.discard(run_id) if self._shutdown or self._new_runs or len( run_ids) == 0: break else: logging.warning("Received invalid message %s", data) self._sse_client = None # fall back to polling if Server-Send Event based approach failed if len(run_ids) != 0 and not self._shutdown: self._fall_back() def _fall_back(self): logging.info("Fall back to polling.") self._web_interface._result_downloader = PollingResultDownloader( self._web_interface, self._result_poll_interval) self._web_interface._result_downloader.start() self.shutdown(wait=False) def start(self): self._new_runs = True if self._sse_client: self._sse_client.resp.close() else: future = self._state_receive_executor.submit( self._start_sse_connection) future.add_done_callback( self._log_future_exception_and_fallback) def shutdown(self, wait=True): self._shutdown = True if self._sse_client: self._sse_client.resp.close() self._state_receive_executor.shutdown(wait=wait)
dt_start = datetime.datetime.now() print(dt_start) print("start \n") remaining_time = mklist_list[0] * mklist_list[1] print("making a device list, please wait about", remaining_time, "second") executor = ThreadPoolExecutor(max_workers=2) #並列処理のためのスレッドを作成 addr_list_info = [] #デバイススキャン側のスレッド情報を格納するList addr_list = executor.submit(Search_btAddr, mklist_list[0], mklist_list[1]) addr_list_info.append(addr_list) #デバイススキャンの情報をListに追加 executor.submit(remaining_timer, addr_list.running(), remaining_time) executor.shutdown() #スレッドを閉じる print(addr_list.result(), "\n") try: data_list = [ addr_list.result(), ] #スレッドで実行した結果を格納 pprint.pprint(data_list) print() ipaddr = IPget() #print(ipaddr,"\n") dt_now = datetime.datetime.now() name = str(dt_now.year) + "_" + str(dt_now.month) + str( dt_now.day) + "_" + str(dt_now.hour) + str(
def streaming_session(enabled_sensors=None, run_signal=None, reads_queue=None, path=None): """ This method reads data from a group of Shimmer devices. :param path: location where to save .csv files with data :param enabled_sensors: list of enabled sensors of the stream :param run_signal: signal to handle thread end :param reads_queue: queue to communicate with PlotsScreen :return: """ # Thread pool for CSV writing pool = ThreadPoolExecutor(10) # 5 types of Shimmer * 2 master_timestamp = -1 # timestamp RTC of the first sensor's first packet (for synchronization purposes) shifts = {} headers = {} # Start Streaming to all sensors for sensor in enabled_sensors: # Retrieve the Header headers[sensor.shimmer_type] = sensor.data_packet_header() # in that context we will edit the packet, so let's tune the fieldnames of the Header headers[sensor.shimmer_type] = headers[ sensor.shimmer_type][3:len(headers[sensor.shimmer_type])] headers[sensor.shimmer_type].insert(0, "Synced RTC Timestamp") # Creating CSV files and printing header and sampling_rate filename = path + "/" + sensor.shimmer_type + '.csv' with open(filename, 'w') as csv_file: writer = csv.DictWriter(csv_file, fieldnames=headers[sensor.shimmer_type]) writer.writeheader() writer.writerow( {headers[sensor.shimmer_type][0]: sensor.sampling_rate}) # start the streaming sensor.start_bt_streaming() for sensor in enabled_sensors: # Let's compute SHIFTS (time difference) between master sensor and the other (for synchronization purposes) data = sensor.read_data_packet_bt(calibrated=True) # If the master timestamp isn't set yet if master_timestamp == -1: master_timestamp = data[1] if sensor.shimmer_type not in shifts.keys(): shifts[sensor.shimmer_type] = data[2] - master_timestamp # Let's loop while run_signal is set. It will be unset when user will close the plot window while run_signal.is_set(): read = {} for sensor in enabled_sensors: # Read a group of available packets n_of_packets, packets = sensor.read_data_packet_extended( calibrated=True) if n_of_packets > 0: # For the plot we are interested only at the last packet of the group packet = packets[-1].copy() # Let's synchronize the timestamp synced_timestamp = packet[2] - shifts[sensor.shimmer_type] # Engineering the packet packet.pop(0) # removing local timestamp packet.pop(0) # removing timestamp rtc of the first packet packet.pop( 0 ) # removing timestamp rtc of the current packet (not synched) packet.insert(0, synced_timestamp) # Put the packet into 'read' that's a dict that will go into the queue read[sensor.shimmer_type] = packet # Send all packets to CSV writer for .csv writing pool.submit(csv_writer.csv_writer, sensor.shimmer_type, headers[sensor.shimmer_type], shifts[sensor.shimmer_type], packets, path) else: read[sensor.shimmer_type] = None # Check if is all empty all_empty = True for key, value in read.items(): if value: all_empty = False if not all_empty: # This line make sure that in the queue there is only one element with # the latest reads with reads_queue.mutex: reads_queue.queue.clear() reads_queue.put(read) # Stop streaming for sensor in enabled_sensors: sensor.stop_bt_streaming() pool.shutdown(wait=True)
sub_task_obj.result = task_result sub_task_obj.save() if __name__ == '__main__': base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(base_dir) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "fortress.settings") import django django.setup() from web import models if len(sys.argv) == 1: exit('task id not provided.') else: task_id = sys.argv[1] #task_obj_id task_obj = models.Task.objects.get(id=task_id) print('task runner', type(task_obj), task_obj) pool = ThreadPoolExecutor(10) if task_obj.task_type == 'cmd': for sub_task_obj in task_obj.tasklogdetail_set.all(): # pool = ThreadPoolExecutor(10) pool.submit(ssh_cmd, sub_task_obj) else: task_data = json.loads(task_obj.content) print('-->task_data', type(task_data), task_data) for sub_task_obj in task_obj.tasklogdetail_set.all(): pool.submit(file_transfer, sub_task_obj, task_data) pool.shutdown(wait=True)
# urlretrieve(imagelink, imagepathname) print('开始时间:%s' % datetime.now()) for item in urls: # if item== urls[0]: # continue page = 1 url = 'https://unsplash.com/napi/topics/%s/photos?page=%d' % (item, page) print("目录: " + item) if page == 1: itemdir = os.path.join('.', imagedownpath) checkdestfile.checkdestdir(item, itemdir) jsondata, end = getjsondata(url, isfirst=True) if not jsondata == None and not end == None: if item == urls[0]: end = int(('%d' % (end))[2:]) downimage(jsondata) page = page + 1 # break while int(page) <= int(end) and int(page) <= 30: url = 'https://unsplash.com/napi/topics/%s/photos?page=%d' % ( item, page) # print(url) jsondata = getjsondata(url) if not json == None: downimage(jsondata) page = page + 1 threadPool.shutdown(wait=True) print('结束时间:%s' % datetime.now())
class Kernel(object): def __init__(self, environment, bundles, configuration_file="config/config.yml", parameters_file="config/parameters.yml"): self.logger = None self.configuration_file = configuration_file self.parameters_file = parameters_file self.bundles = bundles self.environment = environment self.log_handlers = [] self.is_shutdown = False max_workers = cpu_count() * 5 self.thread_pool = ThreadPoolExecutor(max_workers=max_workers) self.running_services = [] self.mediator = Mediator() try: self.configuration = self.load_configuration(environment) # Subscribe bundle events for bundle in self.bundles: if hasattr(bundle, 'event_listeners'): for event_type, listener in bundle.event_listeners: self.mediator.add_listener(event=event_type, listener=listener) if hasattr(bundle, 'event_subscribers'): for subscriber in bundle.event_subscribers: self.mediator.add_subscriber(subscriber=subscriber) # Injection provided by the base system injection_bindings = { Kernel: self, Configuration: self.configuration, EventManager: self.mediator } self.mediator.dispatch(ConfigurationReadyEvent(self.configuration)) # Injection from other bundles for bundle in self.bundles: if hasattr(bundle, 'injection_bindings'): injection_bindings.update(bundle.injection_bindings) # Set this kernel and configuration available for injection def my_config(binder): for key, value in injection_bindings.items(): binder.bind(key, value) inject.configure(my_config) self.mediator.dispatch(InjectorReadyEvent()) for bundle in self.bundles: if hasattr(bundle, 'log_handlers'): self.log_handlers += bundle.log_handlers except Exception as e: logging.exception(e) raise e self.configure_logger(environment=environment) self.register_signals() logging.info("Kernel Ready") self.mediator.dispatch(KernelReadyEvent()) def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): self.shutdown() def register_signals(self): signal.signal(signal.SIGINT, self._signal_handler) signal.signal(signal.SIGTERM, self._signal_handler) def _signal_handler(self, signal, frame): self.shutdown() def run_service(self, service_function, *args): self.running_services.append( self.thread_pool.submit(service_function, *args)) def configure_logger(self, environment): log_level = logging.INFO if environment == Environments.DEVELOPMENT: # Console output ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) self.log_handlers.append(ch) log_level = logging.INFO logging.basicConfig(level=log_level, handlers=self.log_handlers) logging.info("Logger ready") def load_configuration(self, environment): config_mappings = [ bundle.config_mapping for bundle in self.bundles if hasattr(bundle, "config_mapping") ] config = None if len(config_mappings) > 0: c = YmlLoader() config = c.load_config(self.configuration_file, self.parameters_file) try: config = c.build_config(config, config_mappings) except (NoValueException, NodeIsNotConfiguredException, IgnoredFieldException) as ex: print("Configuration error: " + str(ex)) exit() return config def shutdown(self): if not self.is_shutdown: self.is_shutdown = True logging.info("Kernel shutting down") self.mediator.dispatch(KernelShutdownEvent()) self.thread_pool.shutdown() logging.info("Kernel shutdown") def wait(self): """Wait for all services to finish""" for i in self.running_services: i.result()
class Crawler: def __init__(self, starting_url, ending_url): self.ending_url = ending_url self.queue = deque() self.queue.append(starting_url) self.queue.append('level') self.session = requests.Session() self.pool = ThreadPoolExecutor(max_workers=10) def crawl(self, depth): seen = set() level = 0 t1 = time() links_crawled = 0 while True: url = self.queue.popleft() if level > depth: return False print(url, level) if url == self.ending_url: return True if url == 'level': level += 1 self.queue.append('level') self.pool.shutdown(wait=True) self.pool = ThreadPoolExecutor(max_workers=10) continue # Check if already visited if url in seen: continue self.pool.submit(self.get_wiki_links, url) # self.get_wiki_links(url=url) links_crawled += 1 if links_crawled % 100 == 0: print(f'for the {links_crawled} required {time() - t1}') return def get_wiki_links(self, url: str): """ Given a wiki url, put all the wiki sun links in the wue :param url: :return: """ prefix = '/wiki' page = self.session.get('https://en.wikipedia.org' + url) soup = BeautifulSoup(page.text, 'lxml', parse_only=SoupStrainer('a')) for link in soup.find_all(['a'], href=True): wiki_link = link.get('href') if ':' in wiki_link or '.png' in wiki_link: continue if wiki_link.startswith(prefix): self.queue.append(wiki_link) return
class Saver: def __init__(self, fallback_filename=None, exclude_matches=None, variable_transforms=None, save_dtype=None, target_model_init_from_base_model=False): self.variable_transforms = variable_transforms or [] self.exclude_matches = exclude_matches self.variables = None self.save_dtype = save_dtype if fallback_filename is not None: self.set_fallback(fallback_filename) self.target_model_init_from_base_model = target_model_init_from_base_model def set_fallback(self, fallback_filename): self.tpe = ThreadPoolExecutor() if not os.path.exists(fallback_filename): raise FileNotFoundError( "Error loading base model - file not found.") self.fallback_filename = fallback_filename self.fallback_future = self.tpe.submit(joblib.load, fallback_filename) self.fallback_ = None @property def fallback(self): if self.fallback_ is None: self.fallback_ = self.fallback_future.result() self.fallback_future = None self.tpe.shutdown() if self.target_model_init_from_base_model: if self.variables is None: self.variables = dict() for k, v in self.fallback_.items(): self.variables['model/target/' + k] = v return self.fallback_ def get_saver_hook(self, estimator, keep_best_model, steps_per_epoch, early_stopping_steps, eval_frequency, cache_weights_to_file): return SaverHook(self, estimator=estimator, keep_best_model=keep_best_model, steps_per_epoch=steps_per_epoch, early_stopping_steps=early_stopping_steps, eval_frequency=eval_frequency, cache_weights_to_file=cache_weights_to_file) def save(self, finetune_obj, path, mkdir=True): if self.variables is None: raise FinetuneError("Cowardly refusing to save default model.") if self.exclude_matches is not None: variables = { k: v for k, v in self.variables.items() if self.exclude_matches not in k } else: variables = self.variables names, values = variables.keys(), variables.values() if isinstance(path, str): folder = os.path.dirname(path) os.makedirs(folder, exist_ok=True) if self.save_dtype is not None: LOGGER.info("Saving with {} precision.".format( self.save_dtype.__name__)) values = [a.astype(self.save_dtype) for a in values] var_names_reduced, vals_reduced = self.remove_unchanged( names, values, self.fallback) var_dict = dict(zip(var_names_reduced, vals_reduced)) assert len(vals_reduced) == len(var_names_reduced) == len(var_dict) joblib.dump((var_dict, finetune_obj), path) def load(self, path): self.variables, finetune_obj = joblib.load(path) finetune_obj.config = get_config(error_on_invalid_keywords=False, **dict(finetune_obj.config)) return finetune_obj def get_scaffold_init_fn(self): def init_fn(scaffold, session, model_portion=None, refresh_base_model=False): var_loader = BatchedVarLoad() self.var_val = [] if self.variables is not None: variables_sv = self.variables else: variables_sv = dict() all_vars = tf.global_variables() zero_out_adapters = False if (model_portion != "entire_model" ): # we must be loading in the case of two separate estimators all_vars, zero_out_adapters = self.subset_to_load( model_portion, refresh_base_model, all_vars) global_step_var = tf.train.get_global_step() for var in all_vars: if global_step_var is not None and global_step_var.name == var.name: continue name = var.name saved_var = None if name in variables_sv.keys(): saved_var = variables_sv[name] elif name in self.fallback.keys(): saved_var = self.fallback[name] elif (self.target_model_init_from_base_model and name.startswith("model/target/") and name.replace( "model/target/", "") in self.fallback.keys()): saved_var = self.fallback[name.replace( "model/target/", "")] if zero_out_adapters and "adapter" in name: var_loader.add(var, np.zeros(var.get_shape().as_list())) if saved_var is not None: for func in self.variable_transforms: saved_var = func(name, saved_var) var_loader.add(var, saved_var) var_loader.run(session) return init_fn def subset_to_load(self, model_portion, refresh_base_model, all_vars): assert model_portion in [ "featurizer", "target", "whole_featurizer", ], "Must be using separate estimators if loading before graph creation" base = [v for v in all_vars if "target" not in v.name] zero_out_adapters = False if ( model_portion == "whole_featurizer" ): # load every weight in featurizer - used to initialize and for loading without adapters to_load = base adapters = [v for v in base if "adapter" in v.name] zero_out_adapters = True elif (model_portion == "featurizer" ): # update featurizer, loading adapters and scaling weights norm_variable_scopes = ["b:0", "g:0", "beta:0", "gamma:0"] to_load = (base if refresh_base_model else [ v for v in base if "target" not in v.name and ("adapter" in v.name or any( scope in v.name for scope in norm_variable_scopes)) ]) elif model_portion == "target": # update target model weights to_load = [v for v in all_vars if "target" in v.name] return to_load, zero_out_adapters def remove_unchanged(self, variable_names, variable_values, fallback_vars): skips = [] for var_val, var_name in zip(variable_values, variable_names): skip = False for fb_var_name, fb_var in fallback_vars.items(): if fb_var_name == var_name: for func in self.variable_transforms: fb_var = func(var_name, fb_var) if np.allclose(fb_var, var_val): skip = True break skips.append(skip) return ( [var for skip, var in zip(skips, variable_names) if not skip], [ var_val for skip, var_val in zip(skips, variable_values) if not skip ], )
class HttpClient(object): """A http based client for submitting Spark-based jobs to a Livy backend. Parameters ---------- url_str : string Livy server url to create a new session or the url of an existing session load_defaults : boolean, optional This parameter decides if the default config needs to be loaded Default is True conf_dict : dict, optional The key-value pairs in the conf_dict will be loaded to the config Default is None Examples -------- Imports needed to create an instance of HttpClient >>> from livy.client import HttpClient 1) Creates a client that is loaded with default config as 'load_defaults' is True by default >>> client = HttpClient("http://example:8998/") 2) Creates a client that does not load default config, but loads config that are passed in 'config_dict' >>> config_dict = {'spark.app.name', 'Test App'} >>> client = HttpClient("http://example:8998/", load_defaults=False, >>> config_dict=config_dict) """ _CONFIG_SECTION = 'env' _LIVY_CLIENT_CONF_DIR = "LIVY_CLIENT_CONF_DIR" def __init__(self, url, load_defaults=True, conf_dict=None): uri = urlparse(url) self._config = ConfigParser() self._load_config(load_defaults, conf_dict) self._job_type = 'pyspark' match = re.match(r'(.*)/sessions/([0-9]+)', uri.path) if match: base = ParseResult(scheme=uri.scheme, netloc=uri.netloc, path=match.group(1), params=uri.params, query=uri.query, fragment=uri.fragment) self._set_uri(base) self._conn = _LivyConnection(base, self._config) self._session_id = int(match.group(2)) self._reconnect_to_existing_session() else: self._set_uri(uri) session_conf_dict = dict(self._config.items(self._CONFIG_SECTION)) self._conn = _LivyConnection(uri, self._config) self._session_id = self._create_new_session( session_conf_dict).json()['id'] self._executor = ThreadPoolExecutor(max_workers=1) self._stopped = False self.lock = threading.Lock() def submit(self, job): """ Submits a job for execution to the spark cluster. Parameters ---------- job : function The function must accept a single parameter, which is an instance of JobContext. Returns ------- job_handle : an instance of the class JobHandle A handle that can be used to monitor the job Examples ------- >>> def simple_spark_job(context): >>> elements = [10, 20, 30, 40, 50] >>> return context.sc.parallelize(elements, 2).count() >>> client.submit(simple_spark_job) """ return self._send_job('submit-job', job) def run(self, job): """ Asks the remote context to run a job immediately. Normally, the remote context will queue jobs and execute them based on how many worker threads have been configured. This method will run the submitted job in the same thread processing the RPC message, so that queueing does not apply. It's recommended that this method only be used to run code that finishes quickly. This avoids interfering with the normal operation of the context. Parameters ---------- job : function The function must accept a single parameter, which is an instance of JobContext. Spark jobs can be created with the help of JobContext, which exposes the Spark libraries. Returns ------- future : concurrent.futures.Future A future to monitor the status of the job Examples ------- >>> def simple_job(context): >>> return "hello" >>> client.run(simple_job) """ return self._send_job("run-job", job) def add_file(self, file_uri): """ Adds a file to the running remote context. Note that the URL should be reachable by the Spark driver process. If running the driver in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist on that node (and not on the client machine). Parameters ---------- file_uri : string String representation of the uri that points to the location of the file Returns ------- future : concurrent.futures.Future A future to monitor the status of the job Examples ------- >>> client.add_file("file:/test_add.txt") >>> # Example job using the file added using add_file function >>> def add_file_job(context): >>> from pyspark import SparkFiles >>> def func(iterator): >>> with open(SparkFiles.get("test_add.txt")) as testFile: >>> fileVal = int(testFile.readline()) >>> return [x * fileVal for x in iterator] >>> return context.sc.parallelize([1, 2, 3, 4]) >>> .mapPartitions(func).collect() >>> client.submit(add_file_job) """ return self._add_file_or_pyfile_job("add-file", file_uri) def add_jar(self, file_uri): """ Adds a jar file to the running remote context. Note that the URL should be reachable by the Spark driver process. If running the driver in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist on that node (and not on the client machine). Parameters ---------- file_uri : string String representation of the uri that points to the location of the file Returns ------- future : concurrent.futures.Future A future to monitor the status of the job Examples ------- >>> client.add_jar("file:/test_package.jar") """ return self._add_file_or_pyfile_job("add-jar", file_uri) def add_pyfile(self, file_uri): """ Adds a .py or .zip to the running remote context. Note that the URL should be reachable by the Spark driver process. If running the driver in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist on that node (and not on the client machine). Parameters ---------- file_uri : string String representation of the uri that points to the location of the file Returns ------- future : concurrent.futures.Future A future to monitor the status of the job Examples ------- >>> client.add_pyfile("file:/test_package.egg") >>> # Example job using the file added using add_pyfile function >>> def add_pyfile_job(context): >>> # Importing module from test_package.egg >>> from test.pyfile_test import TestClass >>> test_class = TestClass() >>> return test_class.say_hello() >>> client.submit(add_pyfile_job) """ return self._add_file_or_pyfile_job("add-pyfile", file_uri) def upload_file(self, file_path): """ Upload a file to be passed to the Spark application. Parameters ---------- file_path : string File path of the local file to be uploaded. Returns ------- future : concurrent.futures.Future A future to monitor the status of the job Examples ------- >>> client.upload_file("/test_upload.txt") >>> # Example job using the file uploaded using upload_file function >>> def upload_file_job(context): >>> from pyspark import SparkFiles >>> def func(iterator): >>> with open(SparkFiles.get("test_upload.txt")) as testFile: >>> fileVal = int(testFile.readline()) >>> return [x * fileVal for x in iterator] >>> return context.sc.parallelize([1, 2, 3, 4]) >>> .mapPartitions(func).collect() >>> client.submit(add_file_job) """ return self._upload_file_or_pyfile("upload-file", open(file_path, 'rb')) def upload_pyfile(self, file_path): """ Upload a .py or .zip dependency to be passed to the Spark application. Parameters ---------- file_path : string File path of the local file to be uploaded. Returns ------- future : concurrent.futures.Future A future to monitor the status of the job Examples ------- >>> client.upload_pyfile("/test_package.egg") >>> # Example job using the file uploaded using upload_pyfile function >>> def upload_pyfile_job(context): >>> # Importing module from test_package.egg >>> from test.pyfile_test import TestClass >>> test_class = TestClass() >>> return test_class.say_hello() >>> client.submit(upload_pyfile_job) """ return self._upload_file_or_pyfile("upload-pyfile", open(file_path, 'rb')) def stop(self, shutdown_context): """ Stops the remote context. The function will return immediately and will not wait for the pending jobs to get completed Parameters ---------- shutdown_context : Boolean Whether to shutdown the underlying Spark context. If false, the context will keep running and it's still possible to send commands to it, if the backend being used supports it. """ with self.lock: if not self._stopped: self._executor.shutdown(wait=False) try: if shutdown_context: session_uri = "/" + str(self._session_id) headers = {'X-Requested-By': 'livy'} self._conn.send_request("DELETE", session_uri, headers=headers) except Exception: raise Exception(traceback.format_exc()) self._stopped = True def _set_uri(self, uri): if uri is not None and uri.scheme in ('http', 'https'): self._config.set(self._CONFIG_SECTION, 'livy.uri', uri.geturl()) else: url_exception = uri.geturl if uri is not None else None raise ValueError('Cannot create client - Uri not supported - ', url_exception) def _set_conf(self, key, value): if value is not None: self._config.set(self._CONFIG_SECTION, key, value) else: self._delete_conf(key) def _delete_conf(self, key): self._config.remove_option(self._CONFIG_SECTION, key) def _set_multiple_conf(self, conf_dict): for key, value in conf_dict.items(): self._set_conf(key, value) def _load_config(self, load_defaults, conf_dict): self._config.add_section(self._CONFIG_SECTION) if load_defaults: self._load_default_config() if conf_dict is not None and len(conf_dict) > 0: self._set_multiple_conf(conf_dict) def _load_default_config(self): config_dir = os.environ.get(self._LIVY_CLIENT_CONF_DIR) if config_dir is not None: config_files = os.listdir(config_dir) default_conf_files = ['spark-defaults.conf', 'livy-client.conf'] for default_conf_file in default_conf_files: if default_conf_file in config_files: self._load_config_from_file(config_dir, default_conf_file) def _load_config_from_file(self, config_dir, config_file): path = os.path.join(config_dir, config_file) data = "[" + self._CONFIG_SECTION + "]\n" + \ open(path, encoding='utf-8').read() self._config.readfp(StringIO(data)) def _create_new_session(self, session_conf_dict): data = {'kind': 'pyspark', 'conf': session_conf_dict} response = self._conn.send_request('POST', "/", headers=self._conn._JSON_HEADERS, data=data) return response def _reconnect_to_existing_session(self): reconnect_uri = "/" + str(self._session_id) + "/connect" self._conn.send_request('POST', reconnect_uri, headers=self._conn._JSON_HEADERS) def _send_job(self, command, job): pickled_job = cloudpickle.dumps(job) base64_pickled_job = base64.b64encode(pickled_job).decode('utf-8') base64_pickled_job_data = \ {'job': base64_pickled_job, 'jobType': self._job_type} handle = JobHandle(self._conn, self._session_id, self._executor) handle._start(command, base64_pickled_job_data) return handle def _add_file_or_pyfile_job(self, command, file_uri): data = {'uri': file_uri} suffix_url = "/" + str(self._session_id) + "/" + command return self._executor.submit(self._add_or_upload_resource, suffix_url, data=data, headers=self._conn._JSON_HEADERS) def _upload_file_or_pyfile(self, command, open_file): files = {'file': open_file} suffix_url = "/" + str(self._session_id) + "/" + command return self._executor.submit(self._add_or_upload_resource, suffix_url, files=files) def _add_or_upload_resource(self, suffix_url, files=None, data=None, headers=None): return self._conn.send_request('POST', suffix_url, files=files, data=data, headers=headers).content
class WebInterface: """ The WebInterface is a executor like class for the submission of runs to the VerifierCloud """ def __init__(self, web_interface_url, user_pwd, svn_branch='trunk', svn_revision='HEAD', thread_count=1, result_poll_interval=2, user_agent=None, version=None): """ Creates a new WebInterface object. The given svn revision is resolved (e.g. 'HEAD' -> 17495). @param web_interface_url: the base URL of the VerifierCloud's web interface @param user_pwd: user name and password in the format '<user_name>:<password>' or none if no authentification is required @param svn_branch: the svn branch name or 'trunk', defaults to 'trunk' @param svn_revision: the svn revision number or 'HEAD', defaults to 'HEAD' @param thread_count: the number of threads for fetching results in parallel @param result_poll_interval: the number of seconds to wait between polling results """ if not (1 <= thread_count <= MAX_SUBMISSION_THREADS): sys.exit( "Invalid number {} of client threads, needs to be between 1 and {}." .format(thread_count, MAX_SUBMISSION_THREADS)) if not 1 <= result_poll_interval: sys.exit("Poll interval {} is too small, needs to be at least 1s.". format(result_poll_interval)) if not web_interface_url[-1] == '/': web_interface_url += '/' default_headers = {'Connection': 'Keep-Alive'} if user_agent: default_headers['User-Agent'] = \ '{}/{} (Python/{} {}/{})'.format(user_agent, version, platform.python_version(), platform.system(), platform.release()) urllib.parse.urlparse(web_interface_url) # sanity check self._web_interface_url = web_interface_url logging.info('Using VerifierCloud at %s', web_interface_url) self._connection = requests.Session() self._connection.headers.update(default_headers) self._connection.verify = '/etc/ssl/certs' if user_pwd: self._connection.auth = (user_pwd.split(":")[0], user_pwd.split(":")[1]) self._base64_user_pwd = base64.b64encode( user_pwd.encode("utf-8")).decode("utf-8") else: self._base64_user_pwd = None self._unfinished_runs = {} self._unfinished_runs_lock = threading.Lock() self._downloading_result_futures = {} self._download_attempts = {} self.thread_count = thread_count self._executor = ThreadPoolExecutor(thread_count) self._thread_local = threading.local() self._hash_code_cache = {} self._group_id = str(random.randint(0, 1000000)) self._read_hash_code_cache() self._resolved_tool_revision(svn_branch, svn_revision) self._tool_name = self._request_tool_name() try: self._result_downloader = SseResultDownloader( self, result_poll_interval) except: self._result_downloader = PollingResultDownloader( self, result_poll_interval) def _read_hash_code_cache(self): if not os.path.isfile(HASH_CODE_CACHE_PATH): return with open(HASH_CODE_CACHE_PATH, mode='r') as hashCodeCacheFile: for line in hashCodeCacheFile: tokens = line.strip().split('\t') if len(tokens) == 3: self._hash_code_cache[(tokens[0], tokens[1])] = tokens[2] def _write_hash_code_cache(self): directory = os.path.dirname(HASH_CODE_CACHE_PATH) try: os.makedirs(directory, exist_ok=True) with tempfile.NamedTemporaryFile(dir=directory, delete=False) as tmpFile: for (path, mTime), hashValue in self._hash_code_cache.items(): line = (path + '\t' + mTime + '\t' + hashValue + '\n').encode() tmpFile.write(line) os.renames(tmpFile.name, HASH_CODE_CACHE_PATH) except OSError as e: logging.warning("Could not write hash-code cache file to %s: %s", HASH_CODE_CACHE_PATH, e.strerror) def _resolved_tool_revision(self, svn_branch, svn_revision): path = "tool/version?svnBranch=" + svn_branch + "&revision=" + svn_revision (resolved_svn_revision, _) = self._request("GET", path) self._svn_branch = svn_branch self._svn_revision = resolved_svn_revision.decode("UTF-8") def _request_tool_name(self): path = "tool/name" (tool_name, _) = self._request("GET", path) return tool_name.decode("UTF-8") def tool_revision(self): return self._svn_branch + ':' + self._svn_revision def tool_name(self): return self._tool_name def _get_sha1_hash(self, path): path = os.path.abspath(path) mTime = str(os.path.getmtime(path)) if ((path, mTime) in self._hash_code_cache): return self._hash_code_cache[(path, mTime)] else: with open(path, 'rb') as file: hashValue = hashlib.sha1(file.read()).hexdigest() self._hash_code_cache[(path, mTime)] = hashValue return hashValue def _create_and_add_run_future(self, run_id): result = RunResultFuture(self, run_id) with self._unfinished_runs_lock: self._unfinished_runs[run_id] = result return result def submit_witness_validation(self, witness_path, program_path, configuration=None, user_pwd=None): """ Submits a single witness validation run to the VerifierCloud. @note: flush() should be called after the submission of the last run. @param witness_path: path to the file containing the witness @param program_path: path to the file containing the program @param configuration: name of configuration (optional) @param user_pwd: overrides the user name and password given in the constructor (optional) """ # collect parameters params = {} with open(witness_path, 'rb') as witness_file: params['errorWitnessText'] = witness_file.read() with open(program_path, 'rb') as program_file: params['programText'] = program_file.read() if configuration: params['configuration'] = configuration # prepare request headers = { "Content-Type": "application/x-www-form-urlencoded", "Content-Encoding": "deflate", "Accept": "text/plain" } paramsCompressed = zlib.compress( urllib.parse.urlencode(params, doseq=True).encode('utf-8')) path = "runs/witness_validation/" (run_id, _) = self._request("POST", path, paramsCompressed, headers, user_pwd=user_pwd) run_id = run_id.decode("UTF-8") logging.debug('Submitted witness validation run with id %s', run_id) return self._create_and_add_run_future(run_id) def submit(self, run, limits, cpu_model, result_files_pattern=None, meta_information=None, \ priority='IDLE', user_pwd=None, svn_branch=None, svn_revision=None, result_files_patterns=[], required_files=[]): """ Submits a single run to the VerifierCloud. @note: flush() should be called after the submission of the last run. @param run: The input for the run: command line options (run.options), source files (run.sourcefiles), property file (run.propertyfile), identifier for error messages (run.identifier) @param limits: dict of limitations for the run (memlimit, timelimit, corelimit, softtimelimit) @param cpu_model: substring of CPU model to use or 'None' for no restriction @param result_files_pattern: the result is filtered with the given glob pattern, '**' is no restriction and None or the empty string do not match any file. @param meta_information: meta information about the submitted run as JSON string @param priority: the priority of the submitted run, defaults to 'IDLE' @param user_pwd: overrides the user name and password given in the constructor (optional) @param svn_branch: overrids the svn branch given in the constructor (optional) @param svn_revision: overrides the svn revision given in the constructor (optional) @param result_files_patterns: list of result_files_pattern (optional) @param required_files: list of additional file required to execute the run (optional) @raise WebClientError: if the HTTP request could not be created @raise HTTPError: if the HTTP request was not successful """ if result_files_pattern: if result_files_patterns: raise ValueError( "Cannot specify result_files_pattern and result_files_patterns " "at the same time.") result_files_patterns = [result_files_pattern] return self._submit(run, limits, cpu_model, required_files, result_files_patterns, meta_information, priority, user_pwd, svn_branch, svn_revision) def _submit(self, run, limits, cpu_model, required_files, result_files_patterns, meta_information, priority, user_pwd, svn_branch, svn_revision, counter=0): params = [] opened_files = [ ] # open file handles are passed to the request library for programPath in run.sourcefiles: norm_path = self._normalize_path_for_cloud(programPath) params.append(('programTextHash', (norm_path, self._get_sha1_hash(programPath)))) for required_file in required_files: norm_path = self._normalize_path_for_cloud(required_file) params.append( ('requiredFileHash', (norm_path, self._get_sha1_hash(required_file)))) params.append(('svnBranch', svn_branch or self._svn_branch)) params.append(('revision', svn_revision or self._svn_revision)) if run.propertyfile: file = self._add_file_to_params(params, 'propertyText', run.propertyfile) opened_files.append(file) if MEMLIMIT in limits: params.append(('memoryLimitation', str(limits[MEMLIMIT]))) if TIMELIMIT in limits: params.append(('timeLimitation', str(limits[TIMELIMIT]))) if SOFTTIMELIMIT in limits: params.append(('softTimeLimitation', str(limits[SOFTTIMELIMIT]))) if CORELIMIT in limits: params.append(('coreLimitation', str(limits[CORELIMIT]))) if cpu_model: params.append(('cpuModel', cpu_model)) if result_files_patterns: for pattern in result_files_patterns: params.append(('resultFilesPattern', pattern)) else: params.append(('resultFilesPattern', '')) if priority: params.append(('priority', priority)) (invalidOption, files) = self._handle_options(run, params, limits) opened_files.extend(files) if invalidOption: raise WebClientError('Command {0} contains option "{1}" that is not usable with the webclient. '\ .format(run.options, invalidOption)) params.append(('groupId', str(self._group_id))) if meta_information: params.append(('metaInformation', meta_information)) # prepare request headers = {"Accept": "text/plain"} path = "runs/" (run_id, statusCode) = self._request("POST", path, files=params, headers=headers, \ expectedStatusCodes=[200, 412], user_pwd=user_pwd) for opened_file in opened_files: opened_file.close() # program files or required files given as hash value are not known by the cloud system if statusCode == 412 and counter < 1: headers = { "Content-Type": "application/octet-stream", "Content-Encoding": "deflate" } filePath = "files/" # upload all used program files for programPath in run.sourcefiles: with open(programPath, 'rb') as programFile: compressedProgramText = zlib.compress( programFile.read(), 9) self._request('POST', filePath, data=compressedProgramText, headers=headers,\ expectedStatusCodes=[200, 204], user_pwd=user_pwd) # upload all required files for required_file_path in required_files: with open(required_file_path, 'rb') as required_file: compressed_required_file = zlib.compress( required_file.read(), 9) self._request('POST', filePath, data=compressed_required_file, headers=headers,\ expectedStatusCodes=[200, 204], user_pwd=user_pwd) # retry submission of run return self._submit(run, limits, cpu_model, required_files, result_files_patterns, meta_information, priority, user_pwd, svn_branch, svn_revision, counter + 1) else: run_id = run_id.decode("UTF-8") logging.debug('Submitted run with id %s', run_id) return self._create_and_add_run_future(run_id) def _handle_options(self, run, params, rlimits): opened_files = [] config = None # TODO use code from CPAchecker module, it add -stats and sets -timelimit, # instead of doing it here manually, too if self._tool_name == "CPAchecker": params.append(("option", "statistics.print=true")) if 'softtimelimit' in rlimits: params.append( ("option", "limits.time.cpu=" + str(rlimits['softtimelimit']) + "s")) if run.options: i = iter(run.options) while True: try: option = next(i) if len(option) == 0: continue if option == "-heap": params.append(('heap', next(i))) elif option == "-stack": params.append(('stack', next(i))) elif option == "-noout": params.append(("option", "output.disable=true")) elif option == "-outputpath": params.append(("option", "output.path=" + next(i))) elif option == "-logfile": params.append(("option", "log.file=" + next(i))) elif option == "-nolog": params.append(("option", "log.level=OFF")) params.append(("option", "log.consoleLevel=OFF")) elif option == "-stats": # ignore, is always set by this script pass elif option == "-disable-java-assertions": params.append(('disableJavaAssertions', 'true')) elif option == "-java": params.append(("option", "language=JAVA")) elif option == "-32": params.append( ("option", "analysis.machineModel=Linux32")) elif option == "-64": params.append( ("option", "analysis.machineModel=Linux64")) elif option == "-entryfunction": params.append( ("option", "analysis.entryFunction=" + next(i))) elif option == "-timelimit": params.append(("option", "limits.time.cpu=" + next(i))) elif option == "-skipRecursion": params.append( ("option", "cpa.callstack.skipRecursion=true")) params.append(("option", "analysis.summaryEdges=true")) elif option == "-cbmc": params.append( ("option", "analysis.checkCounterexamples=true")) params.append( ("option", "counterexample.checker=CBMC")) elif option == "-preprocess": params.append( ("option", "parser.usePreprocessor=true")) elif option == "-generateReport": params.append(('generateReport', 'true')) elif option == "-spec": spec_path = next(i) file = self._add_file_to_params( params, "specificationText", spec_path) opened_files.append(file) elif option == "-config": configPath = next(i) tokens = configPath.split('/') if (tokens[0] == "config" and len(tokens) == 2): config = tokens[1].split('.')[0] params.append(('configuration', config)) else: params.append( ("option", "configuration.file=" + configPath)) elif option == "-setprop": params.append(("option", next(i))) elif option[0] == '-': if config: raise WebClientError( "More than one configuration: '{}' and '{}'". format(config, option[1:])) else: params.append(('configuration', option[1:])) config = option[1:] else: return (option, opened_files) except StopIteration: break return (None, opened_files) def _add_file_to_params(self, params, name, path): norm_path = self._normalize_path_for_cloud(path) file = open(path, 'rb') params.append((name, (norm_path, file))) return file def _normalize_path_for_cloud(self, path): norm_path = os.path.normpath(path) if '..' in norm_path or os.path.isabs(norm_path): norm_path = os.path.basename(norm_path) return norm_path def flush_runs(self): """ Starts the execution of all previous submitted runs in the VerifierCloud. The web interface groups runs and submits them to the VerifierCloud only from time to time. This method forces the web interface to do this immediately and starts downloading of results. @return: the ids of the RunCollections created since the last flush request """ headers = { "Content-Type": "application/x-www-form-urlencoded", "Connection": "Keep-Alive" } params = {"groupId": self._group_id} path = "runs/flush" (response, _) = self._request("POST", path, data=params, headers=headers, expectedStatusCodes=[200, 204]) run_collections = response.decode('utf-8').split("\n") if len(run_collections) == 0: logging.warning( 'No runs were submitted to the VerifierCloud before or a rate limit is hit.' ) else: logging.info('Submitted %s run collection: %s', len(run_collections), ",".join(run_collections)) self._result_downloader.start() return run_collections def _is_finished(self, run_id): headers = {"Accept": "text/plain"} path = "runs/" + run_id + "/state" try: (state, _) = self._request("GET", path, headers=headers) state = state.decode('utf-8') if state == "FINISHED": logging.debug('Run %s finished.', run_id) if state == "UNKNOWN": logging.debug( 'Run %s is not known by the webclient, trying to get the result.', run_id) return state except requests.HTTPError as e: logging.warning('Could not get run state %s: %s', run_id, e.response) return False def _download_result(self, run_id): # download result as zip file headers = {"Accept": "application/zip"} path = "runs/" + run_id + "/result" (zip_content, _) = self._request("GET", path, headers=headers) return zip_content def _download_result_async(self, run_id): def callback(downloaded_result): run_id = self._downloading_result_futures.pop(downloaded_result) exception = downloaded_result.exception() if not exception: with self._unfinished_runs_lock: result_future = self._unfinished_runs.pop(run_id, None) if result_future: result_future.set_result(downloaded_result.result()) else: logging.info('Could not get result of run %s: %s', run_id, downloaded_result.exception()) # client error if type(exception) is HTTPError and exception.response and \ 400 <= exception.response.status_code and exception.response.status_code <= 499: attempts = self._download_attempts.pop(run_id, 1) if attempts < 10: self._download_attempts[run_id] = attempts + 1 self._download_result_async(run_id) else: self._run_failed(run_id) else: # retry it self._download_result_async(run_id) if run_id not in self._downloading_result_futures.values( ): # result is not downloaded future = self._executor.submit(self._download_result, run_id) self._downloading_result_futures[future] = run_id future.add_done_callback(callback) def _run_failed(self, run_id): run_result_future = self._unfinished_runs.pop(run_id, None) if run_result_future: logging.warning('Execution of run %s failed.', run_id) run_result_future.set_exception( WebClientError("Execution failed.")) def shutdown(self): """ Cancels all unfinished runs and stops all internal threads. """ self._result_downloader.shutdown() if len(self._unfinished_runs) > 0: logging.info("Stopping tasks on server...") stop_executor = ThreadPoolExecutor(max_workers=5 * self.thread_count) stop_tasks = set() with self._unfinished_runs_lock: for runId in self._unfinished_runs.keys(): stop_tasks.add(stop_executor.submit(self._stop_run, runId)) self._unfinished_runs[runId].set_exception( WebClientError("WebInterface was stopped.")) self._unfinished_runs.clear() for task in stop_tasks: task.result() stop_executor.shutdown(wait=True) logging.info("Stopped all tasks.") self._write_hash_code_cache() self._executor.shutdown(wait=True) self._connection.close() def _stop_run(self, run_id): with self._unfinished_runs_lock: self._unfinished_runs.pop(run_id, None) path = "runs/" + run_id try: self._request("DELETE", path, expectedStatusCodes=[200, 204, 404]) except HTTPError as e: logging.info("Stopping of run %s failed: %s", run_id, e.reason) def _request(self, method, path, data=None, headers=None, files=None, expectedStatusCodes=[200], user_pwd=None): url = self._web_interface_url + path if user_pwd: auth = (user_pwd.split(":")[0], user_pwd.split(":")[1]) else: auth = None counter = 0 while (counter < 5): counter += 1 # send request try: response = self._connection.request(method, url, data=data, files=files, headers=headers, auth=auth) except Exception as e: if (counter < 5): logging.debug("Exception during %s request to %s: %s", method, path, e) sleep(1) continue else: raise if response.status_code in expectedStatusCodes: return (response.content, response.status_code) else: message = "" if response.status_code == 401: message = 'Error 401: Permission denied. Please check the URL given to --cloudMaster and specify credentials if necessary.' elif response.status_code == 404: message = 'Error 404: Not found. Please check the URL given to --cloudMaster.' elif response.status_code == 503: message = 'Error 503: Service Unavailable.' if counter < 5: logging.debug(message) sleep(60) continue else: message += response.content.decode('UTF-8') logging.warning(message) raise requests.HTTPError(path, message, response=response)
print('执行下载任务') print(page) fullurl = 'http://blog.jobbole.com/all-posts/page/%s/' % str(page) #发起请求 req_headers = { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36', } response = requests.get(fullurl, headers=req_headers) if response.status_code == 200: print('数据加入成功') #2.将获取到的页面源码数据,返回 return response.text, response.status_code def download_done(future): print(future.result) if __name__ == '__main__': #实例化一个线程池 #max_workers:在线程池中要创建的线程的数量 thread_pool = ThreadPoolExecutor(max_workers=10) for page in range(1, 101): handler = thread_pool.submit(download_page_data, page) handler.add_done_callback(download_done) thread_pool.shutdown() #->实际上调用了join()方法
print('modifying company code: {0} from {1} to {2}'.format( comp[0], exist_comp, comp[1])) cur.execute('INSERT INTO fund_company VALUES(?, ?)', ( comp[0], comp[1], )) except Exception as err: print('get_fund_companies error when INSERT INTO') print(err) raise print('init companies finished...') pass def get_fund_real_time(name): pass def init(): init_database() get_fund_companies() get_all_funds() print("hello") init() for future in futures: future.result() thread_pool.shutdown(True) sql.close()
class MessageHandler: def __init__(self, client): self.app = App() self.client = client self.state = None self.executors = ThreadPoolExecutor(1) # end def def destroy(self): try: self.app.destroy() self.executors.shutdown(False) except: logger.exception('While destroying MessageHandler') finally: self.client.handlers.pop(self.user.id) shutil.rmtree(self.app.output_path, ignore_errors=True) # end try # end def @asyncio.coroutine async def send(self, *contents): for text in contents: if text: # await self.client.send_typing(self.user) async with self.user.typing(): await self.user.send(text) # end if # end for # end def @asyncio.coroutine async def process(self, message): self.message = message self.user = message.author if not self.state: await self.send('-' * 80 + '\n' + ('Hello %s\n' % self.user.name) + '*Lets make reading lightnovels great again!*\n' + '-' * 80 + '\n') self.state = self.get_novel_url # end if await self.state() # end def async def get_novel_url(self): await self.send( 'I recognize these two categories:\n' '- Profile page url of a lightnovel.\n' '- A query to search your lightnovel.', 'What are you looking for?') self.state = self.handle_novel_url # end def async def handle_novel_url(self): try: self.app.user_input = self.message.content.strip() self.app.init_search() except: await self.send( 'Sorry! I only know these sources:\n' + '\n'.join(['- %s' % x for x in crawler_list.keys()]), 'Enter something again.') # end try if len(self.app.user_input) < 4: await self.send('Your query is too short') return # end if if self.app.crawler: await self.send('Got your page link') await self.get_novel_info() else: await self.send( 'Searching %d sources for "%s"\n' % (len(self.app.crawler_links), self.app.user_input), 'Please do not type anything before I reply!') await self.display_novel_selection() # end if # end def async def display_novel_selection(self): async with self.user.typing(): self.app.search_novel() if len(self.app.search_results) == 0: await self.send('No novels found for "%s"' % self.app.user_input) return # end if if len(self.app.search_results) == 1: self.selected_novel = self.app.search_results[0] await self.display_sources_selection() return # end if await self.send( ('Found %d novels:\n' % len(self.app.search_results)) + '\n'.join([ '%d. **%s** `%d sources`' % (i + 1, item['title'], len(item['novels'])) for i, item in enumerate(self.app.search_results) ]) + '\n' + 'Enter name or index of your novel.\n' + 'Send `!cancel` to stop this session.') self.state = self.handle_novel_selection # end def async def handle_novel_selection(self): text = self.message.content.strip() if text.startswith('!cancel'): await self.get_novel_url() return # end if async with self.user.typing(): match_count = 0 selected = None for i, res in enumerate(self.app.search_results): if str(i + 1) == text: selected = res match_count += 1 elif text.isdigit() or len(text) < 3: pass elif res['title'].lower().find(text) != -1: selected = res match_count += 1 # end if # end for if match_count != 1: await self.send( 'Sorry! You should select *one* novel from the list (%d selected).' % match_count) await self.display_novel_selection() return # end if self.selected_novel = selected await self.display_sources_selection() # end def async def display_sources_selection(self): async with self.user.typing(): await self.send( ('**%s** is found in %d sources:\n' % (self.selected_novel['title'], len(self.selected_novel['novels']))) + '\n'.join([ '%d. <%s> %s' % (i + 1, item['url'], item['info'] if 'info' in item else '') for i, item in enumerate(self.selected_novel['novels']) ]) + '\n' + 'Enter index or name of your source.\n' + 'Send `!cancel` to stop this session.') self.state = self.handle_sources_to_search # end def async def handle_sources_to_search(self): if len(self.selected_novel['novels']) == 1: novel = self.selected_novel['novels'][0] await self.handle_search_result(novel) return # end if text = self.message.content.strip() if text.startswith('!cancel'): await self.get_novel_url() return # end if match_count = 0 selected = None for i, res in enumerate(self.selected_novel['novels']): if str(i + 1) == text: selected = res match_count += 1 elif text.isdigit() or len(text) < 3: pass elif res['url'].lower().find(text) != -1: selected = res match_count += 1 # end if # end for if match_count != 1: await self.send( 'Sorry! You should select *one* source from the list (%d selected).' % match_count) await self.display_sources_selection() return # end if await self.handle_search_result(selected) # end def async def handle_search_result(self, novel): await self.send('Selected: %s' % novel['url']) self.app.init_crawler(novel['url']) await self.get_novel_info() # end def async def get_novel_info(self): if not self.app.crawler: await self.send('Could not find any crawler to get your novel') self.state = self.get_novel_info return # end if # TODO: Handle login here await self.send('Getting information about your novel...') async with self.user.typing(): self.app.get_novel_info() # Setup output path good_name = os.path.basename(self.app.output_path) output_path = os.path.abspath( os.path.join('.discord_bot_output', str(self.user.id), good_name)) if os.path.exists(output_path): shutil.rmtree(output_path, ignore_errors=True) # end if os.makedirs(output_path, exist_ok=True) self.app.output_path = output_path # Get chapter range await self.send( 'It has %d volumes and %d chapters.' % (len(self.app.crawler.volumes), len(self.app.crawler.chapters)) ) await self.display_range_selection() # end def async def display_range_selection(self): await self.send('\n'.join([ 'Now you can send the following commands to modify what to download:', '- To download everything send `!all` or pass `!cancel` to stop.', '- Send `!last` followed by a number to download last few chapters. ' 'If it does not followed by a number, last 50 chapters will be downloaded.', '- Similarly you can send `!first` followed by a number to get first few chapters.', '- Send `!volume` followed by volume numbers to download.', '- To download a range of chatpers, Send `!chapter` followed by ' + 'two chapter numbers or urls separated by *space*. ' + ('Chapter number must be between 1 and %d, ' % len(self.app.crawler.chapters)) + ('and chapter urls should be from <%s>.' % (self.app.crawler.home_url)) ])) self.state = self.handle_range_selection # end def async def handle_range_selection(self): text = self.message.content.strip() if text.startswith('!cancel'): await self.get_novel_url() return # end if if text.startswith('!all'): self.app.chapters = self.app.crawler.chapters[:] elif text.startswith('!first'): text = text[len('!first'):].strip() n = int(text) if text.isdigit() else 50 n = 50 if n < 0 else n self.app.chapters = self.app.crawler.chapters[:n] elif text.startswith('!last'): text = text[len('!last'):].strip() n = int(text) if text.isdigit() else 50 n = 50 if n < 0 else n self.app.chapters = self.app.crawler.chapters[-n:] elif text.startswith('!volume'): text = text[len('!volume'):].strip() selected = re.findall(r'\d+', text) await self.send('Selected volumes: ' + ', '.join(selected), ) selected = [int(x) for x in selected] self.app.chapters = [ chap for chap in self.app.crawler.chapters if selected.count(chap['volume']) > 0 ] elif text.startswith('!chapter'): text = text[len('!chapter'):].strip() pair = text.split(' ') if len(pair) == 2: def resolve_chapter(name): cid = 0 if name.isdigit(): cid = int(str) else: cid = self.app.crawler.get_chapter_index_of(name) # end if return cid - 1 # end def first = resolve_chapter(pair[0]) second = resolve_chapter(pair[1]) if first > second: second, first = first, second # end if if first >= 0 or second < len(self.app.crawler.chapters): self.app.chapters = self.app.crawler.chapters[first:second] # end if # end if if len(self.app.chapters) == 0: await self.send('Chapter range is not valid. Please try again') return # end if else: await self.send( 'Sorry! I did not recognize your input. Please try again') return # end if if len(self.app.chapters) == 0: await self.send( 'You have not selected any chapters. Please select at least one' ) return # end if await self.send('Get your range selection') await self.display_output_selection() # end def async def display_output_selection(self): await self.send('\n'.join([ 'Now you can send the following commands choose book format you want to download:', '- Output format now supported is docx,mobi,pdf,rtf,txt,azw3,fb2,lit,lrf,oeb,pdb,rb,snb,tcr,epub,text,web', '- To download everything send `!all` or pass `!cancel` to stop.', '- or pass supported ootput format after ! command like example bellow', '- Send `!mobi` to download in mobi format. ', '- Send `!pdf` to download in pdf format.', '- Send `!{supported output format}', ])) self.state = self.handle_output_selection # end def async def handle_output_selection(self): text = self.message.content.strip() if text.startswith('!cancel'): await self.get_novel_url() return # end if output_format = text[1:] if text.startswith('!all'): self.app.output_formats = None else: self.app.output_formats = {} if output_format in available_formats: for x in available_formats: if x == output_format: self.app.output_formats[x] = True else: self.app.output_formats[x] = False # end if # end for await self.send('I will generate e-book in %s format' % output_format) else: await self.send( 'Sorry! I did not recognize your input. Please try again') return # end if # end if await self.send( 'Received your request. Starting download...\n' + 'Send anything to view status.\nSend `!cancel` to stop it.') self.status = ['', ''] self.state = self.report_download_progress try: self.executors.submit(self.start_download) except Exception: logger.exception('Download failure: %s', self.user.id) # end try # end def def start_download(self): self.app.pack_by_volume = False self.status = ['**%s**' % self.app.crawler.novel_title] self.status.append('Downloading %d chapters...' % len(self.app.chapters)) self.app.start_download() self.status.append('Binding books...') self.app.bind_books() self.status[-1] = 'Book binding completed.' self.status.append('Compressing output folder...') self.app.compress_output() self.status[-1] = 'Compressed output folder.' self.status.append('Uploading files...') for archive in self.app.archived_outputs: asyncio.run_coroutine_threadsafe(self.upload_file(archive), self.client.loop).result() # end for self.destroy() # end def async def upload_file(self, archive): file_size = os.stat(archive).st_size if file_size > 7.99 * 1024 * 1024: await self.send('File %s exceeds 8MB. Uploading To Google Drive.' % os.path.basename(archive)) description = 'Generated By : Discord Bot Ebook Smelter' link_id = upload(archive, description) if link_id: await self.send('https://drive.google.com/open?id=%s' % link_id ) else: await self.send('Failed to upload to google drive') # end if else: k = 0 while (file_size > 1024 and k < 3): k += 1 file_size /= 1024.0 # end while await self.send('Uploading %s [%d%s] ...' % (os.path.basename(archive), int(file_size * 100) / 100.0, ['B', 'KB', 'MB', 'GB'][k])) async with self.user.typing(): # await message.channel.send('Hello', file=discord.File('cool.png', 'testing.png')) await self.user.send('Here you go ! ', file=discord.File( open(archive, 'rb'), os.path.basename(archive))) # end if # end def async def report_download_progress(self): text = self.message.content.strip() if text == '!cancel': await self.send('Closing the session') self.destroy() await self.send( 'Session is now closed. Type *anything* to create a new one.') return # end if async with self.user.typing(): if self.app.progress < len(self.app.chapters): self.status[ 1] = '%d out of %d chapters has been downloaded.' % ( self.app.progress, len(self.app.chapters)) else: self.status[1] = 'Download complete.' # end if await self.send('\n'.join(self.status).strip() + '\n\n' + 'Send `!cancel` to stop')
class RemoteShell(): def __init__(self, max_async_threads=2): """ :type args: int :param args: maximun number of async command executors """ self.ssh = paramiko.SSHClient() self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) self.executor = ThreadPoolExecutor(max_workers=max_async_threads) self.futures = [] def connect(self, auth, address, port): """ Creates an ssh session to a remote host :type auth: :py:class:`margaritashotgun.auth.AuthMethods` :param auth: Authentication object :type address: str :param address: remote server address :type port: int :param port: remote server port """ self.username = auth.username self.address = address self.port = port try: logger.debug(("{0}: paramiko client connecting to " "{0}:{1} with {2}".format(address, port, auth.method))) if auth.method == AuthMethods.key: self.connect_with_key(auth.username, auth.key, address, port) elif auth.method == AuthMethods.password: self.connect_with_password(auth.username, auth.password, address, port) else: raise AuthenticationMethodMissingError() logger.debug(("{0}: paramiko client connected to " "{0}:{1}".format(address, port))) except (AuthenticationException, SSHException, SocketError) as ex: raise SSHConnectionError("{0}:{1}".format(address, port), ex) def connect_with_password(self, username, password, address, port): """ Create an ssh session to a remote host with a username and password :type username: str :param username: username used for ssh authentication :type password: str :param password: password used for ssh authentication :type address: str :param address: remote server address :type port: int :param port: remote server port """ self.ssh.connect(username=username, password=password, hostname=address, port=port) def connect_with_key(self, username, key, address, port): """ Create an ssh session to a remote host with a username and rsa key :type username: str :param username: username used for ssh authentication :type key: :py:class:`paramiko.key.RSAKey` :param key: paramiko rsa key used for ssh authentication :type address: str :param address: remote server address :type port: int :param port: remote server port """ self.ssh.connect(hostname=address, port=port, username=username, pkey=key) def execute(self, command): """ Executes command on remote hosts :type command: str :param command: command to be run on remote host """ logger.debug('{0}: executing "{1}"'.format(self.address, command)) stdin, stdout, stderr = self.ssh.exec_command(command) return dict(zip(['stdin', 'stdout', 'stderr'], [stdin, stdout, stderr])) def execute_async(self, command, callback=None): """ Executes command on remote hosts without blocking :type command: str :param command: command to be run on remote host :type callback: function :param callback: function to call when execution completes """ logger.debug(('{0}: execute async "{1}"' 'with callback {2}'.format(self.address, command, callback))) future = self.executor.submit(self.execute, command) if callback is not None: future.add_done_callback(callback) return future def decode(self, stream, encoding='utf-8'): """ Convert paramiko stream into a string :type stream: :param stream: stream to convert :type encoding: str :param encoding: stream encoding """ data = stream.read().decode(encoding).strip("\n") if data != "": logger.debug(('{0}: decoded "{1}" with encoding ' '{2}'.format(self.address, data, encoding))) return data def upload_file(self, local_path, remote_path): """ Upload a file from the local filesystem to the remote host :type local_path: str :param local_path: path of local file to upload :type remote_path: str :param remote_path: destination path of upload on remote host """ logger.debug("{0}: uploading {1} to {0}:{2}".format( self.address, local_path, remote_path)) try: sftp = self.ssh.open_sftp() sftp.put(local_path, remote_path) sftp.close() except SSHException as ex: logger.warn(("{0}: LiME module upload failed with exception:" "{1}".format(self.address, ex))) def cleanup(self): """ Release resources used during shell execution """ for future in self.futures: future.cancel() self.executor.shutdown(wait=10) self.ssh.close()
def pagedownload(baseurl, start): def gethtml(url): headers = {'User-Agent': 'Mozilla/5.0'} page1 = urllib.request.Request(url, headers=headers) page = urllib.request.urlopen(page1) basehtml = page.read().decode("UTF-8") return basehtml basehtml = gethtml(baseurl) def basereg(html): reg = r'(data-id=")([0-9]+?)(")' all = re.compile(reg) alllist = re.findall(all, html) return alllist baseimgurls = basereg(basehtml) def download(baseurl, start, num): url = "https://danbooru.donmai.us/posts/" + str(baseurl) # print('当前图片爬取地址 '+url) os.makedirs('./image/', exist_ok=True) def gethtml(url): headers = {'User-Agent': 'Mozilla/5.0'} page1 = urllib.request.Request(url, headers=headers) page = urllib.request.urlopen(page1) basehtml = page.read().decode("UTF-8") return basehtml html = gethtml(url) def reg(html): reg = r'(id="image")(.+?)(src=")(.+?)(")' all = re.compile(reg) alllist = re.findall(all, html) return alllist imgurls = reg(html) def urllib_download(): for imgurl in imgurls: from urllib.request import urlretrieve IMAGE_URL = imgurl[3] IMAGE_URL = IMAGE_URL.replace("/sample", "", 1) IMAGE_URL = IMAGE_URL.replace("sample-", "", 1) IMAGE_URL = IMAGE_URL.replace("jpg", "jpg?download=1", 1) def cbk(a, b, c): '''''回调函数 @a:已经下载的数据块 @b:数据块的大小 @c:远程文件的大小 ''' per = 100.0 * a * b / c if per > 100: per = 100 print('%.2f%%' % per) try: opener = urllib.request.build_opener() opener.addheaders = [('User-agent', 'Mozilla/5.0')] urllib.request.install_opener(opener) urlretrieve(IMAGE_URL, './image/danimg' + str(baseurl) + '.jpg') print('第' + str(start - 1) + '页 ' + '第' + str(num) + '张图 下载成功') except: IMAGE_URL = IMAGE_URL.replace("jpg", "png?download=1", 1) opener = urllib.request.build_opener() opener.addheaders = [('User-agent', 'Mozilla/5.0')] urllib.request.install_opener(opener) urlretrieve(IMAGE_URL, './image/danimg' + str(baseurl) + '.png') print('第' + str(start - 1) + '页 ' + '第' + str(num) + '张图 下载成功') else: pass urllib_download() length = len(baseimgurls) if (length == 0): print('第' + str(start - 1) + '页 无数据') else: print('第' + str(start - 1) + '页 共' + str(length) + '张图片') num = 1 pool = ThreadPoolExecutor(max_workers=11) for baseimgurl in baseimgurls: pool.submit(download, baseimgurl[1], start, num) num += 1 pool.shutdown()
articleInfo['coverImage'] = '暂无图片' p_as = article.xpath('.//div[@class="post-meta"]/p[1]//a') if len(p_as) > 2: # tag类型 articleInfo['tag'] = p_as[1].xpath('./text()')[0] # 评论量 articleInfo['commentNum'] = p_as[2].xpath('./text()')[0] else: # tag类型 articleInfo['tag'] = p_as[1].xpath('./text()')[0] # 评论量 articleInfo['commentNum'] = '0' # 简介 articleInfo['content'] = article.xpath( './/span[@class="excerpt"]/p/text()')[0] # 时间 articleInfo['publishTime'] = ''.join( article.xpath('.//div[@class="post-meta"]/p[1]/text()')).replace( '\n', '').replace(' ', '').replace('\r', '').replace('·', '') print(articleInfo) if __name__ == '__main__': pool = ThreadPoolExecutor(max_workers=10) for i in range(1, 201): handler = pool.submit(down_load_data, i) #设置回调方法 handler.add_done_callback(download_done) #执行shutdown()实质是执行了join()方法 pool.shutdown()
def scrape(self): """function that scrapes job posting from monster and pickles it""" log_info(f'jobfunnel monster to pickle running @ {self.date_string}') # get the search url search = self.get_search_url() # get the html data, initialize bs4 with lxml request_html = self.s.get(search, headers=self.headers) # create the soup base soup_base = BeautifulSoup(request_html.text, self.bs4_parser) # scrape total number of results, and calculate the # pages needed num_res = soup_base.find('h2', 'figure').text.strip() num_res = int(re.findall(r'(\d+)', num_res)[0]) log_info(f'Found {num_res} monster results for query=' f'{self.query}') pages = int(ceil(num_res / self.max_results_per_page)) # scrape soups for all the pages containing jobs it found page_url = f'{search}&start={pages}' log_info(f'getting monster pages 1 to {pages} : {page_url}') jobs = BeautifulSoup( self.s.get(page_url, headers=self.headers).text, self.bs4_parser). \ find_all('div', attrs={'class': 'flex-row'}) job_soup_list = [] job_soup_list.extend(jobs) # id regex quantifiers id_regex = re.compile(r'/((?:[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f' r']{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})|\d+)') # make a dict of job postings from the listing briefs for s in job_soup_list: # init dict to store scraped data job = dict([(k, '') for k in MASTERLIST_HEADER]) # scrape the post data job['status'] = 'new' try: # jobs should at minimum have a title, company and location job['title'] = s.find('h2', attrs={ 'class': 'title'}).text.strip() job['company'] = s.find( 'div', attrs={'class': 'company'}).text.strip() job['location'] = s.find('div', attrs={ 'class': 'location'}).text.strip() except AttributeError: continue # no blurb is available in monster job soups job['blurb'] = '' # tags are not supported in monster job['tags'] = '' try: job['date'] = s.find('time').text.strip() except AttributeError: job['date'] = '' # captures uuid or int ids, by extracting from url instead try: job['link'] = str(s.find('a', attrs={ 'data-bypass': '******'}).get('href')) job['id'] = id_regex.findall(job['link'])[0] except AttributeError: job['id'] = '' job['link'] = '' job['query'] = self.query job['provider'] = self.provider # key by id self.scrape_data[str(job['id'])] = job # apply job pre-filter before scraping blurbs super().pre_filter(self.scrape_data, self.provider) # stores references to jobs in list to be used in blurb retrieval scrape_list = [i for i in self.scrape_data.values()] # converts job date formats into a standard date format post_date_from_relative_post_age(scrape_list) threads = ThreadPoolExecutor(max_workers=8) # checks if delay is set or not, then extracts blurbs from job links if self.delay_config is not None: # calls super class to run delay specific threading logic super().delay_threader(scrape_list, self.get_blurb_with_delay, self.parse_blurb, threads) else: # start time recording start = time() # maps jobs to threads and cleans them up when done threads.map(self.search_joblink_for_blurb, scrape_list) threads.shutdown() # end and print recorded time end = time() print(f'{self.provider} scrape job took {(end - start):.3f}s')
def apply_activities(experiment: Experiment, configuration: Configuration, secrets: Secrets, pool: ThreadPoolExecutor, journal: Journal, dry: bool = False) -> List[Run]: with controls(level="method", experiment=experiment, context=experiment, configuration=configuration, secrets=secrets) as control: result = [] runs = [] method = experiment.get("method", []) wait_for_background_activities = True try: for run in run_activities(experiment, configuration, secrets, pool, dry): runs.append(run) if journal["status"] in ["aborted", "failed", "interrupted"]: break except SystemExit as x: # when we got a signal for an ungraceful exit, we can decide # not to wait for background activities. Their statuses will # remain failed. wait_for_background_activities = x.code != 30 # see exit.py raise finally: background_activity_timeout = None if wait_for_background_activities and pool: logger.debug("Waiting for background activities to complete") pool.shutdown(wait=True) elif pool: harshly_terminate_pending_background_activities(pool) logger.debug( "Do not wait for the background activities to finish " "as per signal") background_activity_timeout = 0.2 pool.shutdown(wait=False) for index, run in enumerate(runs): if not run: continue if isinstance(run, dict): result.append(run) else: try: # background activities result.append( run.result(timeout=background_activity_timeout)) except TimeoutError: # we want an entry for the background activity in our # results anyway, we won't have anything meaningful # to say about it result.append({ "activity": method[index], "status": "failed", "output": None, "duration": None, "start": None, "end": None, "exception": None }) # now let's ensure the journal has all activities in their correct # order (background ones included) journal["run"] = result control.with_state(result) return result
# t2.start() # t3.start() # t4.start() for i in range(0, 1): # give_packet() # time.sleep(3) packet_id = str( sql.read_mysql( sql= 'select order_no from activity.rp_give_red_packet order by created_at desc' )[0]) print(packet_id) time.sleep(1) p = ThreadPoolExecutor(5) p.submit(draw, './data_1.txt') p.shutdown(wait=True) time.sleep(3) amount = sql.read_mysql( sql= 'select draw_amount from activity.rp_give_red_packet where order_no = {}' .format(packet_id)) draw_sum = sql.read_mysql( sql= 'select sum(amount) from activity.rp_draw_red_packet where order_no = {}' .format(packet_id)) sum = sql.read_mysql( sql= 'select count(*) from activity.rp_draw_red_packet where order_no = {}' .format(packet_id)) sum_rp = sql.read_mysql( sql=
def run(links): executor = ThreadPoolExecutor(max_workers=thread_count) futures = [executor.submit(work, link) for link in links ] # Not bothering about results of futures for time being. executor.shutdown(wait=True)
class TaskSupervisor: timeout_message = '{supervisor_id} task {task_id}: ' + \ '{target} started in {time_spent:.3f} seconds. ' + \ 'Increase pool size or decrease number of workers' def __init__(self, supervisor_id=None): self.poll_delay = default_poll_delay self.timeout_warning = default_timeout_warning self.timeout_warning_func = None self.timeout_critical = default_timeout_critical self.timeout_critical_func = None self.id = supervisor_id if supervisor_id else str(uuid.uuid4()) self._active_threads = set() self._active_mps = set() self._active = False self._main_loop_active = False self._started = threading.Event() self._lock = threading.Lock() self._max_threads = {} self._max_mps = {} self._schedulers = {} self._tasks = {} self._Qt = {} self._Qmp = {} self.default_aloop = None self.default_async_job_scheduler = None self.mp_pool = None self.daemon = False self._processors_stopped = {} self.aloops = {} self.async_job_schedulers = {} self.set_thread_pool(pool_size=thread_pool_default_size, reserve_normal=default_reserve_normal, reserve_high=default_reserve_high, max_size=None) def set_thread_pool(self, **kwargs): for p in ['pool_size', 'reserve_normal', 'reserve_high']: if p in kwargs: setattr(self, 'thread_' + p, int(kwargs[p])) self._max_threads[TASK_LOW] = self.thread_pool_size self._max_threads[ TASK_NORMAL] = self.thread_pool_size + self.thread_reserve_normal thc= self.thread_pool_size + \ self.thread_reserve_normal + self.thread_reserve_high self._max_threads[TASK_HIGH] = thc self._prespawn_threads = kwargs.get('min_size', 0) max_size = kwargs.get('max_size') if not max_size: max_size = thc if self.thread_pool_size else \ thread_pool_default_size if self._prespawn_threads == 'max': self._prespawn_threads = max_size elif max_size < self._prespawn_threads: raise ValueError( 'min pool size ({}) can not be larger than max ({})'.format( self._prespawn_threads, max_size)) self.thread_pool = ThreadPoolExecutor( max_workers=max_size, thread_name_prefix='supervisor_{}_pool'.format(self.id)) if self._max_threads[TASK_HIGH] > max_size: logger.warning( ('supervisor {} executor thread pool max size ({}) is ' + 'lower than reservations ({})').format( self.id, max_size, self._max_threads[TASK_HIGH])) def set_mp_pool(self, **kwargs): for p in ['pool_size', 'reserve_normal', 'reserve_high']: setattr(self, 'mp_' + p, int(kwargs.get(p, 0))) self._max_mps[TASK_LOW] = self.mp_pool_size self._max_mps[TASK_NORMAL] = self.mp_pool_size + self.mp_reserve_normal self._max_mps[TASK_HIGH] = self.mp_pool_size + \ self.mp_reserve_normal + self.mp_reserve_high if not self.mp_pool: self.create_mp_pool(processes=self._max_mps[TASK_HIGH]) def timeout_warning_func(task): pass def timeout_critical_func(task): pass def _higher_queues_busy(self, tt, task_priority): if tt == TT_THREAD: q = self._Qt elif tt == TT_MP: q = self._Qmp if task_priority == TASK_NORMAL: return not q[TASK_HIGH].empty() elif task_priority == TASK_LOW: return not q[TASK_HIGH].empty() or not q[TASK_NORMAL].empty() else: return False def spawn_thread(self, target, args=(), kwargs={}): return self.thread_pool.submit(target, *args, **kwargs) def put_task(self, target, args=(), kwargs={}, callback=None, priority=TASK_NORMAL, delay=None, tt=TT_THREAD, task_id=None, worker=None, _send_task_id=True): if not self._started.is_set() or not self._active or target is None: return ti = Task(tt, task_id, priority=priority, target=target, args=args, kwargs=kwargs, callback=callback, delay=delay, worker=worker, _send_task_id=_send_task_id) ti.time_queued = time.time() ti._tqueued = time.perf_counter() with self._lock: self._tasks[ti.id] = ti if priority == TASK_CRITICAL: self.mark_task_started(ti) asyncio.run_coroutine_threadsafe(self._start_task(ti), loop=self.event_loop) else: if tt == TT_THREAD: q = self._Qt[priority] else: q = self._Qmp[priority] asyncio.run_coroutine_threadsafe(q.put(ti), loop=self.event_loop) return ti async def _task_processor(self, queue, priority, tt): logger.debug('supervisor {} task processor {}/{} started'.format( self.id, tt, priority)) while True: task = await queue.get() if task is None: break if tt == TT_THREAD: pool_size = self.thread_pool_size elif tt == TT_MP: pool_size = self.mp_pool_size if pool_size: self._lock.acquire() try: if tt == TT_THREAD: mx = self._max_threads[priority] elif tt == TT_MP: mx = self._max_mps[priority] while (self._get_active_count(tt) >= mx or self._higher_queues_busy(tt, priority)): self._lock.release() await asyncio.sleep(self.poll_delay) self._lock.acquire() finally: self._lock.release() self.mark_task_started(task) self.event_loop.create_task(self._start_task(task)) logger.debug('supervisor {} task processor {}/{} finished'.format( self.id, tt, _priorities[priority])) self._processors_stopped[(tt, priority)].set() def get_task(self, task_id): with self._lock: return self._tasks.get(task_id) def create_mp_pool(self, *args, **kwargs): if args or kwargs: self.mp_pool = multiprocessing.Pool(*args, **kwargs) else: self.mp_pool = multiprocessing.Pool( processes=multiprocessing.cpu_count()) def register_scheduler(self, scheduler): if not self._started.is_set(): return False asyncio.run_coroutine_threadsafe(self._Q.put( (RQ_SCHEDULER, scheduler, time.time())), loop=self.event_loop) return True def create_async_job(self, scheduler=None, **kwargs): if scheduler is None: scheduler = self.default_async_job_scheduler elif isinstance(scheduler, str): scheduler = self.async_job_schedulers[scheduler] return scheduler.create_threadsafe(**kwargs) def cancel_async_job(self, scheduler=None, job=None): if job: if scheduler is None: scheduler = self.default_async_job_scheduler elif isinstance(scheduler, str): scheduler = self.async_job_schedulers[scheduler] scheduler.cancel(job) else: logger.warning('supervisor {} async job cancellation ' + 'requested but job not specified'.format(self.id)) def register_sync_scheduler(self, scheduler): with self._lock: self._schedulers[scheduler] = None return True def unregister_sync_scheduler(self, scheduler): with self._lock: try: del self._schedulers[scheduler] return True except: return False def unregister_scheduler(self, scheduler): with self._lock: if scheduler not in self._schedulers: return False else: self._schedulers[scheduler][1].cancel() del self._schedulers[scheduler] return True def _get_active_count(self, tt): if tt == TT_THREAD: return len(self._active_threads) elif tt == TT_MP: return len(self._active_mps) def create_aloop(self, name, daemon=False, start=True, default=False): if name == '__supervisor__': raise RuntimeError('Name "__supervisor__" is reserved') with self._lock: if name in self.aloops: logger.error('supervisor {} loop {} already exists'.format( self.id, name)) return False l = ALoop(name, supervisor=self) l.daemon = daemon l.poll_delay = self.poll_delay with self._lock: self.aloops[name] = l if start: l.start() if default: self.set_default_aloop(l) return l def create_async_job_scheduler(self, name, aloop=None, start=True, default=False): """ Create async job scheduler (aiosched.scheduler) ALoop must always be specified or default ALoop defined """ if name == '__supervisor__': raise RuntimeError('Name "__supervisor__" is reserved') with self._lock: if name in self.async_job_schedulers: logger.error( 'supervisor {} async job_scheduler {} already exists'. format(self.id, name)) return False l = AsyncJobScheduler(name) if aloop is None: aloop = self.default_aloop elif not isinstance(aloop, ALoop): aloop = self.get_aloop(aloop) loop = aloop.get_loop() with self._lock: self.async_job_schedulers[name] = l if default: self.set_default_async_job_scheduler(l) if start: l.set_loop(loop) l._aloop = aloop aloop.background_task(l.scheduler_loop()) else: l.set_loop(loop) return l def set_default_aloop(self, aloop): self.default_aloop = aloop def set_default_async_job_scheduler(self, scheduler): self.default_async_job_scheduler = scheduler def get_aloop(self, name=None, default=True): with self._lock: if name is not None: return self.aloops.get(name) elif default: return self.default_aloop def start_aloop(self, name): with self._lock: if name not in self.aloops: logger.error('supervisor {} loop {} not found'.format( self.id, name)) return False else: self.aloops[name].start() return True def stop_aloop(self, name, wait=True, cancel_tasks=False, _lock=True): if _lock: self._lock.acquire() try: if name not in self.aloops: logger.error('supervisor {} loop {} not found'.format( self.id, name)) return False else: self.aloops[name].stop(wait=wait, cancel_tasks=cancel_tasks) return True finally: if _lock: self._lock.release() def get_info(self, tt=None, aloops=True, schedulers=True, async_job_schedulers=True): class SupervisorInfo: pass result = SupervisorInfo() with self._lock: result.id = self.id result.active = self._active result.started = self._started.is_set() for p in ['pool_size', 'reserve_normal', 'reserve_high']: if tt == TT_THREAD or tt is None or tt is False: setattr(result, 'thread_' + p, getattr(self, 'thread_' + p)) if self.mp_pool and (tt == TT_MP or tt is None or tt is False): setattr(result, 'mp_' + p, getattr(self, 'mp_' + p)) if tt == TT_THREAD or tt is None or tt is False: if not tt is False: result.thread_tasks = list(self._active_threads) result.thread_tasks_count = len(self._active_threads) if tt == TT_MP or tt is None or tt is False: if not tt is False: result.mp_tasks = list(self._active_mps) result.mp_tasks_count = len(self._active_mps) if aloops: result.aloops = self.aloops.copy() if schedulers: result.schedulers = self._schedulers.copy() if async_job_schedulers: result.async_job_schedulers = self.async_job_schedulers.copy() if tt != False: result.tasks = {} for n, v in self._tasks.items(): if tt is None or v.tt == tt: result.tasks[n] = v return result def get_aloops(self): with self._lock: return self.aloops.copy() def get_schedulers(self): with self._lock: return self._schedulers.copy() def get_tasks(self, tt=None): result = {} with self._lock: for n, v in self._tasks.items(): if tt is None or v.tt == tt: result[n] = v return result def mark_task_started(self, task): with self._lock: if task.tt == TT_THREAD: self._active_threads.add(task.id) if debug: logger.debug( ('supervisor {} new task {}: {}, {}' + ' thread pool size: {} / {}').format( self.id, task.id, task.target, _priorities[task.priority], len(self._active_threads), self.thread_pool_size)) elif task.tt == TT_MP: self._active_mps.add(task.id) if debug: logger.debug( ('supervisor {} new task {}: {}, {}' + ' mp pool size: {} / {}').format( self.id, task.id, task.target, _priorities[task.priority], len(self._active_mps), self.mp_pool_size)) async def _start_task(self, task): with self._lock: task.time_started = time.time() task._tstarted = time.perf_counter() if not task.delay: task.mark_started() if task.delay: task.status = TASK_STATUS_DELAYED await asyncio.sleep(task.delay) task.mark_started() if task.tt == TT_THREAD: self.thread_pool.submit(task.target, *task.args, **task.kwargs) elif task.tt == TT_MP: self.mp_pool.apply_async(task.target, task.args, task.kwargs, task.callback) time_spent = task._tstarted - task._tqueued if time_spent > self.timeout_critical: logger.critical( self.timeout_message.format(supervisor_id=self.supervisor.id, task_id=task_id, target=task.target, time_spent=time_spent)) self.timeout_critical_func(task) elif time_spent > self.timeout_warning: logger.warning( self.timeout_message.format(supervisor_id=self.supervisor.id, task_id=task_id, target=task.target, time_spent=time_spent)) self.timeout_warning_func(task) def mark_task_completed(self, task=None, task_id=None): with self._lock: if task is None: try: task = self._tasks[task_id] except: raise LookupError('supervisor {} task {} not found'.format( self.id, task_id)) task_id = task.id tt = task.tt if tt == TT_THREAD: if task_id in self._active_threads: self._active_threads.remove(task_id) if debug: logger.debug(('supervisor {} removed task {}:' + ' {}, thread pool size: {} / {}').format( self.id, task_id, task, len(self._active_threads), self.thread_pool_size)) task.mark_completed() del self._tasks[task_id] elif tt == TT_MP: if task_id in self._active_mps: self._active_mps.remove(task_id) if debug: logger.debug(('supervisor {} removed task {}:' + ' {} mp pool size: {} / {}').format( self.id, task_id, task, len(self._active_mps), self.mp_pool_size)) task.mark_completed() del self._tasks[task_id] return True def start(self, daemon=None): def _prespawn(): pass self._active = True self._main_loop_active = True t = threading.Thread( name='supervisor_{}_event_loop'.format(self.id), target=self._start_event_loop, daemon=daemon if daemon is not None else self.daemon) t.start() for i in range(self._prespawn_threads): self.thread_pool.submit(_prespawn) self._started.wait() def block(self): while self._started.is_set(): time.sleep(0.1) async def _launch_scheduler_loop(self, scheduler): try: t = scheduler.worker_loop.create_task(scheduler.loop()) with self._lock: self._schedulers[scheduler] = (scheduler, t) if hasattr(scheduler, 'extra_loops'): for l in scheduler.extra_loops: scheduler.worker_loop.create_task(getattr(scheduler, l)()) await t except CancelledError: pass except Exception as e: logger.error(e) async def _main_loop(self): self._Q = asyncio.queues.Queue() for p in (TASK_LOW, TASK_NORMAL, TASK_HIGH): self._Qt[p] = asyncio.queues.Queue() self._processors_stopped[(TT_THREAD, p)] = asyncio.Event() self.event_loop.create_task( self._task_processor(self._Qt[p], p, TT_THREAD)) if self.mp_pool: for p in (TASK_LOW, TASK_NORMAL, TASK_HIGH): self._Qmp[p] = asyncio.queues.Queue() self._processors_stopped[(TT_MP, p)] = asyncio.Event() self.event_loop.create_task( self._task_processor(self._Qmp[p], p, TT_MP)) self._started.set() logger.info('supervisor {} event loop started'.format(self.id)) while self._main_loop_active: data = await self._Q.get() try: if data is None: break r, res, t_put = data if r == RQ_SCHEDULER: if debug: logger.debug('supervisor {} new scheduler {}'.format( self.id, res)) asyncio.run_coroutine_threadsafe( self._launch_scheduler_loop(res), loop=res.worker_loop) finally: self._Q.task_done() for i, t in self._processors_stopped.items(): await t.wait() logger.info('supervisor {} event loop finished'.format(self.id)) def _start_event_loop(self): if self._active: self.event_loop = asyncio.new_event_loop() asyncio.set_event_loop(self.event_loop) mp = ', mp pool: {} + {} RN + {} RH'.format( self.mp_pool_size, self.mp_reserve_normal, self.mp_reserve_high) if hasattr(self, 'mp_pool_size') else '' logger.info( ('supervisor {} started, thread pool: ' + '{} + {} RN + {} RH{}').format(self.id, self.thread_pool_size, self.thread_reserve_normal, self.thread_reserve_high, mp)) try: self.event_loop.run_until_complete(self._main_loop()) except CancelledError: logger.warning('supervisor {} loop had active tasks'.format( self.id)) def _cancel_all_tasks(self): with self._lock: for task in asyncio.Task.all_tasks(loop=self.event_loop): task.cancel() def _stop_schedulers(self, wait=True): with self._lock: schedulers = self._schedulers.copy() for s in schedulers: s.stop(wait=wait) def _stop_async_job_schedulers(self, wait=True): with self._lock: schedulers = self.async_job_schedulers.copy().items() for i, s in schedulers: try: s.stop(wait=wait) except: pass def stop(self, wait=True, stop_schedulers=True, cancel_tasks=False): self._active = False if isinstance(wait, bool): to_wait = None else: to_wait = time.perf_counter() + wait if stop_schedulers: self._stop_async_job_schedulers(wait) self._stop_schedulers(True if wait else False) if debug: logger.debug('supervisor {} schedulers stopped'.format( self.id)) with self._lock: for i, l in self.aloops.items(): self.stop_aloop(i, wait=wait, cancel_tasks=cancel_tasks, _lock=False) if debug: logger.debug('supervisor {} async loops stopped'.format( self.id)) if (to_wait or wait is True) and not cancel_tasks: while True: with self._lock: if not self._tasks: break time.sleep(self.poll_delay) if to_wait and time.perf_counter() > to_wait: break if debug: logger.debug('supervisor {} no task in queues'.format(self.id)) if to_wait or wait is True: if debug: logger.debug( 'supervisor {} waiting for tasks to finish'.format( self.id)) while True: if not self._active_threads: break if to_wait and time.perf_counter() > to_wait: logger.warning( ('supervisor {} wait timeout, ' + 'skipping, hope threads will finish').format(self.id)) break time.sleep(self.poll_delay) if cancel_tasks: self._cancel_all_tasks() if debug: logger.debug('supervisor {} remaining tasks canceled'.format( self.id)) if to_wait or wait is True: while True: with self._lock: if (not self._active_threads and not self._active_mps) or ( to_wait and time.perf_counter() > to_wait): break time.sleep(self.poll_delay) if debug: logger.debug('supervisor {} no active threads/mps'.format(self.id)) if debug: logger.debug('supervisor {} stopping event loop'.format(self.id)) asyncio.run_coroutine_threadsafe(self._Q.put(None), loop=self.event_loop) for p in (TASK_LOW, TASK_NORMAL, TASK_HIGH): asyncio.run_coroutine_threadsafe(self._Qt[p].put(None), loop=self.event_loop) if self.mp_pool: for p in (TASK_LOW, TASK_NORMAL, TASK_HIGH): asyncio.run_coroutine_threadsafe(self._Qmp[p].put(None), loop=self.event_loop) self._main_loop_active = False if wait is True or to_wait: while True: if to_wait and time.perf_counter() > to_wait: logger.warning( 'supervisor {} wait timeout, canceling all tasks'. format(self.id)) self._cancel_all_tasks() break else: can_break = True for t in asyncio.Task.all_tasks(self.event_loop): if not t.cancelled() and not t.done(): can_break = False break if can_break: break time.sleep(self.poll_delay) with self._lock: for i, v in self._tasks.items(): v.status = TASK_STATUS_CANCELED self._started.clear() self.thread_pool.shutdown() logger.info('supervisor {} stopped'.format(self.id))
class RiotApiScheduler(object): MAX_QSIZE = 25 REQ_PER_MIN = 50 _DEFAULT_DUPE_BACKOFF = 1 def __init__(self): self._timer = threading.Timer(0, self._run) self._lock = threading.Lock() self._executor = ThreadPoolExecutor(max_workers=5) self._queue = Queue(maxsize=RiotApiScheduler.MAX_QSIZE) self._is_running = False self._backoff = None # guarded by lock self._next_sleep = 60.0 / RiotApiScheduler.REQ_PER_MIN # initial sleep period def _process_request(self, req): try: try: req.execute() except RiotRateLimitException as e: print "!! [API Rate Limit] Type: %r, Retry after: %r" % ( e.limit_type, e.retry_after) with self._lock: if self._backoff == None: self._backoff = e.retry_after else: self.backoff = RiotApiScheduler._DEFAULT_DUPE_BACKOFF req.mark_invalid() except RiotApiException as e: print "!! [API Exception] Code: %r for url: %r" % ( e.status_code, e.url) req.mark_invalid() except Exception as e: print "!! [RANDOM EXCEPTION] %r" % e except Exception as e: print "!! [RANDOM EXCEPTION] %r" % e def _run(self): # TODO optimize to take better advantage of api limit # Pull requests off per rate limit and schedule them with thread pool executor try: req = self._queue.get(False) self._executor.submit(self._process_request, req) except Empty: pass if self._is_running: delay = self._next_sleep with self._lock: if self._backoff: delay = self._backoff self._backoff = None self._timer = threading.Timer(delay, self._run) self._timer.start() return def add_request(self, req, timeout=1): # should block if full, raises Full exception if times out while self._is_running: try: self._queue.put(req, True, timeout) return except Full: continue def start(self): self._is_running = True self._timer.start() def stop(self): self._is_running = False self._executor.shutdown(wait=True) self._timer.cancel()
class Worker(Server): """ Worker Node Workers perform two functions: 1. **Serve data** from a local dictionary 2. **Perform computation** on that data and on data from peers Additionally workers keep a Center informed of their data and use that Center to gather data from other workers when necessary to perform a computation. You can start a worker with the ``dworker`` command line application:: $ dworker scheduler-ip:port **State** * **data:** ``{key: object}``: Dictionary mapping keys to actual values * **active:** ``{key}``: Set of keys currently under computation * **ncores:** ``int``: Number of cores used by this worker process * **executor:** ``concurrent.futures.ThreadPoolExecutor``: Executor used to perform computation * **local_dir:** ``path``: Path on local machine to store temporary files * **center:** ``rpc``: Location of center or scheduler. See ``.ip/.port`` attributes. * **name:** ``string``: Alias * **services:** ``{str: Server}``: Auxiliary web servers running on this worker * **service_ports:** ``{str: port}``: Examples -------- Create centers and workers in Python: >>> from distributed import Center, Worker >>> c = Center('192.168.0.100', 8787) # doctest: +SKIP >>> w = Worker(c.ip, c.port) # doctest: +SKIP >>> yield w._start(port=8788) # doctest: +SKIP Or use the command line:: $ dcenter Start center at 127.0.0.1:8787 $ dworker 127.0.0.1:8787 Start worker at: 127.0.0.1:8788 Registered with center at: 127.0.0.1:8787 See Also -------- distributed.center.Center: """ def __init__(self, center_ip, center_port, ip=None, ncores=None, loop=None, local_dir=None, services=None, service_ports=None, name=None, **kwargs): self.ip = ip or get_ip() self._port = 0 self.ncores = ncores or _ncores self.data = dict() self.loop = loop or IOLoop.current() self.status = None self.local_dir = local_dir or tempfile.mkdtemp(prefix='worker-') self.executor = ThreadPoolExecutor(self.ncores) self.thread_tokens = Queue( ) # https://github.com/tornadoweb/tornado/issues/1595#issuecomment-198551572 for i in range(self.ncores): self.thread_tokens.put_nowait(i) self.center = rpc(ip=center_ip, port=center_port) self.active = set() self.name = name if not os.path.exists(self.local_dir): os.mkdir(self.local_dir) if self.local_dir not in sys.path: sys.path.insert(0, self.local_dir) self.services = {} self.service_ports = service_ports or {} for k, v in (services or {}).items(): if isinstance(k, tuple): k, port = k else: port = 0 self.services[k] = v(self) self.services[k].listen(port) self.service_ports[k] = self.services[k].port handlers = { 'compute': self.compute, 'gather': self.gather, 'compute-stream': self.compute_stream, 'run': self.run, 'get_data': self.get_data, 'update_data': self.update_data, 'delete_data': self.delete_data, 'terminate': self.terminate, 'ping': pingpong, 'health': self.health, 'upload_file': self.upload_file } super(Worker, self).__init__(handlers, **kwargs) @gen.coroutine def _start(self, port=0): self.listen(port) self.name = self.name or self.address for k, v in self.services.items(): v.listen(0) self.service_ports[k] = v.port logger.info(' Start worker at: %20s:%d', self.ip, self.port) for k, v in self.service_ports.items(): logger.info(' %16s at: %20s:%d' % (k, self.ip, v)) logger.info('Waiting to connect to: %20s:%d', self.center.ip, self.center.port) while True: try: resp = yield self.center.register(ncores=self.ncores, address=(self.ip, self.port), keys=list(self.data), services=self.service_ports, name=self.name) break except (OSError, StreamClosedError): logger.debug("Unable to register with scheduler. Waiting") yield gen.sleep(0.5) if resp != 'OK': raise ValueError(resp) logger.info(' Registered to: %20s:%d', self.center.ip, self.center.port) self.status = 'running' def start(self, port=0): self.loop.add_callback(self._start, port) def identity(self, stream): return { 'type': type(self).__name__, 'id': self.id, 'center': (self.center.ip, self.center.port) } @gen.coroutine def _close(self, report=True, timeout=10): if report: yield gen.with_timeout(timedelta(seconds=timeout), self.center.unregister(address=(self.ip, self.port)), io_loop=self.loop) self.center.close_streams() self.stop() self.executor.shutdown() if os.path.exists(self.local_dir): shutil.rmtree(self.local_dir) for k, v in self.services.items(): v.stop() self.status = 'closed' self.stop() @gen.coroutine def terminate(self, stream, report=True): yield self._close(report=report) raise Return('OK') @property def address(self): return '%s:%d' % (self.ip, self.port) @property def address_tuple(self): return (self.ip, self.port) @gen.coroutine def gather(self, stream=None, who_has=None): who_has = { k: [coerce_to_address(addr) for addr in v] for k, v in who_has.items() if k not in self.data } try: result = yield gather_from_workers(who_has) except KeyError as e: logger.warn("Could not find data", e) raise Return({'status': 'missing-data', 'keys': e.args}) else: self.data.update(result) raise Return({'status': 'OK'}) @gen.coroutine def _ready_task(self, function=None, key=None, args=(), kwargs={}, task=None, who_has=None): diagnostics = {} if who_has: local_data = {k: self.data[k] for k in who_has if k in self.data} who_has = { k: set(map(coerce_to_address, v)) for k, v in who_has.items() if k not in self.data } try: logger.info("gather %d keys from peers: %s", len(who_has), str(who_has)) diagnostics['transfer-start'] = time() other = yield gather_from_workers(who_has) diagnostics['transfer-stop'] = time() data = merge(local_data, other) except KeyError as e: logger.warn("Could not find data for %s", key) raise Return({ 'status': 'missing-data', 'keys': e.args, 'key': key }) else: data = {} transfer_time = 0 try: start = default_timer() if task is not None: task = loads(task) if function is not None: function = loads(function) if args: args = loads(args) if kwargs: kwargs = loads(kwargs) diagnostics['deserialization'] = default_timer() - start except Exception as e: logger.warn("Could not deserialize task", exc_info=True) raise Return(assoc(error_message(e), 'key', key)) if task is not None: assert not function and not args and not kwargs function = execute_task args = (task, ) # Fill args with data args2 = pack_data(args, data) kwargs2 = pack_data(kwargs, data) raise Return({ 'status': 'OK', 'function': function, 'args': args2, 'kwargs': kwargs2, 'diagnostics': diagnostics, 'key': key }) @gen.coroutine def executor_submit(self, key, function, *args, **kwargs): """ Safely run function in thread pool executor We've run into issues running concurrent.future futures within tornado. Apparently it's advantageous to use timeouts and periodic callbacks to ensure things run smoothly. This can get tricky, so we pull it off into an separate method. """ token = yield self.thread_tokens.get() job_counter[0] += 1 i = job_counter[0] # logger.info("%s:%d Starts job %d, %s", self.ip, self.port, i, key) future = self.executor.submit(function, *args, **kwargs) pc = PeriodicCallback( lambda: logger.debug("future state: %s - %s", key, future._state), 1000) pc.start() try: if sys.version_info < (3, 2): yield future else: while not future.done() and future._state != 'FINISHED': try: yield gen.with_timeout(timedelta(seconds=1), future, io_loop=self.loop) break except gen.TimeoutError: logger.info("work queue size: %d", self.executor._work_queue.qsize()) logger.info("future state: %s", future._state) logger.info("Pending job %d: %s", i, future) finally: pc.stop() self.thread_tokens.put(token) result = future.result() logger.info("Finish job %d, %s", i, key) raise gen.Return(result) @gen.coroutine def compute_stream(self, stream): with log_errors(): logger.debug("Open compute stream") bstream = BatchedSend(interval=10, loop=self.loop) bstream.start(stream) @gen.coroutine def process(msg): try: result = yield self.compute(report=False, **msg) bstream.send(result) except Exception as e: logger.exception(e) bstream.send(assoc(error_message(e), 'key', msg.get('key'))) with log_errors(): while True: try: msgs = yield read(stream) except StreamClosedError: break if not isinstance(msgs, list): msgs = [msgs] for msg in msgs: op = msg.pop('op', None) if op == 'close': break elif op == 'compute-task': self.loop.add_callback(process, msg) else: logger.warning("Unknown operation %s, %s", op, msg) yield bstream.close() logger.info("Close compute stream") @gen.coroutine def compute(self, stream=None, function=None, key=None, args=(), kwargs={}, task=None, who_has=None, report=True): """ Execute function """ self.active.add(key) # Ready function for computation msg = yield self._ready_task(function=function, key=key, args=args, kwargs=kwargs, task=task, who_has=who_has) if msg['status'] != 'OK': try: self.active.remove(key) except KeyError: pass raise Return(msg) else: function = msg['function'] args = msg['args'] kwargs = msg['kwargs'] # Log and compute in separate thread result = yield self.executor_submit(key, apply_function, function, args, kwargs) result['key'] = key result.update(msg['diagnostics']) if result['status'] == 'OK': self.data[key] = result.pop('result') if report: response = yield self.center.add_keys(address=(self.ip, self.port), keys=[key]) if not response == 'OK': logger.warn('Could not report results to center: %s', str(response)) else: logger.warn( " Compute Failed\n" "Function: %s\n" "args: %s\n" "kwargs: %s\n", str(funcname(function))[:1000], str(args)[:1000], str(kwargs)[:1000], exc_info=True) logger.debug("Send compute response to scheduler: %s, %s", key, msg) try: self.active.remove(key) except KeyError: pass raise Return(result) @gen.coroutine def run(self, stream, function=None, args=(), kwargs={}): function = loads(function) if args: args = loads(args) if kwargs: kwargs = loads(kwargs) try: result = function(*args, **kwargs) except Exception as e: logger.warn( " Run Failed\n" "Function: %s\n" "args: %s\n" "kwargs: %s\n", str(funcname(function))[:1000], str(args)[:1000], str(kwargs)[:1000], exc_info=True) response = error_message(e) else: response = { 'status': 'OK', 'result': dumps(result), } raise Return(response) @gen.coroutine def update_data(self, stream, data=None, report=True): data = valmap(loads, data) self.data.update(data) if report: response = yield self.center.add_keys(address=(self.ip, self.port), keys=list(data)) assert response == 'OK' info = { 'nbytes': {k: sizeof(v) for k, v in data.items()}, 'status': 'OK' } raise Return(info) @gen.coroutine def delete_data(self, stream, keys=None, report=True): for key in keys: if key in self.data: del self.data[key] logger.info("Deleted %d keys", len(keys)) if report: logger.debug("Reporting loss of keys to center") yield self.center.remove_keys(address=self.address, keys=list(keys)) raise Return('OK') def get_data(self, stream, keys=None): return {k: dumps(self.data[k]) for k in keys if k in self.data} def upload_file(self, stream, filename=None, data=None, load=True): out_filename = os.path.join(self.local_dir, filename) if isinstance(data, unicode): data = data.encode() with open(out_filename, 'wb') as f: f.write(data) f.flush() if load: try: name, ext = os.path.splitext(filename) if ext in ('.py', '.pyc'): logger.info("Reload module %s from .py file", name) name = name.split('-')[0] reload(import_module(name)) if ext == '.egg': sys.path.append(out_filename) pkgs = pkg_resources.find_distributions(out_filename) for pkg in pkgs: logger.info("Load module %s from egg", pkg.project_name) reload(import_module(pkg.project_name)) if not pkgs: logger.warning("Found no packages in egg file") except Exception as e: logger.exception(e) return {'status': 'error', 'exception': dumps(e)} return {'status': 'OK', 'nbytes': len(data)} def health(self, stream=None): """ Information about worker """ d = { 'active': len(self.active), 'stored': len(self.data), 'time': time() } try: import psutil mem = psutil.virtual_memory() d.update({ 'cpu': psutil.cpu_percent(), 'memory': mem.total, 'memory-percent': mem.percent }) try: net_io = psutil.net_io_counters() d['network-send'] = net_io.bytes_sent - self._last_net_io.bytes_sent d['network-recv'] = net_io.bytes_recv - self._last_net_io.bytes_recv except AttributeError: pass self._last_net_io = net_io try: disk_io = psutil.disk_io_counters() d['disk-read'] = disk_io.read_bytes - self._last_disk_io.read_bytes d['disk-write'] = disk_io.write_bytes - self._last_disk_io.write_bytes except (AttributeError, RuntimeError): disk_io = None self._last_disk_io = disk_io except ImportError: pass return d
def executor(): executor = ThreadPoolExecutor(max_workers=1) yield executor executor.shutdown()
class HostPolicy(LoggedThread, Terminable): """ At a regular interval, this thread triggers system reconfiguration by sampling host and guest data, evaluating the policy and reporting the results to all enabled Controller plugins. """ def __init__(self, config: DictConfig, libvirt_iface, host_monitor: Monitor, guest_manager: GuestManager, shared_terminable: Terminable = None): LoggedThread.__init__(self, daemon=True) Terminable.__init__(self, shared_terminable=shared_terminable) self.policy_lock = threading.RLock() self.host_monitor = host_monitor self.guest_manager = guest_manager self.config = config self.interval: float = max(self.config.get('policy', 'interval'), 1.) self.grace_period: float = min( self.config.get('policy', 'grace-period'), self.interval) self.inquiry_timeout: float = min( self.config.get('policy', 'inquiry-timeout'), self.interval) self.resources = [ r.strip() for r in self.config.get('policy', 'resources') ] self.policy = ClassImporter(allocators).get_class( config.get('policy', 'allocator'))(self.resources) self.controllers = [] self.properties = { 'libvirt_iface': libvirt_iface, 'host_monitor': host_monitor, 'guest_manager': guest_manager, 'config': config } self.policy_data_loggers = {} self.client_executor = None self.get_controllers() def get_controllers(self): """ Initialize the Controllers called for in the config file. """ controllers_importer = ClassImporter(controllers) for resource_name in self.resources: try: controller_name = self.config.get( 'policy', f'{resource_name}-controller') controller_class = controllers_importer.get_class( controller_name) self.logger.debug("Loaded %s controller for resource %s", controller_name, resource_name) except Exception as e: self.logger.exception( "Unable to import controller for resource '%s': %s", resource_name, e) continue try: self.controllers.append( controller_class(resource_name, self.properties)) except Exception as e: self.logger.exception( "Unable to instantiate controller for resource '%s': %s", resource_name, e) def get_policy_data_logger(self, entity: MonitorDataEntity): source = entity.prop('source') data_logger = self.policy_data_loggers.get(source, None) if data_logger is None: data_logger = DataLogger('policy', source) self.policy_data_loggers[source] = data_logger return data_logger def notify_guest(self, entity: MonitorDataEntity, grace_period: Union[int, float, None] = None): alloc = {} for r in self.resources: alloc[r] = entity.get_control(r) try: msg_obj = MessageNotify({ 'alloc': alloc, 'grace-period': grace_period }) timeout = grace_period if grace_period is None: timeout = self.inquiry_timeout entity.prop('guest-client').send_receive_message(msg_obj, timeout=timeout) except socket.timeout as e: if self.should_run: self.logger.error("Could not notify guest: %s", e) except Exception as ex: if self.should_run: self.logger.exception("Failed to notify guest %s: %s", entity.prop('source'), ex) def inquire_guest(self, entity: MonitorDataEntity, grace_period: Union[int, float, None] = None): last_alloc = entity.get_var('last_control', {}) try: msg_obj = MessageInquiry({ 'last-alloc': last_alloc, 'grace-period': grace_period, 'timeout': self.inquiry_timeout }) return entity.prop('guest-client').send_receive_message( msg_obj, timeout=self.inquiry_timeout) except socket.timeout as e: if self.should_run: self.logger.error("Could not inquire guest: %s", e) except Exception as ex: if self.should_run: self.logger.exception("Failed to inquire guest: %s", ex) def parallel_for_each_guest(self, func, guest_list: List[MonitorDataEntity], results_var_key=None, **kwargs): jobs = [] for g in guest_list: jobs.append(self.client_executor.submit(func, g, **kwargs)) for g, j in zip(guest_list, jobs): res = j.result() if results_var_key: g.set_var(results_var_key, res) def do_controls(self): """ Sample host and guest data, process the rule set and feed the results into each configured Controller. """ if not self.should_run: return # collect data host: MonitorDataEntity = self.host_monitor.interrogate() if host is None: return guest_list: List[MonitorDataEntity] = list( self.guest_manager.interrogate().values()) # send inquiry to the clients regarding the next allocation inquiry_time = time.time() self.parallel_for_each_guest(self.inquire_guest, guest_list, results_var_key='inquiry', grace_period=self.grace_period) if not self.should_run: return try: with self.policy_lock: self.policy.apply_policy(host, guest_list) except Exception as e: self.logger.exception("Exception while applying policy: %s", e) return if not self.should_run: return # send notification to the clients on the next allocation notify_start = time.time() remaining_grace = max( 0., self.grace_period - (notify_start - inquiry_time)) self.parallel_for_each_guest(self.notify_guest, guest_list, grace_period=remaining_grace) notify_end = time.time() if not self.should_run: return # log and store control and variable of host and guest for entity in (host, *guest_list): data_logger = self.get_policy_data_logger(entity) data_logger.append_data({'notify': entity.controls}, notify_start, notify_end) remaining_grace = max(0., self.grace_period - (time.time() - inquiry_time)) self.terminable_sleep(remaining_grace) if not self.should_run: return policy_start = time.time() for c in self.controllers: c.apply_control(host, guest_list) policy_end = time.time() # log and store control and variable of host and guest for entity in (host, *guest_list): data_logger = self.get_policy_data_logger(entity) data_logger.append_data( { 'controls': entity.controls, 'variables': entity.variables }, policy_start, policy_end) entity.store_variables() if not self.should_run: return # send notification to the clients on the applied allocation self.parallel_for_each_guest(self.notify_guest, guest_list, grace_period=None) def logged_run(self) -> None: self.client_executor = ThreadPoolExecutor( thread_name_prefix=f"{self.logger_name}-dispatcher") try: while self.should_run: start = time.time() self.do_controls() self.terminable_sleep(self.interval - (time.time() - start)) finally: self.client_executor.shutdown()