def _connect_rover_fired(self): """Handle callback for HTTP rover connections. """ if not self.device_uid: msg = "\nDevice ID not found!\n\nConnection requires a valid Piksi device ID." self._prompt_setting_error(msg) return if not self.http: self._prompt_networking_error("\nNetworking disabled!") return try: _base_pragma = self.base_pragma if not self.http.connect_write(self.link, self.whitelist, pragma=_base_pragma): msg = ("\nUnable to connect to Skylark!\n\n" "Please check that you have a network connection.") self._prompt_networking_error(msg) self.http.close() self.connected_rover = False return self.connected_rover = True print "Connected as a base station!" executor = ThreadPoolExecutor(max_workers=2) executor.submit(self._retry_read) except: self.connected_rover = False import traceback print traceback.format_exc()
def search(rootTitle, destTitle, session, maxDepth=-1): ''' Given the title of a wikipedia page, find how many 'hops' it takes to get to the given destination page. Uses the breadth first search algorithm. ''' visited = set() consumerQueue = queue.Queue(MAX_CONSUMER_QUEUE_SIZE) consumerQueue.put(page.Page(rootTitle, session, 0)) producerQueue = queue.Queue() executor = ThreadPoolExecutor(2) # Start thread to consume pages consumerArgs = [destTitle, consumerQueue, producerQueue, visited] consumerFuture = executor.submit(consumer, *consumerArgs) # Start thread to produce child pages and add them to consumer queue producerArgs = [consumerQueue, producerQueue, session] executor.submit(producer, *producerArgs) return consumerFuture.result()
def start(self): left_lng = 103.9213455517 top_lat = 30.7828453209 right_lng = 104.2178123382 bottom_lat = 30.4781772402 offset = 0.002 if os.path.isfile(self.db_name): os.remove(self.db_name) try: with sqlite3.connect(self.db_name) as c: c.execute('''CREATE TABLE mobike (Time DATETIME, bikeIds VARCHAR(12), bikeType TINYINT,distId INTEGER,distNum TINYINT, type TINYINT, x DOUBLE, y DOUBLE)''') except Exception as ex: pass executor = ThreadPoolExecutor(max_workers=250) print("Start") self.total = 0 lat_range = np.arange(top_lat, bottom_lat, -offset) for lat in lat_range: lng_range = np.arange(left_lng, right_lng, offset) for lon in lng_range: self.total += 1 executor.submit(self.get_nearby_bikes, (lat, lon)) executor.shutdown() self.group_data()
class DriveSink: def __init__(self, parser_args): self.cloud_drive = CloudDrive(parser_args) self.sync_progress = SyncProgress() self.executor = ThreadPoolExecutor(max_workers=12) self.root_node = CloudNode( self.cloud_drive, parser_args.skipmd5, frozenset(parser_args.exclude_dirs.lower().split( ",")) if parser_args.exclude_dirs else None, frozenset(parser_args.extensions.lower().split(",")), parser_args.source, self.cloud_drive.get_pictures_root_node_id(), self.sync_progress) def upload(self): futures = deque([self.executor.submit(self.root_node.upload)]) while futures: current = futures.popleft().result() for child in current.children(): futures.append(self.executor.submit(child.upload)) print("") self.sync_progress.report_and_wait_for_uploads() def download(self): # parallel download NYI pass
class RemoteHelloConsumer(object): def __init__(self): self._helloservice = None self._name = "Python" self._msg = "Hello Java" self._executor = ThreadPoolExecutor() @Validate def _validate(self, bundle_context): # call it! resp = self._helloservice.sayHello(self._name + "Sync", self._msg) print( "{0} IHello service consumer received sync response: {1}".format( self._name, resp ) ) # call sayHelloAsync which returns Future and we add lambda to print # the result when done self._executor.submit( self._helloservice.sayHelloAsync, self._name + "Async", self._msg ).add_done_callback( lambda f: print("async response: {0}".format(f.result())) ) print("done with sayHelloAsync method") # call sayHelloAsync which returns Future and we add lambda to print # the result when done self._executor.submit( self._helloservice.sayHelloPromise, self._name + "Promise", self._msg, ).add_done_callback( lambda f: print("promise response: {0}".format(f.result())) ) print("done with sayHelloPromise method")
def handle(self, *args, **options): #return start_date = datetime.date(2013, 11, 24) #start_date = datetime.date(2015, 4, 29) now = datetime.date.today()+datetime.timedelta(days=1) #now = start_date + datetime.timedelta(days=3) day = start_date threads = [] executor = ThreadPoolExecutor(max_workers=32) while day < now: print (day) try: pass #gain(day.strftime('%Y-%m-%d')) #t = Thread(target=gain, args=(day.strftime('%Y-%m-%d'),)) #threads.append( t ) #t.start() #time.sleep(1) executor.submit (gain, day.strftime('%Y-%m-%d')) except Exception as e: print(e) pass day += datetime.timedelta(days=1) #for x in threads: # print (x) # x.join()
class ActiveVoice: def __init__(self): # See https://msdn.microsoft.com/en-us/library/ms723602%28v=vs.85%29.aspx for doc self._voice = win32com.client.Dispatch("SAPI.SpVoice") self._executor = ThreadPoolExecutor(max_workers=1) def shutdown(self): self._executor.shutdown(False) def speak(self, text, volume=DEFAULT_VOLUME, rate=DEFAULT_RATE): volume = max(MIN_VOLUME, min(MAX_VOLUME, volume)) rate = max(MIN_RATE, min(MAX_RATE, rate)) def do_speak(): self._voice.Volume = volume self._voice.Rate = rate self._voice.Speak(text) self._executor.submit(do_speak) def wait(self, time_in_sec): self._executor.submit(time.sleep, time_in_sec) def list_avatars(self): # pointless, only one on Windows 10! avatars = [] for index, token in enumerate(self._voice.GetVoices("", "")): avatars.append( '%d: %s' % (index+1, token.GetDescription()) ) return avatars
def main(): formatter = logging.Formatter( '[%(asctime)s] %(name)s<%(levelname)s> %(message)s') handler = logging.StreamHandler() handler.setFormatter(formatter) logger.addHandler(handler) logger.setLevel(logging.INFO) args = get_parser().parse_args() if args.subworker and args.worker != 'process': logger.error('using subworkers requires worker set to be "process"') return if args.worker == 'thread': from concurrent.futures import ThreadPoolExecutor e = ThreadPoolExecutor(args.concurrency) for _ in range(args.concurrency): e.submit(run_worker, args.odq, args.queue, args.worker) elif args.worker == 'process': from concurrent.futures import ProcessPoolExecutor e = ProcessPoolExecutor(args.concurrency) for _ in range(args.concurrency): e.submit(run_worker, args.odq, args.queue, args.worker, args.subworker, args.subconcurrency) elif args.worker == 'gevent': from gevent import monkey monkey.patch_all() from gevent.pool import Pool pool = Pool(args.concurrency) for _ in range(args.concurrency): pool.spawn(run_worker, args.odq, args.queue, args.worker) pool.join()
class ElasticDataSink: def __init__(self, name, conn, model_identifier, workers=5, bound=10000): self.name = name self.conn = conn self.model_identifier = model_identifier self.queue = Queue(maxsize=bound) self.pool = ThreadPoolExecutor(max_workers=workers) def start(self): self.model_identifier.model_class.init(using=self.conn) def __sink_item(self): try: item = self.queue.get_nowait() save_status = item.save(using=self.conn) if not save_status: logger.error("Error saving the item to the sink") else: logger.info("item saved to the sink") except Empty as e: logger.warn("sink queue is empty") logger.warn(e) def sink_item(self, item): assert isinstance(item, self.model_identifier.model_class), \ " item must be instance of " + self.model_identifier.model_class try: self.queue.put(item, timeout=10) self.pool.submit(self.__sink_item) except Full as e: logger.error("sink queue is full") logger.error(e)
class AsyncClient(object): """Client which uses the base to be more performant. This client uses Futures with a ThreadPoolExecutor. This allows requests to be executed asynchronously. Asynchronous execution with multiple Clients enables requests to be processed in parallel and with pipeline execution at the server, which can drastically improve achievable interoperability rate as observed at the client. Note that methods return Future objects. Users should handle the response and errors appropriately. If serial request execution is desired, ensure the Future response or error is received prior to making another request. """ def __init__(self, url, username, password, timeout=1): """Create a new AsyncClient and login. Args: url: Base URL of interoperability server (e.g., http://localhost:8000) username: Interoperability username password: Interoperability password timeout: Individual session request timeout (seconds) """ self.client = Client(url, username, password, timeout) self.server_info_executor = ThreadPoolExecutor(max_workers=1) self.uas_telemetry_executor = ThreadPoolExecutor(max_workers=1) self.obstacles_executor = ThreadPoolExecutor(max_workers=1) def get_server_info(self): """GET server information, to be displayed to judges. Returns: Future object which contains the return value or error from the underlying Client. """ return self.server_info_executor.submit(self.client.get_server_info) def post_telemetry(self, telem): """POST new telemetry. Args: telem: Telemetry object containing telemetry state. Returns: Future object which contains the return value or error from the underlying Client. """ return self.uas_telemetry_executor.submit(self.client.post_telemetry, telem) def get_obstacles(self): """GET obstacles. Returns: Future object which contains the return value or error from the underlying Client. """ return self.obstacles_executor.submit(self.client.get_obstacles)
class BaseDataLayer(Layer): def setup(self, bottom, top): param = eval(self.param_str_) self.batch_size_ = param['batch_size'] self.data_setup(bottom, top) top[0].reshape(*self.data_.shape) self.executor_ = ThreadPoolExecutor(max_workers=1) self.thread_ = self.executor_.submit(self.internal_thread_entry) def reshape(self, bottom, top): pass def forward(self, bottom, top): self.thread_.result() top[0].reshape(*self.data_.shape) top[0].data[...] = self.data_ self.thread_ = self.executor_.submit(self.internal_thread_entry) def data_setup(self, bottom, top): raise NotImplementedError() def internal_thread_entry(self): raise NotImplementedError() def __del__(self): self.thread_.result() self.executor_.shutdown() super(self.__class__, self).__del__()
def main(): """Main method executed when run""" if len(argv) != 2: print("Usage: flymirror.py [rules_file]") return if not file_exists(argv[1]): print("Error: rules file", argv[1], "does not exist.") return config = read_config(argv[1]) URLS.put(config.start) # Start the loops in another thread # include 2 extra threads for the loopers pool = ThreadPoolExecutor(int(config.workers) + 2) perfprint("[START]") pool.submit(download_loop, pool) pool.submit(handle_response_loop, pool, config) # Join on both the queues at once (Yeah, this is hacky -- may break in later versions) while URLS.unfinished_tasks or RESPONSES.unfinished_tasks: sleep(0.3) # Shut everything down (may take 1 second) DONE.put(True) pool.shutdown() perfprint("[END]")
def main(): # Update configuration from the local file from .configuration import Setting Setting.read_cfg_from_file() # Print instance information print("Node name: {0}\nNode address: {1}".format(Setting.get_node_name(), Setting.get_node_addr())) # Reset data in the database from .meta_storage import MetaStorage meta_storage = MetaStorage() meta_storage.drop_database() meta_storage.close_connection() print("Clear data in the database complete.") # Reset data in the local storage from general.services import Services if not Services.is_folder_exist(Setting.get_local_storage()): Services.t_print(Setting.get_local_storage() + " does not exist! (Local Storage).") # Get file from the folder import glob import os files = glob.glob(Setting.get_local_storage() + "*") for file in files: os.remove(file) print("Clear {0} files in the local storage complete.".format(len(files))) # Create a thread for running REST service from concurrent.futures import ThreadPoolExecutor pool = ThreadPoolExecutor() pool.submit(run_rest_service)
def main(cmd, args): logging.basicConfig( level=logging.INFO, format=("%(relativeCreated)04d %(process)05d %(threadName)-10s " "%(levelname)-5s %(msg)s")) pool = Pool(max_workers=1) arg1 = tmp_fname arg2 = 'arg2' stage_file_name = cmd if (glob_stage == 1): f = pool.submit(check_output, ["ec-perl", "-w", stage_file_name, args, arg2], shell=True) print "stage 1 started. Initiate call to" + stage_file_name print "Stage is " + str(glob_stage) else: f = pool.submit(check_output, ["ec-perl", stage_file_name, " ", args, arg2], shell=True) print "stage 2 started. Initiate call to" + stage_file_name + " " + args + " " + arg2 + "'" print "Stage is " + str(glob_stage) f.add_done_callback(callback) pool.shutdown(wait=False)
class CreateQueue: ''' Queue of creation requests ''' def __init__(self): self.create_executor = ThreadPoolExecutor(max_workers=32) self.tasks = [] self.lock = threading.Lock() self.init() def init(self): for creation in models.QueuedCreation.objects.all(): req = VmCreationRequest(creation.owner.name, creation.stats, creation.resource.id) self.tasks.append( Task(created=util.datetime_to_unix(creation.created), request=req, params=creation.params)) def queue(self, request, params): if len(self.tasks) > settings.QUEUE_LIMIT: raise errors.QueueFull() models.QueuedCreation(owner=models.Account.objects.get(name=request.owner), stats=request.stats, resource=models.DerivativeResource.objects.get(id=request.res_id), params=params).save() self.tasks.append(Task(time.time(), request, params)) def maybe_unqueue(self, spawner): # TODO: this may be too expensive if queue is really big with self.lock: tasks = list(self.tasks) tasks.sort(key=lambda k: -k.current_priority) for task in tasks: if spawner.check_estimates(task.request): logging.info('submitting queued task %s', task.request) self._submit(spawner, task) self.tasks.remove(task) break def start(self): pass def _requeue(self, task): self.tasks.append(task) def _submit(self, spawner, task): def execute(): try: response = spawner.create_vm_if_possible(task.request) logging.info('sandbox created (response=%s)', response) if not response: self._requeue(task) else: task.finish(response) except: logging.exception('create vm from queue') raise self.create_executor.submit(execute).add_done_callback(lambda val: None)
def echo_server(addr): pool = ThreadPoolExecutor(128) sock = socket(AF_INET, SOCK_STREAM) sock.bind(addr) sock.listen(5) while True: client_sock, client_addr = sock.accept() pool.submit(echo_client, client_sock, client_addr)
class ThreadPool(object): def __init__(self): self.pool = ThreadPoolExecutor(128) def execute(self, func, *args, **kwargs): LOG.info(_('execute %s in a thread pool') % (func.__name__)) self.pool.submit(func, *args, **kwargs)
def compress_dir(in_dir, out_dir): if not out_dir.exists(): out_dir.mkdir() executor = ThreadPoolExecutor(4) for file in (f for f in in_dir.iterdir() if f.suffix == '.bmp'): out_file = (out_dir / file.name).with_suffix('.rle') executor.submit(compress_image, str(file), str(out_file))
def pool_run(self): """ 通过线程池跑 :return: """ pool = ThreadPoolExecutor(max_workers=10) for i in range(10): pool.submit(self.run, i)
def __iter__(self): pool = ThreadPoolExecutor(1) it = iter(self.it) future = pool.submit(self._next, it) while True: result = future.result() future = pool.submit(self._next, it) yield result
class Core(object): def __init__(self, driver, mapping, store, default_state): self.driver = driver self.mapping = mapping self.store = store self.default_state = default_state self.logger = logging.getLogger(__name__) self.executor = ThreadPoolExecutor(max_workers=1) def set_pdu_outlet_command(self, pdu, outlet, command): self.logger.info("PDU '{}', outlet '{}' has new command: '{}'".format( pdu, outlet, command) ) self.store[(pdu, outlet)] = FINAL_STATES[command] try: device = self._get_device(pdu, outlet) self.logger.debug( "Found server '{}' on PDU '{}' outlet '{}'".format( device, pdu, outlet) ) self.executor.submit(self._switch_power, command, device) except KeyError: pass def _switch_power(self, command, device): if command == POWER_ON: self.driver.power_on(device) elif command == POWER_OFF: self.driver.power_off(device) elif command == REBOOT: self.driver.power_off(device) self.driver.power_on(device) else: self.logger.error("Unknown power command: {}".format(command)) return def get_pdu_outlet_state(self, pdu, outlet): try: return self.store[(pdu, outlet)] except KeyError: pass try: device = self._get_device(pdu, outlet) power_state = self.driver.get_power_state(device) except KeyError: power_state = self.default_state self.store[(pdu, outlet)] = power_state return power_state def _get_device(self, pdu, outlet): return self.mapping[(pdu, outlet)]
def echo_server(addr): print('Serwer Echo działa pod adresem', addr) pool = ThreadPoolExecutor(128) sock = socket(AF_INET, SOCK_STREAM) sock.bind(addr) sock.listen(5) while True: client_sock, client_addr = sock.accept() pool.submit(echo_client, client_sock, client_addr)
def main(): # Update configuration from the local file from .configuration import Setting Setting.read_configuration_from_file() # Create a thread for running REST service from concurrent.futures import ThreadPoolExecutor pool = ThreadPoolExecutor() pool.submit(run_rest_service)
def __init__(self, ordererStub, entity, directory, nodeAdminTuple, timeout = 600): pool = ThreadPoolExecutor(1) StreamHelper.__init__(self) self.nodeAdminTuple = nodeAdminTuple self.directory = directory self.entity = entity # Set the UpdateMessage and start the stream sendGenerator = self.createSendGenerator(timeout) self.replyGenerator = ordererStub.Deliver(sendGenerator, timeout + 1) pool.submit(self._start_receive)
def search(self, query): n_days = (self.until - self.since).days tp = ThreadPoolExecutor(max_workers=self.n_threads) for i in range(0, n_days): since_query = self.since + datetime.timedelta(days=i) until_query = self.since + datetime.timedelta(days=(i + 1)) day_query = "%s since:%s until:%s" % (query, since_query.strftime("%Y-%m-%d"), until_query.strftime("%Y-%m-%d")) tp.submit(self.perform_search, day_query) tp.shutdown(wait=True)
class Listener: BUF_SIZE = 4096 PORT = 3439 def __init__(self): self.pool = ThreadPoolExecutor(max_workers=1) self.queue = queue.Queue() self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) self.socket.bind(('', Listener.PORT)) handler = threading.Thread(target=self.handle) handler.daemon = True handler.start() self.listen() def listen(self): while True: try: (buf, address) = self.socket.recvfrom(Listener.BUF_SIZE) self.queue.put((buf, address)) self.pool.submit(print, (buf, address)) except KeyboardInterrupt: traceback.print_exc() self.queue.join() self.socket.close() sys.exit(0) def handle(self): instances = {} counts = {} while True: while not self.queue.empty(): try: buf, _ = self.queue.get() items = buf.split(','.encode('utf8')) instances[items[0]] = (items[1], items[2], items[3]) counts[items[0]] = 5 finally: self.queue.task_done() for key in instances.keys(): counts[key] -= 1 if counts[key] == 0: del instances[key] del counts[key] # print('Instances: {0}'.format(instances)) # print('Counts: {0}'.format(counts)) time.sleep(5)
def console_main(): """ This function handles all the console action. """ setproctitle('image-scraper') scraper = ImageScraper() scraper.get_arguments() print("\nImageScraper\n============\nRequesting page....\n") try: scraper.get_html() except PageLoadError as err: if err.status_code is None: print("ImageScraper is unable to acces the internet.") else: print("Page failed to load. Status code: {0}".format(err.status_code)) sys.exit() scraper.get_img_list() if len(scraper.images) == 0: sys.exit("Sorry, no images found.") if scraper.no_to_download is None: scraper.no_to_download = len(scraper.images) print("Found {0} images: ".format(len(scraper.images))) try: scraper.process_download_path() except DirectoryAccessError: print("Sorry, the directory can't be accessed.") sys.exit() except DirectoryCreateError: print("Sorry, the directory can't be created.") sys.exit() if scraper.dump_urls: for img_url in scraper.images: print(img_url) status_flags = {'count': 0, 'percent': 0.0, 'failed': 0, 'under_min_or_over_max_filesize': 0} widgets = ['Progress: ', Percentage(), ' ', Bar(marker=RotatingMarker()), ' ', ETA(), ' ', FileTransferSpeed()] pbar = ProgressBar(widgets=widgets, maxval=100).start() pool = ThreadPoolExecutor(max_workers=scraper.nthreads) status_lock = threading.Lock() for img_url in scraper.images: if status_flags['count'] == scraper.no_to_download: break pool.submit(download_worker_fn, scraper, img_url, pbar, status_flags, status_lock) status_flags['count'] += 1 pool.shutdown(wait=True) pbar.finish() print("\nDone!\nDownloaded {0} images\nFailed: {1}\n".format( status_flags['count']-status_flags['failed']-status_flags['under_min_or_over_max_filesize'], status_flags['failed'])) return
def test_watcher(self): executor = ThreadPoolExecutor(max_workers=1) f = executor.submit(lambda: 42) w = FutureWatcher(f) def spies(w): return SimpleNamespace( done=QSignalSpy(w.done), finished=QSignalSpy(w.finished), result=QSignalSpy(w.resultReady), error=QSignalSpy(w.exceptionReady), cancelled=QSignalSpy(w.cancelled) ) spy = spies(w) self.assertTrue(spy.done.wait()) self.assertEqual(list(spy.done), [[f]]) self.assertEqual(list(spy.finished), [[f]]) self.assertEqual(list(spy.result), [[42]]) self.assertEqual(list(spy.error), []) self.assertEqual(list(spy.cancelled), []) f = executor.submit(lambda: 1/0) w = FutureWatcher(f) spy = spies(w) self.assertTrue(spy.done.wait()) self.assertEqual(list(spy.done), [[f]]) self.assertEqual(list(spy.finished), [[f]]) self.assertEqual(len(spy.error), 1) self.assertIsInstance(spy.error[0][0], ZeroDivisionError) self.assertEqual(list(spy.result), []) self.assertEqual(list(spy.cancelled), []) ev = threading.Event() # block the executor to test cancellation executor.submit(lambda: ev.wait()) f = executor.submit(lambda: 0) w = FutureWatcher(f) self.assertTrue(f.cancel()) ev.set() spy = spies(w) self.assertTrue(spy.done.wait()) self.assertEqual(list(spy.done), [[f]]) self.assertEqual(list(spy.finished), []) self.assertEqual(list(spy.error), []) self.assertEqual(list(spy.result), []) self.assertEqual(list(spy.cancelled), [[f]])
class Processor(object): PROCESS_TIME_LIMIT = 30 EXCEPTION_LIMIT = 3 executor = ThreadPoolExecutor(max_workers=5) def __init__(self, newtask_queue, resultdb, projectdb, enable_stdout_capture=True): self.newtask_queue = newtask_queue self.resultdb = resultdb self.enable_stdout_capture = enable_stdout_capture self.project_manager = ProjectManager(projectdb, dict( enable_stdout_capture=self.enable_stdout_capture, )) self.executor = ThreadPoolExecutor(max_workers=5) def handle_result(self, task, result): '''Deal one response result''' start_time = time.time() response = rebuild_response(result) follows = [] try: assert 'taskid' in task, 'need taskid in task' project_name = task['project_name'] project_id = task['project_id'] project_data = self.project_manager.get(project_name, project_id) assert project_data, "No such project!" if project_data.get('exception'): logger.error(project_data.get('exception_log')) else: logger.info('Sccceed in getting project data.') results, follows, db_name, coll_name = project_data['instance'].run_task( project_data['module'], task, response ) return (results, follows, db_name, coll_name) except Exception as e: logger.exception(e) process_time = time.time() - start_time logger.info('Process time cost: %s' % str(process_time)) return (None, None, None, None) def put_tasks(self, tasks): self.executor.submit(self.newtask_queue.put, *tasks) def put_results(self, results, db_name, coll_name, task, query=None): self.executor.submit(self.save2db, results, db_name, coll_name, task, query=query) def save2db(self, results, db_name, coll_name, task, query=None): self.resultdb.update(db_name, coll_name, results, task, query=query)
def on_message(self, parameters): input_p = json.loads(parameters) # Decode JSON dictionary into a python dictionary to get key information. pool = ThreadPoolExecutor(max_workers=settingmain.MAX_WORKERS) if input_p['task'] == 'uniprotparser': # Input key to look for result. result = yield pool.submit(uptasks.get_results, input_p['key']) pool.shutdown() self.write_message(result) if input_p['task'] == 'uniprotparser2': # This statement have no function as of yet. result = yield pool.submit(uniparser.uniprot_parser, input_p) pool.shutdown() self.write_message(json.encode(result))
class Datafeed(object): def __init__(self): self.markets = [] self.observers = [] self.depths = {} self.init_markets(config.markets) self.init_observers(config.observers) self.threadpool = ThreadPoolExecutor(max_workers=10) def init_markets(self, _markets): logging.debug("init_markets:%s" % _markets) self.market_names = _markets markets = create_markets(_markets) for market_name, market in markets.items(): if self.get_market(market_name): continue self.markets.append(market) def init_observers(self, _observers): logging.debug("init_observers:%s" % _observers) self.observer_names = _observers for observer_name in _observers: try: exec('import observers.' + observer_name.lower()) observer = eval('observers.' + observer_name.lower() + '.' + observer_name + '()') self.observers.append(observer) except (ImportError, AttributeError) as e: print("%s observer name is invalid: Ignored (you should check your config file)" % (observer_name)) print(e) def register_observer(self, _observer): logging.debug("register_observer:%s" % _observer) self.observers.append(_observer) def get_market(self, market_name): for market in self.markets: if market.name == market_name: return market return None def observer_tick(self): for observer in self.observers: observer.tick(self.depths) def tick(self): self.print_tickers() self.observer_tick() def __get_market_depth(self, market, depths): depth = market.get_depth() if depth: depths[market.name] = depth def update_depths(self): depths = {} futures = [] for market in self.markets: futures.append(self.threadpool.submit(self.__get_market_depth, market, depths)) wait(futures, timeout=20) return depths def print_tickers(self): for market in self.markets: logging.debug("ticker: " + market.name + " - " + str(market.get_ticker())) def replay_history(self, directory): import os import json import pprint files = os.listdir(directory) files.sort() for f in files: depths = json.load(open(directory + '/' + f, 'r')) self.depths = {} for market in self.market_names: if market in depths: self.depths[market] = depths[market] self.tick() def update_balance(self): pass def terminate(self): for observer in self.observers: observer.terminate() for market in self.markets: market.terminate() def _run_loop(self, is_feed = True): if not is_feed and len(self.observers) == 0: print('empty observers') return if is_feed and len(self.markets) == 0: print('empty markets') return # signal.signal(signal.SIGINT, sigint_handler) #以下那句在windows python2.4不通过,但在freebsd下通过 signal.signal(signal.SIGHUP, sigint_handler) signal.signal(signal.SIGTERM, sigint_handler) kafka_topic = config.kafka_topic bootstrap_servers = config.bootstrap_servers try: if is_feed: producer = KafkaProducer(bootstrap_servers=bootstrap_servers, # key_serializer=str.encode, value_serializer=lambda v: json.dumps(v).encode('utf-8')) else: consumer = KafkaConsumer(kafka_topic, value_deserializer=lambda m: json.loads( m.decode('utf-8')), bootstrap_servers=bootstrap_servers) except Exception as ex: logging.warn("exception depths:%s" % ex) traceback.print_exc() return if is_feed: while True: self.depths = self.update_depths() future = producer.send(kafka_topic, value=self.depths) # Block for 'synchronous' sends try: record_metadata = future.get(timeout=20) logging.info(record_metadata) except Exception as ex: logging.warn("exception in producer:%s" % ex) traceback.print_exc() continue if is_sigint_up: # 中断时需要处理的代码 logging.info("exit in producer") self.terminate() break sys.stdout.write(".") sys.stdout.flush() time.sleep(config.refresh_rate) else: try: for message in consumer: if is_sigint_up: # 中断时需要处理的代码 logging.info("exit in consumer begin") self.terminate() break # print (message) # message value and key are raw bytes -- decode if necessary! # e.g., for unicode: `message.value.decode('utf-8')` logging.info ("datafeed: %s:%d:%d: key=%s" % (message.topic, message.partition, message.offset, message.key)) # print(message) self.update_balance() self.depths = message.value self.tick() if is_sigint_up: # 中断时需要处理的代码 logging.info("exit in consumer end") self.terminate() break logging.info('consumer done...') # self.tick() except Exception as ex: logging.warn("exception in consumer:%s" % ex) traceback.print_exc() self.terminate() return logging.info('app exist.') def run_loop(self): is_feed = False self._run_loop(is_feed)
""" 使用ThreadPoolExecutor向线程池进行任务提交者更 方便的从被调用函数中获取返回值 """ from concurrent.futures import ThreadPoolExecutor import urllib.request def fetch_url(url): u = urllib.request.urlopen(url) data = u.read() return data pool = ThreadPoolExecutor(10) # 提交到线程池 a = pool.submit(fetch_url, 'http://www.python.org') b = pool.submit(fetch_url, 'http://www.pypy.org') # 获得的handle对象会处理所有的阻塞与协作,然后从工作线程中返回数据 x = a.result() y = b.result() print(x) print(y)
class UpdateTask(TaskQueue.Task): def __init__(self, ui, data_hub, data_center, force: bool): super(UpdateTask, self).__init__('UpdateTask') self.__ui = ui self.__force = force self.__data_hub = data_hub self.__data_center = data_center self.__quit = False # Thread pool self.__patch_count = 0 self.__apply_count = 0 self.__future = None self.__pool = ThreadPoolExecutor(max_workers=1) # Parameters self.uri = '' self.identities = [] self.clock = Clock(False) self.progress = ProgressRate() def in_work_package(self, uri: str) -> bool: return self.uri == uri def set_work_package(self, uri: str, identities: list or str or None): if isinstance(identities, str): identities = [identities] self.uri = uri self.identities = identities # ---------------------------------------------------------------------------------------- # def __build_click_table(self): # self.__clock_table = {} # for uri, identities in self.__update_pack: # self.__clock_table[uri] = Clock() # # def __build_progress_table(self): # self.__progress_table = {} # for uri, identities in self.__update_pack: # progress_rate = ProgressRate() # self.__progress_table[uri] = progress_rate # if identities is not None: # progress_rate.set_progress(uri, 0, len(identities)) # for identity in identities: # progress_rate.set_progress([uri, identity], 0, 1) # else: # progress_rate.set_progress(uri, 0, 1) def run(self): print('Update task start.') self.__patch_count = 0 self.__apply_count = 0 try: # Catch "pymongo.errors.ServerSelectionTimeoutError: No servers found yet" exception and continue. self.__execute_update() except Exception as e: print('Update got Exception: ') print(e) print('Continue...') finally: if self.__future is not None: self.__future.cancel() print('Update task finished.') def quit(self): self.__quit = True def identity(self) -> str: return self.uri # ------------------------------------- Task ------------------------------------- def __execute_update(self): self.clock.reset() self.progress.reset() self.progress.set_progress(self.uri, 0, len(self.identities) if self.identities is not None else 1) identities = self.identities if self.identities is not None else [None] for identity in identities: while (self.__patch_count - self.__apply_count > 20) and not self.__quit: time.sleep(0.5) continue if self.__quit: break print('------------------------------------------------------------------------------------') if identity is not None: # Optimise: Update not earlier than listing date. listing_date = self.__data_hub.get_data_utility().get_securities_listing_date(identity, default_since()) if self.__force: since, until = listing_date, now() else: since, until = self.__data_center.calc_update_range(self.uri, identity) since = max(listing_date, since) time_serial = (since, until) else: time_serial = None patch = self.__data_center.build_local_data_patch(self.uri, identity, time_serial, force=self.__force) self.__patch_count += 1 print('Patch count: ' + str(self.__patch_count)) self.__future = self.__pool.submit(self.__execute_persistence, self.uri, identity, patch) if self.__future is not None: print('Waiting for persistence task finish...') self.__future.result() self.clock.freeze() # self.__ui.task_finish_signal[UpdateTask].emit(self) def __execute_persistence(self, uri: str, identity: str, patch: tuple) -> bool: try: if patch is not None: self.__data_center.apply_local_data_patch(patch) if identity is not None: self.progress.set_progress([uri, identity], 1, 1) self.progress.increase_progress(uri) except Exception as e: print('e') return False finally: self.__apply_count += 1 print('Persistence count: ' + str(self.__apply_count)) return True
try: os.mkdir(config['certs_folder']) except FileExistsError as e: print('mkdir failed:\n\t', e) exit(0) init_trust_certs(config['trusted_certs_folder']) count = 0 if os.path.isfile(config['extract_from_folder']): count += 1 extract_file(config['extract_from_folder'], count) elif os.path.isdir(config['extract_from_folder']): p = ThreadPoolExecutor() #线程池 for root, _, files in os.walk(config['extract_from_folder']): for f in files: ext = os.path.splitext(f)[1] f = os.path.join(root, f) if ext == '.pcap': count += 1 p.submit(extract_file, f, count) # extract_file(f) p.shutdown() else: print(f"{config['extract_from_folder']} not found.") print("total count:", count) print("time consumed:", time.time() - start) # print(sum) # extract_file('pcaps/1.pcap')
def get_page(url): print('<%s> is getting [%s]' % (os.getpid(), url)) response = requests.get(url) time.sleep(random.randint(1, 7)) return {'url': url, 'text': response.text} # 解析网页 def parse_page(res): res = res.result() print("res", res) print("<%s> parse [%s]" % (os.getpid(), res['url'])) with open('content.txt', 'a') as f: parse_res = "[url]: %s [size]: %s\n" % (res['url'], len(res['text'])) f.write(parse_res) if __name__ == '__main__': urls = [ 'https://www.baidu.com', 'https://www.python.org', 'https://www.openstack.org', 'https://help.github.com/', 'http://www.sina.com.cn/' ] start = time.time() # pool = ThreadPoolExecutor() pool = ThreadPoolExecutor() for url in urls: pool.submit(get_page, url).add_done_callback(parse_page) pool.shutdown() print("主进程PID:%s,运行时间:%s" % (os.getpid(), time.time() - start))
class HttpClient(object): """A http based client for submitting Spark-based jobs to a Livy backend. Parameters ---------- url_str : string Livy server url to create a new session or the url of an existing session load_defaults : boolean, optional This parameter decides if the default config needs to be loaded Default is True conf_dict : dict, optional The key-value pairs in the conf_dict will be loaded to the config Default is None Examples -------- Imports needed to create an instance of HttpClient >>> from livy.client import HttpClient 1) Creates a client that is loaded with default config as 'load_defaults' is True by default >>> client = HttpClient("http://example:8998/") 2) Creates a client that does not load default config, but loads config that are passed in 'config_dict' >>> config_dict = {'spark.app.name', 'Test App'} >>> client = HttpClient("http://example:8998/", load_defaults=False, >>> config_dict=config_dict) """ _CONFIG_SECTION = 'env' _LIVY_CLIENT_CONF_DIR = "LIVY_CLIENT_CONF_DIR" def __init__(self, url, load_defaults=True, conf_dict=None): uri = urlparse(url) self._config = ConfigParser() self._load_config(load_defaults, conf_dict) match = re.match(r'(.*)/sessions/([0-9]+)', uri.path) if match: base = ParseResult(scheme=uri.scheme, netloc=uri.netloc, path=match.group(1), params=uri.params, query=uri.query, fragment=uri.fragment) self._set_uri(base) self._conn = _LivyConnection(base, self._config) self._session_id = int(match.group(2)) self._reconnect_to_existing_session() else: self._set_uri(uri) session_conf_dict = dict(self._config.items(self._CONFIG_SECTION)) self._conn = _LivyConnection(uri, self._config) self._session_id = self._create_new_session( session_conf_dict).json()['id'] self._executor = ThreadPoolExecutor(max_workers=1) self._stopped = False self.lock = threading.Lock() def submit(self, job): """ Submits a job for execution to the spark cluster. Parameters ---------- job : function The function must accept a single parameter, which is an instance of JobContext. Returns ------- job_handle : an instance of the class JobHandle A handle that can be used to monitor the job Examples ------- >>> def simple_spark_job(context): >>> elements = [10, 20, 30, 40, 50] >>> return context.sc.parallelize(elements, 2).count() >>> client.submit(simple_spark_job) """ return self._send_job('submit-job', job) def run(self, job): """ Asks the remote context to run a job immediately. Normally, the remote context will queue jobs and execute them based on how many worker threads have been configured. This method will run the submitted job in the same thread processing the RPC message, so that queueing does not apply. It's recommended that this method only be used to run code that finishes quickly. This avoids interfering with the normal operation of the context. Parameters ---------- job : function The function must accept a single parameter, which is an instance of JobContext. Spark jobs can be created with the help of JobContext, which exposes the Spark libraries. Returns ------- future : concurrent.futures.Future A future to monitor the status of the job Examples ------- >>> def simple_job(context): >>> return "hello" >>> client.run(simple_job) """ return self._send_job("run-job", job) def add_file(self, file_uri): """ Adds a file to the running remote context. Note that the URL should be reachable by the Spark driver process. If running the driver in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist on that node (and not on the client machine). Parameters ---------- file_uri : string String representation of the uri that points to the location of the file Returns ------- future : concurrent.futures.Future A future to monitor the status of the job Examples ------- >>> client.add_file("file:/test_add.txt") >>> # Example job using the file added using add_file function >>> def add_file_job(context): >>> from pyspark import SparkFiles >>> def func(iterator): >>> with open(SparkFiles.get("test_add.txt")) as testFile: >>> fileVal = int(testFile.readline()) >>> return [x * fileVal for x in iterator] >>> return context.sc.parallelize([1, 2, 3, 4]) >>> .mapPartitions(func).collect() >>> client.submit(add_file_job) """ return self._add_file_or_pyfile_job("add-file", file_uri) def add_jar(self, file_uri): """ Adds a jar file to the running remote context. Note that the URL should be reachable by the Spark driver process. If running the driver in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist on that node (and not on the client machine). Parameters ---------- file_uri : string String representation of the uri that points to the location of the file Returns ------- future : concurrent.futures.Future A future to monitor the status of the job Examples ------- >>> client.add_jar("file:/test_package.jar") """ return self._add_file_or_pyfile_job("add-jar", file_uri) def add_pyfile(self, file_uri): """ Adds a .py or .zip to the running remote context. Note that the URL should be reachable by the Spark driver process. If running the driver in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist on that node (and not on the client machine). Parameters ---------- file_uri : string String representation of the uri that points to the location of the file Returns ------- future : concurrent.futures.Future A future to monitor the status of the job Examples ------- >>> client.add_pyfile("file:/test_package.egg") >>> # Example job using the file added using add_pyfile function >>> def add_pyfile_job(context): >>> # Importing module from test_package.egg >>> from test.pyfile_test import TestClass >>> test_class = TestClass() >>> return test_class.say_hello() >>> client.submit(add_pyfile_job) """ return self._add_file_or_pyfile_job("add-pyfile", file_uri) def upload_file(self, file_path): """ Upload a file to be passed to the Spark application. Parameters ---------- file_path : string File path of the local file to be uploaded. Returns ------- future : concurrent.futures.Future A future to monitor the status of the job Examples ------- >>> client.upload_file("/test_upload.txt") >>> # Example job using the file uploaded using upload_file function >>> def upload_file_job(context): >>> from pyspark import SparkFiles >>> def func(iterator): >>> with open(SparkFiles.get("test_upload.txt")) as testFile: >>> fileVal = int(testFile.readline()) >>> return [x * fileVal for x in iterator] >>> return context.sc.parallelize([1, 2, 3, 4]) >>> .mapPartitions(func).collect() >>> client.submit(add_file_job) """ return self._upload_file_or_pyfile("upload-file", open(file_path, 'rb')) def upload_pyfile(self, file_path): """ Upload a .py or .zip dependency to be passed to the Spark application. Parameters ---------- file_path : string File path of the local file to be uploaded. Returns ------- future : concurrent.futures.Future A future to monitor the status of the job Examples ------- >>> client.upload_pyfile("/test_package.egg") >>> # Example job using the file uploaded using upload_pyfile function >>> def upload_pyfile_job(context): >>> # Importing module from test_package.egg >>> from test.pyfile_test import TestClass >>> test_class = TestClass() >>> return test_class.say_hello() >>> client.submit(upload_pyfile_job) """ return self._upload_file_or_pyfile("upload-pyfile", open(file_path, 'rb')) def stop(self, shutdown_context): """ Stops the remote context. The function will return immediately and will not wait for the pending jobs to get completed Parameters ---------- shutdown_context : Boolean Whether to shutdown the underlying Spark context. If false, the context will keep running and it's still possible to send commands to it, if the backend being used supports it. """ with self.lock: if not self._stopped: self._executor.shutdown(wait=False) try: if shutdown_context: session_uri = "/" + str(self._session_id) headers = {'X-Requested-By': 'livy'} self._conn.send_request("DELETE", session_uri, headers=headers) except: raise Exception(traceback.format_exc()) self._stopped = True def _set_uri(self, uri): if uri is not None and uri.scheme in ('http', 'https'): self._config.set(self._CONFIG_SECTION, 'livy.uri', uri.geturl()) else: url_exception = uri.geturl if uri is not None else None raise ValueError('Cannot create client - Uri not supported - ', url_exception) def _set_conf(self, key, value): if value is not None: self._config.set(self._CONFIG_SECTION, key, value) else: self._delete_conf(key) def _delete_conf(self, key): self._config.remove_option(self._CONFIG_SECTION, key) def _set_multiple_conf(self, conf_dict): for key, value in conf_dict.items(): self._set_conf(key, value) def _load_config(self, load_defaults, conf_dict): self._config.add_section(self._CONFIG_SECTION) if load_defaults: self._load_default_config() if conf_dict is not None and len(conf_dict) > 0: self._set_multiple_conf(conf_dict) def _load_default_config(self): config_dir = os.environ.get(self._LIVY_CLIENT_CONF_DIR) if config_dir is not None: config_files = os.listdir(config_dir) default_conf_files = ['spark-defaults.conf', 'livy-client.conf'] for default_conf_file in default_conf_files: if default_conf_file in config_files: self._load_config_from_file(config_dir, default_conf_file) def _load_config_from_file(self, config_dir, config_file): path = os.path.join(config_dir, config_file) data = "[" + self._CONFIG_SECTION + "]\n" + \ open(path, encoding='utf-8').read() self._config.readfp(StringIO(data)) def _create_new_session(self, session_conf_dict): data = {'kind': 'pyspark', 'conf': session_conf_dict} response = self._conn.send_request('POST', "/", headers=self._conn._JSON_HEADERS, data=data) return response def _reconnect_to_existing_session(self): reconnect_uri = "/" + str(self._session_id) + "/connect" self._conn.send_request('POST', reconnect_uri, headers=self._conn._JSON_HEADERS) def _send_job(self, command, job): pickled_job = cloudpickle.dumps(job) base64_pickled_job = base64.b64encode(pickled_job).decode('utf-8') base64_pickled_job_data = {'job': base64_pickled_job} handle = JobHandle(self._conn, self._session_id, self._executor) handle._start(command, base64_pickled_job_data) return handle def _add_file_or_pyfile_job(self, command, file_uri): data = {'uri': file_uri} suffix_url = "/" + str(self._session_id) + "/" + command return self._executor.submit(self._add_or_upload_resource, suffix_url, data=data, headers=self._conn._JSON_HEADERS) def _upload_file_or_pyfile(self, command, open_file): files = {'file': open_file} suffix_url = "/" + str(self._session_id) + "/" + command return self._executor.submit(self._add_or_upload_resource, suffix_url, files=files) def _add_or_upload_resource(self, suffix_url, files=None, data=None, headers=None): return self._conn.send_request('POST', suffix_url, files=files, data=data, headers=headers).content
class JobExecutor: # The behaviour with blocking_submit == True is that the submit will block after queuing a number of jobs. # In the case of a storage write for example this ensures that we don't enqueue to many blocks at once and # so use up all available memory. # In the other case there is no limit on the number of submitted jobs. But the number of simultaneous # outstanding results is limited. # In the case of a storage read for example this ensures that we don't have to many outstanding read blocks # at once and so use up all available memory. def __init__(self, *, workers: int, blocking_submit: bool, name: str) -> None: self._name = name self._executor = ThreadPoolExecutor(max_workers=workers, thread_name_prefix=name) self._futures: List[Future] = [] self._blocking_submit = blocking_submit # Set the queue limit to two times the number of workers plus one to ensure that there are always # enough jobs available even when all futures finish at the same time. self._semaphore = BoundedSemaphore(2 * workers + 1) def submit(self, function: Callable) -> None: if self._blocking_submit: self._semaphore.acquire() def execute_with_release(): try: return function() finally: self._semaphore.release() self._futures.append(self._executor.submit(execute_with_release)) else: def execute_with_acquire(): self._semaphore.acquire() return function() self._futures.append(self._executor.submit(execute_with_acquire)) # This is tricky to implement as we need to make sure that we don't hold a reference to the completed Future anymore. # Indeed it's so tricky that older Python versions had the same problem. See https://bugs.python.org/issue27144. def get_completed(self, timeout: int = None) -> Iterator[Any]: for future in concurrent.futures.as_completed(self._futures, timeout=timeout): self._futures.remove(future) if not self._blocking_submit and not future.cancelled(): self._semaphore.release() try: result = future.result() except Exception as exception: result = exception del future yield result def shutdown(self) -> None: if len(self._futures) > 0: logger.warning('Job executor "{}" is being shutdown with {} outstanding jobs, cancelling them.'.format( self._name, len(self._futures))) for future in self._futures: future.cancel() logger.debug('Job executor "{}" cancelled all outstanding jobs.'.format(self._name)) if not self._blocking_submit: # Get all jobs so that the semaphore gets released and still waiting jobs can complete for _ in self.get_completed(): pass logger.debug('Job executor "{}" read results for all outstanding jobs.'.format(self._name)) self._executor.shutdown() def wait_for_all(self) -> None: concurrent.futures.wait(self._futures)
def main() -> None: """Main application entry point.""" parser = argparse.ArgumentParser() parser.add_argument('--ticket-path', default='tickets/eng/dev/aws', type=str, help="Vanadium ticket used to access Grail resources.") parser.add_argument('--install', action='store_true') parser.add_argument('--install-adhoc-2', action='store_true') parser.add_argument( '--adhoc-name', default='', type=str, help="Name of the adhoc instance (may be a glob pattern)") parser.add_argument('--release', default='', type=str, help="""Copy gqljupyter and jupyter.py to S3. The arg is the local path of the gqljupyter binary""") parser.add_argument('--port', default=8888, type=int) args = parser.parse_args() if args.install_adhoc_2: install_jupyter_kernel() sys.exit(0) config = Config(adhoc_addr=get_adhoc_addr(args.adhoc_name), ticket_path=args.ticket_path) if args.install: install_jupyter(config) sys.exit(0) if args.release: if os.environ.get('USER') != 'ysaito': raise Exception('--release should be done only by ysaito') sha256_got = check_output(['sha256sum', args.release]) with tempfile.NamedTemporaryFile(mode='w') as fd: fd.write(sha256_got) fd.flush() s3_cp(config, fd.name, 's3://grail-ysaito/gql/gqljupyter.sha256') s3_cp(config, args.release, 's3://grail-ysaito/gql/gqljupyter') s3_cp(config, __file__, 's3://grail-ysaito/gql/jupyter.py') sys.exit(0) if os.environ.get('USER') == 'ubuntu': logging.error('This script must run on your desktop/laptop, not adhoc') sys.exit(1) if not validate_gql_binary(config): install_jupyter(config) if not validate_gql_binary(config): raise Exception("Failed to install gql") subprocess.call([ 'ssh', 'ubuntu@' + config.adhoc_addr, "ps awx | grep jupyter | awk '{print $1}' | xargs kill -9" ]) pool = ThreadPoolExecutor(max_workers=128) r = pool.submit(start_browser, args.port) logging.info('Starting jupyter notebook at localhost:%s', args.port) subprocess.check_call([ 'ssh', '-L', '%d:127.0.0.1:%d' % (args.port, args.port), 'ubuntu@' + config.adhoc_addr, 'export V23_CREDENTIALS=/home/ubuntu/.v23; ', '/home/ubuntu/.local/bin/jupyter', 'notebook', '--ip=127.0.0.1', '--port=%d' % (args.port, ), '--no-browser', '--NotebookApp.token=', '--NotebookApp.password=' ])
class Downloader: from ..TuiTools.Bar import DataTransformBar def __init__(self, url: str, num: int, name: str = '', proxy: str = '', referer: str = '', output_error: bool = False, failed2exit: bool = False, exit_if_exist: bool = False, disableStatus: bool = False, disableParallel: bool = False): """ qs普通文件下载引擎 Qs general file download engine :param url: 文件url :param num: 线程数量 """ signal.signal(signal.SIGINT, self._kill_self) info_flag = True self.url, self.num, self.output_error, self.proxies = url, num, output_error, {} self.exit_if_exist = exit_if_exist self.enabled = True if not disableStatus: with qs_default_console.status('Getting file info..' if user_lang != 'zh' else '获取文件信息中..'): self.url, self.name, r = get_fileinfo(url, proxy, referer) else: self.url, self.name, r = get_fileinfo(url, proxy, referer) if not (self.url and self.name and r): info_flag = False qs_default_console.print(qs_error_string if failed2exit else qs_warning_string, 'Get File information failed, please check network!' if user_lang != 'zh' else '获取文件信息失败,请检查网络!') if failed2exit: self.enabled = False return if self.name and '.' not in self.name: self.name = os.path.basename(url) if name: self.name = name if os.path.exists(self.name) and self.exit_if_exist: self.enabled = False return if proxy: self.proxies = { 'http': 'http://'+proxy, 'https': 'https://'+proxy } self.headers = headers if referer: self.headers['Referer'] = referer if not self.url: qs_default_console.print(qs_error_string, self.name) raise Exception('Connection Error!' if user_lang != 'zh' else '连接失败!') try: if not info_flag: raise KeyError self.size = int(r.headers['content-length']) self.main_progress = Downloader.DataTransformBar() self.dl_id = self.main_progress.add_task('Download', filename=self.name, start=False) self.main_progress.update(self.dl_id, total=self.size) if self.size < 5e6 or disableParallel: qs_default_console.print(qs_info_string, 'FILE SIZE' if user_lang != 'zh' else '文件大小' , size_format(self.size)) self.size = -self.size else: self.fileBlock = GetBlockSize(self.size) except KeyError: self.size = -1 self.main_progress = Downloader.DataTransformBar(False) self.dl_id = self.main_progress.add_task('Download', filename=self.name, start=False) if self.size > 0: self.pool = ThreadPoolExecutor(max_workers=self.num) self.futures, self.fileLock = [], Lock() self.job_queue = queue.Queue() if os.path.exists(self.name + '.qs_dl'): self.ctn_file = open(self.name + '.qs_dl', 'r+') self.ctn = [int(i) for i in self.ctn_file.read().strip().split()] else: self.ctn_file = open(self.name + '.qs_dl', 'w') self.ctn = [] self.writers = FileWriters(self.name, max(2, int(core_num / 2)), "rb+" if self.ctn else "wb") qs_default_console.print(qs_info_string, 'FILE SIZE' if user_lang != 'zh' else '文件大小' , size_format(self.size, align=True)) qs_default_console.print(qs_info_string, 'THRAED NUM' if user_lang != 'zh' else '线程数量' , '%7d' % num) def _kill_self(self, signum, frame): """ 终止下载任务,保存断点 Terminate the download and save the breakpoint :param signum: 信号 :param frame: frame :return: None """ if self.size > 0: self.main_progress.stop_task(self.dl_id) self.main_progress.stop() qs_default_console.print(qs_info_string, 'Get Ctrl-C, exiting...' if user_lang != 'zh' else '捕获Ctrl-C, 正在退出...') self.ctn_file.close() self.pool.shutdown(wait=False) self.writers.wait() qs_default_console.print(qs_info_string, 'Deal Done!' if user_lang != 'zh' else '处理完成!') os._exit(0) def _dl(self, start): """ 执行文件块下载任务 Perform the file block download task :param start: 文件块起始偏移量 :return: None """ try: _sz = min(start + self.fileBlock, self.size - 1) _headers = self.headers.copy() _headers['Range'] = 'bytes={}-{}'.format(start, _sz) _content = b'' for chunk in get(self.url, headers=_headers, timeout=50, proxies=self.proxies).iter_content(65536): _content += chunk self.main_progress.advance(self.dl_id, sys.getsizeof(chunk)) self.writers.new_job(_content, start) except Exception as e: if self.output_error: qs_default_console.print(qs_error_string, repr(e)) self.job_queue.put(start) else: self.fileLock.acquire() self.ctn.append(start) self.ctn_file.write('%d\n' % start) self.fileLock.release() def _single_dl(self): """ 无法并行下载或无需并行下载时采用串行下载 Parallel downloads are not possible or serial downloads are not required :return: None """ r = get(self.url, stream=True, proxies=self.proxies, headers=self.headers) flag = self.size != -1 if flag: self.main_progress.start_task(self.dl_id) else: self.main_progress.update(self.dl_id, total=-1) with open(self.name, 'wb') as f: for chunk in r.iter_content(32768): f.write(chunk) self.main_progress.advance(self.dl_id, sys.getsizeof(chunk)) def run(self): """ 规划下载任务并开始下载 qs可以有效应对网络问题对下载任务造成的影响,如: 1.下载过程中切换代理 2.链路异常 并确保下载任务顺利完成 :return: None """ if not self.enabled: return self.name if self.exit_if_exist else '' self.main_progress.start() if self.size > 0: self.main_progress.start_task(self.dl_id) if not self.ctn: with open(self.name, "wb") as fp: fp.truncate(self.size) self.main_progress.advance(self.dl_id, self.fileBlock*len(self.ctn)) for i in range(0, self.size, self.fileBlock): if self.ctn and i in self.ctn: continue else: self.job_queue.put(i) retry_cnt = 0 while not self.job_queue.empty(): self.futures.clear() while not self.job_queue.empty(): cur = self.job_queue.get() if cur not in self.ctn: self.futures.append(self.pool.submit(self._dl, cur)) wait(self.futures) if not self.job_queue.empty() and retry_cnt > 2: qs_default_console.print(qs_warning_string, 'Exists File Block Lost, Retrying after 0.5 sec' if user_lang != 'zh' else '存在文件块丢失,0.5秒后重试') time.sleep(0.5) retry_cnt += 1 self.writers.wait() self.ctn_file.close() os.remove(self.name + '.qs_dl') else: self._single_dl() self.main_progress.stop() qs_default_console.print(qs_info_string, self.name, 'download done!' if user_lang != 'zh' else '下载完成!') return self.name
# concurrent的案例 from concurrent.futures import ThreadPoolExecutor import time def return_future(msg): time.sleep(3) return msg # 创建一个线程池,设定最多线程池个数 pool = ThreadPoolExecutor(max_workers=2) # 往线程池加入2个task f1 = pool.submit(return_future, 'hello') f2 = pool.submit(return_future, 'world') # 等待执行完毕 print(f1.done()) time.sleep(3) print(f2.done()) # 结果 print(f1.result()) print(f2.result())
class WavDown(WavDownBasic): def __init__(self, seqid, list_wav_url, callback_url, consumer_logger): # 创建线程池 self.executor = ThreadPoolExecutor(WavDownBasic.PoolNum) self.consumer_logger = consumer_logger # 批次号,在消息队列里获取 self.seqid = seqid self.list_wav_url = list_wav_url self.callback_url = callback_url # todo 文件夹加时间戳 # 创建md5对象 m = hashlib.md5() m.update((str(self.seqid) + str(time.time())).encode()) self.filename = m.hexdigest() # 创建一个下载文件夹暂存文件 self.download_temppath = os.path.join(WavDownBasic.share_path, str(self.filename)) self.consumer_logger.info("下载文件夹暂存文件 {}".format( self.download_temppath)) self.consumer_logger.info("创建打包文件名 {}".format(self.filename)) os.makedirs(self.download_temppath, exist_ok=True) def run(self): """ 主执行函数,执行批量下载 并打包 :param list_Wav_url: 下载url列表 :return res_json: 返回打包zip文件全路径 """ print("@@@@@@@@@@@@@@@@@@@@@@run ", self.list_wav_url) future_list = [] for urlnode in self.list_wav_url: url = urlnode.strip() name = urlnode.split("/")[-1] res_flag = re.match('http', url) if res_flag is not None: future = self.executor.submit(self.do_down_wav, url, name) else: future = self.executor.submit(self.do_copy_wav, url, name) future_list.append(future) all_n = len(future_list) n = 0 while n != all_n: n = 0 for future in future_list: if future.done(): n = n + 1 time.sleep(3) schedule = int(n * 100 / all_n) self.consumer_logger.info("schedule {}".format(schedule)) # 返回结果放在一个字典中 res_json_schedule = { "task_id": str(self.seqid), "download": "", "schedule": str(schedule) } # todo 结果检验,异常处理 if n != all_n: self.notify_schedule(send_json_schedule=res_json_schedule) self.executor.shutdown() self.consumer_logger.info('''begin make_archive {}'''.format( self.download_temppath)) t1 = time.time() # 第一个参数是归档文件名称,第二个参数是指定的格式,不仅是支持zip,第三个参数是要压缩文件/文件夹的路径 # shutil.make_archive(os.path.join(WavDownBasic.share_path, self.filename), 'zip', self.download_temppath) self.consumer_logger.info( '''zip -r -q -o {zippath} {download_temppath}'''.format( zippath=os.path.join(WavDownBasic.share_path, self.filename + ".zip "), download_temppath=self.download_temppath)) subprocess.run( '''zip -r -q -o {zippath} {download_temppath}'''.format( zippath=os.path.join(WavDownBasic.share_path, self.filename + ".zip"), download_temppath=self.download_temppath), shell=True, cwd=WavDownBasic.share_path) t2 = time.time() self.consumer_logger.debug( '''end make_archive {download_temppath} ,cost time {costtime}'''. format(download_temppath=self.download_temppath, costtime=(t2 - t1) / 60)) t1 = time.time() # 删除暂存文件夹 time.sleep(1) self.consumer_logger.debug('''begin del {}'''.format( self.download_temppath)) # shutil.rmtree(self.download_temppath, ignore_errors=True) subprocess.run('''rm -rf {download_temppath}'''.format( download_temppath=self.download_temppath), shell=True, cwd=WavDownBasic.share_path) t2 = time.time() self.consumer_logger.debug( '''end del {download_temppath} ,cost time {costtime}'''.format( download_temppath=self.download_temppath, costtime=(t2 - t1) / 60)) download_url = "http://{domain_name}:{LocalPort}/{filename}".format( domain_name=WavDownBasic.domain_name, LocalPort=WavDownBasic.LocalPort, filename=self.filename + ".zip") self.consumer_logger.info( "task_id: {task_id} download_url: {download_url}".format( task_id=self.seqid, download_url=download_url)) # 返回结果放在一个字典中 res_json = { "task_id": str(self.seqid), "download": str(download_url), "schedule": "100" } return res_json def do_down_wav(self, url, name): """ 下载单个网络上录音文件 :param url: :param name: :return: """ # print("下载地址为:",url,"文件名为:",name) try: self.consumer_logger.debug( "begin download wav {url} {name}".format(url=url, name=name)) res = requests.get(url=url) if res.status_code == 200: self.consumer_logger.debug( "begin write wav {name} to localfile".format(url=url, name=name)) with open(os.path.join(self.download_temppath, name), mode='bw') as f: f.write(res.content) self.consumer_logger.debug( "end write wav {name} to localfile".format(url=url, name=name)) else: self.consumer_logger.error( "[ERROR],wav文件下载异常,可能找不到 wav文件 {}".format(url)) except Exception as e: self.consumer_logger.error( "[ERROR],wav文件下载异常,可能无法访问 远程服务器{e} {url}".format(e=str(e), url=str(url))) def do_copy_wav(self, filepath, name): """ 下载单个网络上录音文件 :param url: :param name: :return: """ des_path = os.path.join(self.download_temppath, name) sou_path = filepath if os.path.exists(sou_path): print("sou_path为:", sou_path, "des_path 为:", des_path) try: self.consumer_logger.debug( "begin copy wav {filepath} {name}".format( filepath=filepath, name=name)) shutil.copy(sou_path, des_path) except Exception as e: self.consumer_logger.error( "[ERROR], wav文件copy异常,可能无法访问 远程服务器{}".format(str(e))) else: self.consumer_logger.error("[ERROR], wav文件copy异常,文件不存在") def notify_schedule(self, send_json_schedule): """ 发生进度情况给 回调通知地址 :param send_json_schedule: 需要发生的消息体 json Post请求 :return: """ try: res = requests.post(url=self.callback_url, json=send_json_schedule) res_content_json = json.loads(res.text) self.consumer_logger.debug("schedule notify code = {}".format( res_content_json['code'])) if int(res_content_json['code']) != 1: self.consumer_logger.error("schedule notify error! {}".format( res_content_json['msg'])) else: self.consumer_logger.debug("schedule notify OK! {}".format( res_content_json['msg'])) except Exception as e: self.consumer_logger.error( "schedule notify error! Network anomaly {}".format(str(e)))
from concurrent.futures import ThreadPoolExecutor def crawlPageDate(url): print(url, kwargs) if __name__ == "__main__": pool = ThreadPoolExecutor(max_workers=8) for page in range(1, 57): # 往线程池中提交任务 url = 'https: // www.meishij.net / chufang / diy / jiangchangcaipu /? & page =' + str( page) pool.submit(crawlPageDate, url)
delta = price - underlying_prices[symbol] pnl = Pnl(delta, day, u_delta, u_gamma, u_theta) data = pd.DataFrame({ "coin": [symbol], "price": [price], "day": [day], "pnl": [pnl] }) return data ################################################ # pull initial data on app start get_new_data() # for the data refresh executor = ThreadPoolExecutor(max_workers=1) executor.submit(get_new_data_every) data = pd.read_csv("input.csv") data = data.to_dict() address = data.get("address")[0] symbol = data.get("coin")[0] price = data.get("price")[0] day = data.get("day")[0] # run the calculator X = trigger_calculator(address, symbol, price, day)
name = input('enter your name: ') pygame.init() pygame.display.set_caption("Pandemic") screen = pygame.display.set_mode((1000, 800)) channel = ServerChannel(ip, 1066) chat_channel = ChatChannel(ip, 1099) server_state = channel.read_new_state() scene = Lobby(screen, channel, chat_channel, server_state, name) game_started = False state_fetch_executor = ThreadPoolExecutor(max_workers=1) state_fetch_future = state_fetch_executor.submit(channel.read_new_state) running = True while running: if state_fetch_future.done(): server_state = state_fetch_future.result() state_fetch_future = state_fetch_executor.submit( channel.read_new_state) if server_state.is_game_mode and not game_started: scene = Game(screen, channel, server_state) game_started = True events = pygame.event.get() scene.update(server_state, events) scene.draw()
class Executor(object): def __init__(self, env, pool, config, io, parallel=None): self._env = env self._io = io self._dry_run = False self._enabled = True self._verbose = False self._authenticator = Authenticator(config, self._io) self._chef = Chef(config, self._env) self._chooser = Chooser(pool, self._env) if parallel is None: parallel = config.get("installer.parallel", True) if parallel and not (PY2 and WINDOWS): # This should be directly handled by ThreadPoolExecutor # however, on some systems the number of CPUs cannot be determined # (it raises a NotImplementedError), so, in this case, we assume # that the system only has one CPU. try: self._max_workers = cpu_count() + 4 except NotImplementedError: self._max_workers = 5 else: self._max_workers = 1 self._executor = ThreadPoolExecutor(max_workers=self._max_workers) self._total_operations = 0 self._executed_operations = 0 self._executed = {"install": 0, "update": 0, "uninstall": 0} self._skipped = {"install": 0, "update": 0, "uninstall": 0} self._sections = OrderedDict() self._lock = threading.Lock() self._shutdown = False @property def installations_count(self): # type: () -> int return self._executed["install"] @property def updates_count(self): # type: () -> int return self._executed["update"] @property def removals_count(self): # type: () -> int return self._executed["uninstall"] def supports_fancy_output(self): # type: () -> bool return self._io.supports_ansi() and not self._dry_run def disable(self): self._enabled = False return self def dry_run(self, dry_run=True): self._dry_run = dry_run return self def verbose(self, verbose=True): self._verbose = verbose return self def execute(self, operations): # type: (Operation) -> int self._total_operations = len(operations) for job_type in self._executed: self._executed[job_type] = 0 self._skipped[job_type] = 0 if operations and (self._enabled or self._dry_run): self._display_summary(operations) # We group operations by priority groups = itertools.groupby(operations, key=lambda o: -o.priority) self._sections = OrderedDict() for _, group in groups: tasks = [] serial_operations = [] for operation in group: if self._shutdown: break # Some operations are unsafe, we mus execute them serially in a group # https://github.com/python-poetry/poetry/issues/3086 # https://github.com/python-poetry/poetry/issues/2658 # # We need to explicitly check source type here, see: # https://github.com/python-poetry/poetry-core/pull/98 is_parallel_unsafe = operation.job_type == "uninstall" or ( operation.package.develop and operation.package.source_type in {"directory", "git"}) if not operation.skipped and is_parallel_unsafe: serial_operations.append(operation) continue tasks.append( self._executor.submit(self._execute_operation, operation)) try: wait(tasks) for operation in serial_operations: wait([ self._executor.submit(self._execute_operation, operation) ]) except KeyboardInterrupt: self._shutdown = True if self._shutdown: # Cancelling further tasks from being executed [task.cancel() for task in tasks] self._executor.shutdown(wait=True) break return 1 if self._shutdown else 0 def _write(self, operation, line): if not self.supports_fancy_output( ) or not self._should_write_operation(operation): return if self._io.is_debug(): with self._lock: section = self._sections[id(operation)] section.write_line(line) return with self._lock: section = self._sections[id(operation)] section.output.clear() section.write(line) def _execute_operation(self, operation): try: if self.supports_fancy_output(): if id(operation) not in self._sections: if self._should_write_operation(operation): with self._lock: self._sections[id(operation)] = self._io.section() self._sections[id(operation)].write_line( " <fg=blue;options=bold>•</> {message}: <fg=blue>Pending...</>" .format(message=self.get_operation_message( operation), ), ) else: if self._should_write_operation(operation): if not operation.skipped: self._io.write_line( " <fg=blue;options=bold>•</> {message}".format( message=self.get_operation_message( operation), ), ) else: self._io.write_line( " <fg=default;options=bold,dark>•</> {message}: " "<fg=default;options=bold,dark>Skipped</> " "<fg=default;options=dark>for the following reason:</> " "<fg=default;options=bold,dark>{reason}</>".format( message=self.get_operation_message(operation), reason=operation.skip_reason, )) try: result = self._do_execute_operation(operation) except EnvCommandError as e: if e.e.returncode == -2: result = -2 else: raise # If we have a result of -2 it means a KeyboardInterrupt # in the any python subprocess, so we raise a KeyboardInterrupt # error to be picked up by the error handler. if result == -2: raise KeyboardInterrupt except Exception as e: try: from clikit.ui.components.exception_trace import ExceptionTrace if not self.supports_fancy_output(): io = self._io else: message = " <error>•</error> {message}: <error>Failed</error>".format( message=self.get_operation_message(operation, error=True), ) self._write(operation, message) io = self._sections.get(id(operation), self._io) with self._lock: trace = ExceptionTrace(e) trace.render(io) io.write_line("") finally: with self._lock: self._shutdown = True except KeyboardInterrupt: try: message = " <warning>•</warning> {message}: <warning>Cancelled</warning>".format( message=self.get_operation_message(operation, warning=True), ) if not self.supports_fancy_output(): self._io.write_line(message) else: self._write(operation, message) finally: with self._lock: self._shutdown = True def _do_execute_operation(self, operation): method = operation.job_type operation_message = self.get_operation_message(operation) if operation.skipped: if self.supports_fancy_output(): self._write( operation, " <fg=default;options=bold,dark>•</> {message}: " "<fg=default;options=bold,dark>Skipped</> " "<fg=default;options=dark>for the following reason:</> " "<fg=default;options=bold,dark>{reason}</>".format( message=operation_message, reason=operation.skip_reason, ), ) self._skipped[operation.job_type] += 1 return 0 if not self._enabled or self._dry_run: self._io.write_line( " <fg=blue;options=bold>•</> {message}".format( message=operation_message, )) return 0 result = getattr(self, "_execute_{}".format(method))(operation) if result != 0: return result message = " <fg=green;options=bold>•</> {message}".format( message=self.get_operation_message(operation, done=True), ) self._write(operation, message) self._increment_operations_count(operation, True) return result def _increment_operations_count(self, operation, executed): with self._lock: if executed: self._executed_operations += 1 self._executed[operation.job_type] += 1 else: self._skipped[operation.job_type] += 1 def run_pip(self, *args, **kwargs): # type: (...) -> int try: self._env.run_pip(*args, **kwargs) except EnvCommandError as e: output = decode(e.e.output) if ("KeyboardInterrupt" in output or "ERROR: Operation cancelled by user" in output): return -2 raise return 0 def get_operation_message(self, operation, done=False, error=False, warning=False): base_tag = "fg=default" operation_color = "c2" source_operation_color = "c2" package_color = "c1" if error: operation_color = "error" elif warning: operation_color = "warning" elif done: operation_color = "success" if operation.skipped: base_tag = "fg=default;options=dark" operation_color += "_dark" source_operation_color += "_dark" package_color += "_dark" if operation.job_type == "install": return "<{}>Installing <{}>{}</{}> (<{}>{}</>)</>".format( base_tag, package_color, operation.package.name, package_color, operation_color, operation.package.full_pretty_version, ) if operation.job_type == "uninstall": return "<{}>Removing <{}>{}</{}> (<{}>{}</>)</>".format( base_tag, package_color, operation.package.name, package_color, operation_color, operation.package.full_pretty_version, ) if operation.job_type == "update": return "<{}>Updating <{}>{}</{}> (<{}>{}</{}> -> <{}>{}</>)</>".format( base_tag, package_color, operation.initial_package.name, package_color, source_operation_color, operation.initial_package.full_pretty_version, source_operation_color, operation_color, operation.target_package.full_pretty_version, ) return "" def _display_summary(self, operations): installs = 0 updates = 0 uninstalls = 0 skipped = 0 for op in operations: if op.skipped: skipped += 1 continue if op.job_type == "install": installs += 1 elif op.job_type == "update": updates += 1 elif op.job_type == "uninstall": uninstalls += 1 if not installs and not updates and not uninstalls and not self._verbose: self._io.write_line("") self._io.write_line("No dependencies to install or update") return self._io.write_line("") self._io.write_line("<b>Package operations</b>: " "<info>{}</> install{}, " "<info>{}</> update{}, " "<info>{}</> removal{}" "{}".format( installs, "" if installs == 1 else "s", updates, "" if updates == 1 else "s", uninstalls, "" if uninstalls == 1 else "s", ", <info>{}</> skipped".format(skipped) if skipped and self._verbose else "", )) self._io.write_line("") def _execute_install(self, operation): # type: (Install) -> None return self._install(operation) def _execute_update(self, operation): # type: (Update) -> None return self._update(operation) def _execute_uninstall(self, operation): # type: (Uninstall) -> None message = " <fg=blue;options=bold>•</> {message}: <info>Removing...</info>".format( message=self.get_operation_message(operation), ) self._write(operation, message) return self._remove(operation) def _install(self, operation): package = operation.package if package.source_type == "directory": return self._install_directory(operation) if package.source_type == "git": return self._install_git(operation) if package.source_type == "file": archive = self._prepare_file(operation) elif package.source_type == "url": archive = self._download_link(operation, Link(package.source_url)) else: archive = self._download(operation) operation_message = self.get_operation_message(operation) message = " <fg=blue;options=bold>•</> {message}: <info>Installing...</info>".format( message=operation_message, ) self._write(operation, message) args = ["install", "--no-deps", str(archive)] if operation.job_type == "update": args.insert(2, "-U") return self.run_pip(*args) def _update(self, operation): return self._install(operation) def _remove(self, operation): package = operation.package # If we have a VCS package, remove its source directory if package.source_type == "git": src_dir = self._env.path / "src" / package.name if src_dir.exists(): safe_rmtree(str(src_dir)) try: return self.run_pip("uninstall", package.name, "-y") except CalledProcessError as e: if "not installed" in str(e): return 0 raise def _prepare_file(self, operation): package = operation.package message = " <fg=blue;options=bold>•</> {message}: <info>Preparing...</info>".format( message=self.get_operation_message(operation), ) self._write(operation, message) archive = Path(package.source_url) if not Path(package.source_url).is_absolute() and package.root_dir: archive = package.root_dir / archive archive = self._chef.prepare(archive) return archive def _install_directory(self, operation): from poetry.factory import Factory package = operation.package operation_message = self.get_operation_message(operation) message = " <fg=blue;options=bold>•</> {message}: <info>Building...</info>".format( message=operation_message, ) self._write(operation, message) if package.root_dir: req = os.path.join(str(package.root_dir), package.source_url) else: req = os.path.realpath(package.source_url) args = ["install", "--no-deps", "-U"] pyproject = PyProjectTOML(os.path.join(req, "pyproject.toml")) if pyproject.is_poetry_project(): # Even if there is a build system specified # some versions of pip (< 19.0.0) don't understand it # so we need to check the version of pip to know # if we can rely on the build system legacy_pip = self._env.pip_version < self._env.pip_version.__class__( 19, 0, 0) package_poetry = Factory().create_poetry( pyproject.file.path.parent) if package.develop and not package_poetry.package.build_script: from poetry.masonry.builders.editable import EditableBuilder # This is a Poetry package in editable mode # we can use the EditableBuilder without going through pip # to install it, unless it has a build script. builder = EditableBuilder(package_poetry, self._env, NullIO()) builder.build() return 0 elif legacy_pip or package_poetry.package.build_script: from poetry.core.masonry.builders.sdist import SdistBuilder # We need to rely on creating a temporary setup.py # file since the version of pip does not support # build-systems # We also need it for non-PEP-517 packages builder = SdistBuilder(package_poetry) with builder.setup_py(): if package.develop: args.append("-e") args.append(req) return self.run_pip(*args) if package.develop: args.append("-e") args.append(req) return self.run_pip(*args) def _install_git(self, operation): from poetry.core.vcs import Git package = operation.package operation_message = self.get_operation_message(operation) message = " <fg=blue;options=bold>•</> {message}: <info>Cloning...</info>".format( message=operation_message, ) self._write(operation, message) src_dir = self._env.path / "src" / package.name if src_dir.exists(): safe_rmtree(str(src_dir)) src_dir.parent.mkdir(exist_ok=True) git = Git() git.clone(package.source_url, src_dir) reference = package.source_resolved_reference if not reference: reference = package.source_reference git.checkout(reference, src_dir) # Now we just need to install from the source directory package._source_url = str(src_dir) return self._install_directory(operation) def _download(self, operation): # type: (Operation) -> Path link = self._chooser.choose_for(operation.package) return self._download_link(operation, link) def _download_link(self, operation, link): package = operation.package archive = self._chef.get_cached_archive_for_link(link) if archive is link: # No cached distributions was found, so we download and prepare it try: archive = self._download_archive(operation, link) except BaseException: cache_directory = self._chef.get_cache_directory_for_link(link) cached_file = cache_directory.joinpath(link.filename) # We can't use unlink(missing_ok=True) because it's not available # in pathlib2 for Python 2.7 if cached_file.exists(): cached_file.unlink() raise # TODO: Check readability of the created archive if not link.is_wheel: archive = self._chef.prepare(archive) if package.files: hashes = {f["hash"] for f in package.files} hash_types = {h.split(":")[0] for h in hashes} archive_hashes = set() for hash_type in hash_types: archive_hashes.add("{}:{}".format( hash_type, FileDependency( package.name, Path(archive.path) if isinstance(archive, Link) else archive, ).hash(hash_type), )) if archive_hashes.isdisjoint(hashes): raise RuntimeError( "Invalid hashes ({}) for {} using archive {}. Expected one of {}." .format( ", ".join(sorted(archive_hashes)), package, archive.name, ", ".join(sorted(hashes)), )) return archive def _download_archive(self, operation, link): # type: (Operation, Link) -> Path response = self._authenticator.request("get", link.url, stream=True, io=self._sections.get( id(operation), self._io)) wheel_size = response.headers.get("content-length") operation_message = self.get_operation_message(operation) message = " <fg=blue;options=bold>•</> {message}: <info>Downloading...</>".format( message=operation_message, ) progress = None if self.supports_fancy_output(): if wheel_size is None: self._write(operation, message) else: from clikit.ui.components.progress_bar import ProgressBar progress = ProgressBar(self._sections[id(operation)].output, max=int(wheel_size)) progress.set_format(message + " <b>%percent%%</b>") if progress: with self._lock: progress.start() done = 0 archive = self._chef.get_cache_directory_for_link(link) / link.filename archive.parent.mkdir(parents=True, exist_ok=True) with archive.open("wb") as f: for chunk in response.iter_content(chunk_size=4096): if not chunk: break done += len(chunk) if progress: with self._lock: progress.set_progress(done) f.write(chunk) if progress: with self._lock: progress.finish() return archive def _should_write_operation(self, operation): # type: (Operation) -> bool if not operation.skipped: return True return self._dry_run or self._verbose
class WebInterface: """ The WebInterface is a executor like class for the submission of runs to the VerifierCloud """ def __init__( self, web_interface_url, user_pwd, revision="trunk:HEAD", thread_count=1, result_poll_interval=2, user_agent=None, version=None, ): """ Creates a new WebInterface object. The given svn revision is resolved (e.g. 'HEAD' -> 17495). @param web_interface_url: the base URL of the VerifierCloud's web interface @param user_pwd: user name (and password) in the format '<user_name>[:<password>]' or none if no authentification is required @param revision: the svn revision string, defaults to 'trunk:HEAD' @param thread_count: the number of threads for fetching results in parallel @param result_poll_interval: the number of seconds to wait between polling results """ # this attribute is used to communicate shutdown to the futures # of the run submission in case the user cancels while the run submission is still ongoing: self.active = True if not (1 <= thread_count <= MAX_SUBMISSION_THREADS): sys.exit( "Invalid number {} of client threads, needs to be between 1 and {}.".format( thread_count, MAX_SUBMISSION_THREADS ) ) if not 1 <= result_poll_interval: sys.exit( "Poll interval {} is too small, needs to be at least 1s.".format( result_poll_interval ) ) if not web_interface_url[-1] == "/": web_interface_url += "/" default_headers = {"Connection": "Keep-Alive"} if user_agent: default_headers["User-Agent"] = "{}/{} (Python/{} {}/{})".format( user_agent, version, platform.python_version(), platform.system(), platform.release(), ) urllib.parse.urlparse(web_interface_url) # sanity check self._web_interface_url = web_interface_url logging.info("Using VerifierCloud at %s", web_interface_url) self._connection = requests.Session() # increase the pool size a bit to get rid of warnings on aborting with SIGINT: self._connection.mount( "https://", requests.adapters.HTTPAdapter(pool_maxsize=100) ) self._connection.headers.update(default_headers) try: cert_paths = ssl.get_default_verify_paths() cert_path = cert_paths.cafile or cert_paths.capath # both might be None except AttributeError: # not available on old Python cert_path = None # make sure that certificate verification is enabled self._connection.verify = cert_path or True if user_pwd: self._connection.auth = self.getUserAndPassword(user_pwd) self._unfinished_runs = {} self._unfinished_runs_lock = threading.Lock() self._downloading_result_futures = {} self._download_attempts = {} self.thread_count = thread_count self._executor = ThreadPoolExecutor(thread_count) self._thread_local = threading.local() self._hash_code_cache = {} self._group_id = str(random.randint(0, 1000000)) # noqa: S311 self._read_hash_code_cache() self._revision = self._request_tool_revision(revision) self._tool_name = self._request_tool_name() if re.match("^.*:[0-9]*$", revision) and revision != self._revision: logging.warning( "Using %s version %s, which is different than the requested version %s!", self._tool_name, self._revision, revision, ) else: logging.info("Using %s version %s.", self._tool_name, self._revision) if HAS_SSECLIENT: self._result_downloader = SseResultDownloader(self, result_poll_interval) else: self._result_downloader = PollingResultDownloader( self, result_poll_interval ) def getUserAndPassword(self, user_pwd): # split only once, password might contain special char ':' tokens = user_pwd.split(":", maxsplit=1) if len(tokens) == 2 and all(tokens): user, password = tokens else: user = user_pwd password = getpass("Please enter password for user '" + user + "': ") return user, password def _read_hash_code_cache(self): if not os.path.isfile(HASH_CODE_CACHE_PATH): return with open(HASH_CODE_CACHE_PATH, mode="r") as hashCodeCacheFile: for line in hashCodeCacheFile: tokens = line.strip().split("\t") if len(tokens) == 3: self._hash_code_cache[(tokens[0], tokens[1])] = tokens[2] def _write_hash_code_cache(self): # make snapshot of hash code cache to avoid concurrent modification. # We reset self._hash_code_cache to {} to save memory and because # it will not be needed any more (this method is only called upon shutdown). hash_code_cache = self._hash_code_cache self._hash_code_cache = {} directory = os.path.dirname(HASH_CODE_CACHE_PATH) try: os.makedirs(directory, exist_ok=True) with tempfile.NamedTemporaryFile(dir=directory, delete=False) as tmpFile: for (path, mTime), hashValue in hash_code_cache.items(): line = path + "\t" + mTime + "\t" + hashValue + "\n" tmpFile.write(line.encode()) os.renames(tmpFile.name, HASH_CODE_CACHE_PATH) except OSError as e: logging.warning( "Could not write hash-code cache file to %s: %s", HASH_CODE_CACHE_PATH, e.strerror, ) def _request_tool_revision(self, revision): path = "tool/version_string?revision=" + revision (resolved_svn_revision, _) = self._request("GET", path) return resolved_svn_revision.decode("UTF-8") def _request_tool_name(self): path = "tool/name" (tool_name, _) = self._request("GET", path) return tool_name.decode("UTF-8") def tool_revision(self): return self._revision def tool_name(self): return self._tool_name def _get_sha256_hash(self, path): path = os.path.abspath(path) mTime = str(os.path.getmtime(path)) if (path, mTime) in self._hash_code_cache: return self._hash_code_cache[(path, mTime)] else: with open(path, "rb") as opened_file: hashValue = hashlib.sha256(opened_file.read()).hexdigest() self._hash_code_cache[(path, mTime)] = hashValue return hashValue def _create_and_add_run_future(self, run_id): result = RunResultFuture(self, run_id) with self._unfinished_runs_lock: self._unfinished_runs[run_id] = result return result def submit_witness_validation( self, witness_path, program_path, configuration=None, user_pwd=None ): """ Submits a single witness validation run to the VerifierCloud. @note: flush() should be called after the submission of the last run. @param witness_path: path to the file containing the witness @param program_path: path to the file containing the program @param configuration: name of configuration (optional) @param user_pwd: overrides the user name and password given in the constructor (optional) """ # collect parameters params = {} with open(witness_path, "rb") as witness_file: params["witnessText"] = witness_file.read() with open(program_path, "rb") as program_file: params["programText"] = program_file.read() if configuration: params["configuration"] = configuration # prepare request headers = { "Content-Type": "application/x-www-form-urlencoded", "Content-Encoding": "deflate", "Accept": "text/plain", } paramsCompressed = zlib.compress( urllib.parse.urlencode(params, doseq=True).encode("utf-8") ) path = "runs/witness_validation/" (response, _) = self._request( "POST", path, paramsCompressed, headers, user_pwd=user_pwd ) try: run_id = response.decode("UTF-8") except UnicodeDecodeError as e: raise WebClientError( "Malformed response from server while submitting witness-validation run:\n{}".format( response ) ) from e logging.debug("Submitted witness validation run with id %s", run_id) return self._create_and_add_run_future(run_id) def submit( self, run, limits, cpu_model, result_files_pattern=None, meta_information=None, priority="IDLE", user_pwd=None, revision=None, result_files_patterns=(), required_files=(), ): """ Submits a single run to the VerifierCloud. @note: flush() should be called after the submission of the last run. @param run: The input for the run: command line options (run.options), source files (run.sourcefiles), property file (run.propertyfile), identifier for error messages (run.identifier) @param limits: dict of limitations for the run (memlimit, timelimit, corelimit, softtimelimit) @param cpu_model: substring of CPU model to use or 'None' for no restriction @param result_files_pattern: the result is filtered with the given glob pattern, '**' is no restriction and None or the empty string do not match any file. @param meta_information: meta information about the submitted run as JSON string @param priority: the priority of the submitted run, defaults to 'IDLE' @param user_pwd: overrides the user name and password given in the constructor (optional) @param revision: overrides the revision given in the constructor (optional) @param result_files_patterns: list of result_files_pattern (optional) @param required_files: list of additional file required to execute the run (optional) @raise WebClientError: if the HTTP request could not be created @raise UserAbortError: if the user already requested shutdown on this instance @raise HTTPError: if the HTTP request was not successful """ if not self.active: raise UserAbortError("User interrupt detected while submitting runs.") if result_files_pattern: if result_files_patterns: raise ValueError( "Cannot specify result_files_pattern and result_files_patterns " "at the same time." ) result_files_patterns = [result_files_pattern] return self._submit( run, limits, cpu_model, required_files, result_files_patterns, meta_information, priority, user_pwd, revision, ) def _submit( self, run, limits, cpu_model, required_files, result_files_patterns, meta_information, priority, user_pwd, revision, counter=0, ): params = [] opened_files = [] # open file handles are passed to the request library for programPath in run.sourcefiles: norm_path = self._normalize_path_for_cloud(programPath) params.append( ("programTextHash", (norm_path, self._get_sha256_hash(programPath))) ) for required_file in get_files(required_files): norm_path = self._normalize_path_for_cloud(required_file) params.append( ("requiredFileHash", (norm_path, self._get_sha256_hash(required_file))) ) params.append(("revision", revision or self._revision)) if run.propertyfile: property_file = self._add_file_to_params( params, "propertyText", run.propertyfile ) opened_files.append(property_file) if MEMLIMIT in limits: params.append(("memoryLimitation", str(limits[MEMLIMIT]))) if TIMELIMIT in limits: params.append(("timeLimitation", str(limits[TIMELIMIT]))) if SOFTTIMELIMIT in limits: params.append(("softTimeLimitation", str(limits[SOFTTIMELIMIT]))) if CORELIMIT in limits: params.append(("coreLimitation", str(limits[CORELIMIT]))) if cpu_model: params.append(("cpuModel", cpu_model)) if result_files_patterns: for pattern in result_files_patterns: params.append(("resultFilesPattern", pattern)) else: params.append(("resultFilesPattern", "")) if priority: params.append(("priority", priority)) (invalidOption, files) = self._handle_options(run, params, limits) opened_files.extend(files) if invalidOption: raise WebClientError( 'Command {0} contains option "{1}" that is not usable with the webclient. '.format( run.options, invalidOption ) ) params.append(("groupId", str(self._group_id))) if meta_information: params.append(("metaInformation", meta_information)) # prepare request headers = {"Accept": "text/plain"} path = "runs/" (response, statusCode) = self._request( "POST", path, files=params, headers=headers, expectedStatusCodes=[200, 412], user_pwd=user_pwd, ) for opened_file in opened_files: opened_file.close() # program files or required files given as hash value are not known by the cloud system if statusCode == 412: if counter >= 1: raise WebClientError( "Files still missing on server for run {0} even after uploading them:\n{1}".format( run.identifier, response ) ) headers = { "Content-Type": "application/octet-stream", "Content-Encoding": "deflate", } filePath = "files/" # upload all used program files for programPath in run.sourcefiles: with open(programPath, "rb") as programFile: compressedProgramText = zlib.compress(programFile.read(), 9) self._request( "POST", filePath, data=compressedProgramText, headers=headers, expectedStatusCodes=[200, 204], user_pwd=user_pwd, ) # upload all required files for required_file_path in required_files: with open(required_file_path, "rb") as required_file: compressed_required_file = zlib.compress(required_file.read(), 9) self._request( "POST", filePath, data=compressed_required_file, headers=headers, expectedStatusCodes=[200, 204], user_pwd=user_pwd, ) # retry submission of run return self._submit( run, limits, cpu_model, required_files, result_files_patterns, meta_information, priority, user_pwd, revision, counter + 1, ) else: try: run_id = response.decode("UTF-8") except UnicodeDecodeError as e: raise WebClientError( "Malformed response from server while submitting run {0}:\n{1}".format( run.identifier, response ) ) from e if not VALID_RUN_ID.match(run_id): raise WebClientError( "Malformed response from server while submitting run {0}:\n{1}".format( run.identifier, run_id ) ) logging.debug("Submitted run with id %s", run_id) return self._create_and_add_run_future(run_id) def _handle_options(self, run, params, rlimits): opened_files = [] config = None # TODO use code from CPAchecker module, it add -stats and sets -timelimit, # instead of doing it here manually, too if self._tool_name == "CPAchecker": params.append(("option", "statistics.print=true")) if "softtimelimit" in rlimits: params.append( ("option", "limits.time.cpu=" + str(rlimits["softtimelimit"]) + "s") ) task_options = getattr(run, "task_options", None) if isinstance(task_options, dict) and task_options.get("language") == "C": data_model = task_options.get("data_model") if data_model: data_model_option = {"ILP32": "Linux32", "LP64": "Linux64"}.get( data_model ) if data_model_option: params.append( ("option", "analysis.machineModel=" + data_model_option) ) else: raise WebClientError( "Unsupported data_model '{}' defined for task '{}'".format( data_model, run.identifier ) ) if run.options: i = iter(run.options) disableAssertions = False while True: try: option = next(i) if len(option) == 0: continue if option == "-heap": params.append(("heap", next(i))) elif option == "-stack": params.append(("stack", next(i))) elif option == "-noout": params.append(("option", "output.disable=true")) elif option == "-outputpath": params.append(("option", "output.path=" + next(i))) elif option == "-logfile": params.append(("option", "log.file=" + next(i))) elif option == "-nolog": params.append(("option", "log.level=OFF")) params.append(("option", "log.consoleLevel=OFF")) elif option == "-stats": # ignore, is always set by this script pass elif option == "-disable-java-assertions": disableAssertions = True elif option == "-java": params.append(("option", "language=JAVA")) elif option == "-32": params.append(("option", "analysis.machineModel=Linux32")) elif option == "-64": params.append(("option", "analysis.machineModel=Linux64")) elif option == "-entryfunction": params.append(("option", "analysis.entryFunction=" + next(i))) elif option == "-timelimit": params.append(("option", "limits.time.cpu=" + next(i))) elif option == "-skipRecursion": params.append(("option", "cpa.callstack.skipRecursion=true")) params.append(("option", "analysis.summaryEdges=true")) elif option == "-cbmc": params.append(("option", "analysis.checkCounterexamples=true")) params.append(("option", "counterexample.checker=CBMC")) elif option == "-clang": params.append(("option", "parser.useClang=true")) elif option == "-preprocess": params.append(("option", "parser.usePreprocessor=true")) elif option == "-generateReport": params.append(("generateReport", "true")) elif option == "-sourcepath": params.append(("option", "java.sourcepath=" + next(i))) elif option in ["-cp", "-classpath"]: params.append(("option", "java.classpath=" + next(i))) elif option == "-spec": spec_path = next(i) spec_file = self._add_file_to_params( params, "specificationText", spec_path ) opened_files.append(spec_file) elif option == "-config": configPath = next(i) tokens = configPath.split("/") if tokens[0] == "config" and len(tokens) == 2: config = tokens[1].split(".")[0] params.append(("configuration", config)) else: params.append( ("option", "configuration.file=" + configPath) ) elif option == "-setprop": params.append(("option", next(i))) elif option == "-benchmark": params.append(("option", "coverage.enabled=true")) params.append(("option", "output.disable=true")) params.append(("option", "statistics.memory=false")) disableAssertions = True elif option[0] == "-": if config: raise WebClientError( "More than one configuration: '{}' and '{}'".format( config, option[1:] ) ) else: params.append(("configuration", option[1:])) config = option[1:] else: return (option, opened_files) except StopIteration: break if disableAssertions: params.append(("disableJavaAssertions", "true")) return (None, opened_files) def _add_file_to_params(self, params, name, path): norm_path = self._normalize_path_for_cloud(path) opened_file = open(path, "rb") params.append((name, (norm_path, opened_file))) return opened_file def _normalize_path_for_cloud(self, path): norm_path = os.path.normpath(path) if ".." in norm_path or os.path.isabs(norm_path): norm_path = os.path.basename(norm_path) return norm_path.replace("\\", "/") def flush_runs(self): """ Starts the execution of all previous submitted runs in the VerifierCloud. The web interface groups runs and submits them to the VerifierCloud only from time to time. This method forces the web interface to do this immediately and starts downloading of results. @return: the ids of the RunCollections created since the last flush request """ headers = { "Content-Type": "application/x-www-form-urlencoded", "Connection": "Keep-Alive", } params = {"groupId": self._group_id} path = "runs/flush" (response, _) = self._request( "POST", path, data=params, headers=headers, expectedStatusCodes=[200, 204] ) run_collections = response.decode("utf-8").split("\n") if len(run_collections) == 0: logging.warning( "No runs were submitted to the VerifierCloud before or a rate limit is hit." ) else: logging.info( "Submitted %s run collection: %s", len(run_collections), ",".join(run_collections), ) self._result_downloader.start() return run_collections def _is_finished(self, run_id): headers = {"Accept": "text/plain"} path = "runs/" + run_id + "/state" try: (state, _) = self._request("GET", path, headers=headers) state = state.decode("utf-8") if state == "FINISHED": logging.debug("Run %s finished.", run_id) if state == "UNKNOWN": logging.debug( "Run %s is not known by the webclient, trying to get the result.", run_id, ) return state except requests.HTTPError as e: logging.warning( "Could not get state for run %s: %s\n%s", run_id, getattr(e, "reason", ""), e.response.content or "", ) return False def _download_result(self, run_id): # download result as zip file headers = {"Accept": "application/zip"} path = "runs/" + run_id + "/result" (zip_content, _) = self._request("GET", path, headers=headers) return zip_content def _download_result_async(self, run_id): def callback(downloaded_result): run_id = self._downloading_result_futures.pop(downloaded_result) exception = downloaded_result.exception() if not exception: with self._unfinished_runs_lock: result_future = self._unfinished_runs.pop(run_id, None) if result_future: result_future.set_result(downloaded_result.result()) else: attempts = self._download_attempts.pop(run_id, 1) logging.info( "Could not get result of run %s on attempt %d: %s", run_id, attempts, exception, ) if attempts < 10: self._download_attempts[run_id] = attempts + 1 self._download_result_async(run_id) else: self._run_failed(run_id) if run_id not in self._downloading_result_futures.values(): # result is not downloaded future = self._executor.submit(self._download_result, run_id) self._downloading_result_futures[future] = run_id future.add_done_callback(callback) def _run_failed(self, run_id): run_result_future = self._unfinished_runs.pop(run_id, None) def getErrorForRun(run_id): headers = {"Accept": "text/plain"} path = "runs/" + run_id + "/failurecause" (error_msg, _) = self._request("GET", path, headers=headers) return error_msg if run_result_future: error_msg = getErrorForRun(run_id) logging.warning("Execution of run %s failed. Error(%s)", run_id, error_msg) run_result_future.set_exception( WebClientError("Execution failed. Error({})".format(error_msg)) ) def shutdown(self): """ Cancels all unfinished runs and stops all internal threads. """ self.active = False self._result_downloader.shutdown() if len(self._unfinished_runs) > 0: logging.info("Stopping tasks on server...") stop_executor = ThreadPoolExecutor(max_workers=5 * self.thread_count) stop_tasks = set() with self._unfinished_runs_lock: for runId in self._unfinished_runs.keys(): stop_tasks.add(stop_executor.submit(self._stop_run, runId)) self._unfinished_runs[runId].set_exception( UserAbortError( "Run was canceled because user requested shutdown." ) ) self._unfinished_runs.clear() for task in stop_tasks: task.result() stop_executor.shutdown(wait=True) logging.info("Stopped all tasks.") self._write_hash_code_cache() self._executor.shutdown(wait=True) self._connection.close() def _stop_run(self, run_id): with self._unfinished_runs_lock: self._unfinished_runs.pop(run_id, None) path = "runs/" + run_id try: self._request("DELETE", path, expectedStatusCodes=[200, 204, 404]) except HTTPError as e: reason = e.reason if hasattr(e, "reason") else "<unknown>" content = ( e.response.content if hasattr(e, "response") and hasattr(e.response, "content") else "<unknown>" ) logging.info( "Stopping of run %s failed: %s\n%s\n%s", run_id, e, reason, content or "", ) def _request( self, method, path, data=None, headers=None, files=None, expectedStatusCodes=(200,), user_pwd=None, ): url = self._web_interface_url + path if user_pwd: auth = (user_pwd.split(":")[0], user_pwd.split(":")[1]) else: auth = None counter = 0 while counter < 5: counter += 1 # send request try: response = self._connection.request( method, url, data=data, files=files, headers=headers, auth=auth ) except Exception as e: if counter < 5: logging.debug( "Exception during %s request to %s: %s", method, path, e ) sleep(1) continue else: raise if response.status_code in expectedStatusCodes: return (response.content, response.status_code) else: if response.status_code == 401: message = "Permission denied. Please check the URL given to --cloudMaster and specify credentials if necessary." elif response.status_code == 404: message = "Not found. Please check the URL given to --cloudMaster." elif response.status_code == 503: message = "Service Unavailable." if counter < 5: logging.debug(message) sleep(60) continue else: message = response.text # HTTPError.request is automatically filled with response.request so no need to pass it. # Also HTTPError extends IOError, so there is a constructor IOError(errno, strerror, filename) raise requests.HTTPError( response.status_code, message, path, response=response )
class FtpServer: limitasyon = 1000000 def __init__(self, port): self.port = port self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.sock.bind(('localhost', port)) self.clients = {} self.client_cwd = {} self.client_data = {} self.executor = ThreadPoolExecutor(10) self.init() def init(self): with open('password.txt') as file: content = file.readlines() lines = [line.strip() for line in content] for line in lines: parts = line.split(':', 2) cwd = os.getcwd().replace('\\', '/') self.clients[parts[0]] = parts[1] self.client_cwd[parts[0]] = cwd self.client_data[parts[0]] = [] def send(self, data, address): self.sock.sendto(data, address) def listen(self): while True: data, address = self.sock.recvfrom(self.limitasyon) received_data = data.decode() self.executor.submit(self.handle, received_data, address) def handle(self, received_data, address): ftp_message = json.loads(received_data) method = ftp_message['method'] credentials = ftp_message['credentials'] data = ftp_message['data'] result = ftp_message['result'] if credentials['username'] not in self.clients: response_data = create_response(ftp_message['session_id'], method, 404) self.send(response_data, address) return password = self.clients[credentials['username']] if password != credentials['password']: response_data = create_response(ftp_message['session_id'], method, 403) self.send(response_data, address) return client_cwd = self.client_cwd[credentials['username']] if method == 'List': file_data = "" prefix = "" files = [f for f in os.listdir(client_cwd)] for f in files: file_data += prefix file_data += f prefix = ";" response_data = create_response(ftp_message['session_id'], method, 200, content=file_data) self.send(response_data, address) elif method == 'cd': folder = data['header'] if folder == '.': result_code = 200 elif folder == '..': dirs = client_cwd.replace('\\', '/').split('/') if len(dirs) > 2: dirs.pop() client_cwd = '/'.join(dirs) self.client_cwd[credentials['username']] = client_cwd result_code = 200 else: result_code = 403 else: dirs = client_cwd.replace('\\', '/').split('/') dirs.append(folder) cwd = '/'.join(dirs) if os.path.isdir(cwd): client_cwd = cwd self.client_cwd[credentials['username']] = client_cwd result_code = 200 else: result_code = 404 response_data = create_response(ftp_message['session_id'], method, result_code, header=folder) self.send(response_data, address) # Handle 'get' command : get the file from the server elif method == 'Get': file_name = data['header'] file_path = client_cwd + '/' + file_name if os.path.isfile(file_path): did = 1 with open(file_path, mode='rb') as file: while True: file_content = file.read(1024) if not file_content: break b64_content = base64.b64encode(file_content).decode() response_data = create_response( ftp_message['session_id'], method, 200, data_id=did, is_last=False, header=file_name, content=b64_content) self.send(response_data, address) did += 1 sleep(0.05) response_data = create_response(ftp_message['session_id'], method, 200, data_id=did, is_last=True, header=file_name, content="") self.send(response_data, address) else: response_data = create_response(ftp_message['session_id'], method, 404, header=file_name) self.send(response_data, address) elif method == 'Put': file_name = data['header'] did = data['data_id'] if did == 1: self.client_data[credentials['username']] = [] if data['is_last']: content = bytearray() file_name = data['header'] file_path = client_cwd + '/' + file_name for c in self.client_data[credentials['username']]: content += c write_file(file_path, content) else: self.client_data[credentials['username']].append( base64.b64decode(data['content'])) response_data = create_response(ftp_message['session_id'], method, 200, data_id=data['data_id'], header=file_name) self.send(response_data, address) # Not implemented command else: response_data = create_response(ftp_message['session_id'], method, 501) self.send(response_data, address)
class SseResultDownloader: def __init__(self, web_interface, result_poll_interval): logging.debug("Server-Send Events are used to get state of runs.") self._web_interface = web_interface self._run_finished_url = web_interface._web_interface_url + "runs/finished" self._result_poll_interval = result_poll_interval self._sse_client = None self._shutdown = False self._new_runs = False self._state_receive_executor = ThreadPoolExecutor(max_workers=1) def _log_future_exception_and_fallback(self, result): e = result.exception() if e is not None: if ( self._shutdown and isinstance(e, AttributeError) and str(e) == "'NoneType' object has no attribute 'read'" ): # This is harmless, it occurs because SSEClient reads on closed connection. logging.debug("Error during result processing:", exc_info=True) else: logging.warning("Error during result processing:", exc_info=True) if not self._shutdown: self._fall_back() def _should_reconnect(self, error): if self._new_runs: return False elif ( type(error) == HTTPError and error.response is not None and error.response.status >= 400 and error.response.status < 500 ): logging.debug("Exception in SSE connection: %s", error) return False else: return True def _start_sse_connection(self): while self._new_runs: run_ids = set(self._web_interface._unfinished_runs.keys()) self._new_runs = False # nothing to do if len(run_ids) == 0: return params = [] for run_id in run_ids: params.append(("run", run_id)) headers = {"Accept-Encoding": "UTF-8"} logging.debug("Creating Server-Send Event connection.") try: self._sse_client = ShouldReconnectSeeClient( self._run_finished_url, self._should_reconnect, session=self._web_interface._connection, headers=headers, data=params, ) except Exception as e: logging.warning("Creating SSE connection failed: %s", e) self._fall_back() return for message in self._sse_client: if self._shutdown: self._sse_client.resp.close() break data = message.data tokens = data.split(" ") if len(tokens) == 2: run_id = tokens[0] state = tokens[1] if state == "FINISHED": if run_id in run_ids: logging.debug("Run %s finished.", run_id) self._web_interface._download_result_async(run_id) elif state == "UNKNOWN": logging.debug( "Run %s is not known by the webclient, trying to get the result.", run_id, ) self._web_interface._download_async(run_id) elif state == "ERROR": self._web_interface._run_failed(run_id) else: logging.warning( "Received unknown run state %s for run %s.", state, run_id, ) run_ids.discard(run_id) if self._shutdown or self._new_runs or len(run_ids) == 0: break else: logging.warning("Received invalid message %s", data) self._sse_client = None # fall back to polling if Server-Send Event based approach failed if len(run_ids) != 0 and not self._shutdown: self._fall_back() def _fall_back(self): logging.info("Fall back to polling.") self._web_interface._result_downloader = PollingResultDownloader( self._web_interface, self._result_poll_interval ) self._web_interface._result_downloader.start() self.shutdown(wait=False) def start(self): self._new_runs = True if self._sse_client: self._sse_client.resp.close() else: future = self._state_receive_executor.submit(self._start_sse_connection) future.add_done_callback(self._log_future_exception_and_fallback) def shutdown(self, wait=True): self._shutdown = True self._state_receive_executor.shutdown(wait=wait)
class PipelineModel: """Controls IO (camera, video file, recording, saving frames). Methods run in worker threads.""" def __init__(self, update_view, camera_config_file=None, rgbd_video=None, device=None): """Initialize. Args: update_view (callback): Callback to update display elements for a frame. camera_config_file (str): Camera configuration json file. rgbd_video (str): RS bag file containing the RGBD video. If this is provided, connected cameras are ignored. device (str): Compute device (e.g.: 'cpu:0' or 'cuda:0'). """ self.update_view = update_view if device: self.device = device.lower() else: self.device = 'cuda:0' if o3d.core.cuda.is_available() else 'cpu:0' self.o3d_device = o3d.core.Device(self.device) self.video = None self.camera = None self.flag_capture = False self.cv_capture = threading.Condition() # condition variable self.recording = False # Are we currently recording self.flag_record = False # Request to start/stop recording if rgbd_video: # Video file self.video = o3d.t.io.RGBDVideoReader.create(rgbd_video) self.rgbd_metadata = self.video.metadata self.status_message = f"Video {rgbd_video} opened." else: # RGBD camera now = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') filename = f"{now}.bag" self.camera = o3d.t.io.RealSenseSensor() if camera_config_file: with open(camera_config_file) as ccf: self.camera.init_sensor(o3d.t.io.RealSenseSensorConfig( json.load(ccf)), filename=filename) else: self.camera.init_sensor(filename=filename) self.camera.start_capture(start_record=False) self.rgbd_metadata = self.camera.get_metadata() self.status_message = f"Camera {self.rgbd_metadata.serial_number} opened." log.info(self.rgbd_metadata) # RGBD -> PCD self.extrinsics = o3d.core.Tensor.eye(4, dtype=o3d.core.Dtype.Float32, device=self.o3d_device) self.intrinsic_matrix = o3d.core.Tensor( self.rgbd_metadata.intrinsics.intrinsic_matrix, dtype=o3d.core.Dtype.Float32, device=self.o3d_device) self.depth_max = 3.0 # m self.pcd_stride = 2 # downsample point cloud, may increase frame rate self.flag_normals = False self.flag_save_rgbd = False self.flag_save_pcd = False self.pcd_frame = None self.rgbd_frame = None self.executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix='Capture-Save') self.flag_exit = False @property def max_points(self): """Max points in one frame for the camera or RGBD video resolution.""" return self.rgbd_metadata.width * self.rgbd_metadata.height @property def vfov(self): """Camera or RGBD video vertical field of view.""" return np.rad2deg(2 * np.arctan(self.intrinsic_matrix[1, 2].item() / self.intrinsic_matrix[1, 1].item())) def run(self): """Run pipeline.""" n_pts = 0 frame_id = 0 t1 = time.perf_counter() if self.video: self.rgbd_frame = self.video.next_frame() else: self.rgbd_frame = self.camera.capture_frame( wait=True, align_depth_to_color=True) pcd_errors = 0 while (not self.flag_exit and (self.video is None or # Camera (self.video and not self.video.is_eof()))): # Video if self.video: future_rgbd_frame = self.executor.submit(self.video.next_frame) else: future_rgbd_frame = self.executor.submit( self.camera.capture_frame, wait=True, align_depth_to_color=True) if self.flag_save_pcd: self.save_pcd() self.flag_save_pcd = False try: self.rgbd_frame = self.rgbd_frame.to(self.o3d_device) self.pcd_frame = o3d.t.geometry.PointCloud.create_from_rgbd_image( self.rgbd_frame, self.intrinsic_matrix, self.extrinsics, self.rgbd_metadata.depth_scale, self.depth_max, self.pcd_stride, self.flag_normals) depth_in_color = self.rgbd_frame.depth.colorize_depth( self.rgbd_metadata.depth_scale, 0, self.depth_max) except RuntimeError: pcd_errors += 1 if self.pcd_frame.is_empty(): log.warning(f"No valid depth data in frame {frame_id})") continue n_pts += self.pcd_frame.point['positions'].shape[0] if frame_id % 60 == 0 and frame_id > 0: t0, t1 = t1, time.perf_counter() log.debug( f"\nframe_id = {frame_id}, \t {(t1-t0)*1000./60:0.2f}" f"ms/frame \t {(t1-t0)*1e9/n_pts} ms/Mp\t") n_pts = 0 frame_elements = { 'color': self.rgbd_frame.color.cpu(), 'depth': depth_in_color.cpu(), 'pcd': self.pcd_frame.cpu(), 'status_message': self.status_message } self.update_view(frame_elements) if self.flag_save_rgbd: self.save_rgbd() self.flag_save_rgbd = False self.rgbd_frame = future_rgbd_frame.result() with self.cv_capture: # Wait for capture to be enabled self.cv_capture.wait_for( predicate=lambda: self.flag_capture or self.flag_exit) self.toggle_record() frame_id += 1 if self.camera: self.camera.stop_capture() else: self.video.close() self.executor.shutdown() log.debug(f"create_from_depth_image() errors = {pcd_errors}") def toggle_record(self): if self.camera is not None: if self.flag_record and not self.recording: self.camera.resume_record() self.recording = True elif not self.flag_record and self.recording: self.camera.pause_record() self.recording = False def save_pcd(self): """Save current point cloud.""" now = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') filename = f"{self.rgbd_metadata.serial_number}_pcd_{now}.ply" # Convert colors to uint8 for compatibility self.pcd_frame.point['colors'] = (self.pcd_frame.point['colors'] * 255).to(o3d.core.Dtype.UInt8) self.executor.submit(o3d.t.io.write_point_cloud, filename, self.pcd_frame, write_ascii=False, compressed=True, print_progress=False) self.status_message = f"Saving point cloud to {filename}." def save_rgbd(self): """Save current RGBD image pair.""" now = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') filename = f"{self.rgbd_metadata.serial_number}_color_{now}.jpg" self.executor.submit(o3d.t.io.write_image, filename, self.rgbd_frame.color) filename = f"{self.rgbd_metadata.serial_number}_depth_{now}.png" self.executor.submit(o3d.t.io.write_image, filename, self.rgbd_frame.depth) self.status_message = ( f"Saving RGBD images to {filename[:-3]}.{{jpg,png}}.")
class ma10CrossMa20Pair(BasePair): def __init__(self, coin_type, time_type,binance,stop_loss): self.coin_type = coin_type self.time_type = time_type self.threadpool = ThreadPoolExecutor(max_workers=1) self.strategy_type = config.MaCrossAverage self.stop_loss = stop_loss self.binance = binance #市场竞价-操作 def __market_broker(self,buy_flag, amount, coin_type,rate_close,rate_average,average_line): if(buy_flag): data=self.binance.get_order_book(symbol=coin_type,limit=5) bidPrice=data["bids"][0][0] two_str1=data["bids"][0][0][8:] two_str2=data["bids"][1][0][8:] bool_flag=re.match("00", two_str1) and re.match("00", two_str2) bidPrice=float(bidPrice)*(1+0.005) if(bool_flag != None): bidPrice=round(bidPrice,6) else: bidPrice=round(bidPrice,8) self.buy(bidPrice,coin_type,rate_close,rate_average,average_line,amount=amount) else: data=self.binance.get_order_book(symbol=coin_type,limit=5) askPrice=data["asks"][0][0] two_str1=data["asks"][0][0][8:] two_str2=data["asks"][1][0][8:] bool_flag=re.match("00", two_str1) and re.match("00", two_str2) askPrice=float(askPrice)*(1-0.005) if(bool_flag != None): askPrice=round(askPrice,6) else: askPrice=round(askPrice,8) self.sell(askPrice,average_line,amount=amount,coin_type=coin_type) def get_opportunity_ma10CrossMa20(self,data): #不顾一切 ma10,ma20,ma60=self.get_maLiner(data) jincha=CROSS(ma10,ma20) sicha=CROSS(ma20,ma10) jincha_flag=jincha.series[-1] and ma60.series[-1]>ma60.series[-2] sicha_flag=sicha.series[-1] if(jincha_flag ): futures = [] futures.append(self.threadpool.submit(self.__market_broker,jincha_flag, config.min_tx_volume, self.coin_type,rate_close=0,rate_average=0,average_line=0)) elif(sicha_flag): futures = [] futures.append(self.threadpool.submit(self.__market_broker,jincha_flag, config.min_tx_volume, self.coin_type,rate_close=0,rate_average=0,average_line=0)) def sell(self, askPirce,average_line,coin_type, amount=config.min_tx_volume): #对数据库进行更新操作 # 创建session对象: session = DBSession() #查询有没有买单的 #时间 update_date=time.strftime('%Y-%m-%d %H:%M:%S') order_result = session.query(Order).filter(Order.type == 0).filter(Order.coin_type == coin_type). \ filter(Order.end_date < update_date).filter(Order.time_type == self.time_type). \ filter(Order.strategy_type == self.strategy_type) count=order_result.count() if(count>0): #说明有成交的买单,那么就可以进行卖出操作了 #更新数据库 order=order_result.one() order.sell_price = askPirce order.buy_price=float(order.buy_price) order.profit_rate = askPirce/order.buy_price -1 order.sell_fee = (askPirce * amount)*0.001 order.profit = (float(order.sell_price)-float(order.buy_price))*(float(order.amount))- \ (float(order.buy_fee) + float(order.sell_fee)) order.type = 1 order.ask_order_id = "test_ask_004" # 添加到session: message="%s:%s执行下单操作成功,卖价为%.8f,数量为%.2f"%(self.coin_type,self.time_type,askPirce,amount) logger.warn(message) if(order.profit_rate>self.stop_loss and order.profit_rate<0): #亏损在0-1%中间就不要动 # 关闭session: session.close() return #亏损超过1%,卖出 # 提交即保存到数据库: if(config.market_game_start == 1): for pair_map in config.binance_game_coin_pairs: if(pair_map["symbol"] == self.coin_type): #实体竞技 try: balance_data_list=self.binance.get_account()["balances"] asset=self.coin_type.replace("BTC","") for value_map in balance_data_list: if(value_map["asset"]==asset): free=value_map["free"] free=float(free) amount=min(free,amount) break result=self.binance.order_limit_sell(symbol=self.coin_type, quantity=amount, price=askPirce ) order.ask_order_id = result["orderId"] message="下单 %s"%(order.ask_order_id) logger.warn(message) except Exception as e: traceback.print_exc() logger.fatal(e) session.commit() # 关闭session: session.close() def double_cross(self, average_line,average_data,bool_data): bool_data = NumericSeries(bool_data) ref_bool_data = REF(bool_data,1) select_bool = bool_data >ref_bool_data select_bool=select_bool.series[99-20:] ret=np.where(select_bool==True) size=ret[0].size if(size<2): return False average_line=average_line[100-20:] first_one_position = ret[0][-1] first_one = average_line[first_one_position] second_one_position = ret[0][-2] second_one = average_line[second_one_position] if(first_one>second_one and select_bool[-1]==True): return True return False def get_maLiner(self, data): X = np.array(data) my_close_list=np.array(X[:,4],dtype=np.float) my_close_list = pd.Series(my_close_list) close_data=np.array(my_close_list) close_data = NumericSeries(close_data) ma10=MA(close_data,10) ma20=MA(close_data,20) ma60=MA(close_data,60) return ma10,ma20,ma60
class NewDataHolder(object): def __init__(self): self.__data = {} self.__ignore = [] self.__buyed = [] self.__selled = [] self.__setting = Config() self.__tpe = ThreadPoolExecutor(5) self.__engine = create_engine(self.__setting.get_DBurl()) def get_buyed(self): return self.__buyed def add_buyed(self, code): self.__buyed.append(code) self.__tpe.submit(self.saveData, self.__data[code].get_data()) def get_selled(self): return self.__selled def add_selled(self, code, save=False): self.__selled.append(code) if save: self.__tpe.submit(self.saveData, self.__data[code].get_data()) def get_ignore(self): return self.__ignore def add_ignore(self, code, save=False): self.__ignore.append(code) if save: self.__tpe.submit(self.saveData, self.__data[code].get_data()) def get_data(self): return self.__data def addDataHandler(self, row): code = row['code'] if code in self.get_buyed() or code in self.get_ignore(): return if code in self.__data and self.__data[code].len() > 0: self.__data[code].add_Line(row) else: self.__data[code] = NewStock(code, row) if float(row['pre_close']) != 0 and (float(row['price']) - float( row['pre_close'])) / float(row['pre_close']) * 100 >= 9.9: MyLog.info('[%s] reach 10' % code) save = False if float(row['price']) != float(row['open']): save = True self.add_ignore(code, save=save) def addSellDataHandler(self, row): code = row['code'] if code in self.get_selled(): return if code in self.__data and self.__data[code].len() > 0: self.__data[code].add_Line(row) else: self.__data[code] = NewStock(code, row) def addData(self, df): df.apply(self.addDataHandler, axis=1) def addSellData(self, df): df.apply(self.addSellDataHandler, axis=1) def saveData(self, data): try: line = data[0] code = line['code'] df = pd.DataFrame(data) df.to_sql('live_' + code, con=self.__engine, if_exists='replace', index=False) MyLog.info('[%s] save data' % code) except Exception as e: MyLog.error('[%s %s] save [%s] data error \n' % (line['date'], line['time'], code)) MyLog.error(str(e) + '\n')
"//div[@class='user-tab active tab get-list']//i/text()"))) user_info['liked_num'] = int(''.join( share_douyin_content.xpath( "//div[@class='like-tab tab get-list']//i/text()"))) print(user_info) insert_mysql(user_info) # 处理抖音用户信息的主方法 def handle_douyin_web_share(user_id): share_douyin_url = 'https://www.douyin.com/share/user/{0}'.format(user_id) share_web_header = {'User-Agent': UserAgent().chrome} share_douyin_response = requests.get(url=share_douyin_url, headers=share_web_header) share_douyin_content = handle_font_decode(share_douyin_response.text) item_insert(share_douyin_content) # 创建队列,利用多线程请求数据库的抖音ID queue_list = Queue() pool = ThreadPoolExecutor(max_workers=20) for i in range(100): for j in range(100): share_id = get_random_id()['share_id'] queue_list.put(share_id) while queue_list.qsize() > 0: try: pool.submit(handle_douyin_web_share, queue_list.get()) except Exception as e: print(e.args)
class QA_Tdx_Executor(): def __init__(self, thread_num=2, *args, **kwargs): self.thread_num = thread_num self._queue = queue.Queue(maxsize=200) self.api_no_connection = TdxHq_API() self._api_worker = Thread(target=self.api_worker, args=(), name='API Worker') self._api_worker.start() self.executor = ThreadPoolExecutor(self.thread_num) def __getattr__(self, item): try: api = self.get_available() func = api.__getattribute__(item) def wrapper(*args, **kwargs): res = self.executor.submit(func, *args, **kwargs) self._queue.put(api) return res return wrapper except: return self.__getattr__(item) def _queue_clean(self): self._queue = queue.Queue(maxsize=200) def _test_speed(self, ip, port=7709): api = TdxHq_API(raise_exception=True, auto_retry=False) #api.need_setup = False _time = datetime.datetime.now() try: with api.connect(ip, port, time_out=0.05): if len(api.get_security_list(0, 1)) > 800: return (datetime.datetime.now() - _time).total_seconds() else: return datetime.timedelta(9, 9, 0).total_seconds() except Exception as e: #print('BAD IP {}, DEL for Reason{}'.format(ip,e)) return datetime.timedelta(9, 9, 0).total_seconds() def get_market(self, code): code = str(code) if code[0] in ['5', '6', '9'] or code[:3] in [ "009", "126", "110", "201", "202", "203", "204" ]: return 1 return 0 def get_level(self, level): if level in ['day', 'd', 'D', 'DAY', 'Day']: level = 9 elif level in ['w', 'W', 'Week', 'week']: level = 5 elif level in ['month', 'M', 'm', 'Month']: level = 6 elif level in ['Q', 'Quarter', 'q']: level = 10 elif level in ['y', 'Y', 'year', 'Year']: level = 11 elif str(level) in ['5', '5m', '5min', 'five']: level = 0 elif str(level) in ['1', '1m', '1min', 'one']: level = 8 elif str(level) in ['15', '15m', '15min', 'fifteen']: level = 1 elif str(level) in ['30', '30m', '30min', 'half']: level = 2 elif str(level) in ['60', '60m', '60min', '1h']: level = 3 return level @property def ipsize(self): return len(self._queue.qsize()) @property def api(self): return self.get_available() def get_available(self): if self._queue.empty() is False: return self._queue.get_nowait() else: Timer(0, self.api_worker).start() return self._queue.get() def api_worker(self): data = [] if self._queue.qsize() < 80: for item in info_ip_list: _sec = self._test_speed(item) if _sec < 0.1: self._queue.put( TdxHq_API(heartbeat=False).connect(ip=item, time_out=0.05)) else: self._queue_clean() Timer(0, self.api_worker).start() Timer(300, self.api_worker).start() def _singal_job(self, context, id_, time_out=0.5): try: _api = self.get_available() __data = context.append( self.api_no_connection.to_df( _api.get_security_quotes([ (self._select_market_code(x), x) for x in code[80 * id_:80 * (id_ + 1)] ]))) __data['datetime'] = datetime.datetime.now() self._queue.put(_api) # 加入注销 return __data except: return self.singal_job(context, id_) def get_realtime(self, code): context = pd.DataFrame() code = [code] if type(code) is str else code try: for id_ in range(int(len(code) / 80) + 1): context = self._singal_job(context, id_) data = context[[ 'datetime', 'code', 'open', 'high', 'low', 'price', 'ask1', 'ask_vol1', 'ask2', 'ask_vol2', 'ask3', 'ask_vol3', 'ask4', 'ask_vol4', 'ask5', 'ask_vol5' ]] return data.set_index('code', drop=False, inplace=False) except: return None def get_realtime_concurrent(self, code): code = [code] if type(code) is str else code try: # for id_ in range(int(len(code) / 80) + 1): data = { self.get_security_quotes([ (self.get_market(x), x) for x in code[80 * pos:80 * (pos + 1)] ]) for pos in range(int(len(code) / 80) + 1) } return (pd.concat([ self.api_no_connection.to_df(i.result()) for i in data ]), datetime.datetime.now()) except: pass def get_security_bar_concurrent(self, code, _type, lens): #code = [code] if type(code) is str else code try: #[api.get_security_bars(level, __select_market_code(str(code)), str(code), (25 - i) * 800, 800) for i in range(26)] data = {[ self.get_security_bars(self.get_level(_type), self.get_market(str(code)), str(code), (25 - i) * 800, 800) for i in range(int(lens / 800) + 1) ]} print([i.result() for i in data]) except: raise Exception def _get_security_bars(self, context, code, _type, lens): try: _api = self.get_available() for i in range(1, int(lens / 800) + 2): context.extend( _api.get_security_bars(self.get_level(_type), self.get_market(str(code)), str(code), (i - 1) * 800, 800)) #print(context) self._queue.put(_api) return context except Exception as e: #print(e) return self._get_security_bars(context, code, _type, lens) def get_security_bars(self, code, _type, lens): code = [code] if type(code) is str else code context = [] try: for item in code: context = self._get_security_bars(context, item, _type, lens) return context except Exception as e: raise e def save_mongo(self): pass
fakeua = {} fakeua['user-agent'] = random.choice(USER_AGENTS) try: sp = bs( rq.get('https://e-hentai.org/favorites.php', cookies=cookies, headers=fakeua).text, 'html.parser') except AttributeError: print("请检查cookie是否配置正确、ip是否被ban") # 以下逻辑仅为获取收藏数,如果发现获取失败,但确认cookies配置正确,可以手动删掉以下逻辑,配置 pagenum=[你的收藏数] favornum = int((sp.find(attrs={ 'name': 'favform' }).p.string.split(' ')[1]).replace(',', '').replace(' ', '')) pagenum = favornum // 50 + 1 urlList = ['https://e-hentai.org/favorites.php'] for i in range(1, pagenum): urlList.append('https://e-hentai.org/favorites.php?page=' + str(i)) # 此处三行表示使用大小为8的线程池并行导出 # 如果发现没有反应(即无法创建线程),可以使用串行版本如下: # for url in urlList: # getfavor(url) pool = ThreadPoolExecutor(8) for url in urlList: pool.submit(getfavor, url) input('task running...\n')
from concurrent.futures import ThreadPoolExecutor import time from log import log log(globals()) def wait_on_b(): log('b function') log('b-->', globals()) time.sleep(5) log(b.result()) return 5 def wait_on_a(): log('a function') log('a-->', globals()) time.sleep(5) log(a.result()) return 6 executor = ThreadPoolExecutor(max_workers=2) a = executor.submit(wait_on_b) b = executor.submit(wait_on_a)
class ZTO(object): def __init__(self): self.dh_list = [] # 保存单号的列表 self.task = None # 线程 self.data_info = [] # 保存获取到的信息,方便导出 self.data_erro = [] # 保存未成功获取的单号 self.pool = ThreadPoolExecutor(10) self.init_gui() def init_gui(self): self.root = tkinter.Tk() self.root_width = 500 # 窗口款多 self.root_height = 480 # 窗口高度 self.init_root_wind() # 初始化主窗口信息 header_frame = tkinter.Frame(self.root) header_frame.pack(fill=tkinter.X, pady=20, padx=20) header_frame_2 = tkinter.Frame(self.root) header_frame_2.pack(fill=tkinter.X, pady=12, padx=20) header_frame_3 = tkinter.Frame(self.root) header_frame_3.pack(fill=tkinter.X, pady=12, padx=50) # zto_session self.zto_session = tkinter.StringVar() tkinter.Label(header_frame, text='zto_session:').pack(side=tkinter.LEFT) self.zto_session_input = tkinter.Entry(header_frame, textvariable=self.zto_session, width=40) self.zto_session_input.pack(padx=12) # session tkinter.Label(header_frame_2, text=' session:').pack(side=tkinter.LEFT) self.session = tkinter.StringVar() self.session_input = tkinter.Entry(header_frame_2, textvariable=self.session, width=40) self.session_input.pack(padx=12) # 导入单号按钮, 短信接口获取, 查询接口获取 # 导入单号按钮 self.import_excel_btn = tkinter.Button(header_frame_3, text='导入单号', command=self.import_excel) self.import_excel_btn.pack(side=tkinter.LEFT, padx=12) self.select_api = tkinter.StringVar() tkinter.Radiobutton(header_frame_3, variable=self.select_api, text='短信接口获取', value=0).pack(side=tkinter.LEFT) tkinter.Radiobutton(header_frame_3, variable=self.select_api, text='查询接口获取', value=1).pack(side=tkinter.LEFT) self.select_api.set(0) # 开始检测按钮 self.start_btn = tkinter.Button(header_frame_3, text='开始获取', command=self.start_get) self.start_btn.pack(side=tkinter.LEFT, padx=12) # 展示信息的表格 # 商品表格的frame self.goods_Frame = tkinter.Frame(self.root) self.goods_Frame.pack(fill=tkinter.X) # 定义中心列表区域 self.info = ttk.Treeview(self.goods_Frame, show="headings", columns=('单号', '联系方式')) self.vbar = ttk.Scrollbar(self.goods_Frame, orient=tkinter.VERTICAL, command=self.info.yview) self.info.configure(yscrollcommand=self.vbar.set) self.info.column("单号", anchor="center", width=280) self.info.column("联系方式", anchor="center", width=200) self.info.heading("单号", text="单号") self.info.heading("联系方式", text="联系方式") self.info.grid(row=0, column=0, sticky=tkinter.NSEW) self.vbar.grid(row=0, column=1, sticky=tkinter.NS) # 导出信息按钮 self.export_excel_btn = tkinter.Button(self.root, text='导出信息', width=40, command=self.export_info) self.export_excel_btn.pack(pady=10) self.root.mainloop() # 初始化主窗口 def init_root_wind(self): self.root.title('中天系统v2.2') self.center_window() # 设置窗口剧中 self.root.resizable(width=False, height=False) # 禁止窗口拉伸 # 设置窗口剧中 def center_window(self): """ 设置窗口剧中 :param width: 窗口的宽度 :param height: 窗口的高度 :return: 自动设置窗口剧中 """ screenwidth = self.root.winfo_screenwidth() screenheight = self.root.winfo_screenheight() size = '%dx%d+%d+%d' % ( self.root_width, self.root_height, (screenwidth - self.root_width) / 2, (screenheight - self.root_height) / 2) self.root.geometry(size) # 导入excel表格 def import_excel(self): excel_file = askopenfilename() if excel_file: if not os.path.isfile(excel_file): messagebox.showinfo('导入失败', '导入失败当前文件不存在!') return if not excel_file.endswith('xlsx'): messagebox.showinfo('导入失败', '导入文件应该是"xlsx"文件!') return self.dh_list = [] workbook = openpyxl.load_workbook(excel_file) worksheet = workbook.active for row in worksheet.rows: value = row[0].value if value: self.dh_list.append(value) # temp_list = [str(row[0].value).strip() for row in worksheet.rows if str(row[0].value)] if self.dh_list: messagebox.showinfo('导入成功', '导入成功,当前导入单号数量: %s' % len(self.dh_list)) print(self.dh_list) return else: messagebox.showinfo('导入失败', '当前文件没有数据!') return # 开始获取每个单号联系方式的按钮事件 def start_get(self): # 判断单号是否导入 if not self.dh_list: messagebox.showerror('Erro', '请先导入单号!') return # 判断session是否输入 if not self.zto_session.get() or not self.session.get(): tkinter.messagebox.showerror('ERRO', 'zto_session 或 session未输入!') return # 判断是否有任务在执行 if self.task: if self.task.isAlive(): tkinter.messagebox.showerror('ERRO', '当前有任务正在进行,请等待任务结束') return if messagebox.askyesno('开始?', '确定开始获取?'): self.task = Thread(target=self.inner_task) self.task.start() # 线程内的函数 def inner_task(self): try: for _ in map(self.info.delete, self.info.get_children("")): pass # export_dh_list = [self.dh_list[i:i + 500] for i in range(0, len(self.dh_list), 500)] export_dh_list = self.dh_list self.data_info = [] self.data_erro = [] zto_session = self.zto_session.get().strip() session = self.session.get().strip() func = self.get_info_msg if int(self.select_api.get()) == 1: func = self.get_info result = [self.pool.submit(func, zto_session, session, dh) for dh in export_dh_list] wait(result) messagebox.showinfo('成功', '全部单号获取完毕,请导出查看') except Exception as e: messagebox.showerror('出现BUG', 'BUG信息:%s' % e) self.data_info = [] self.data_erro = [] # 导出信息按钮事件 def export_info(self): try: t = time.strftime('%Y-%m-%d-%H%M%S', time.localtime()) diretory = askdirectory() if diretory: success_file = os.path.join(diretory, '%s-success.xlsx' % t) workbook = openpyxl.Workbook() worksheet = workbook.active for dd in self.data_info: worksheet.append(dd) workbook.save(success_file) erro_file = os.path.join(diretory, '%s-erro.xlsx' % t) erro_workbook = openpyxl.Workbook() erro_sheet = erro_workbook.active for dd in self.data_erro: erro_sheet.append(dd) erro_workbook.save(erro_file) messagebox.showinfo('导出成功', '成功:%s\n失败:%s' % (success_file, erro_file)) except Exception as e: messagebox.showerror('BUG', '导出文件出现BUG:%s' % e) # 程序核心, 获取指定单号的联系人 def get_info(self, zto_session, session, dh): headers = { 'X-Canvas-Fingerprint': 'a5830a699863251ddd8fb1c4af8e336v', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36' } cookies = { 'com.zto.sessionid': zto_session, 'SESSION': session } try: response = requests.get( url='https://sso.zto.com/security-services/billtrack/billinfo-query-preauth?bill_id=%s&type=A014' % dh, headers=headers, cookies=cookies, timeout=10) time.sleep(0.1) print(response.text) # {"error":"no_perm_query:not_scan_node","error_description":"运单不属于贵网点","message":"该运单未在贵网点进行到件、发件、中转扫描或不是发放给贵网点使用的,不能进行「订单信息」查询操作。\n如您是客服中心或服务网点用户,请向您的主管确认您所在的网点具有此运单流经路由网点的查询权限。"} # {"ticket":"4He3WNFTEeiX3QBQVoRltw"} # {"error":"you_need_login_first","error_description":"需要登录","message":"您需要登录才可以继续此操作。"} data = json.loads(response.text) if data.get('error', None): info = data.get('error_description') self.data_erro.append([dh, info]) self.info.insert('', 0, values=(dh, info)) return dh, info headers_2 = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36', '_type': 'A014', '_token': data.get('ticket', '') } try: response = requests.get(url='https://newbill.zt-express.com/order-query/get?billCode=%s' % dh, headers=headers_2, cookies=cookies, timeout=10) time.sleep(0.1) print(dh, response.text) except TooManyRedirects as e: self.data_erro.append([dh, 'SESSION失效']) self.info.insert('', 0, values=(dh, 'SESSION失效')) return dh, 'SESSION失效' data2 = json.loads(response.text) if data2['status']: if data2.get('result', None): ii = data2['result'][0]['receiveInfo'] if ii == "R11:没有获取到收件人信息": self.data_erro.append([dh, ii]) self.info.insert('', 0, values=(dh, ii)) else: self.info.insert('', 0, values=(dh, ii)) self.data_info.append([dh, ii]) return dh, ii else: self.data_erro.append([dh, '暂无数据']) self.info.insert('', 0, values=(dh, '暂无数据')) return dh, '暂无数据' else: self.data_erro.append([dh, response.text]) self.info.insert('', 0, values=(dh, response.text)) return dh, data2 except Exception as e: info = '查询接口获取单号:%s出错:%s\n' % e self.data_erro.append([dh, info]) return dh, str(e) # 短信接口获取联系方式 def get_info_msg(self, zto_session, session, dh): try: headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36' } data = {"billCode": dh} cookies = { 'com.zto.sessionid': zto_session, 'SESSION': session } try: time.sleep(0.1) response = requests.post(url='https://sms.zto.com/mobile/billcode', json=data, cookies=cookies, timeout=10, headers=headers) except TooManyRedirects as e: self.data_erro.append([dh, 'SESSION失效']) self.info.insert('', 0, values=(dh, 'SESSION失效')) return dh, 'SESSION失效' info = json.loads(response.text) print(dh, info) # {"resultData":null,"requestStatus":false,"StatusMessage":"无法正确获取手机号"} # {"resultData":{"RecMobile":"13916437644","RecName":""},"requestStatus":true,"StatusMessage":"成功获取"} if info.get('requestStatus'): ii = info['resultData']['RecMobile'] self.data_info.append([dh, ii]) self.info.insert('', 0, values=(dh, ii)) return dh, ii else: self.data_erro.append([dh, '无法正确获取手机号']) self.info.insert('', 0, values=(dh, '无法正确获取手机号')) return dh, '无法正确获取手机号' except Exception as e: info = '短信接口获取单号:%s时候出错%s\n' % (dh, e) self.data_erro.append([dh, info]) return dh, str(e)
def main(): parser = argparse.ArgumentParser( description='Produce a single message taken from input') parser.add_argument('--server', type=str, metavar='HOST', default='localhost', help='Kafka bootstrap-server address') parser.add_argument('--port', type=int, metavar='PORT', default=9092, help='Kafka bootstrap-server port') parser.add_argument('--client', type=str, default='ch-kafka-python', help='custom client id for this producer') parser.add_argument('--topic', type=str, required=True, help='name of Kafka topic to store in') parser.add_argument('--retries', type=int, default=0, help='number of retries to send on failure') parser.add_argument('--multiply', type=int, default=1, help='multiplies incoming string many times') parser.add_argument('--repeat', type=int, default=1, help='send same (multiplied) message many times') mode_group = parser.add_mutually_exclusive_group() mode_group.add_argument('--jobs', type=int, default=multiprocessing.cpu_count(), help='number of concurrent jobs') mode_group.add_argument('--delay', type=int, metavar='SECONDS', default=0, help='delay before sending next message') args = parser.parse_args() config = { 'bootstrap_servers': f'{args.server}:{args.port}', 'client_id': args.client, 'retries': args.retries, } client = kafka.KafkaProducer(**config) message = sys.stdin.buffer.read() * args.multiply def send(num): if args.delay > 0: time.sleep(args.delay) client.send(topic=args.topic, value=message) print( f'iteration {num}: sent a message multiplied {args.multiply} times' ) if args.delay > 0: args.jobs = 1 pool = ThreadPoolExecutor(max_workers=args.jobs) for num in range(args.repeat): pool.submit(send, num) pool.shutdown() client.flush() client.close()
class EvaluationRunner(object): """ Class for efficiently evaluating an algorithm across multiple threads for every mixture in a dataset. If the algorithm requires something that must happen in a single thread, then the execution of the algorithm can happen in two steps - once on a blocking thread, and then continue on a parallel thread by specifying use_blocking_executor as True: .. code-block:: python # run on a blocking executor (e.g. on one GPU), in self.blocking_func features = algorithm.extract_features() # run the rest on the main executor, using the result self.blocking_func # this is in self.run_func algorithm.run(features=features) algorithm.make_audio_signals() This class is built assuming that you are using NUSSL style algorithms for source separation. If you're evaluating on other tasks (e.g. sound classification or music transcription), then you can use new testers or edit this class to your liking. If you're editing the class, your new code likely belongs in either main_func, run_func, or blocking_func. Args: testers ([type]): [description] algorithm_config ([type]): [description] dataset ([type]): [description] output_path ([type]): [description] max_workers (int, optional): [description]. Defaults to 1. use_blocking_executor (bool, optional): [description]. Defaults to False. """ def __init__(self, testers, algorithm_config, dataset, output_path, max_workers=1, use_blocking_executor=False): self.evaluation_executor = ThreadPoolExecutor(max_workers=max_workers) self.main_executor = ThreadPoolExecutor(max_workers=max_workers) self.dataset = dataset self.testers = testers self.algorithm_config = algorithm_config self.use_blocking_executor = use_blocking_executor self.output_path = os.path.join(output_path, 'results') os.makedirs(self.output_path, exist_ok=True) self.AlgorithmClass = getattr(algorithms, algorithm_config['class']) dummy_mixture = self.dataset.load_audio_files(self.dataset.files[0])[0] if self.use_blocking_executor: blocking_algorithm_config = copy.deepcopy( self.algorithm_config['args']) if 'use_cuda' in inspect.getargspec(self.AlgorithmClass).args: blocking_algorithm_config['use_cuda'] = True self.algorithm_config['args']['use_cuda'] = False args = inspect.getfullargspec(self.AlgorithmClass)[0] if 'extra_modules' in args: blocking_algorithm_config['extra_modules'] = model.extras self.blocking_algorithm = self.AlgorithmClass( dummy_mixture, **blocking_algorithm_config) self.blocking_executor = ThreadPoolExecutor(max_workers=1) def blocking_func(self, file_path): """ [summary] [extended_summary] Args: file_path ([type]): [description] Returns: [type]: [description] """ mixture, _, _ = self.dataset.load_audio_files(file_path) self.blocking_algorithm.set_audio_signal(mixture) self.blocking_algorithm._compute_spectrograms() features = self.blocking_algorithm.extract_features() return {'features': features} def log_scores(self, scores): """ [summary] [extended_summary] Args: scores ([type]): [description] """ for key in scores: if key != 'permutation': logging_str = f"{key}: " for metric in scores[key]: logging_str += f"{metric} => {np.mean(scores[key][metric])}, " logging.info(logging_str) def run_func(self, file_path, data=None): """ [summary] [extended_summary] Args: file_path ([type]): [description] data ([type], optional): [description]. Defaults to None. """ mixture, sources, labels = self.dataset.load_audio_files(file_path) classes = self.dataset.options['source_labels'] labels = [classes[np.argmax(l)] for l in labels] args = inspect.getfullargspec(self.AlgorithmClass)[0] algorithm_args = copy.deepcopy(self.algorithm_config['args']) if 'extra_modules' in args: algorithm_args['extra_modules'] = model.extras algorithm = self.AlgorithmClass(mixture, **algorithm_args) if data: for key in data: if key not in inspect.getargspec(algorithm.run).args: data.pop(key) algorithm.run(**data) else: algorithm.run() estimates = algorithm.make_audio_signals() tester_args = { 'true_sources_list': sources, 'estimated_sources_list': estimates, 'source_labels': labels } all_scores = [] try: for tester in self.testers: TestClass = tester[0] kwargs = tester[1] args = {} for k in tester_args: if k in inspect.getargspec(TestClass).args: args[k] = tester_args[k] args.update(kwargs) evaluator = TestClass(**args) scores = evaluator.evaluate() self.log_scores(scores) all_scores.append(scores) path_to_yml = os.path.join( self.output_path, os.path.splitext(os.path.basename(file_path))[0] + '.yml') logging.info(path_to_yml) with open(path_to_yml, 'w') as f: yaml.dump(all_scores, f) except: logging.exception() def main_func(self, file_path): """ [summary] [extended_summary] Args: file_path ([type]): [description] """ data = None if self.use_blocking_executor: task = self.blocking_executor.submit(self.blocking_func, file_path) data = task.result() self.evaluation_executor.submit(self.run_func, file_path, data) def run(self): """ [summary] [extended_summary] """ for file_path in self.dataset.files: self.main_executor.submit(self.main_func, file_path) self.main_executor.shutdown(wait=True) self.evaluation_executor.shutdown(wait=True)