def main(args): logdirs = tf.gfile.Glob(args.logdirs) print(len(logdirs), 'logdirs.') assert logdirs tf.gfile.MakeDirs(args.outdir) pool = multiprocessing.Pool(args.workers) lock = multiprocessing.Lock() pool.map(functools.partial(process_logdir, lock=lock, args=args), logdirs)
def test_api(client_key): # 1. create test image data and both processing and result queues urls = ['https://demo.restb.ai/images/demo/demo-1.jpg', 'https://demo.restb.ai/images/demo/demo-2.jpg', 'https://demo.restb.ai/images/demo/demo-3.jpg', 'https://demo.restb.ai/images/demo/demo-4.jpg', 'https://demo.restb.ai/images/demo/demo-5.jpg', 'https://demo.restb.ai/images/demo/demo-6.jpg'] queue = mp.Queue() image_id = 1 for url in urls: for model in __MODELS.keys(): queue.put(dict(id=image_id, url=url, model=model)) image_id += 1 results = mp.Queue() # 2. Pick which API endpoint to use (US vs. EU) url = __URL_US # 3. Define concurrency specific objects # stats objects lock_stats = mp.Lock() counter = mp.Value('i', 0) avg_req_time = mp.Value('f', 0) time_start = mp.Value('f', 999999999999999) time_end = mp.Value('f', 0) # 4. Spawn processes/threads to process the images in the queue pool = [] for i in range(__requests_per_second): # pass in necessary parameters to thread, including client key, etc. p = mp.Process(target=image_process_thread, args=(url, client_key, queue, results, lock_stats, counter, avg_req_time, time_start, time_end)) pool.append(p) p.start() # 5. clean-up after queue has been processed with "poison pill" while not queue.empty(): # wait for queue to be processed time.sleep(1) for i in pool: # seed shutdown messages / poison pills queue.put(dict(id=-1, url='shutdown', model='shutdown')) for p in pool: # enforce clean shutdown of threads p.join() # 6. finally, return accumulated results total = time_end.value - time_start.value print('[{requests}] requests processed in [{seconds}] seconds with average time [{time}] ms, total throughput: [{throughput}] rps'.format( requests=counter.value, seconds=str(round(total / 1000.0, 1)), time=str(round(avg_req_time.value / counter.value, 0)), throughput=str(round(counter.value / (total / 1000.0), 2)) )) return results
import fnmatch import functools import multiprocessing.dummy as multiprocessing import os import re import sys import traceback # import imageio import numpy as np import skimage.io import tensorflow as tf from tensorboard.backend.event_processing import (plugin_event_multiplexer as event_multiplexer) lock = multiprocessing.Lock() def safe_print(*args, **kwargs): with lock: print(*args, **kwargs) def create_reader(logdir): reader = event_multiplexer.EventMultiplexer() reader.AddRun(logdir, 'run') reader.Reload() return reader def extract_values(reader, tag):
TYC_HOST = 'https://www.tianyancha.com' SOGOU = [ '6b-f2--5-----ec---98-------034d1a-7--', '18-------b9fd7-c--203-------46-a-5e--', '-6-0--3----d-ea---f43b-7-c-8-219----5', '-70d--ac----f6-e-4b5--9-----3----21-8', '--3-1--45-c-7--2-a-e--b-8-6--0d----f9', '----3d658----b4------a01c9-27-e---f--', '8---d-7--4-f---2e9--5-1---63--b--ca-0', 'd49---5---c-6b----a-3------207--8f1-e', '7---65-------0-----f--9--12-4e8cbad-3', '1-8--fl52--9----e-d4-b-a-c--3--0--7-6'] BREAK_EVENT = mt.Event() _LOCK = mt.Lock() _DONE = mt.Value('i', 0) _FAIL = mt.Value('i', 0) _EMPTY = mt.Value('i', 0) logger = src.util.loginit.get_logger('tyc2') def get_login(): url = 'https://www.tianyancha.com/cd/login.json' login_json = {'mobile': '13606181270', 'cdpassword': '******', 'loginway': 'PL', 'autoLogin': True} """
class Runner(object): """ Summary: 执行模块 Attributes: __cases: case列表, case为组成webApiprocessor的参数元祖 __tasks: 任务列表, 将待执行的任务放入该列表 __results: 结果列表, 多线程收集结果的队列 """ _lock = multithreading.Lock() RUNNING = False # 当前任务是否正在运行的标记 def __init__(self, cases=None): """ :param cases: case 列表 """ super(Runner, self).__init__() self.__cases = cases self.__tasks = multithreading.JoinableQueue() self.__results = multithreading.Queue() def init_cases(self, cases): """ 初始化case列表 :param cases: :return: """ self.__cases = cases def producing(self): """ Summary: 任务生产者,生产任务 :return: """ if self.__cases: case_processors = map(interface_process.WebApiProcessor, self.__cases) map(self.__tasks.put, case_processors) return self.__tasks log.logger.error("任务列表为空") return None def worker(self): """ Summary: 消费 :return: """ while True: web_processor = self.__tasks.get() case_id = web_processor.get_case_id() case_name = web_processor.get_case_name() url = web_processor.get_case_url() log.logger.info("即将处理case:{},URL为{}".format(case_name, url)) web_processor.make_request() log.logger.info("完成处理case:{},URL为{}".format(case_name, url)) status = True # web_processor.assert_all_json_value() tests_result = web_processor.get_javascript_tests_result() if tests_result is None: # test_result为空 with self._lock: html_plugin.HTML_report_manager.add_error(case_name, error=u'返回数据为非json格式') log.logger.info("case:{}完成报告写入".format(case_name)) self.__results.put((case_id, case_name, "", False)) else: test_count = len(tests_result) if test_count == 0: with self._lock: html_plugin.HTML_report_manager.add_error(case_name, error=u'没有指定测试字段') log.logger.info("case:{}完成报告写入".format(case_name)) self.__results.put((case_id, case_name, "", False)) for result_key in tests_result: result_key_uni = result_key.decode('utf8') try: assert tests_result[result_key] status = True with self._lock: # 写入报告时,需要同步多线程 html_plugin.HTML_report_manager.add_success(case_name, result_key_uni, test_count) except AssertionError: # 验证失败 status = False with self._lock: html_plugin.HTML_report_manager.add_failure(case_name, result_key_uni, test_count) finally: log.logger.info("case:{}完成报告写入".format(case_name)) self.__results.put((case_id, case_name, result_key, status)) self.__tasks.task_done() # 任务池每完成一个任务,标记一次 return def create_consumers(self, customer_count=0): """ Summary: 创建消费者 :return: """ consumer_count = customer_count or multithreading.cpu_count() consumer_list = list() for _ in range(consumer_count): curr_consumer = multithreading.Process( target=self.worker, args=()) curr_consumer.daemon = True # 设置为守护进程,主进程结束则子进程结束 consumer_list.append(curr_consumer) curr_consumer.start() return consumer_list def get_results(self): """ Summary: :return: """ html_plugin.HTML_report_manager.finalize() # 报告结尾, 完成报告写入, 创建报告(包括时间戳) return self.__results def run(self, thread_num=3): """ 用例执行,在当前进程内同步执行 :param thread_num: 并发的线程数 :return: report_create_time: 测试报告创建时间 """ self.RUNNING = True self.create_consumers(thread_num) tasks = self.producing() tasks.join() results = self.get_results() report_create_time = html_plugin.HTML_report_manager.get_report_create_time() while not results.empty(): result = results.get_nowait() case_id, case_name, case_attr, status = result log.logger.info("第{}个case:\"{}\"的结果属性\"{}\"验证{}".format( case_id, case_name, case_attr, '成功' if status else '失败')) self.RUNNING = False return report_create_time def run_async(self, thread_num=3): """ 用例执行, 开启新的线程异步执行 """ new_thread = multithreading.Process(target=self.run, args=(thread_num,)) new_thread.start()
bookUrls = getBookUrls(url) safePrint(u'found %d books !\n' % len(bookUrls)) for bookName, bookUrl in filterBooksByUser(bookUrls): downloadBook(os.path.join(u'Books', bookName), bookUrl, servers) def filterBooksByUser(bookUrls): for name, url in bookUrls: safePrint(name + u'\n') while True: keyword = raw_input(u'input keyword(regex) to filter books:\n').decode( sys.getfilesystemencoding()) names = [name for name, url in bookUrls if re.search(keyword, name)] for name in names: safePrint(name + u'\n') if raw_input(u'is the book list ok?(y/n):\n') == 'y': break book2Url = dict(bookUrls) return [(name, book2Url[name]) for name in names] if __name__ != '__main__': exit(0) if len(sys.argv) < 2: print('Usage : python %s url' % sys.argv[0]) exit(0) g_gLock = dummy.Lock() g_pool = dummy.Pool(16) downloadBooks(sys.argv[1])
# vim:fileencoding=utf-8 from __future__ import unicode_literals import urllib2, urllib import multiprocessing.dummy as dummy import os, sys g_netHost = 'localhost:8080' g_netDirPath = '' g_localDirPath = '' g_lock = dummy.Lock() g_downloadedCount = 0 g_filePathList = [] def saveContent2File(filePath, ifile): filePath = g_localDirPath + filePath[len(g_netDirPath):] dirs, fname = os.path.split(filePath) if not os.path.isdir(dirs): os.makedirs(dirs) BUF_SIZE = 2**20 with file(filePath, 'wb') as ofile: while True: data = ifile.read(BUF_SIZE) if not data: break ofile.write(data) def downloadFile(filePath):
def __init__(self, maxsize=0, calls=1, per=1.0, fuzz=0): """ A thread safe priority queue with a given maximum size and rate limit. Prioritized items should be tuples of form (priority, item), with priority lowest first. Priority determines the order of items returned by get(). If `maxsize` is <= 0, the queue size is infinite (see `queue.LifoQueue`). The rate limit is described as `calls` `per` time window, with `per` measured in seconds. The default rate limit is 1 call per second. If `per` is <= 0, the rate limit is infinite. To avoid immediately filling the whole queue at startup, an extra randomized wait period can be configured with `fuzz`. This will cause the RateLimitQueue to wait between 0 and `fuzz` seconds before putting the object in the queue. Fuzzing only occurs if there is no rate limit waiting to be done. Parameters ---------- maxsize : int, optional, default 0 The number of slots in the queue, <=0 for infinite. calls : int, optional, default 1 The number of call per time unit `per`. Must be at least 1. per : float, optional, default 1.0 The time window for tracking calls, in seconds, <=0 for infinite rate limit. fuzz: float, options, default 0 The maximum length (in seconds) of fuzzed extra sleep, <=0 for no fuzzing Examples -------- Basic usage: >>> rlq = RateLimitPriorityQueue() >>> rlq.put((2, 'second')) >>> rlq.put((1, 'first')) >>> rlq.get() (1, 'first') >>> rlq.get() (2, 'second') A rate limit of 3 calls per 5 seconds: >>> rlq = RateLimitPriorityQueue(calls=3, per=5) A queue with the default 1 call per second, with a maximum size of 3: >>> rlq = RateLimitPriorityQueue(3) A queue of infinite size and rate limit, equivalent to queue.Queue(): >>> rlq = RateLimitPriorityQueue(per=0) A queue with wait time fuzzing up to 1 second so that the queue cannot be filled immediately directly after instantiation: >>> rlq = RateLimitPriorityQueue(fuzz=1) """ if calls < 1: raise ValueError("`calls` must be an integer >= 1") super().__init__(maxsize) self.calls = int(calls) self.per = float(per) self.fuzz = float(fuzz) self._call_log = queue.Queue(maxsize=self.calls) self._pending_get = mp.Lock()
def __init__(self, process: Process, underlying=dummy.Queue): self.process = process self._underlying = underlying() self._head = 1 self.lock = dummy.Lock()
def __init__(self, process: Process): self.value = 1 self.process = process self.lock = dummy.Lock()