class MKDockerClient(docker.DockerClient): '''a docker.DockerClient that caches containers and node info''' API_VERSION = "auto" _DEVICE_MAP_LOCK = multiprocessing.Lock() def __init__(self, config): super(MKDockerClient, self).__init__(config['base_url'], version=MKDockerClient.API_VERSION) all_containers = self.containers.list(all=True) if config['container_id'] == "name": self.all_containers = dict([(c.attrs["Name"].lstrip('/'), c) for c in all_containers]) elif config['container_id'] == "long": self.all_containers = dict([(c.attrs["Id"], c) for c in all_containers]) else: self.all_containers = dict([(c.attrs["Id"][:12], c) for c in all_containers]) self._env = {"REMOTE": os.getenv("REMOTE", "")} self._container_stats = {} self._device_map = None self.node_info = self.info() def device_map(self): with self._DEVICE_MAP_LOCK: if self._device_map is not None: return self._device_map self._device_map = {} for device in os.listdir('/sys/block'): with open('/sys/block/%s/dev' % device) as handle: self._device_map[handle.read().strip()] = device return self._device_map @staticmethod def iter_socket(sock, descriptor): '''iterator to recv data from container socket ''' header = sock.recv(8) while header: actual_descriptor, length = struct.unpack('>BxxxL', header) while length: data = sock.recv(length) length -= len(data) LOGGER.debug("Received data: %r", data) if actual_descriptor == descriptor: yield data header = sock.recv(8) def get_stdout(self, exec_return_val): '''read stdout from container process ''' if isinstance(exec_return_val, tuple): # it's a tuple since version 3.0.0 exit_code, sock = exec_return_val if exit_code not in (0, None): return '' else: sock = exec_return_val return ''.join(self.iter_socket(sock, 1)) def run_agent(self, container): '''run checkmk agent in container''' result = container.exec_run(['check_mk_agent'], environment=self._env, socket=True) return self.get_stdout(result) def get_container_stats(self, container_key): '''return cached container stats''' try: return self._container_stats[container_key] except KeyError: pass container = self.all_containers[container_key] if not container.status == "running": return self._container_stats.setdefault(container_key, None) stats = container.stats(stream=False) return self._container_stats.setdefault(container_key, stats)
def __init__(self, fd): self.reader = multiprocessing.connection.Connection(fd, writable=False) self.rlock = multiprocessing.Lock()
from utils import * from trainer import Trainer from model import * from sklearn.preprocessing import LabelBinarizer import numpy as np import multiprocessing as mp import time from sklearn.metrics import confusion_matrix lock = mp.Lock() counter = mp.Value('i', 0) def main(): # load 參數 config = get_args() # load 資料 X_train, y_train, X_val, y_val, X_test, y_test = load_data(config) # normalize X X_train, X_val, X_test = np.array(X_train) / 255, np.array( X_val) / 255, np.array(X_test) / 255 # 把 y 的 string 做成 one hot encoding 形式 label_encoder = LabelBinarizer() y_train_onehot = label_encoder.fit_transform(y_train) y_val_onehot = label_encoder.transform(y_val) y_test_onehot = label_encoder.transform(y_test) # 印一些有的沒的 print('X_train size:', len(X_train)) max_x = 0 for x in X_train: if max_x < len(x):
def __init__(self): super().__init__() self.note = Note("夜不语诡异档案") self.connection = database.open_connection() self.lock = mp.Lock()
with lock: fs = open(f, 'a+') n = 10 while n > 1: fs.write("Lockd acquired via with\n") n -= 1 fs.close() def worker_no_with(lock, f): lock.acquire() try: fs = open(f, 'a+') n = 10 while n > 1: fs.write("Lock acquired directly\n") n -= 1 fs.close() finally: lock.release() if __name__ == '__main__': lock = multiprocessing.Lock() f = 'file.txt' w = multiprocessing.Process(target=worker_with, args=(lock, f)) nw = multiprocessing.Process(target=worker_no_with, args=(lock, f)) w.start() nw.start() print "end!"
class BaseStore(MutableMapping): """The Base class for Daemon stores""" _kind = '' _status_model = StoreStatus _lock = multiprocessing.Lock() def __init__(self): self._logger = JinaLogger(self.__class__.__name__, **vars(jinad_args)) self.status = self.__class__._status_model() def add(self, *args, **kwargs) -> DaemonID: """Add a new element to the store. This method needs to be overridden by the subclass .. #noqa: DAR101""" raise NotImplementedError def update(self, *args, **kwargs) -> DaemonID: """Updates the element to the store. This method needs to be overridden by the subclass .. #noqa: DAR101""" raise NotImplementedError def delete(self, *args, **kwargs) -> DaemonID: """Deletes an element from the store. This method needs to be overridden by the subclass .. #noqa: DAR101""" raise NotImplementedError def clear(self) -> None: """Deletes an element from the store. This method needs to be overridden by the subclass .. #noqa: DAR101""" raise NotImplementedError def __iter__(self): return iter(self.status.items) def __len__(self): return len(self.status.items) def __repr__(self) -> str: return str(self.status.dict()) def keys(self) -> Sequence['DaemonID']: """Get keys in the store :return: Keys in the local store """ return self.status.items.keys() def values(self) -> Sequence[Union['WorkspaceItem', 'ContainerItem']]: """Get values in the store :return: Values in the local store """ return self.status.items.values() def items( self, ) -> Sequence[Tuple['DaemonID', Union['WorkspaceItem', 'ContainerItem']]]: """Get items in the store :return: Items in the local store """ return self.status.items.items() def __getitem__(self, key: DaemonID) -> Union['WorkspaceItem', 'ContainerItem']: """Fetch a Container/Workspace object from the store :param key: the key (DaemonID) of the object :return: the value of the object """ return self.status.items[key] def __setitem__(self, key: DaemonID, value: StoreItem) -> None: """Add a Container/Workspace object to the store :param key: the key (DaemonID) of the object :param value: the value to be assigned """ self.status.items[key] = value self.status.num_add += 1 self.status.time_updated = datetime.now() def __delitem__(self, key: DaemonID) -> None: """Release a Container/Workspace object from the store :param key: the key (DaemonID) of the object .. #noqa: DAR201""" self.status.items.pop(key) self.status.num_del += 1 self.status.time_updated = datetime.now() def __setstate__(self, state: Dict): self._logger = JinaLogger(self.__class__.__name__, **vars(jinad_args)) now = datetime.now() self.status = self._status_model(**state) self.status.time_updated = now def __getstate__(self) -> Dict: return self.status.dict() @classmethod def dump(cls, func) -> Callable: """Dump store as a pickle to local workspace :param func: function to be wrapped :return: decorator for dump """ def wrapper(self, *args, **kwargs): r = func(self, *args, **kwargs) filepath = os.path.join(__root_workspace__, f'{self._kind}.store') if Path(filepath).is_file(): shutil.copyfile(filepath, f'{filepath}.backup') with open(filepath, 'wb') as f: pickle.dump(self, f) return r return wrapper @classmethod def load(cls) -> 'BaseStore': """Load store from a pickle in local workspace :return: Store from local or empty store """ filepath = os.path.join(__root_workspace__, f'{cls._kind}.store') if Path(filepath).is_file() and os.path.getsize(filepath) > 0: with open(filepath, 'rb') as f: return pickle.load(f) else: return cls() def reset(self) -> None: """Calling :meth:`clear` and reset all stats """ self.clear() self.status = self._status_model()
def __init__(self): self.lock = multiprocessing.Lock()
def __init__(self): self.cpu = 0. self.wall = 0. self.lock = multiprocessing.Lock()
def fill_samples( meta: Meta, dataset: Dataset, max_samples=SAMPLES_PER_LANGUAGE, n_jobs=-1, filter_lang: T.Optional[str] = None, query_extra: T.Optional[str] = None, ): if n_jobs == -1: n_jobs = multiprocessing.cpu_count() q_req = multiprocessing.Queue(maxsize=20 * 400) q_resp = multiprocessing.Queue(maxsize=20 * 400) lock = multiprocessing.Lock() workers = [] for n in range(1, n_jobs + 1): p = multiprocessing.Process( name="harvest-worker-%d" % n, target=_worker, args=(meta, dataset, lock, q_req, q_resp), ) p.start() workers.append(p) # Search repos needed_samples = count_missing_samples(meta, dataset, max_samples) if filter_lang: needed_samples = {filter_lang: needed_samples[filter_lang]} seen_repos = get_seen_repos() all_repos = {} for lang, n_need in needed_samples.items(): if not n_need: continue all_repos[lang] = search_repos(lang=lang, n=n_need * 5, query_extra=query_extra, seen_repos=seen_repos) random.shuffle(all_repos[lang]) while True: n_added = 0 for lang, n_need in needed_samples.items(): if not n_need: continue repos = [r for r in all_repos[lang] if r not in seen_repos] repos_next, repos_remaining = repos[:n_need], repos[n_need:] logger.info(f"trying to get {len(repos_next)} repos for {lang}") for repo in repos_next: seen_repos.add(repo) q_req.put((repo, lang)) n_added += 1 if n_added == 0: logger.info("no more jobs to add") break for _ in range(n_added): repo, lang, ok = q_resp.get() if ok: needed_samples[lang] -= 1 q_req.close() q_resp.close() for p in workers: p.join()
def __init__(self): self._log = multiprocessing.get_logger() self._lock = multiprocessing.Lock()
def compute_voxel_depth(xyz, inner, outer, S2E_mat, method='equivolume', n_jobs=4, dtype=None, lock=None): ''' Parameters ---------- method : str "equivolume" "equidistance" Notes ----- 1. Unfortunately, dtype=np.float32 doesn't work for high density surface meshes (because the element area becomes zero in some locations, which is invalid). Although it does work for ordinary meshes, it is unnecessary in that case. ''' if isinstance(xyz, six.string_types): xyz = io.Mask(xyz, kind='full').xyz xyz = xyz.astype(dtype) if isinstance(S2E_mat, six.string_types): S2E_mat = afni.get_S2E_mat(S2E_mat, mat='S2B') S2E_mat = S2E_mat.astype(dtype) if method == 'equivolume': method = 'equivolume_inside' if lock is None: lock = multiprocessing.Lock() min_depth, max_depth = -0.2, 1.2 n_depths = round((max_depth - min_depth) / 0.1) + 1 alphas = np.linspace(min_depth, max_depth, n_depths, dtype=dtype) print('>> Compute intermediate meshes...') verts, faces = compute_intermediate_mesh(inner, outer, alphas, method=method, dtype=dtype) # 234s n_faces = faces.shape[0] LPI2RAI = np.array([-1, -1, 1], dtype=dtype) verts = np.dot(S2E_mat[:, :3], (verts * LPI2RAI).transpose( 0, 2, 1)).transpose(1, 2, 0) + S2E_mat[:, 3] face_xyz = (verts[:, faces[:, 0], :] + verts[:, faces[:, 1], :] + verts[:, faces[:, 2], :]).reshape(-1, 3) / 3 print('>> Construct k-d tree...') # kdt = spatial.cKDTree(face_xyz.reshape(-1,3)) # 171s kdt = spatial.cKDTree(face_xyz) # This is slightly more memory efficient depths = utils.SharedMemoryArray.zeros( xyz.shape[0], dtype=dtype, lock=False) # np.zeros(xyz.shape[0]) print('>> Compute cortical depth...') def compute_depth(ids, depths, xyz, kdt, verts, faces, alphas, n_faces, n_depths, min_depth, max_depth): for k in ids: p = xyz[k] # idx = np.argmin(np.linalg.norm(p - face_xyz, axis=-1)) // faces.shape[0] idx = kdt.query(p)[1] # This is like 4000x faster! fidx = idx % n_faces didx = idx // n_faces if didx == 0: depths[k] = min_depth elif didx == n_depths - 1: depths[k] = max_depth else: A, B, C = verts[didx - 1:didx + 2][:, faces[fidx, :], :].swapaxes( 0, 1) N = np.cross(B - A, C - A) N = N / np.linalg.norm(N, axis=-1, keepdims=True) T = np.sum(A * N, axis=-1) - np.sum(p * N, axis=-1) W = np.abs(T) if T[0] * T[1] < 0: w = W[1] / (W[0] + W[1]) depths[k] = w * alphas[didx - 1] + (1 - w) * alphas[didx] else: w = W[1] / (W[2] + W[1]) depths[k] = w * alphas[didx + 1] + (1 - w) * alphas[didx] with lock: pc = utils.PooledCaller(pool_size=n_jobs) pc( pc.run(compute_depth, ids, depths, xyz, kdt, verts, faces, alphas, n_faces, n_depths, min_depth, max_depth) for ids in pc.idss(len(depths))) return depths
class GmailService(IssueService): APPLICATION_NAME = 'Bugwarrior Gmail Service' SCOPES = 'https://www.googleapis.com/auth/gmail.readonly' DEFAULT_CLIENT_SECRET_PATH = '~/.gmail_client_secret.json' ISSUE_CLASS = GmailIssue CONFIG_PREFIX = 'gmail' AUTHENTICATION_LOCK = multiprocessing.Lock() def __init__(self, *args, **kw): super(GmailService, self).__init__(*args, **kw) self.query = self.config.get('query', 'label:Starred') self.login_name = self.config.get('login_name', 'me') self.client_secret_path = self.get_config_path( 'client_secret_path', self.DEFAULT_CLIENT_SECRET_PATH) credentials_name = clean_filename( self.login_name if self.login_name != 'me' else self.target) self.credentials_path = os.path.join( self.config.data.path, 'gmail_credentials_%s.json' % (credentials_name, )) self.gmail_api = self.build_api() def get_config_path(self, varname, default_path=None): return os.path.expanduser(self.config.get(varname, default_path)) def build_api(self): credentials = self.get_credentials() http = credentials.authorize(httplib2.Http()) return googleapiclient.discovery.build('gmail', 'v1', http=http) def get_credentials(self): """Gets valid user credentials from storage. If nothing has been stored, or if the stored credentials are invalid, the OAuth2 flow is completed to obtain the new credentials. Returns: Credentials, the obtained credential. """ with self.AUTHENTICATION_LOCK: log.info('Starting authentication for %s', self.target) store = oauth2client.file.Storage(self.credentials_path) credentials = store.get() if not credentials or credentials.invalid: log.info("No valid login. Starting OAUTH flow.") flow = oauth2client.client.flow_from_clientsecrets( self.client_secret_path, self.SCOPES) flow.user_agent = self.APPLICATION_NAME flags = oauth2client.tools.argparser.parse_args([]) credentials = oauth2client.tools.run_flow(flow, store, flags) log.info('Storing credentials to %r', self.credentials_path) return credentials def get_labels(self): result = self.gmail_api.users().labels().list( userId=self.login_name).execute() return {label['id']: label['name'] for label in result['labels']} def get_threads(self): thread_service = self.gmail_api.users().threads() result = thread_service.list(userId=self.login_name, q=self.query).execute() return [ thread_service.get(userId='me', id=thread['id']).execute() for thread in result.get('threads', []) ] def annotations(self, issue): sender = issue.extra['last_sender_name'] subj = issue.extra['subject'] issue_url = issue.get_processed_url(issue.extra['url']) return self.build_annotations([(sender, subj)], issue_url) def issues(self): labels = self.get_labels() for thread in self.get_threads(): issue = self.get_issue_for_record(thread, thread_extras(thread, labels)) extra = { 'annotations': self.annotations(issue), } issue.update_extra(extra) yield issue
def __init__(self): self.tlock = threading.Lock() self.mlock = multiprocessing.Lock()
def commencementdujeu(nbjoueurs): decklist = [] cartesonboard = [] keys = [] mqverifT = [] lockverif = mp.Lock() deck = mp.Queue(20) # On crée un deck mélangé for i in range(10): decklist.append(Cards(i + 1, "bleu")) decklist.append(Cards(i + 1, "rouge")) random.shuffle(decklist) # On le convertit ensuite en queue afin de pouvoir le partager à tous les process for card in decklist: deck.put(card) cartesonboard.append(deck.get()) mqtouchejouee1 = sysv_ipc.MessageQueue mqtouchejouee2 = sysv_ipc.MessageQueue mqcards = sysv_ipc.MessageQueue mqhandsize = sysv_ipc.MessageQueue mqaffichage = sysv_ipc.MessageQueue try: mqtouchejouee1 = sysv_ipc.MessageQueue(201, sysv_ipc.IPC_CREX) except sysv_ipc.ExistentialError: sysv_ipc.MessageQueue(201).remove() mqtouchejouee1 = sysv_ipc.MessageQueue(201, sysv_ipc.IPC_CREX) try: mqtouchejouee2 = sysv_ipc.MessageQueue(202, sysv_ipc.IPC_CREX) except sysv_ipc.ExistentialError: sysv_ipc.MessageQueue(202).remove() mqtouchejouee2 = sysv_ipc.MessageQueue(202, sysv_ipc.IPC_CREX) try: mqcards = sysv_ipc.MessageQueue(100, sysv_ipc.IPC_CREX) except sysv_ipc.ExistentialError: sysv_ipc.MessageQueue(100).remove() mqcards = sysv_ipc.MessageQueue(100, sysv_ipc.IPC_CREX) try: mqhandsize = sysv_ipc.MessageQueue(300, sysv_ipc.IPC_CREX) except sysv_ipc.ExistentialError: sysv_ipc.MessageQueue(300).remove() mqhandsize = sysv_ipc.MessageQueue(300, sysv_ipc.IPC_CREX) try: mqaffichage = sysv_ipc.MessageQueue(400, sysv_ipc.IPC_CREX) except sysv_ipc.ExistentialError: sysv_ipc.MessageQueue(400).remove() mqaffichage = sysv_ipc.MessageQueue(400, sysv_ipc.IPC_CREX) # Initialisation des différents process (board et les joueurs) board = Board(deck, cartesonboard, nbjoueurs) for i in range(nbjoueurs): keys.append(i + 1) try: mqverif = sysv_ipc.MessageQueue(keys[i], sysv_ipc.IPC_CREX) mqverifT.append(mqverif) except sysv_ipc.ExistentialError: sysv_ipc.MessageQueue(keys[i]).remove() # Démarrage des process board.start() # print('Lancement de la lecture de clavier non bloquante') touchesjoueurs = [['a', 'z', 'e', 'r', 't', 'y', 'u', 'i', 'o'], ['1', '2', '3', '4', '5', '6', '7', '8', '9']] try: kb = KBHit() while not deck.empty(): if kb.kbhit(): c = kb.getch() if c in touchesjoueurs[0]: mqtouchejouee1.send( str((touchesjoueurs[0].index(c) + 1)).encode()) elif c in touchesjoueurs[1]: mqtouchejouee2.send(c.encode()) else: print("Veuillez entrer une touche valide") mqcards.remove() mqtouchejouee1.remove() mqtouchejouee2.remove() mqhandsize.remove() mqaffichage.remove() for x in mqverifT: x.remove() os.killpg(os.getpgid(board.pid), signal.SIGTERM) except KeyboardInterrupt: print("Le grand nettoyage") mqcards.remove() mqtouchejouee1.remove() mqtouchejouee2.remove() mqhandsize.remove() mqaffichage.remove() for x in mqverifT: x.remove() os.killpg(os.getpgid(board.pid), signal.SIGTERM)
def parallel_try(func, args=None, kwargs=None, nworkers=None, progress=None, progress_interval=0.1, update_interval=0.05, initializer=None, join=True, use_threads=False): """ Execute `func` in parallel until it returns a value. Return that value. `func` the function to call `args` the arguments to `func` `kwargs` the keyword arguments to `func` `nworkers` the number of worker processes (/threads) to use `progress` a function to periodically call with the number of tries already performed `progress_interval` the approximate time interval to call the `progress` function `update_interval` the approximate interval in which the worker threads report their status --- a higher values gives better performance, but worse latency (if `join` is True) `initializer` called in each worker process when spawn, with (args, kwargs) as arguments. `initializer` may change args, kwargs. `join` specifies whether to wait for all threads to finish `use_threads` specifies to use threads instead of processes. """ def worker(c_func, c_args, c_kwargs, c_lock, c_done, c_output, c_counter, c_initializer): try: if c_initializer is not None: c_initializer(c_args, c_kwargs) last_update = time.time() iterations = 0 while True: now = time.time() if now - last_update > update_interval: last_update = now with c_lock: if c_done.is_set(): return c_counter.value += iterations iterations = 0 ret = c_func(*c_args, **c_kwargs) iterations += 1 if ret is None: continue with c_lock: if c_done.is_set(): return c_output.put(ret) c_done.set() except KeyboardInterrupt: pass if args is None: args = () if kwargs is None: kwargs = {} if nworkers is None: nworkers = multiprocessing.cpu_count() p_done = multiprocessing.Event() p_lock = multiprocessing.Lock() p_counter = multiprocessing.RawValue('i', 0) p_input = multiprocessing.Queue() processes = [] constr = threading.Thread if use_threads else multiprocessing.Process try: for i in xrange(nworkers): process = constr(target=worker, args=(func, args, kwargs, p_lock, p_done, p_input, p_counter, initializer)) processes.append(process) process.start() if progress is None: p_done.wait() else: while not p_done.is_set(): p_done.wait(progress_interval) progress(p_counter.value) if join: for process in processes: process.join() except KeyboardInterrupt: for process in processes: process.terminate() raise return p_input.get()
class RemoteManager(object): NODES = {} LOCK = mp.Lock() PORT_ID = mp.Value('i', 8700) MASTER_IP = None __instance = None def __new__(cls): # Singleton if cls.__instance is None: cls.__instance = object.__new__(cls) cls.MASTER_IP = get_ip() cls.start_local_node() return cls.__instance @classmethod def get_master_node(cls): return cls.NODES[cls.MASTER_IP] @classmethod def start_local_node(cls): port = cls.get_port_id() with warning_filter(): remote = Remote(cls.MASTER_IP, port, local=True) with cls.LOCK: cls.NODES[cls.MASTER_IP] = remote @classmethod def launch_each(cls, launch_fn, *args, **kwargs): for node in cls.NODES.values(): node.submit(launch_fn, *args, **kwargs) @classmethod def get_remotes(cls): return list(cls.NODES.values()) @classmethod def upload_files(cls, files, **kwargs): if isinstance(files, str): files = [files] for node in cls.NODES.values(): node.upload_files(files, **kwargs) @classmethod def add_remote_nodes(cls, ip_addrs): ip_addrs = [ip_addrs] if isinstance(ip_addrs, str) else ip_addrs remotes = [] for node_ip in ip_addrs: if node_ip in cls.NODES.keys(): logger.warning('Already added remote {}'.format(node_ip)) continue port = cls.get_port_id() remote = Remote(node_ip, port) with cls.LOCK: cls.NODES[node_ip] = remote remotes.append(remote) return remotes @classmethod def shutdown(cls): for node in cls.NODES.values(): node.shutdown() cls.NODES = {} cls.__instance = None @classmethod def get_port_id(cls): with cls.LOCK: cls.PORT_ID.value += 1 return cls.PORT_ID.value def __enter__(self): return self def __exit__(self, *args): for node in cls.NODES.values(): node.shutdown() def __repr__(self): reprstr = self.__class__.__name__ + '(\n' for node in self.NODES.values(): reprstr += '{}, \n'.format(node) reprstr += ')\n' return reprstr
#!/usr/bin/python2.7 #coding:utf-8 import multiprocessing manager = multiprocessing.Manager() # targets target_lock = multiprocessing.Lock() # done_targets = [] # undone_targets =[] done_targets = manager.list() undone_targets = manager.list() willdone_targets = manager.list() # main scan task # scan_task_dict just like # {'pid': 3280, # 'scanID': None, # 'subtargets': {}, # 'target': 'http://www.leesec.com/'} scan_task_dict = {} scan_task_dict_lock = multiprocessing.Lock() # for each sub scan task
def __init__(self, name, config, prepped_frame_queue, mqtt_client, mqtt_config, ha_config): self.name = name self.config = config self.detected_objects = [] self.recent_frames = {} self.rtsp_url = get_rtsp_url(self.config['rtsp']) self.regions = self.config.get('regions', {}) self.frame_shape = get_frame_shape(self.rtsp_url) self.mqtt_client = mqtt_client # compute the flattened array length from the shape of the frame flat_array_length = self.frame_shape[0] * self.frame_shape[ 1] * self.frame_shape[2] # create shared array for storing the full frame image data self.shared_frame_array = mp.Array(ctypes.c_uint8, flat_array_length) # create shared value for storing the frame_time self.shared_frame_time = mp.Value('d', 0.0) # Lock to control access to the frame self.frame_lock = mp.Lock() # Condition for notifying that a new frame is ready self.frame_ready = mp.Condition() # Condition for notifying that objects were parsed self.objects_parsed = mp.Condition() # shape current frame so it can be treated as a numpy image self.shared_frame_np = tonumpyarray(self.shared_frame_array).reshape( self.frame_shape) self.capture_process = None # for each region, create a separate thread to resize the region and prep for detection self.detection_prep_threads = [] for region in self.config.get('regions', []): # set a default threshold of 0.5 if not defined if not 'threshold' in region: region['threshold'] = 0.5 if not isinstance(region['threshold'], float): print('Threshold is not a float. Setting to 0.5 default.') region['threshold'] = 0.5 self.detection_prep_threads.append( FramePrepper(self.name, self.shared_frame_np, self.shared_frame_time, self.frame_ready, self.frame_lock, region['size'], region['x_offset'], region['y_offset'], region['threshold'], prepped_frame_queue)) # start a thread to store recent motion frames for processing self.frame_tracker = FrameTracker(self.shared_frame_np, self.shared_frame_time, self.frame_ready, self.frame_lock, self.recent_frames) self.frame_tracker.start() # start a thread to store the highest scoring recent person frame self.best_person_frame = BestPersonFrame(self.objects_parsed, self.recent_frames, self.detected_objects) self.best_person_frame.start() # start a thread to expire objects from the detected objects list self.object_cleaner = ObjectCleaner(self.objects_parsed, self.detected_objects) self.object_cleaner.start() # start a thread to publish object scores (currently only person) if mqtt_client: self.mqtt_topic_prefix = '{}/{}'.format( mqtt_config.get(CONF_MQTT_TOPIC, ""), self.name) mqtt_publisher = MqttPublisher(self.name, self.mqtt_client, self.mqtt_topic_prefix, self.objects_parsed, self.detected_objects) mqtt_publisher.start() if ha_config: ha_publisher = HomeAssistantPublisher( self.name, ha_config[CONF_HOME_ASSISTANT_URL], ha_config[CONF_HOME_ASSISTANT_TOKEN], self.objects_parsed, self.detected_objects) ha_publisher.start() # create a watchdog thread for capture process self.watchdog = CameraWatchdog(self) # load in the mask for person detection if 'mask' in self.config: self.mask = cv2.imread("/config/{}".format(self.config['mask']), cv2.IMREAD_GRAYSCALE) else: self.mask = np.zeros((self.frame_shape[0], self.frame_shape[1], 1), np.uint8) self.mask[:] = 255
def plot(self): """ This function will plot analysis data as a funciton of the number of images. uses multiprocessing to speed things up :return: void """ a = analysis_concurrent(self.y_start, self.y_stop, self.x_start, self.x_stop, self.selection) trunc_list = [] cpu_count = 10 #multiprocessing.cpu_count() temp_list = [] for i in range(0, cpu_count): if i == cpu_count - 1: temp_list = self.tif_list[( i * len(self.tif_list) // cpu_count):(((1 + i) * len(self.tif_list) // cpu_count) + (len(self.tif_list) % cpu_count))] temp_list.insert(0, i) else: temp_list = self.tif_list[(i * len(self.tif_list) // cpu_count):((1 + i) * len(self.tif_list) // cpu_count)] temp_list.insert(0, i) trunc_list.append(temp_list) # print(self.check_lists(self.tif_list, trunc_list, cpu_count)) process_list = [] x = range(0, len(self.tif_list)) y = [] q = multiprocessing.Queue() #a = multiprocessing.Array() l = multiprocessing.Lock() #p = multiprocessing.Process(a.x_and_y_vals, args=(l,)) for i in range(0, cpu_count): process_list.append( multiprocessing.Process(target=a.x_and_y_vals, args=(l, q, trunc_list[i]))) for process in process_list: process.start() for process in process_list: y.append(q.get()) process.join() # for i in range(0,cpu_count): # y.append(q.get()) y = self.selectionSort(y) flattened_y = [val for sublist in y for val in sublist] assert len(flattened_y) == len(self.tif_list) plt.scatter(x, flattened_y) plt.xlabel("file num") plt.ylabel(self.selection) # plt.xscale() plt.show()
# -*- coding: utf-8 -*- """ Created on Wed Apr 11 08:56:17 2018 @author: busby """ import multiprocessing as mp import time def job(v, num, l): l.acquire() for _ in range(10): time.sleep(0.1) v.value += num print(v.value) l.release() if __name__ == '__main__': l = mp.Lock() v = mp.Value('i', 0) p1 = mp.Process(target=job, args=(v, 1, l)) p2 = mp.Process(target=job, args=(v, 3, l)) p1.start() p2.start() p1.join() p2.join()
def __init__(self, logger_manager, filename=None, cam_ip=None, width=1920, height=1200, auto_expose=True, auto_balance=True): self.logger = logger_manager.get_logger("BQ_Cam") self.sched_run = None if filename is None: # 设置相机模式标识。 self.mode = 0 # 相机模式分辨率自设,默认1920*1200 (self.w, self.h) = (width, height) self.cam_ip = cam_ip self.auto_expose = auto_expose self.auto_balance = auto_balance self.logger.debug("网络相机模式,IP: {}, (w, h): ({}, {}).".format( cam_ip, self.w, self.h)) else: # 设置文件模式标识。 self.mode = 1 # 文件模式获取分辨率 (self.w, self.h) = gige.get_resolution(filename) self.logger.debug("文件读取模式: {}, (w, h): ({}, {}).".format( filename, self.w, self.h)) self.width = self.w self.height = self.h # 准备进程锁和进程间共享空间。 self.process_lock = multiprocessing.Lock() self.array_temp = np.ones(shape=(self.h * self.w * 3), dtype=np.ubyte) self.shared_array = RawArray(ctypes.c_ubyte, self.array_temp) self.shared_value = RawValue(ctypes.c_uint, 0) self.logger.debug("进程共享内存准备完毕。") if self.mode == 0: self.sched_run = schedrun.SchedRun( func=gige.get_frame_from_camera, args=( self.shared_array, self.shared_value, self.process_lock, False, ), init_func=gige.init_camera, init_args=(self.cam_ip, self.w, self.h, self.auto_expose, self.auto_balance), clean_func=gige.close_camera, clean_args={}, interval=0.0, init_interval=0.0) else: self.sched_run = schedrun.SchedRun(func=gige.get_frame_from_file, args=( self.shared_array, self.shared_value, self.process_lock, ), init_func=gige.init_file, init_args=(filename, ), clean_func=gige.close_file, clean_args={}, interval=0.025, init_interval=0.0) self.logger.debug("帧读取进程启动。") #while self.shared_value != 0: for i in range(10): if self.shared_value == 0: self.logger.debug("帧读取进程没准备好,等待100ms。") # 相机进程还没有准备好。 time.sleep(0.1) if self.shared_value == 0: return None
The solution: Have one thread (process via multiprocessing) poll the buttons at 60 Hz and generate events via multiprocessing's shared queues. We can also do rendering in this process (Note: this is starting to sound like an X11 server ...) """ import multiprocessing import time import logging import lcd import random import time logging.basicConfig(level=logging.INFO) LCD_WIDTH, LCD_HEIGHT = lcd.LCD_WIDTH, lcd.LCD_HEIGHT framebuffer_lock = multiprocessing.Lock() framebuffer = multiprocessing.Array('b', LCD_WIDTH * LCD_HEIGHT, lock=False) framebuffer_needs_redraw = multiprocessing.Value('b') eventqueue = multiprocessing.Queue() def server_main(): """ Read input and render at a fixed frame rate of 60 Hz. """ ALL_KEYS = [ fakelcd.KEY_LEFT, fakelcd.KEY_RIGHT, fakelcd.KEY_UP, fakelcd.KEY_DOWN, fakelcd.KEY_CENTER ] logger = logging.getLogger('server') logger.info('starting up')
def main(args): # variables fp = None sequence = "" cline = "" lreads = [] rreads = [] lheaders = [] header = "" result = 0 seq_temp = "" fout_stub = args.stub ltemp = [] dmap = {} using_map = False min_seq_len = 0 made_reads = 0 num_keys = 0 num_threads = 0 num_tasks = 0 i = 0 j = 0 result = [] # create result queue for parallel threads # processes = [] qtasks = mp.Queue() qresult = mp.Queue() lock = mp.Lock() fai = args.infile + ".fai" dfai = {} if args.pseudo_map and args.prob_map: sys.stderr.write("Error: you can't specify both --probability-map and --pseudo-map\n") return 1 # # check input file # if not file_exists(args.infile): sys.stderr.write("Error: input file doesn't exist!\n") return(1) #} # # check for fasta index # if not file_exists(fai): sys.stderr.write("> Cannot find fasta index, rebuilding now...\n") p1 = Popen("samtools faidx {:s}".format(args.infile).split()) p1.wait() #} # hash fasta index sys.stderr.write("> Hashing fasta index...") fin = open(fai, "r") for szl in fin: ll = szl.strip().split("\t") dfai[ll[0]] = list(map(int, ll[1:])) #} fin.close() sys.stderr.write("done\n") # # set minimum reference length based on read length or fragment length if args.make_paired: min_seq_len = args.frag_len * 3 else: min_seq_len = args.read_length * 3 #} # # check if there is a map file # if args.map is not None or args.make_random_map > 0: if args.map is not None: # hash map file using_map = True fin = open(args.map, "r") for szl in fin: ll = szl.strip().split("\t") if len(ll) > 1: dmap[ll[0]] = float(ll[1]) else: dmap[ll[0]] = args.coverage #} #} fin.close() else: # # Generate a random set of N features where N = args.make_random_map. # # make a new dict of fasta ref ids that are longer than the minimum # sequence length dfai_temp = {} for rid in dfai.keys(): if dfai[rid][0] > min_seq_len: # keep it dfai_temp[rid] = dfai[rid][0] #} #} if len(dfai_temp.keys()) < args.make_random_map: # not enough references are long enough to make the requested number of random # refs sys.stderr.write("Warning: only {:d} references are long enough to sample\n") args.make_random_map = len(dfai_temp.keys()) #} # make a random set rset = sample(dfai_temp.keys(), args.make_random_map) dcounts = [] if args.pseudo_map: dcounts = make_multinom_counts(args.total_reads, args.make_random_map) temp = sorted(dcounts) temp.reverse() dcounts = temp #print dcounts dmap = {} i = 0 for rid in rset: if args.prob_map: dmap[rid] = dfai[rid][0] elif args.pseudo_map: if args.make_paired: dmap[rid] = dcounts[i]*args.read_length*2.0/dfai[rid][0] else: dmap[rid] = dcounts[i]*args.read_length*1.0/dfai[rid][0] i += 1 else: dmap[rid] = args.coverage #} #} #} # check list against the fasta index for rid in dmap.keys(): if rid in dfai: # check length against minimum length if dfai[rid][0] < min_seq_len: # remove feature dmap.pop(rid, None) sys.stderr.write("Warning: removing {:s} from simulation because it's too short ({:d})\n".format(rid, dfai[rid][0])) #} #} #} if args.prob_map: # values dmap are intended to be proportions of reads. first we have to # make them sum to one and convert them to total read counts n_sum = 0 for rid in dmap.keys(): n_sum += dmap[rid] n_ratio = 1.0/n_sum for rid in dmap.keys(): dmap[rid] = dmap[rid]*n_ratio * args.total_reads if dmap[rid] < 1: dmap[rid] = 1 # scale by read length (or read length x2 for paired) if args.make_paired: dmap[rid] *= args.read_length*2 else: dmap[rid] *= args.read_length # now the total base count is converted to coverage based on the # feature's length (from the fasta index) dmap[rid] = dmap[rid]/dfai[rid][0] #} #} elif args.make_random_map == -1: # for this mode all transcripts that are long enough to be sampled will be used and a semi-realistic count # distribution will be randomly generated. sys.stderr.write("> generating multinomial count distribution...") # first figure out which references can generate reads. this will be the dmap dmap = {} count_prob = [] for rid in sorted(dfai.keys()): if dfai[rid][0] > min_seq_len: # keep it dmap[rid] = dfai[rid][0] count_prob.append(10**r.rnorm(1, 2.5, 1)[0]-0.01) if count_prob[-1] < 0: count_prob[-1] = 0 # generate random multinomial count distribution count_levels = r.rmultinom(1, args.total_reads, count_prob) # copy count_levels into the dmap and convert to coverages (they get converted back into counts # in the read generating functions) i =0 for rid in sorted(dmap.keys()): if args.make_paired: dmap[rid] = count_levels[i]*args.read_length*2.0/dfai[rid][0] else: dmap[rid] = count_levels[i]*args.read_length*1.0/dfai[rid][0] i += 1 sys.stderr.write("done\n") else: # no map file so we'll take all features for rid in dfai.keys(): dmap[rid] = args.coverage #} #} # # split off extention from input file # ltemp = os.path.basename(args.infile).split(".") # fout_stub = ".".join(ltemp[0:(len(ltemp)-1)]) # # open output file(s) if args.make_paired: fp1 = open(fout_stub + ".1.fq","w") fp2 = open(fout_stub + ".2.fq","w") else: fp = open(fout_stub + ".fq","w") sys.stderr.write("> Generating reads...\n") # # open fasta file and get to work # fpin = open(args.infile,"r") klist = dmap.keys() num_keys = len(klist) i = 0 num_tasks = 0 processes = [] # make workers for i in range(args.num_threads): mp.Process(target=worker, args=(qtasks, qresult, lock)).start() # load up the tasks in the queue so the workers have something to do for rid in dmap.keys(): if dfai[rid][0] >= min_seq_len and dmap[rid] > 0: # print "sending %s to workers" % rid # pull sequence sequence = fetch_ref_seq(fpin, dfai, rid).upper() # push into queue if args.make_paired: qtasks.put((make_paired_reads, (rid, sequence, args.read_length, dmap[rid], args.frag_len, args.frag_sdev, args.error_rate))) else: qtasks.put((make_reads, (rid, sequence, args.read_length, dmap[rid], args.error_rate))) # count tasks num_tasks += 1 # retrieve results and print out for i in range(num_tasks): result = qresult.get() # print "received %d" % i if args.make_paired: write_paired_reads(fp1, fp2, result) else: write_reads(fp, result) # tell the processes to stop for i in range(args.num_threads): # print "stopping..." qtasks.put('STOP') fpin.close() if args.make_paired: fp1.close() fp2.close() else: fp.close() return 0
REQUIRED_CONFIG_KEYS = { 'tap': ['bucket', 'start_date'], 'target': [ 'account', 'dbname', 'user', 'password', 'warehouse', 's3_bucket', 'stage', 'file_format', ], } LOCK = multiprocessing.Lock() def tap_type_to_target_type(csv_type): """Data type mapping from S3 csv to Snowflake""" return { 'integer': 'INTEGER', 'number': 'FLOAT', 'string': 'VARCHAR', 'boolean': 'VARCHAR', # The guess sometimes can be wrong, we'll use varchar for now. 'date': 'VARCHAR', # The guess sometimes can be wrong, we'll use varchar for now. 'date_override': 'TIMESTAMP_NTZ', # Column type to use when date_override defined in YAML
MANIPULATION = "ON" #ON/OFF Manipulation of preferences # Thresholds # To = 0.25 #Overlap time (hours) Tl = 5 #Off-routing (percentage) Tm = 1 #Meal size (kilograms) Ta = 20 #Extra payload (percentage) Tpm = 20 #Default perishable food travel distance (kilometers) Tpnm = 5 #Default perishable food travel distance (kilometers) Tnp = 100 #Default non-perishable food travel distance (kilometers) Td = 2 #Process advance start threshold for donors (hours) Tr = 3 #Process advance start threshold for receivers (hours) Tw = 10 #Match acceptance window (minutes) # Do not change # LOCK = multiprocessing.Lock() #Multiprocessing lock CPU_COUNT = multiprocessing.cpu_count() #Logical CPUs MEMORY = math.ceil(psutil.virtual_memory().total / (1024.**3)) #RAM capacity DATA_LOAD_LOCATION = os.path.dirname(sys.argv[0]) + "/" #Data load location DATA_STORE_LOCATION = os.path.dirname(sys.argv[0]) + "/" #Data store location ## Function definitions ## # Print with lock # def print_locked(*content, sep=" ", end="\n"): store = DATA_STORE_LOCATION with open(store + "_Log_File.txt", "a") as log_file:
pass turn_flag.value = 1 gyro_arr = [] if stop_key == True: break #main bus = smbus.SMBus(1) address = 0x68 # via i2cdetect bus.write_byte_data(address, power_mgmt_1, 0) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) #s.connect((HOST,PORT)) mutex = mp.Lock() distance = mp.Value("d", 0) turn = mp.Value("i", 0) turn_flag = mp.Value("i", 0) dis_flag = mp.Value("i", 0) p = mp.Process(target=get_bes, args=(mutex, distance, dis_flag)) p1 = mp.Process(target=check_turning, args=(mutex, turn, turn_flag)) p.start() p1.start() try: while True: #print(mp.current_process())
def __init__(self, fd): self.writer = multiprocessing.connection.Connection(fd, readable=False) self.wlock = multiprocessing.Lock() # Why bb.event needs this I have no idea self.event = self
def color_trace_multi(inputs, outputs, colors, processcount, tracemasksdir, quantization='mc', dither=None, remap=None, stack=False, prescale=2, despeckle=2, smoothcorners=1.0, optimizepaths=0.2): """color trace input images with specified options inputs: list of input paths, source png files outputs: list of output paths, dest svg files colors: number of colors to quantize to, 0 for no quantization processcount: number of process to launch for image processing quantization: color quantization algorithm to use: - 'mc' = median-cut (default, for few colors, uses pngquant) - 'as' = adaptive spatial subdivision (uses imagemagick, may result in fewer colors) - 'nq' = neuquant (for lots of colors, uses pngnq) dither: dithering algorithm to use. (Remember, final output is affected by despeckle.) None: the default, performs no dithering 'floydsteinberg': available with 'mc', 'as', and 'nq' 'riemersma': only available with 'as' palette: source of custom palette image for color reduction (overrides colors and quantization) stack: whether to stack color traces (recommended for more accurate output) despeckle: supress speckles of this many pixels smoothcorners: corner smoothing: 0 for no smoothing, 1.334 for max optimizepaths: Bezier curve optimization: 0 for least, 5 for most """ tmp = tempfile.mkdtemp() # create a two job queues # q1 = scaling + color reduction q1 = multiprocessing.JoinableQueue() # q2 = isolation + tracing q2 = multiprocessing.JoinableQueue() # create a manager to share the layers between processes manager = multiprocessing.Manager() layers = [] for i in range(min(len(inputs), len(outputs))): layers.append(manager.list()) # and make a lock for reading and modifying layers layers_lock = multiprocessing.Lock() # create a shared memory counter of completed and total tasks for measuring progress progress = multiprocessing.Value('i', 0) if colors is not None: # this is only an estimate because quantization can result in less colors # than in the "colors" variable. This value is corrected by q1 tasks to converge # on the real total. total = multiprocessing.Value('i', len(layers) * colors) elif remap is not None: # get the number of colors in the palette image palettesize = len(make_palette(remap)) # this is only an estimate because remapping can result in less colors # than in the remap variable. This value is corrected by q1 tasks to converge # on the real total. total = multiprocessing.Value('i', len(layers) * palettesize) else: #argparse should have caught this raise Exception( "One of the arguments 'colors' or 'remap' must be specified") # create and start processes processes = [] for i in range(processcount): p = multiprocessing.Process(target=process_worker, args=(q1, q2, progress, total, layers, layers_lock, locals())) p.name = "color_trace worker #" + str(i) p.start() processes.append(p) try: # so for each input and (dir-appended) output... for index, (i, o) in enumerate(zip(inputs, outputs)): verbose(i, ' -> ', o) # add a job to the first job queue q1.put({ 'input': i, 'output': o, 'findex': index, 'tracemasksdir': tracemasksdir }) # show progress until all jobs have been completed # while progress.value < total.value: # sys.stdout.write("\r%.1f%%" % (progress.value / total.value * 100)) # sys.stdout.flush() # time.sleep(0.25) sys.stdout.write("\rTracing complete!\n") # join the queues just in case progress is wrong q1.join() q2.join() except (Exception, KeyboardInterrupt) as e: # shut down subproesses for p in processes: p.terminate() shutil.rmtree(tmp) raise e # close all processes for p in processes: p.terminate()
# --*--coding:utf-8 # Author:cnn from time import sleep import multiprocessing g_num = 0 # 创建进程锁 mutex = multiprocessing.Lock() # 多任务--进程,第二种实现方式 class MutiProcess(object): def update(self, num): global g_num for i in range(0, num + 1): # 上锁 mutex.acquire() g_num += i mutex.release() print(g_num) sleep(1) def main(self): m1 = multiprocessing.Process(target=self.update, args=(100, )) m2 = multiprocessing.Process(target=self.update, args=(200, )) m1.start() m2.start() if __name__ == '__main__': mu = MutiProcess()
def main(): options = get_args() label = options.label if options.label else options.config prefix = '.{}.{}'.format(options.scheduler, options.agent.lower()) config_dir = os.path.abspath(options.config_dir) config_file = os.path.join(config_dir, options.config + '.sumocfg') additional_file = os.path.join(config_dir, label + prefix + '.additional.xml') log_dir = os.path.abspath(options.log_dir) log_file = os.path.join(log_dir, label + prefix + '.summary.log') trip_file = os.path.join(log_dir, label + prefix + '.trip.xml') tls_switch_file = os.path.join(log_dir, label + prefix + '.tlsSwitch.xml') if options.save_output: debug_file = os.path.join(log_dir, label + prefix + '.log') logging.basicConfig(filename=debug_file, level=logging.DEBUG) with open(options.info_file, 'r') as fp: info = json.load(fp) with open(additional_file, 'w') as fp: fp.write('<additional>') for tlsID in info: fp.write( '<timedEvent type="SaveTLSSwitchStates" source="{}" dest="{}"/>' .format(tlsID, tls_switch_file)) fp.write('</additional>') sumo_server = multiprocessing.Process(target=start_sumo, args=(config_file, log_file, trip_file, additional_file, options.port)) sumo_server.start() logger.debug("Loaded tlsinfo") sets = [] for sample_set in options.samples: with open(sample_set, 'r') as fp: samples = pickle.load(fp) sets.append(samples) processes = [] manager = multiprocessing.Manager() shared_results = manager.dict() shared_lock = multiprocessing.Lock() registry = adasco.registry.Registry() agent_module = importlib.import_module('adasco.agents.{}.{}'.format( options.scheduler, options.agent.lower())) agent_class = getattr(agent_module, '{}Agent'.format(options.agent)) for tls in info: shared_results[tls] = manager.dict() Y = {int(phase): value for phase, value in info[tls]['Y'].items()} Gmin = { int(phase): value for phase, value in info[tls]['Gmin'].items() } Gmax = { int(phase): value for phase, value in info[tls]['Gmax'].items() } request_queue = multiprocessing.JoinableQueue() response_queue = multiprocessing.JoinableQueue() process = agent_class( ID=info[tls]['id'], phases=info[tls]['phases'], Y=Y, Gmin=Gmin, Gmax=Gmax, incoming_edges=info[tls]['incoming_edges'], outgoing_edges=info[tls]['outgoing_edges'], edge_lengths=info[tls]['edge_lengths'], turn_proportions=info[tls]['turn_proportions'], upstream_agents=info[tls]['upstream_agents'], startup_lost_time=info[tls]['startup_lost_time'], free_flow_speed=info[tls]['free_flow_speed'], headway=info[tls]['headway'], saturation_flow_rate=info[tls]['saturation_flow_rate'], time_resolution=info[tls]['time_resolution'], sampling_interval=info[tls]['sampling_interval'], merging_threshold=info[tls]['merging_threshold'], horizon_extension=info[tls]['horizon_extension'], extension_threshold=info[tls]['extension_threshold'], minimum_extension=info[tls]['minimum_extension'], shared_results_directory=shared_results, shared_lock=shared_lock, publish_directory=shared_results[tls], coordinate=options.coordinate, request_queue=request_queue, response_queue=response_queue, sample_count=options.sample_count, timelimit=options.timelimit, samples=sets) processes.append(process) first_phase = info[tls]['phases'][0] entry = adasco.registry.Entry(process, request_queue, response_queue, None, 0, Gmin[first_phase]) registry.append(entry) master = adasco.master.Master(options.port, registry) processes.append(master) for process in processes: process.start() for process in processes: process.join()