class ImageLogger(object): def __init__(self, dev_id): config = ConfigParser.ConfigParser() config.read(path.dirname(path.realpath(__file__))+"/condor.ini") storage_root = config.get('condor', 'storage') self.storage = path.join(storage_root, str(dev_id)) if not path.isdir(self.storage): mkdir(self.storage) self.l = Lock() def image(self, img, name =''): self.l.acquire() if not name: name = str(uuid.uuid4()) imgpath = path.join(self.storage, "%s.png" % (name)) if path.isfile(imgpath): i = 1 imgpath = path.join(self.storage, "%s-%s.png" % (name, i)) while path.isfile(imgpath): i += 1 imgpath = path.join(self.storage, "%s-%s.png" % (name, i)) cv2.imwrite(imgpath,img) self.l.release()
class ClientProcess(Process): def __init__(self): Process.__init__(self) self.parent, self.child = Pipe() self.state = Lock() self.start() def run(self): while 1: active_clients = self.child.recv() self.state.acquire() client_socket = socket.fromfd(reduction.recv_handle(self.child), socket.AF_INET, socket.SOCK_STREAM) self.client_process(client_socket, active_clients) self.state.release() def client_process(self, sock, active_clients): while 1: data = protocol.recv(sock) if not data: return sock.close() encryption = TripleDESCipher(data["key"]) if data["encryption"] == 1: protocol.send(sock, (active_clients, encryption.encrypt(data["message"]))) else: protocol.send(sock, (active_clients, encryption.decrypt(data["message"]))) def add_task(self): if self.state.acquire(False): self.parent.send(True) self.state.release() return True return False
def get_module(self): lock = Lock() try: lock.acquire() return load_module("hook_chef", self.hook_path) finally: lock.release()
def main(): print("Start main()") lock = Lock() # hoge_lock = Lock() # piyo_lock = Lock() hoge_pipe1, hoge_pipe2 = Pipe() hoge_subprocess = Process(target=hoge, args=(hoge_pipe2, lock)) # hoge_subprocess = Process(target=hoge, args=(hoge_pipe2, hoge_lock)) piyo_pipe1, piyo_pipe2 = Pipe() piyo_subprocess = Process(target=piyo, args=(piyo_pipe2, lock)) # piyo_subprocess = Process(target=piyo, args=(piyo_pipe2, piyo_lock)) hoge_subprocess.start() piyo_subprocess.start() while True: lock.acquire( ) ## If this statement enables, doesn't work this program. hoge_pipe1.send("send from main() to hoge()") string = hoge_pipe1.recv() print("Time:{}, Excute main():{}".format(time.time(), string)) # time.sleep(1) piyo_pipe1.send("send from main() to piyo()") string = piyo_pipe1.recv() print("Time:{}, Excute main():{}".format(time.time(), string)) # time.sleep(1) lock.release( ) ## If this statement enables, doesn't work this program.
class DbTools: def __init__(self, db_name='db.sqlite'): self.DB_Name = db_name self.db = sqlite3.connect(self.DB_Name, check_same_thread=False) self.cur = self.db.cursor() self.lock = Lock() def write(self, q, val, do_commit=True, many=False): self.lock.acquire(True) if many: self.cur.executemany(q, val) last_row_id = None else: last_row_id = self.cur.execute(q, val).lastrowid if do_commit: self.db.commit() self.lock.release() return last_row_id def read(self, q, val=(), f_all=True): self.lock.acquire(True) if f_all: res = self.cur.execute(q).fetchall() if val == () else self.cur.execute(q, val).fetchall() else: res = self.cur.execute(q).fetchone() if val == () else self.cur.execute(q, val).fetchone() self.lock.release() return res
class SMS_split(object): '''split the sms_list for sending ''' def __init__(self,package_size,phone_list): self.phone_list = phone_list self.package_size = package_size self.package = [elem for elem in range(package_size)] self._lock = Lock() def __iter__(self): #the number of sms which already have be splited self.current_spot = 0 return self def next(self): self._lock.acquire() try: if (self.current_spot >= len(self.phone_list)): self.current_spot = len(self.phone_list) raise StopIteration self.package = self.phone_list[self.current_spot : \ self.current_spot+self.package_size] self.current_spot += self.package_size finally: self._lock.release() return self.package def set_package_size(self, package_size): self.package_size = package_size def get_package_size(self): return self.package_size def get_already_send_num(self): return self.current_spot
def determine_violence_make_request(violence_judgement_array): lock = Lock() determinant = 0 try: while True: lock.acquire() if len(violence_judgement_array) == 5: for each_judgement in violence_judgement_array: if each_judgement == 'violence': determinant = determinant + 1 if determinant >= 3: print("post request Occur") with requests.Session() as session: with session.get( url, params={'location': 'police hospital'}) as response: print("response status_code - " + str(response.status_code)) print("response text - " + str(response.text)) determinant = 0 del violence_judgement_array[:] lock.release() except KeyboardInterrupt: pass
class MLock: def __init__(self): self.lock = Lock() self.rlock = RLock() def _acquire(self): # 需要添加对锁获取次数的判断 self.lock.acquire() def _release(self): self.lock.release() def _rlacquire(self): try: self.rlock.acquire() except Exception as e: printlog.err(e) finally: pass def _rlrelase(self): try: self.rlock.release() except Exception as e: printlog.err(e) finally: pass
class XmlBackend(Backend): """ ini file backend for configg """ def __init__(self, path: str): super().__init__(path) self.lock = Lock() def read(self) -> Dict[str, Any]: root = ET.parse(self.path).getroot() data = {} for child in root: data[child.tag] = {} for child2 in child: data[child.tag][child2.tag] = child2.text return data def write(self, data: Dict[str, Any]) -> None: sections = [] for section, section_data in data.items(): xml = "<{}>".format(section) for key, value in section_data.items(): xml += "<{key}>{value}</{key}>".format(key=key, value=value) xml += "</{}>".format(section) sections.append(xml) xml_data = "<config>" for xml in sections: xml_data += xml xml_data += "</config>" self.lock.acquire(False) with open(self.path, "w+") as fp: fp.truncate() fp.seek(0) fp.write(xml_data) self.lock.release()
class CacheLock: def __init__(self): self._cache_lock = Lock() def acquire_lock(self): g.logger.info("Acquiring Lock... {}".format(g.web_protocol)) self._cache_lock.acquire() g.logger.info("Opening Cache") if 'AWS_ACCESS_KEY_ID' not in session: g.logger.info("Can't open cache files, no AWS key available") return 404 log_entries_file = "logEntriesCache.{}.data".format( session['AWS_ACCESS_KEY_ID']) log_streams_file = "logStreamsCache.{}.data".format( session['AWS_ACCESS_KEY_ID']) g.logEntriesCache = shelve.open(log_entries_file, writeback=True) g.logStreamsCache = shelve.open(log_streams_file, writeback=True) g.logger.info("{}".format(g.logEntriesCache)) g.logger.info("{}".format(g.logStreamsCache)) g.logger.info("Acquired {}".format(g.web_protocol)) def release_lock(self): g.logger.info("Release Lock...{}".format(g.web_protocol)) g.logEntriesCache.close() g.logStreamsCache.close() self._cache_lock.release() g.logger.debug("Released {}".format(g.web_protocol))
def launch_workers(outfile, start_index, end_index, score_flag, force, verbose): BASE_URL = "http://www.ign.com/games/all-ajax?startIndex=" # Synchronization Tools num_workers = Semaphore(MAX_NUM_PROCESSES) outfile_lock = Lock() urlopen_lock = Lock() stderr_lock = Lock() print_lock = Lock() # Write the categories if (outfile != None): outfile.write("title,link,platform,publisher,score,date\n") # Launch the workers processes = [] curr_index = start_index; while curr_index <= end_index: curr_url = BASE_URL + str(curr_index) worker = Process(target=open_url_and_parse, args=(outfile, curr_url, score_flag, force, verbose, outfile_lock, urlopen_lock, stderr_lock, print_lock, num_workers)) processes.append(worker) if verbose: print_lock.acquire() print "Launching worker for url: %s" % curr_url print_lock.release() num_workers.acquire() worker.start() curr_index += INDEX_INCREMENT; for p in processes: p.join()
class MyQueue(): def __init__(self, name, maxsize=30): self.queue = Queue() self.name = name self.maxsize = maxsize self.lock = Lock() self.put_lock = Lock() def qput(self, item): # print("queue", self.name, self.queue.qsize()) self.put_lock.acquire() try: while self.queue.qsize() >= self.maxsize: item = self.queue.get() self.queue.put(item) except: pass self.put_lock.release() def qget(self, timeout=None): # print("queue", self.name, self.queue.qsize()) self.lock.acquire() try: item = self.queue.get(timeout=timeout) while self.queue.qsize() > self.maxsize: item = self.queue.get() except: item = None try: self.lock.release() except: pass return item
def worker(q: JoinableQueue, i: int, output, print_lock: Lock, FLAGS: Tuple[Any]) -> None: """Retrieves files from the queue and annotates them.""" if FLAGS.in_memory: with open(FLAGS.alias_db, 'rb') as f: alias_db = pickle.load(f) with open(FLAGS.relation_db, 'rb') as f: relation_db = pickle.load(f) with open(FLAGS.wiki_db, 'rb') as f: wiki_db = pickle.load(f) else: alias_db = SqliteDict(FLAGS.alias_db, flag='r') relation_db = SqliteDict(FLAGS.relation_db, flag='r') wiki_db = SqliteDict(FLAGS.wiki_db, flag='r') annotator = Annotator(alias_db, relation_db, wiki_db, distance_cutoff=FLAGS.cutoff, match_aliases=FLAGS.match_aliases, unmatch=FLAGS.unmatch, prune_clusters=FLAGS.prune_clusters) while True: logger.debug('Worker %i taking a task from the queue', i) json_data = q.get() if json_data is None: break annotation = annotator.annotate(json_data) print_lock.acquire() output.write(json.dumps(annotation) + '\n') print_lock.release() q.task_done() logger.debug('Worker %i finished a task', i)
class Logger(): def __init__(self, filename, filemode='w', *arg, **kwargs): from multiprocessing import Lock from datetime import datetime self.filename = filename self.datetime = datetime self.on = True self.screenon = False self.lock = Lock() self.lock.acquire(True) f = open(self.filename, filemode) f.close() self.lock.release() self.write('Start logging.') def write(self, content): self.lock.acquire(True) if self.on: f = open(self.filename, 'a') if isinstance(content, str): f.write(self.datetime.now().strftime('%Y-%m-%d %H:%M:%S ') + content.strip() + '\n') if self.screenon: print(self.datetime.now().strftime('%Y-%m-%d %H:%M:%S ') + content.strip()).decode('utf-8') else: for i in content: f.write( self.datetime.now().strftime('%Y-%m-%d %H:%M:%S ') + i.strip() + '\n') if self.screenon: print(self.datetime.now().strftime( '%Y-%m-%d %H:%M:%S ') + i.strip()).decode('utf-8') f.close() self.lock.release()
class WSRunner(Process): def __init__(self, url, d): Process.__init__(self) self.url = url self.d = d self.lock = Lock() self.lock.acquire() @asyncio.coroutine def ws(self): websocket = yield from websockets.connect(self.url) update_count = 0 self.lock.release() while update_count < num_loc_updates * num_threads: update = yield from websocket.recv() update_count += 1 j = json.loads(update) lat = float(j['lat']) lng = float(j['lng']) player_id = j['player_id'] self.d[player_id] = (lat, lng) websocket.close() def run(self): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) loop.run_until_complete(self.ws())
def fetch_to_db(process_count=1, from_date=datetime(2011, 1, 1)): setup_tables() unit_handler = UnitHandler() s = get_virk_search(from_date) try: tmp_file = tempfile.NamedTemporaryFile(delete=False) m = IOQueueManager() m.start() queue = m.IOQueue(tmp_file.name) queue_lock = Lock() consumer_partial = functools.partial(consumer_insert, queue_lock=queue_lock, unit_handler=unit_handler) processes = [Process(target=consumer_partial, args=(queue,), daemon=True) for _ in range(process_count)] for p in processes: p.start() engine.dispose() # for multiprocessing. producer_scan(s, queue, queue_lock=queue_lock) queue_lock.acquire() for end in range(process_count): queue.put('DONE') queue_lock.release() for p in processes: p.join() finally: os.remove(tmp_file.name) pass return
def say_hello(name='world', **kwargs): """@todo: Docstring for say_hello """ l = Lock() l.acquire() print "Hello, %s" % name, kwargs l.release()
class ReoderBuffer(object): def __init__(self, out_buffer, size=10): self.out_buffer = out_buffer self.size = size self._array = Manager().list(range(self.size)) self._lock = Lock() self._min = 0 self._max = self.size self._count = 0 self.update_control() def update_control(self): self._control = dict() for e, i in enumerate(xrange(self._min, self._max)): self._control.update({i: e}) def put(self, block_num, data): self._lock.acquire() if block_num not in self._control.keys(): self._lock.release() raise IndexError self._array[self._control[block_num]] = data self._count += 1 if self._count == self.size: self.sync() self._lock.release() def sync(self): for i in xrange(self._count): data = self._array[i] self.out_buffer.put(data) self._min += self._count self._max += self._count self._count = 0 self.update_control()
class Database: def __init__(self, filename): self.connection = sqlite3.connect(filename) self.connection.text_factory = lambda x: unicode( x, UTF8, 'ignore') # ignore bad unicode shit self.cursor = self.connection.cursor() self.cursor.executescript(SCHEMA_SQL) self.now = time.strftime('%Y%m%d%H%M%S', time.gmtime()) self.lock = Lock() def commit(self): self.connection.commit() def close(self): self.commit() self.connection.close() def summary(self, url, limit=DETECTOR_HISTORY): params = {'url': url, 'limit': limit} rows = self.cursor.execute(SUMMARY_SQL, params) return rows.fetchall() def insert(self, rowhash): rowhash['run'] = self.now self.lock.acquire() # BEGIN PRIVILEGED CODE self.cursor.execute(INSERT_SQL, rowhash) self.commit() self.lock.release() # END PRIVILEGED CODE
class ReoderBuffer(object): def __init__(self, out_buffer, size=10): self.out_buffer = out_buffer self.size = size self._array = Manager().list(range(self.size)) self._lock = Lock() self._min = 0 self._max = self.size self._count = 0 self.update_control() def update_control(self): self._control = dict() for e, i in enumerate(xrange(self._min, self._max)): self._control.update({i: e}) def put(self, block_num, data): self._lock.acquire() if block_num not in self._control.keys(): self._lock.release() raise IndexError self._array[self._control[block_num]] = data self._count += 1 if self._count == self.size: self.sync() self._lock.release() def sync(self): for i in xrange(self._count): data = self._array[i] self.out_buffer.put(data) self._min += self._count self._max += self._count self._count = 0 self.update_control()
def setup(): """Performs basic setup for the daemon and ADC""" global exitLock logger.debug("Setting up ADC") try: ADC.setup() except RuntimeError as e: logger.critical( "Attempting to start the BBB GPIO library failed. This can be " "due to a number of things, including:" ) logger.critical( "- Too new a kernel. Adafruit BBIO runs on 3.8.13. Downgrades " "to the version this is tested with can be done easily via:") logger.critical( " apt-get install linux-{image,headers}-3.8.13-bone79") logger.critical("- Not running on a BBB") logger.critical("- Conflicting capes") logger.critical("Raw exception: %s", str(e)) return tstat = connect() # TODO: retries logger.debug("Attaching signal handlers") signal.signal(signal.SIGINT, handle_exit) signal.signal(signal.SIGTERM, handle_exit) logger.debug("Building Lock for singal interrupts") exitLock = Lock() exitLock.acquire() logger.debug("Running main loop") return tstat
def add_mention_db(tweet): print(tweet) lock = Lock() db = Collector.mention_tweet_db() c = db[0] conn = db[1] lock.acquire() c.execute( '''INSERT INTO mentioned (tweet, username, tweet_date, tweet_id, tweet_source, user_id, reply_id) VALUES(?,?,?,?,?,?,?)''', [ tweet[0], tweet[1], tweet[2], tweet[3], tweet[4], tweet[5], tweet[6] ]) conn.commit() lock.release()
class Atomic(object): def __init__(self): self._lock = Lock() self._in_transaction = False self._transaction_thread = None def __enter__(self): self.begin_transaction() return self def __exit__(self, exc_type, exc_val, exc_tb): self.end_transaction() return self def in_transaction(self): return self._in_transaction def begin_transaction(self): self._in_transaction = True self._transaction_thread = threading.current_thread() self._lock.acquire() def end_transaction(self): self._in_transaction = False self._transaction_thread = None self._lock.release()
def infinite_gen(self, desired_num=np.iinfo(np.int).max): manager = Manager() my_workers = [] gen_buffer = manager.list() worker_lock = Lock() work_assigned = 0 for i in range(self.num_worker - 1): job_num = desired_num // self.num_worker work_assigned += job_num my_workers.append( general_worker(self.callable_constructor, self.constructor_params, job_num)) # last worker my_workers.append( general_worker(self.callable_constructor, self.constructor_params, desired_num - work_assigned)) processes = [] for i in range(self.num_worker): p = Process(target=my_workers[i].start, args=(worker_lock, gen_buffer)) processes.append(p) p.start() returned_sample_cnt = 0 while returned_sample_cnt < desired_num: while not gen_buffer: time.sleep(0.1) worker_lock.acquire() yield gen_buffer.pop(0) worker_lock.release() returned_sample_cnt += 1 # for i in range(desired_num): # yield self.callable_constructor(*self.constructor_params) for p in processes: p.join()
class _DiscoveryWorkbook: def __init__(self, name): self.l = Lock() self.fname = name self.init = False def new_sheet(self, sname, rows): self.l.acquire() if self.init: w = load_workbook(self.fname) else: w = Workbook() w.remove_sheet(w.worksheets[0]) ws = w.create_sheet(sname) for row in rows: row[0] = '=+"' + str(row[0]).replace('"', '""') + '"' ws.append(row) #simple formatting if self.init: ws.column_dimensions['A'].width = 89 for i in range(1, ws.max_row + 1): ws.cell(row=i, column=1).font = Font(name='Lucida Console') else: ws.column_dimensions['A'].width = 12 ws.column_dimensions['B'].width = 30 for i in range(1, ws.max_row + 1): for j in range(1, ws.max_column + 1): ws.cell(row=i, column=j).font = Font(name='Lucida Console') w.save(self.fname) self.init = True self.l.release()
def batch_copy(file_list): count = 0 count_lock = Lock() # alive indicates if any process is alive alive = False proc_list = [None] * config["MAX_PROCESSES"] while count < len(file_list) or alive: alive = False for i in range(config["MAX_PROCESSES"]): try: # Maybe here will be a TypeError or something if proc_list[i] and proc_list[i].is_alive(): alive = True continue else: proc_list[i].join(0) count_lock.acquire() if count < len(file_list): proc_list[i] = Process(target=copy, args=(file_list[count])) proc_list[i].start() alive = True count += 1 count_lock.release() except Exception: count_lock.acquire() if count < len(file_list): proc_list[i] = Process(target=copy, args=(file_list[count])) alive = True count += 1 proc_list[i].start() count_lock.release() print("Batch copy complete")
def sensor_read_process(raw_shm: shared_memory.SharedMemory, proc_shm: shared_memory.SharedMemory, r_lock: Lock, p_lock: Lock): script_dir = Path(__file__).parent device_file = script_dir.parent.joinpath("um7_A500CNP8.json") assert device_file.exists( ), f"Device file with connection info: {device_file} does not exist!" um7 = UM7Serial(device=str(device_file)) for packet in um7.recv_broadcast(flush_buffer_on_start=False): packet_bytes = bytes(json.dumps(packet.__dict__), encoding='utf-8') assert len( packet_bytes ) <= BUFFER_SIZE, f"Packet cannot be serialized, increase `BUFFER` size at least up to {len(packet_bytes)}" if isinstance(packet, UM7AllRawPacket): r_lock.acquire() raw_shm.buf[:] = b' ' * BUFFER_SIZE raw_shm.buf[:len(packet_bytes)] = packet_bytes r_lock.release() # logging.warning(f"[SR][RAW] -> {packet}") elif isinstance(packet, UM7AllProcPacket): p_lock.acquire() proc_shm.buf[:] = b' ' * BUFFER_SIZE proc_shm.buf[:len(packet_bytes)] = packet_bytes p_lock.release()
class IPClusterPool(object): def __init__(self, workers): self.queue_lock = Lock() self.queue = [] self.workers = {w: None for w in workers} self.terminated = False self.closed = False threading.Thread(target=self._manager).start() def terminate(self): self.terminated = True def join(self): while self.terminated != True: pass def close(self): self.closed = True def apply_async(self, job, *job_args): assert isinstance(job, str), "Job has to be string" try: self.queue_lock.acquire() self.queue.append(IPClusterTask(job, job_args)) return self.queue[-1] except Exception, e: raise e finally:
def producer(self, pres: list, q_facts: mp.Queue, pre_to_offset, lock: mp.Lock): print("Producer started.") num_pres = len(pres) max_threads = 100 num_finished = 0 q_threads = queue.Queue() for j in range(min(max_threads, num_pres)): p = pres[j] lock.acquire() offset = pre_to_offset[p] lock.release() self.add_thread( q_threads, q_facts, ("?x", p, "?z"), 1, offset ) num_finished += 1 while not q_threads.empty(): q_threads.get().join() if num_finished < num_pres: p = pres[num_finished] lock.acquire() offset = pre_to_offset[p] lock.release() self.add_thread( q_threads, q_facts, ("?x", p, "?z"), 1, offset ) num_finished += 1 # all requests has been posted and returned. q_facts.put(None) print("Producer stopped.")
class FeExtractor(nn.Module): def __init__(self, backbone_cfg, distance, delta_dist, device=None): """ Be aware that the underlying embedding space that this feature extractor assumes imploys the cosine distance""" super(FeExtractor, self).__init__() self.embed_model = UNet2D(**backbone_cfg) self.device = device self.distance = distance self.delta_dist = delta_dist self.fwd_mtx = Lock() def forward(self, raw): self.fwd_mtx.acquire() try: ret = self.embed_model(raw.unsqueeze(2)).squeeze(2) finally: self.fwd_mtx.release() if isinstance(self.distance, CosineDistance): ret = ret / torch.clamp(torch.norm(ret, dim=1, keepdim=True), min=1e-10) return ret def get_mean_sp_embedding_chunked(self, embeddings, supix, chunks=1): """ This average reduction scheme implements a weighted averaging where the weight distribution is given by the normed similarity of each sample to the mean in each superpixel """ dev = embeddings.device mask = torch.zeros((int(supix.max()) + 1,) + supix.size(), device=dev).scatter_(0, supix[None], 1) slc_sz = mask.shape[0] // chunks slices = [slice(slc_sz*step, slc_sz*(step+1), 1) for step in range(chunks)] if mask.shape[0] != chunks * slc_sz: slices.append(slice(slc_sz*chunks, mask.shape[0], 1)) sp_embeddings = [self.get_mean_sp_embedding(embeddings, mask[slc]) for slc in slices] return torch.cat(sp_embeddings, dim=0) def get_mean_sp_embedding(self, embeddings, mask): masses = mask.flatten(1).sum(-1) masked_embeddings = embeddings[:, None] * mask[None] means = masked_embeddings.flatten(2).sum(-1) / masses[None] # if isinstance(self.distance, CosineDistance): # means = means / torch.clamp(torch.norm(means, dim=0, keepdim=True), min=1e-10) # normalize since we use cosine distance # probs = self.distance.similarity(masked_embeddings, means[..., None, None], dim=0, kd=False) # probs = probs * mask # probs = probs / probs.flatten(1).sum(-1)[..., None, None] # get the probabilities for the embeddings distribution # sp_embeddings = (masked_embeddings * probs[None]).flatten(2).sum(-1) # return (sp_embeddings / torch.clamp(torch.norm(sp_embeddings, dim=0, keepdim=True), min=1e-10)).T return means.T def get_mean_sp_embedding_sparse(self, embeddings, supix): """ :param embeddings: should have shape = NCDHW :param supix: should be consecutive integers and of shape = NDHW :return: the mean embedding verctor over each label region in supix """ feat = embeddings.permute(1, 0, 2, 3, 4).flatten(1) lbl = supix.flatten() n_lbl = torch.unique(supix).size(0) sp_embeddings = scatter_mean(feat, lbl, dim=1, dim_size=n_lbl) return sp_embeddings
def simulate_session_ch(channel: tuple, nodes, edges, lock: mp.Lock) -> None: '''This function simulates session for current channel.''' lock.acquire() try: # clean auxiliary information for actor in channel: params = nodes[actor] params['result_list'] = list() nodes[actor] = params # simulate dialogue if not Bernoulli_trial(edges[channel]['a']): # channel is not active return # channel is active # determine actors participating as Alice and Bob in the dialogue alice, bob = channel # simulated dialog wA, wB = simulate_dialog(alice, bob, nodes, edges) # update result for each actor alice_node = nodes[alice] alice_node['result_list'].append(wA) nodes[alice] = alice_node bob_node = nodes[bob] bob_node['result_list'].append(wA) nodes[bob] = bob_node finally: lock.release()
def set_params_actor(n: int, nodes, lock: mp.Lock) -> None: '''This function sets parameters for current actor in net.''' lock.acquire() try: params = dict() # set parameters of community actors params['rho'] = 20 if n == 0: params['choice'] = 0 else: params['choice'] = DISCLAIMER # specify initial prefernce densities of community actors if n == 0: params['w'] = np.array([1.0, 0.0], float) elif n == 1: params['w'] = np.array([0.0, 1.0], float) else: params['w'] = uncertainty(nvars) nodes[n] = params finally: lock.release()
def thread(pid: int, q: mp.Queue, loggers: dict, lock: mp.Lock): switch_frequency = 100 epsilon = 0.1 alpha = 1.0 while not q.empty(): try: seed, num_atrs = q.get() set_seed(seed) model = create_model(num_atrs) logger = ContextLogger(track_epochs_internally=True) callbacks = [logger] deltas = {} accuracy = wisconsin_card_sort(model, num_episodes, switch_frequency, epsilon, alpha, auto_switch=True, deltas=deltas, initial_shuffle=True, shuffle=True, callbacks=callbacks, verbose=0) lock.acquire() loggers[(seed, num_atrs)] = [logger.to_dict(), deltas, accuracy] print(f"Runs remaining: {q.qsize()}", end="\r") lock.release() except queue.Empty: return
class IPClusterPool(object): def __init__(self, workers): self.queue_lock = Lock() self.queue = [] self.workers = {w: None for w in workers} self.terminated = False self.closed = False threading.Thread(target=self._manager).start() def terminate(self): self.terminated = True def join(self): while self.terminated != True: pass def close(self): self.closed = True def apply_async(self, job, *job_args): assert isinstance(job, str), "Job has to be string" try: self.queue_lock.acquire() self.queue.append(IPClusterTask(job, job_args)) return self.queue[-1] except Exception, e: raise e finally:
def register_nodes(): if request is None: return "Error: Please supply a valid Node", 400 data = request.json if data is None: return "Error: Please supply a valid Node", 400 if (BootstrapDict['nodeCount'] >= BootstrapDict['N']): return "Error: System is full", 405 lock = Lock() lock.acquire() BootstrapDict['nodeCount'] += 1 BootstrapDictInstance = BootstrapDict.copy() lock.release() node.ring.append({ 'id': BootstrapDictInstance['nodeCount'] - 1, 'ip': data['ip'], 'port': data['port'], 'public_key': data['public_key'], 'balance': 0 }) if (BootstrapDict['nodeCount'] == BootstrapDict['N']): start_new_thread(FirstBroadcast, (node.ring, )) blockchainjson = jsonpickle.encode(blockchain) start_new_thread(MakeFirstTransaction, ( data['public_key'], data['ip'], data['port'], )) return jsonify({'id':BootstrapDictInstance['nodeCount']-1, 'bootstrap_public_key':makeRSAjsonSendable(BootstrapDictInstance['bootstrap_public_key']),\ 'blockchain': blockchainjson, 'block_capacity': BLOCK_CAPACITY,\ 'start_ring': {'id': 0, 'ip': '192.168.1.5', 'port': '5000', 'public_key':makeRSAjsonSendable(BootstrapDictInstance['bootstrap_public_key']), 'balance': 0}\ , 'current_block': jsonpickle.encode(node.current_block), 'NBCs': node.NBCs, 'current_NBCs': node.current_NBCs})
class TestClass(): def __init__(self): self.count = 0 self.lock = Lock() self.child = None def increment(self, l, i): self.lock.acquire() l[i] = i node = self node.count = node.count + 1 while (node.child): node = node.child node.count = node.count + 1 node.child = TestClass() self.lock.release() def get(self): return self.count def getSelf(self): return self def getChild(self): return self.child.child.child.get()
def next_id(self): mutex = Lock() try: # 获取互斥锁后,进程只能在释放锁后下个进程才能进来 mutex.acquire() timestamp = int(self.gen_time()) if timestamp < self.last_timestamp: tmp = self.last_timestamp - timestamp raise Exception( "Clock moved backwards. Refusing to generate id for %d milliseconds" % tmp) if self.last_timestamp == timestamp: self.sequence = (self.sequence + 1) & self.sequence_mask if not self.sequence: timestamp = self.til_next_millis(self.last_timestamp) else: self.sequence = 0 self.last_timestamp = timestamp return ((timestamp - self.twepoch) << self.timestamp_left_shift) | \ (self.data_center_id << self.data_center_id_shift) | \ (self.worker_id << self.worker_id_shift) | self.sequence except Exception as e: print(e) finally: # 互斥锁必须被释放掉 mutex.release()
def traverse(): """以默认配置为基准,每次只调一个参数,m个参数,每个参数n个选项,总共运行m*(n-1)次""" commands = [] nvmlInit() deviceCount = nvmlDeviceGetCount() gpu_usage_list = Manager().list([0 for i in range(deviceCount)]) lock = Lock() default_setting = {k: v[0] for k, v in options.items()} logger.info(F'default setting: {default_setting}') for feature in options: for i, option in enumerate(options[feature]): if i and default_setting[feature] != option: # 默认设置 setting = default_setting setting[feature] = option command = construct_command(setting) commands.append(command) for c in commands: logger.info(F'commands: {c}') while commands: lock.acquire() gpu_id = get_free_gpu_id_and_update(gpu_usage_list) if gpu_id != -1: setting = {} command = commands.pop() setting_str = re.sub(r"[\'\{\}\s]", '', str(setting)) setting_str = setting_str.replace(',', '/').replace(':', '.') P = Process(target=pao, args=(gpu_id, command, setting_str)) P.start() lock.release() sleep(1)
def setup_prometheus_multiproc_dir(lock: multiprocessing.Lock = None): """ Set up prometheus_multiproc_dir for prometheus to work in multiprocess mode, which is required when working with Gunicorn server Warning: for this to work, prometheus_client library must be imported after this function is called. It relies on the os.environ['prometheus_multiproc_dir'] to properly setup for multiprocess mode """ if lock is not None: lock.acquire() try: prometheus_multiproc_dir = config('instrument').get( 'prometheus_multiproc_dir') logger.debug("Setting up prometheus_multiproc_dir: %s", prometheus_multiproc_dir) # Wipe prometheus metrics directory between runs # https://github.com/prometheus/client_python#multiprocess-mode-gunicorn # Ignore errors so it does not fail when directory does not exist shutil.rmtree(prometheus_multiproc_dir, ignore_errors=True) os.makedirs(prometheus_multiproc_dir, exist_ok=True) os.environ['prometheus_multiproc_dir'] = prometheus_multiproc_dir finally: if lock is not None: lock.release()
class ImageLogger(object): def __init__(self, root): self.storage = root if not path.isdir(self.storage): mkdir(self.storage) self.l = Lock() self.fmt = 'jpg' def image(self,dev_id, img, ts=None): self.l.acquire() if ts: name = datetime.datetime.fromtimestamp(ts).replace(microsecond=0).strftime('%Y%m%d%H%M%S') else: name = str(uuid.uuid4()) image_path = path.join(self.storage,str(dev_id)) if not path.isdir(image_path): mkdir(image_path) imgpath = path.join(image_path, "%s.%s" % (name, self.fmt)) i = 0 while path.isfile(imgpath): imgpath = path.join(image_path, "%s-%s.%s" % (name, i, self.fmt)) i += 1 cv2.imwrite(imgpath, img) self.l.release()
class ObjectManager(object): """A thread management object for single object threads""" def __init__(self, input_Q=None, output_Q=None, timeout=10, locking=False, lock=None, **kwargs): super(ObjectManager, self).__init__(**kwargs) self.input = Queue() if input_Q is None else input_Q self.output = Queue() if output_Q is None else output_Q self._timeout = timeout self._locking = locking if lock is None: self._lock = Lock() else: self._lock = lock self.hdr = {} self.hdr["pid"] = current_process().pid self.hdr["STATUS"] = self.NORMAL def __getattr__(self, attr): """Call a method on the underlying threaded object""" def method(*args, **kwargs): """A threaded method""" if self._locking: self._lock.acquire() self.input.put((self.hdr, attr, args, kwargs), timeout=self._timeout) return method ERROR = "ERROR" NORMAL = "NORMAL" @property def duplicator(self): """Arguments required to duplicate this manager""" return { "input_Q": self.input, "output_Q": self.output, "timeout": self._timeout, "locking": self._locking, "lock": self._lock, } def retrieve(self, inputs=False, timeout=None): """Retrieve a return value off the top of the output queue""" timeout = self._timeout if timeout is None else timeout try: hdr, func, args, kwargs, rvalue = self.output.get(timeout=timeout) except QFull, QEmpty: raise ThreadStateError(code=2 ** 3, msg="Subthread Ended") if self._locking: self._lock.release() if hdr["STATUS"] == self.ERROR: self.clear() raise rvalue if inputs: return func, args, kwargs, rvalue, hdr else: return rvalue
def run(self): signal.signal(signal.SIGINT, signal.SIG_IGN) l = Lock() l.acquire() pd = pcapy.open_live(self.iface_name, ethernet.ETHERNET_MTU, 0, 100) pcap_filter = "ether proto %s" % hex(frames.WiwoFrame.ethertype) \ + " and (ether[14:1] = 0x07 or ether[14:1] = 0x08)" pd.setfilter(pcap_filter) l.release() pd.loop(-1, self.frame_handler)
class __generator: def __init__(self): self.lock = Lock() self.lastId = 0 def getId(self): self.lock.acquire() self.lastId += 1 self.lock.release() return self.lastId
def initialize_proposer(self): """ Intializes a proposer process that acts as a proposer Paxos member - creates listening socket for client connections - initializes connections to other server connections - starts main loop for proposer which reads proposal requests off a queue of requests - server_list is a list of pairs (host, port) """ # the msg type need to handle dup proposer_msg_types = [ MESSAGE_TYPE.PREPARE_ACK, MESSAGE_TYPE.PREPARE_NACK, MESSAGE_TYPE.ACCEPT_ACK, ] msg_history = set() def if_dup(msg, msg_history): # handle duplication if msg.msg_type in proposer_msg_types: msg_signature = ( msg.msg_type, msg.value, msg.proposal, msg.r_proposal, msg.client_id, msg.instance, msg.origin_id) if msg_signature in msg_history: # dup, pass # print "dup msg received by proposer!" return True else: msg_history.add(msg_signature) return False # counter for proposer number proposer_num = self.server_id # log file write_lock = Lock() write_lock.acquire() logfile = open("server" + str(self.server_id) + ".txt", "w+") write_lock.release() def send_to_acceptors(msg, server_connections): assert isinstance(msg, message.message) # send the proposal to acceptors for s_socket in server_connections: try: s_socket.sendall(pickle.dumps(msg)) except Exception, e: server_connections.remove(s_socket) print "{}: ERROR - {}".format(self.DEBUG_TAG, e) pass
class DataLoader(object): def __init__(self, params, db, nn_db): self.lock = Lock() self.db = db self.cur = db.length self.im_shape = params['im_shape'] self.nn_shape = params['nn_shape'] self.hist_eq = params['hist_eq'] self.indexes = np.arange(db.length) self.shuffle = params['shuffle'] self.subtract_mean = params['subtract_mean'] if self.subtract_mean: self.mean_img = self.db.read_mean_img(self.im_shape) self.im_shape = params['im_shape'] self.load_nn = params['load_nn'] self.nn_query_size = params['nn_query_size'] if self.load_nn: self.nn_db = nn_db #nn_ignore = 1 if db.db_root == nn_db.db_root else 0 nn_ignore = 0 self.nn = NN(nn_db, params['nn_db_size'], nn_ignore) def load_next_data(self): nid = self.get_next_id() jp, imgs, segs = self.db.read_instance(nid, size=self.im_shape) item = {'jp':jp} for i in xrange(len(imgs)): img = imgs[i] if self.hist_eq: img = correct_hist(img) item.update({'img_' + shape_str(self.im_shape[i]):img.transpose((2,0,1)), 'seg_' + shape_str(self.im_shape[i]): segs[i]}) if self.load_nn: nn_id = self.nn.nn_ids(jp, self.nn_query_size) if hasattr(nn_id, '__len__'): nn_id = random.choice(nn_id) nn_jp, nn_imgs, nn_segs = self.nn_db.read_instance(nn_id, size=self.nn_shape) item.update({'nn_jp':nn_jp}) for i in xrange(len(nn_imgs)): nn_img = nn_imgs[i] if self.hist_eq: nn_img = correct_hist(nn_img) item.update({'nn_img_' + shape_str(self.nn_shape[i]):nn_img.transpose((2,0,1)), 'nn_seg_' + shape_str(self.nn_shape[i]): nn_segs[i]}) return item def get_next_id(self): self.lock.acquire() if self.cur >= len(self.indexes) - 1: self.cur = 0 if self.shuffle: random.shuffle(self.indexes) else: self.cur += 1 self.lock.release() return self.indexes[self.cur]
class KNN(object): def __init__(self): (self.train, self.valid, self.test), _ = load_data(data_path='data/mfcc_{}.npy', theano_shared=False) self.train_x, self.train_y = self.train self.test_x, self.test_y = self.test self.valid_x, self.valid_y = self.valid self.train_y = self.train_y.reshape(self.train_y.shape[0]) self.test_y = self.test_y.reshape(self.test_y.shape[0]) self.accurate = Value('i', 0) self.lck = Lock() # self.unique_artists() # self.centers = len(self.artists_map.keys()) self.neigh = KNeighborsClassifier(weights='distance', n_jobs=-1, p=1) def unique_artists(self): self.artists_map = {} for artist in self.data_y: temp = self.artists_map.get(artist, 0) self.artists_map[artist] = temp + 1 def fit_data(self): self.neigh.fit(self.train_x, self.train_y) def test_accuracy(self, st, en, thread_number, var_x, var_y): for indx in range(st, en): data_pt = var_x[indx] temp = self.neigh.predict(data_pt.reshape(1, -1))[0] if temp == var_y[indx]: self.lck.acquire() # print self.accurate.value, self.accurate.value += + 1 # print self.accurate.value, indx, # print 'Thread: {}'.format(thread_number) self.lck.release() def testing(self, data_x, data_y): self.accurate.value = 0 processes = [] ranges = range(0, data_y.shape[0], data_y.shape[0] // 10) for x in range(len(ranges) - 1): p = Process(target=self.test_accuracy, args=(ranges[x], ranges[x + 1], x, data_x, data_y)) p.start() processes.append(p) _ = map(lambda x: x.join(), processes) self.accuracy_percentage = self.accurate.value * 100. / data_y.shape[0] print 'Accuracy: {}'.format(self.accuracy_percentage) print 'Accurate: {}/{}'.format(self.accurate.value, data_y.shape[0])
class MessageThrottle(object): """ Send at most throttle_interval message per second to zoom clients """ def __init__(self, configuration, clients): self._interval = configuration.throttle_interval self._lock = Lock() self._clients = clients self._message = None self._thread = Thread(target=self._run, name='message_throttler') self._running = True def start(self): self._thread.start() def add_message(self, message): self._lock.acquire() try: if self._message is None: self._message = message else: self._message.combine(message) finally: self._lock.release() def _run(self): while self._running: self._lock.acquire() try: if self._message is not None: logging.debug('Sending message: {0}' .format(self._message.to_json())) for client in self._clients: try: client.write_message(self._message.to_json()) except IndexError: logging.debug('Client closed when trying to send ' 'update.') continue self._message = None except AttributeError as e: logging.exception('Exception in MessageThrottle: {0}'.format(e)) finally: self._lock.release() time.sleep(float(self._interval)) def stop(self): if self._thread.is_alive(): self._running = False self._thread.join()
class DataWriter: def __init__ (self, targetDir, fps=10.0): # Create target directory and pose log if not os.path.exists(targetDir) : os.mkdir (targetDir) self.poseFd = open(targetDir+"/pose.csv", 'w') self.path = targetDir self.currentImage = None self.currentPose = None self.fps = fps self.bridge = CvBridge() self.currentTimestamp = 0.0 # Prepare multiprocessing self.dataEx = Lock() self.process = mp.Process(target=self.start) self.end = False self.process.start() def start (self): rate = rospy.Rate(self.fps) counter = 0 while (self.end == False): if self.currentImage is not None and self.currentPose is not None : self.dataEx.acquire() cpImg = copy (self.currentImage) cpPose = copy (self.currentPose) self.dataEx.release() curImageName = "{0:06d}.jpg".format (counter) cv2.imwrite (self.path + "/"+curImageName, self.currentImage) poseStr = "{:.6f} {} {} {} {} {} {} {}".format( rospy.Time.now().to_sec(), cpPose.position.x, cpPose.position.y, cpPose.position.z, cpPose.orientation.x, cpPose.orientation.y, cpPose.orientation.z, cpPose.orientation.w) self.poseFd.write(poseStr+"\n") counter += 1 rate.sleep() def close (self): self.end = True self.poseFd.close()
def multiprocess_index(file_or_urls,shard_size=10000): lock = Lock() t1 = time.time() total_recs = 0 for file_ref in file_or_urls: reader = pymarc.MARCReader(open(file_ref,'rb'),utf8_handling="xmlcharrefreplace") # error_recs = open('%s-bad.mrc' % file_ref,'wb') print("Starting multiprocess index for %s, sharding by %s" % (file_ref, shard_size)) count = 1 marc_recs = [] while 1: try: rec = next(reader) except ValueError as error: lock.acquire() error_msg = "%s for record %s in %s" % ("ValueError: {0}".format(error), count, file_ref) print(error_msg) logging.error(error_msg) lock.release() count += 1 pass #rec = next(reader) except StopIteration: break if not count%shard_size: marc_recs.append(rec) shard_process = Process(target=index_shard,args=(marc_recs,)) if shard_process is not None: shard_process.start() shard_process.join() else: lock.acquire() error_msg = "Unable to process %s total_recs=%s" % (file_ref,total_recs) print(error_msg) logging.error(error_msg) lock.release() marc_recs = [] else: marc_recs.append(rec) if count%1000: sys.stderr.write(".") else: sys.stderr.write(str(count)) count += 1 total_recs += count t2 = time.time() total_time = (t2 - t1) / 60.0 print("Finished multi-processing %s in %0.3f" % (total_recs, total_time))
class PassiveTestEndpoint(object): def __init__(self): self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.s.bind((HOST, PORT)) self.s.listen(1) self.mutex = Lock() def loop(self): while True: conn, addr = self.s.accept() print addr, 'connected' self.handle_read(conn) def handle_read(self, s): data = None try: resp = s.recv(8192) if not resp: return data = json.loads(resp) except OSError: pass if data: cmd = data.get('cmd') if cmd == "start": self.mutex.acquire() program = data.get('program') args = data.get('args') env = data.get('env') if not child_pid: print "now running", program, args test_start(program, args, env) s.sendall(json.dumps({'status': 'ok'})) else: s.sendall(json.dumps({'status': 'error', 'code': 'busy', 'errmsg': 'test (%s) stillrunning ' % child_pid})) s.close() self.mutex.release() elif cmd == "terminate": self.mutex.acquire() print "terminate" if not child_pid: s.sendall(json.dumps({'status': 'error', 'code': 'no_child', 'errmsg': 'nothing to terminate'})) else: test_terminate() log_content = file('/tmp/test_out', 'r').read() s.sendall(json.dumps({'status': 'ok', 'log': log_content})) s.close() self.mutex.release()
class sync_queue(multiprocessing.queues.Queue): def __init__(self): self.q = Queue() self.lock = Lock() def add(self, item): self.lock.acquire() self.q.put(item) self.lock.release() def remove(self): return self.q.get()
def run(self): signal.signal(signal.SIGINT, signal.SIG_IGN) l = Lock() l.acquire() pd = pcapy.open_live(self.iface_name, ethernet.ETHERNET_MTU, 0, 100) iface_mac_addr = interface.get_string_mac_address(self.iface_name) pcap_filter = "ether dst " \ + iface_mac_addr \ + " and ether proto %s " % hex(frames.WiwoFrame.ethertype) \ + "and not (ether[14:1] = 0x07 or ether[14:1] = 0x08)" pd.setfilter(pcap_filter) l.release() pd.loop(-1, self.frame_handler)
class clientExecutor(Pyro.core.SynchronizedObjBase): def __init__(self): Pyro.core.SynchronizedObjBase.__init__(self) self.continueRunning = True self.mutex = Lock() def continueLoop(self): self.mutex.acquire() return self.continueRunning def serverShutdownCall(self): # receive call from server when all stages are processed self.mutex.acquire() self.continueRunning = False self.mutex.release()
class ProcessManager(): def __init__(self): self.lock = Lock() self.processes = dict() self.logger = logging.getLogger('{0}.{1}'.format(__name__, self.__class__.__name__)) def has(self, name): self.lock.acquire() isExists = True if self.processes.has_key(name) else False self.lock.release() return isExists def add(self, name, process): self.lock.acquire() self.processes[name] = process self.logger.debug('Add process ' + name) self.lock.release() def remove(self, name): self.lock.acquire() if self.processes.has_key(name): del self.processes[name] self.logger.debug('Remove process ' + name) self.lock.release() def kill(self, name): self.lock.acquire() if self.processes.has_key(name): self.processes[name].signalProcess('KILL') del self.processes[name] self.logger.debug('Killed and removed process ' + name) self.lock.release()
class KnowledgeBase(Daemon): def __init__(self, config): set_logging(config) self.config = config self.pidfile = os.path.abspath(config['pidfile']) self.time_lock = Lock() self.teller_queue = JoinableQueue() self.session_factory = get_sasession(self.config) session = self.session_factory() def run(self): if int(self.config['instant_duration']): self.clock = Ticker(self.config, self.session_factory(), self.time_lock, self.teller_queue) self.clock.start() host = self.config['kb_host'] port = int(self.config['kb_port']) nproc = int(self.config['teller_processes']) for n in range(nproc): teller = Teller(self.config, self.session_factory, self.teller_queue) teller.daemon = True teller.start() self.socket = Listener((host, port)) while True: try: client = self.socket.accept() except InterruptedError: return self.time_lock.acquire() self.teller_queue.put(client) self.time_lock.release() def cleanup(self, signum, frame): """cleanup tasks""" nproc = int(self.config['teller_processes']) for n in range(nproc): self.teller_queue.put(None) self.teller_queue.close() try: self.clock.ticking = False except AttributeError: pass self.teller_queue.join() try: self.clock.join() except AttributeError: pass logger.warn('bye from {n}, received signal {p}'.format(n=mp.current_process().name, p=str(signum)))
def main(): global keyAlphabet, configParam, myOutputFile queueLock = Lock() outputLock = Lock() configParam = [] workQueue = Queue() myProcesses= [] orderArray = Array('i',[0,0]) if len(sys.argv) == 4: for e in range(1,4): if sys.argv[e].isdigit(): configParam.append(int(sys.argv[e])) else: print "parameters other than filename have to be integers" sys.exit() else: print "usage : <filename> <shift> <numThread> <length> " sys.exit() myOutputFile = 'crypted_'+`configParam[0]`+'_'+`configParam[1]`+'_'+`configParam[2]`+".txt" try: global myInputFile myInputFile = open('metin.txt','r') (open(myOutputFile,'w')).close() except: print "Opening failed" sys.exit() keyAlphabet = createKey() queueLock.acquire() myText = myInputFile.read(configParam[2]) while myText != "": workQueue.put(myText) myText = myInputFile.read(configParam[2]) queueLock.release() for i in range(0,configParam[1]): process = Process(target=readFile,args=(orderArray,workQueue,queueLock,outputLock,myOutputFile)) process.start() myProcesses.append(process) for p in myProcesses: p.join() myInputFile.close()
class Database(): databaseName = 'data.sqlite3' def __init__(self): self.lock = Lock() self.connection = sqlite3.connect(self.databaseName) self.cursor = self.connection.cursor() def insertItem(self, item): sql = 'INSERT INTO UNIT VALUES(' sql += item.fields[4][1].replace('.', '') sql += ", '" sql += item.fields[0][1] sql += "', '" sql += item.fields[1][1] sql += "', '" sql += item.fields[2][1] sql += "', " sql += item.fields[3][1]; sql += ");" self.lock.acquire() try: logging.debug(sql + '\n') self.cursor.execute(sql) except sqlite3.Error as e: logging.warning(sql + ' ' + e.args[0]) try: self.connection.commit() except: logging.error('SQLITE COMMIT ISSUE') self.lock.release() def doesNotContainItem(self, serial): self.cursor.execute('SELECT * FROM UNIT WHERE ID = ' + serial) return self.cursor.fetchone() == None
class Director(object): def __init__(self, producer, consumer): self.producer = producer self.consumer = consumer self.queue = Queue() self.prod_proc = Process(target = self.produce) self.prod_proc.daemon = True self.lock = Lock() self.done = Value('b') self.done.value = 0 def start(self): self.prod_proc.start() def step(self): self.lock.acquire() done = (self.done.value != 0) self.lock.release() if done: raise Done try: data = self.queue.get(block = True, timeout = 1.0) self.consumer.consume(data) except Empty: pass def stop(self): self.prod_proc.join() def run(self): self.start() while True: try: self.step() except Done: break self.stop() def produce(self): try: while True: data = self.producer.produce() self.queue.put(data) except: self.lock.acquire() self.done.value = 1 self.lock.release() self.queue.close() self.queue.join_thread()
class MPlot(ct.StoppableProcess): def __init__(self, xlim, ylim, channel=1, post_process=None, x_range=None, *args, **kwargs): super(MPlot, self).__init__(*args, **kwargs) self.in_queue = multiprocessing.Queue() self.xlim = xlim self.ylim = ylim self.x_range = x_range if(self.x_range is None): self.x_range = np.arange(self.xlim[0], self.xlim[1]) self.lock = Lock() self.channel = channel self.post_process = post_process if(self.post_process is None): self.post_process = lambda data: data def init_plot(self): self.lock.acquire() self.fig, self.axarr = plt.subplots(self.channel, sharex=True, squeeze=False) for i in range(self.channel): self.axarr[i, 0].hold(False) self.fig.show() self.lock.release() def run(self): self.init_plot() while(self.is_stopped() is False): try: data = self.in_queue.get_nowait() split_data = ast.split_channel(data, self.channel) p_split_data = [self.post_process(sd) for sd in split_data] self.lock.acquire() for i in range(self.channel): self.axarr[i, 0].plot(self.x_range, p_split_data[i]) self.axarr[i, 0].set_xlim(self.xlim) self.axarr[i, 0].set_ylim(self.ylim) self.fig.canvas.draw() self.lock.release() except queue.Empty: pass
def insert_to_db(table, data, visit_info): """Insert event data to database.""" for _ in xrange(0, cm.MAX_WAIT_FOR_DB_CONN): try: l = Lock() l.acquire() with sq.connect(visit_info.out_db, timeout=cm.DB_CONN_TIMOEUT) as conn: cursor = conn.cursor() return insert_to_opened_db(cursor, table, data, visit_info) l.release() except sq.OperationalError: # @UndefinedVariable l.release() sleep(1) cm.print_log(visit_info, "Cannot insert to DB (%s) URL: %s Table: (%s)" % (visit_info.out_db, visit_info.url, table))