# _*_ coding: utf-8 _*_ import sys import multiprocessing as mp import logging from ishub import genLabelFromHtml #filename = sys.argv[1] filename = "head10" #featurefilename = sys.argv[2] featurefilename = "titleWithWeight.txt" lock = mp.Lock() counter = mp.Value('i', 0) # int type,相当于java里面的原子变量 def genLabel(line): global lock, counter with lock: print "row:" + str(counter.value) counter.value += 1 result = genLabelFromHtml(line) return result def multicore(): urlDict = {} with open(featurefilename, 'r') as f: for line in f.readlines(): line = line.split("\t") url = line[0] w = line[1] urlDict[url] = w
def run_clients(options, db_table_set): # Spawn one client for each db.table exit_event = multiprocessing.Event() processes = [] error_queue = SimpleQueue() interrupt_event = multiprocessing.Event() sindex_counter = multiprocessing.Value(ctypes.c_longlong, 0) signal.signal(signal.SIGINT, lambda a, b: abort_export(a, b, exit_event, interrupt_event)) errors = [] try: sizes = get_all_table_sizes(options["host"], options["port"], options["auth_key"], db_table_set) progress_info = [] arg_lists = [] for db, table in db_table_set: progress_info.append( (multiprocessing.Value(ctypes.c_longlong, 0), multiprocessing.Value(ctypes.c_longlong, sizes[(db, table)]))) arg_lists.append( (options["host"], options["port"], options["auth_key"], db, table, options["directory_partial"], options["fields"], options["delimiter"], options["format"], error_queue, progress_info[-1], sindex_counter, exit_event)) # Wait for all tables to finish while len(processes) > 0 or len(arg_lists) > 0: time.sleep(0.1) while not error_queue.empty(): exit_event.set() # Stop rather immediately if an error occurs errors.append(error_queue.get()) processes = [ process for process in processes if process.is_alive() ] if len(processes) < options["clients"] and len(arg_lists) > 0: processes.append( multiprocessing.Process(target=export_table, args=arg_lists.pop(0))) processes[-1].start() update_progress(progress_info) # If we were successful, make sure 100% progress is reported # (rows could have been deleted which would result in being done at less than 100%) if len(errors) == 0 and not interrupt_event.is_set(): print_progress(1.0) # Continue past the progress output line and print total rows processed def plural(num, text, plural_text): return "%d %s" % (num, text if num == 1 else plural_text) print("") print("%s exported from %s, with %s" % (plural(sum([max(0, info[0].value) for info in progress_info]), "row", "rows"), plural(len(db_table_set), "table", "tables"), plural(sindex_counter.value, "secondary index", "secondary indexes"))) finally: signal.signal(signal.SIGINT, signal.SIG_DFL) if interrupt_event.is_set(): raise RuntimeError("Interrupted") if len(errors) != 0: # multiprocessing queues don't handling tracebacks, so they've already been stringified in the queue for error in errors: print("%s" % error[1], file=sys.stderr) if options["debug"]: print("%s traceback: %s" % (error[0].__name__, error[2]), file=sys.stderr) raise RuntimeError("Errors occurred during export")
# Lotería-Prioridades + CLOCK_DWF # # Arizmendi Ramírez Esiel Kevin # Coria Rios Marco Antonio # # Primavera 2020 import multiprocessing import charger import dispatcher from common import MAXPROCESS if __name__ == "__main__": # Creando los elementos en común (memoria, bandera de paro, semáforo) shlist = multiprocessing.Queue() fband = multiprocessing.Value('i', 0) lock = multiprocessing.Lock() # creating new processes p1 = multiprocessing.Process(target=charger.charge, args=(shlist, lock, fband)) p2 = multiprocessing.Process(target=dispatcher.dispatch, args=(shlist, lock, fband)) # running processes p1.start() p2.start() # wait until processes finish p1.join() p2.join()
from ConfigParser import NoOptionError from functools import partial from logging import getLogger from pysolr import SolrError from sqlalchemy import and_ from .util import SIR_EXIT from ctypes import c_bool __all__ = ["reindex", "index_entity", "queue_to_solr", "send_data_to_solr", "_multiprocessed_import", "_index_entity_process_wrapper", "live_index", "live_index_entity"] logger = getLogger("sir") PROCESS_FLAG = multiprocessing.Value(c_bool, True) FAILED = multiprocessing.Value(c_bool, False) STOP = None def reindex(args): """ Reindexes all entity types in args["entity_type"]. If no types are specified, all known entities will be reindexed. :param args: A dictionary with a key named ``entities``. :type args: dict """ # Checking for PROCESS_FLAG before proceeding in case the parent process
global wg, results wg, results = wg_, results_ if __name__ == '__main__': import sys from pathlib import Path if len(sys.argv) < 3: print('Usage: %s <input glob> <func info pickle>' % (sys.argv[0])) print( 'The pickle designates the filename of a pickle file that the computed function metadata will be saved to.' ) sys.exit(1) import multiprocessing as mp wg = mp.Value('i', 0) results = mp.Queue() func_info = {} with mp.Pool(mp.cpu_count(), initializer=init_child, initargs=(wg, results)) as pool: pool.map(process_binary, map(str, Path('.').glob(sys.argv[1]))) while True: time.sleep(0.1) with wg.get_lock(): if wg.value == 0: break while not results.empty():
class FarmWorker(): terminate = multiprocessing.Value('i', 0) def __init__(self, task_queue): self.task_queue = task_queue def __call__(self): #LOG.debug("Start FarmWorker") global snmp_queue global rrd_queue rrd_workers = [RRDWorker() for i in range(RRD_WORKERS_NUM)] rrd_threads = [threading.Thread(target=w) for w in rrd_workers] for t in rrd_threads: t.start() snmp_workers = [SNMPWorker() for i in range(SNMP_WORKERS_NUM)] snmp_threads = [threading.Thread(target=w) for w in snmp_workers] for t in snmp_threads: t.start() while not FarmWorker.terminate.value or self.task_queue.qsize() != 0: try: task = self.task_queue.get(False) except: time.sleep(1) continue task_name = task[0] if task_name == 'farm': farm = task[1] farm_id = farm[0] roles = farm[1] for farm_role_id, servers in roles.iteritems(): for serv in servers: index = serv[0] community = serv[1] host = serv[2] port = serv[3] while True: if FARM_WORKERS_NUM > 1 and snmp_queue.qsize( ) > 300: time.sleep(2) continue try: snmp_queue.put(['server', farm_id, farm_role_id,\ index, community, host, port], False) break except: time.sleep(1) continue elif task_name == 'done': FarmWorker.terminate.value = 1 while True: try: snmp_queue.put(['done'], False) break except: time.sleep(1) continue for t in snmp_threads: t.join() snmp_join_time = time.time() - start_time #LOG.debug('SNMP join time: %s' %snmp_join_time) #put ra into rrd_queue for k, oid_names in ra.iteritems(): k_split = k.split('/') data = {} for oid_name, v in oid_names.iteritems(): if v[1] is not 'U': if type(v[1]) is int: data.update({oid_name: '%i' % (v[1] / v[0])}) elif type(v[1]) is float: data.update({oid_name: '%.2f' % (v[1] / v[0])}) else: data.update({oid_name: v[1]}) while True: try: rrd_queue.put( ['ra', k_split[0], k_split[1], k_split[2], data], False) break except: time.sleep(1) continue #put fa into rrd_queue for k, oid_names in fa.iteritems(): k_split = k.split('/') data = {} for oid_name, v in oid_names.iteritems(): if v[1] is not 'U': if type(v[1]) is int: data.update({oid_name: '%i' % (v[1] / v[0])}) elif type(v[1]) is float: data.update({oid_name: '%.2f' % (v[1] / v[0])}) else: data.update({oid_name: v[1]}) while True: try: rrd_queue.put(['fa', k_split[0], k_split[1], data], False) break except: time.sleep(1) continue #put rs into rrd_queue for k, v in rs.iteritems(): k_split = k.split('/') while True: try: rrd_queue.put( ['rs', k_split[0], k_split[1], k_split[2], v], False) break except: time.sleep(1) continue #put fs into rrd_queue for k, v in fs.iteritems(): k_split = k.split('/') while True: try: rrd_queue.put(['fs', k_split[0], k_split[1], v], False) break except: time.sleep(1) continue while True: try: rrd_queue.put(['done'], False) break except: time.sleep(1) continue for t in rrd_threads: t.join()
def decrement(final, lock): for i in range(100): time.sleep(0.01) lock.acquire() final.value = final.value - 1 lock.release() if __name__ == '__main': ''' While Working with Shared memory by multiple processes the results could be inconsistent .So We use Mutex flavour of Unix Operating system in Python i.e Lock before operating on shared memory and release the lock after working on shared memory variable''' final = multiprocessing.Value('i', 500) lock = multiprocessing.Lock() decr = multiprocessing.Process(target=decrement, args=( final, lock, )) incr = multiprocessing.Process(target=increment, args=( final, lock, )) decr.start() incr.start() decr.join() incr.join() print("Final Value After Increment, Decrement in Shared Variable is:{}", final.value)
def main(): # global bestRoute_forplot cityList = [] data = read_tsp() data_len = len(data) for i in range(data_len): cityList.append(City(x=data[i][1], y=data[i][2])) exitProcessFlag = multiprocessing.Value('b', False) A = [] progress = multiprocessing.Manager().list(A) bestRoute_forplot = multiprocessing.Manager().list(A) #设置迭代次数 generations = 5000 #设置变异率 mutationRate = 0.01 #设置种群大小 data_len = 2*len(data) elite_all_proc = multiprocessing.Manager().list(A) exitProcessFlag.value = True lock = multiprocessing.Lock() p1 = multiprocessing.Process(target=geneticAlgorithm,args=("process1", cityList, data_len, 5, mutationRate, generations,exitProcessFlag, progress, bestRoute_forplot, elite_all_proc,lock)) p2 = multiprocessing.Process(target=geneticAlgorithm,args=("process2", cityList, data_len, 5, mutationRate, generations,exitProcessFlag, progress, bestRoute_forplot, elite_all_proc,lock)) p3 = multiprocessing.Process(target=geneticAlgorithm,args=("process3", cityList, data_len, 5, mutationRate, generations,exitProcessFlag, progress, bestRoute_forplot, elite_all_proc,lock)) p4 = multiprocessing.Process(target=geneticAlgorithm,args=("process4", cityList, data_len, 5, mutationRate, generations,exitProcessFlag, progress, bestRoute_forplot, elite_all_proc,lock)) p1.start() p2.start() p3.start() p4.start() p1.join() p2.join() p3.join() p4.join() progress = progress._getvalue()[0] bestRoute = bestRoute_forplot._getvalue() print(len(bestRoute),len(cityList)) # bestRoute_forplot = cityList #问题根源,没有采用copy的方式,才直接用了= bestRoute_forplot = cityList.copy() for gene_index in range(len(bestRoute)): bestRoute_forplot[bestRoute[gene_index]] = cityList[gene_index] #通过一下函数争端出cityList被修改过 #count = 0 # for i in cityList: # alarm = 0 # for j in cityList: # if i.x==j.x and i.y == j.y: # alarm += 1 # if alarm == 2: # print('Oh,no',j) # print(count) # count+=1 print("This took", time.clock() - start_time, "seconds to calculate.") #画出最优路径的路线图 bestRoute_forplot.append(bestRoute_forplot[0]) x_coords = [] y_coords = [] for individual in bestRoute_forplot: x_coords.append(individual.x) y_coords.append(individual.y) plt.plot(x_coords[0], y_coords[0], 'ro-') plt.plot(x_coords, y_coords, 'rx-') for elem in range(0, len(bestRoute_forplot)): elem_num = elem + 1 point_x = bestRoute_forplot[elem].x point_y = bestRoute_forplot[elem].y if elem<len(bestRoute_forplot)-1: plt.annotate("%d" % elem_num, xy=(point_x, point_y)) else: plt.annotate(" ", xy=(point_x, point_y)) plt.title("GATS_parallel_tackle_elite") plt.show() #画出距离随迭代次数的变化 plt.plot(progress) plt.ylabel('Distance') plt.xlabel('Generation') plt.title("GATS_parallel_tackle_elite") plt.show()
MHsampler.MHsampler.Model = Model artificial_real_parameters = np.ones(no_parameters)*1.5 artificial_observation_without_noise = np.zeros(no_observations) comm_world.Send(artificial_real_parameters, dest=group_leader_ids[0], tag=1) comm_world.Recv(artificial_observation_without_noise, source=group_leader_ids[0],tag=1) G_data = np.zeros([0,no_observations+no_parameters]) G_data = np.vstack([G_data,np.append(artificial_observation_without_noise,artificial_real_parameters)]) # test_solver.solve(aftificial_real_parameters,artificial_observation_without_noise) print('artificial_observation_without_noise:',artificial_observation_without_noise) # Model.SetNoisyObservation(artificial_observation_without_noise) Model.observation = -0.01 # edited print('artificial observation with noise:',Model.observation) ## FINISH MODEL SETTINGS ## START SAMPLING PROCESS shared_finisher = mp.Value('i',0) shared_queue_solver = mp.Queue() shared_queue_surrogate = mp.Queue() shared_queue_updater = mp.Queue() shared_queue_surrogate_solver = mp.Queue() shared_queue_surrogate.put([artificial_real_parameters,artificial_observation_without_noise]) shared_parents_solver = [] shared_children_solver = [] shared_parents_surrogate = [] shared_children_surrogate = [] for i in range(no_chains): new_parent, new_child = mp.Pipe() shared_parents_solver.append(new_parent) shared_children_solver.append(new_child) new_parent, new_child = mp.Pipe() shared_parents_surrogate.append(new_parent)
def test_parameter_result(self): """ This example illustrates how you can get results from IoTPy processes when the processes terminate. The results are stored in a buffer (a multiprocessing.Array) which your non-IoTPy code can read. You can insert data into the IoTPy processes continuously or before the processes are started. In this example output_buffer[j] = 0 + 1 + 2 + ... + j """ print('starting test_parameter_result') print('') print('Output stream s and output_buffer.') print('output_buffer is [0, 1, 3, 6, 10, .., 45]') print('s[j] = output_buffer[j] + 100') print('') # The results of the parallel computation are stored in output_buffer. output_buffer = multiprocessing.Array('i', 20) # The results are in output_buffer[:output_buffer_ptr] output_buffer_ptr = multiprocessing.Value('i', 0) # In this example v represents an element of an input stream. # sum is the sum of all the stream-element values received # by the agent. The state of the agent is sum. # output_buffer and output_buffer_ptr are keyword arguments. @map_e def ff(v, sum, output_buffer, output_buffer_ptr): sum += v output_buffer[output_buffer_ptr.value] = sum output_buffer_ptr.value += 1 return sum, sum # Agent function for process named 'p0' def f(in_streams, out_streams, output_buffer, output_buffer_ptr): ff(in_streams[0], out_streams[0], state=0, output_buffer=output_buffer, output_buffer_ptr=output_buffer_ptr) # Agent function for process named 'p1' def g(in_streams, out_streams): s = Stream('s') map_element(lambda v: v * 2, in_streams[0], s) print_stream(s, 's') # Source thread target for source stream named 'x'. def source_thread_target(procs): for i in range(3): extend_stream(procs, data=list(range(i * 2, (i + 1) * 2)), stream_name='x') time.sleep(0.001) terminate_stream(procs, stream_name='x') # Specification multicore_specification = [ # Streams [('x', 'i'), ('y', 'i')], # Processes [{ 'name': 'p0', 'agent': f, 'inputs': ['x'], 'outputs': ['y'], 'keyword_args': { 'output_buffer': output_buffer, 'output_buffer_ptr': output_buffer_ptr }, 'sources': ['x'] }, { 'name': 'p1', 'agent': g, 'inputs': ['y'], }] ] # Execute processes (after including your own non IoTPy processes) processes, procs = get_processes_and_procs(multicore_specification) source_thread = threading.Thread(target=source_thread_target, args=(procs, )) procs['p0'].threads = [source_thread] for process in processes: process.start() for process in processes: process.join() for process in processes: process.terminate() # Verify that output_buffer can be read by the parent process. print('output_buffer is ', output_buffer[:output_buffer_ptr.value]) print('') print(' ') print('finished test_parameter_result') print(' ')
def _batch_iterator(self, batch_size: int, config: Dict, num_readers, num_batchers, train=False, return_examples=False, return_prediction_target=None, max_seq_len=-1, truncate=False, shuffle=True, seed=42, test_only=False) -> Iterable[Batch]: global batcher_sync_msg batcher_sync_msg = multiprocessing.Value('i', 0) json_enc_queue = multiprocessing.Queue(maxsize=10000) worker_manager_lock = multiprocessing.Value('i', 0) json_loader = multiprocessing.Process( target=json_line_reader, args=(self.file_paths, json_enc_queue, num_readers, shuffle, False, None, True, truncate, seed)) json_loader.daemon = True example_generators = [] worker_result_queue = torch_mp.Queue(maxsize=150) if return_prediction_target is None: return_prediction_target = train for i in range(num_readers): p = multiprocessing.Process( target=example_to_batch, args=(json_enc_queue, worker_result_queue, batch_size, train, config, worker_manager_lock, return_examples, return_prediction_target, max_seq_len, test_only)) p.daemon = True example_generators.append(p) json_loader.start() for p in example_generators: p.start() batch_queue = queue.Queue(maxsize=100) worker_manager_thread = threading.Thread( target=worker_manager, args=(worker_result_queue, batch_queue, num_readers, worker_manager_lock, 100)) worker_manager_thread.start() while True: # t1 = time.time() main_process_queue_get_lock = 1 batch = batch_queue.get() # print(f'[MainThread] local batch queue size {batch_queue.qsize()}', file=sys.stderr) # print(f'{time.time() - t1} took to load a batch', file=sys.stderr) if batch is None: break else: yield batch worker_result_queue.close() # print('start joining...') batcher_sync_msg.value = 1 json_loader.join() # print('json_loader quitted') for p in example_generators: p.join() worker_manager_thread.join() # print('example generators quitted') # print('batchers quiteed') sys.stdout.flush() sys.stderr.flush()
def test_example_passing_data_to_multicore(self): print(' ') print('starting test_example_passing_data_to_multicore') print(' ') total = multiprocessing.Value('f') num = multiprocessing.Value('i') # Values computed from an earlier computation which is not shown. # total and num are passed to the multiprocessing block. total.value = 4.0e-13 num.value = 25 def sine(in_streams, out_streams): map_element(np.sin, dtype_float(in_streams[0]), out_streams[0], name='sine') def cosine(in_streams, out_streams): map_element(np.cos, dtype_float(in_streams[0]), out_streams[0], name='cosine') def tangent(in_streams, out_streams): map_element(np.tan, dtype_float(in_streams[0]), out_streams[0], name='tangent') def coordinate(in_streams, out_streams, total, num): x, sines, cosines, tangents = in_streams def f(lst): return lst[0] / lst[1] def g(lst): error_squared = (lst[0] - lst[1])**2 return error_squared ratios = Stream('ratios') errors = Stream('errors') zip_map(f, [sines, cosines], ratios, name='sine / cosine') zip_map(g, [ratios, tangents], errors, name='compute error') print_stream(errors, 'error') # Source thread target. def source_thread_target(procs): extend_stream(procs, data=np.linspace(0.0, np.pi, 10), stream_name='x') terminate_stream(procs, stream_name='x') multicore_specification = [ # Streams [('x', 'f'), ('sines', 'f'), ('cosines', 'f'), ('tangents', 'f')], # Processes [{ 'name': 'sine', 'agent': sine, 'inputs': ['x'], 'outputs': ['sines'] }, { 'name': 'cosine', 'agent': cosine, 'inputs': ['x'], 'outputs': ['cosines'] }, { 'name': 'tanget', 'agent': tangent, 'inputs': ['x'], 'outputs': ['tangents'] }, { 'name': 'coordinator', 'agent': coordinate, 'inputs': ['x', 'sines', 'cosines', 'tangents'], 'sources': ['x'], 'keyword_args': { 'total': total, 'num': num } }] ] processes, procs = get_processes_and_procs(multicore_specification) thread_0 = threading.Thread(target=source_thread_target, args=(procs, )) procs['coordinator'].threads = [thread_0] for process in processes: process.start() for process in processes: process.join() for process in processes: process.terminate() print(' ') print('finished test_example_passing_data_to_multicore') print(' ')
def test_example_parameters_with_queue(self, DATA=list(range(4)), ADDEND=10, MULTIPLICAND=3, EXPONENT=2): """ This example illustrates integrating processes running non-IoTPy code with processes running IoTPy. The example shows how results generated by IoTPy processes are obtained continuously by non-IoTPy processes through queues. The example also shows how results computed by IoTPy processes are returned to the non-IoTPy calling process when the IoTPy processes terminate In this simple example, (s[j]+ADDEND)*MULTIPLICAND is the j-th value put in the queue, and (s[j]+ADDEND)**EXPONENT is the j-th element of the buffer returned by the multiprocess computation. """ print(' ') print('starting test_example_parameters_with_queue') print(' ') # Values generated continuously by the IoTPy process are read by # the calling non-IoTPy process using this queue. q = multiprocessing.Queue() # The results of the parallel computation are stored in buffer. buffer = multiprocessing.Array('f', 10) # The results are in buffer[:ptr]. # The values in buffer[ptr:] are arbitrary ptr = multiprocessing.Value('i', 0) # The computational function for process p0. # Arguments are: in_streams, out_streams, and additional # arguments. Here ADDEND is an additional argument. def f(in_streams, out_streams, ADDEND): map_element(lambda a: a + ADDEND, in_streams[0], out_streams[0]) # The computational function for process p1 def g(in_streams, out_streams, MULTIPLICAND, EXPONENT, q, buffer, ptr): @sink_e def h(v): q.put(v * MULTIPLICAND) buffer[ptr.value] = v**EXPONENT ptr.value += 1 h(in_streams[0]) def source_thread_target(procs): extend_stream(procs, data=DATA, stream_name='data') terminate_stream(procs, stream_name='data') multicore_specification = [ # Streams [('data', 'f'), ('result', 'f')], # Processes [{ 'name': 'p0', 'agent': f, 'inputs': ['data'], 'outputs': ['result'], 'args': [ADDEND], 'sources': ['data'] }, { 'name': 'p1', 'agent': g, 'inputs': ['result'], 'args': [MULTIPLICAND, EXPONENT, q, buffer, ptr], 'output_queues': [q] }] ] processes, procs = get_processes_and_procs(multicore_specification) source_thread = threading.Thread(target=source_thread_target, args=(procs, )) procs['p0'].threads = [source_thread] for process in processes: process.start() for process in processes: process.join() for process in processes: process.terminate() print('TERMINATED') def get_data_from_output_queue(q): queue_index = 0 while True: v = q.get() if v == '_finished': break else: print('q.get(', queue_index, ') = ', v) queue_index += 1 get_data_from_output_queue(q) # finished_source indicates that this source is finished. No more data will be sent on the # stream called stream_name ('data') in the process with the specified name ('p0'). ## queue_index = 0 ## finished_getting_output = False ## while not finished_getting_output: ## element_from_queue = q.get() ## print ('element_from_queue is ', element_from_queue) ## print ('queue[', queue_index, '] = ', element_from_queue) ## queue_index += 1 ## if element_from_queue == '_finished': ## print ('element_from_queue is finished') ## finished_getting_output = True ## break # Get the results returned in the buffer. print('buffer is ', buffer[:ptr.value]) print(' ') print('finished test_example_parameters_with_queue') print(' ')
import time import multiprocessing def deposit(balance, lock): for i in range(100): time.sleep(0.01) lock.acquire() balance.value = balance.value + 1 lock.release() def withdraw(balance, lock): for i in range(100): time.sleep(0.01) lock.acquire() balance.value = balance.value - 1 lock.release() if __name__ == '__main__': balance = multiprocessing.Value('i', 200) lock = multiprocessing.Lock() d = multiprocessing.Process(target=deposit, args=(balance, lock)) w = multiprocessing.Process(target=withdraw, args=(balance, lock)) d.start() w.start() d.join() w.join() print(balance.value)
def start_workers(actions_map, actions, context, analyzer_config_map, jobs, output_path, skip_handler, metadata, quiet_analyze, capture_analysis_output, timeout, ctu_reanalyze_on_failure, statistics_data, manager): """ Start the workers in the process pool. For every build action there is worker which makes the analysis. """ # Handle SIGINT to stop this script running. def signal_handler(signum, frame): try: pool.terminate() manager.shutdown() finally: sys.exit(128 + signum) signal.signal(signal.SIGINT, signal_handler) # Start checking parallel. checked_var = multiprocessing.Value('i', 1) actions_num = multiprocessing.Value('i', len(actions)) pool = multiprocessing.Pool(jobs, initializer=init_worker, initargs=(checked_var, actions_num)) failed_dir = os.path.join(output_path, "failed") # If the analysis has failed, we help debugging. if not os.path.exists(failed_dir): os.makedirs(failed_dir) success_dir = os.path.join(output_path, "success") # Analysis was successful processing results. if not os.path.exists(success_dir): os.makedirs(success_dir) output_dirs = {'success': success_dir, 'failed': failed_dir} # Construct analyzer env. analyzer_environment = env.extend(context.path_env_extra, context.ld_lib_path_extra) actions, skipped_actions = skip_cpp(actions, skip_handler) analyzed_actions = [ (actions_map, build_action, context, analyzer_config_map, output_path, skip_handler, quiet_analyze, capture_analysis_output, timeout, analyzer_environment, ctu_reanalyze_on_failure, output_dirs, statistics_data) for build_action in actions ] if analyzed_actions: try: # Workaround, equivalent of map. # The main script does not get signal # while map or map_async function is running. # It is a python bug, this does not happen if a timeout is # specified, then receive the interrupt immediately. pool.map_async(check, analyzed_actions, 1, callback=lambda results: worker_result_handler( results, metadata, output_path, context. analyzer_binaries)).get(31557600) pool.close() except Exception: pool.terminate() raise finally: pool.join() else: LOG.info("----==== Summary ====----") for skp in skipped_actions: LOG.debug_analyzer("%s is skipped", skp.source) LOG.info("Total analyzed compilation commands: %d", len(analyzed_actions)) LOG.info("----=================----") if not os.listdir(success_dir): shutil.rmtree(success_dir) if not os.listdir(failed_dir): shutil.rmtree(failed_dir)
# utf-8 # python 3.5 import multiprocessing as mp value = mp.Value('d', 1) array = mp.Array('i', [1, 2, 3])
import re import time import pandas as pd from pyquery import PyQuery as pq import requests import sportsref # time between requests, in seconds THROTTLE_DELAY = 0.5 # variables used to throttle requests across processes throttle_thread_lock = threading.Lock() throttle_process_lock = multiprocessing.Lock() last_request_time = multiprocessing.Value(ctypes.c_longdouble, time.time() - 10 * THROTTLE_DELAY) @sportsref.decorators.cache def get_html(url): """Gets the HTML for the given URL using a GET request. :url: the absolute URL of the desired page. :returns: a string of HTML. """ global last_request_time with throttle_process_lock: with throttle_thread_lock: # sleep until THROTTLE_DELAY secs have passed since last request wait_left = THROTTLE_DELAY - (time.time() - last_request_time.value)
more_for_file_analyzed = True #if len(metal_ligand_dict_by_cutoff[i]) < len(metal_ligand_dict_by_CONECT[i]): with lock: less_than_CONECT_count_by_atom.value += 1 if not less_for_file_analyzed: with lock: less_than_CONECT_count_by_file.value += 1 less_for_file_analyzed = True # Multiprocess if __name__ == '__main__': manager = mp.Manager() lock = mp.Lock() ok_file_count = mp.Value(c_int) error_file_count = mp.Value(c_int) files_with_metal_count = mp.Value(c_int) metal_count = mp.Value(c_int) more_than_CONECT_count_by_atom = mp.Value(c_int) less_than_CONECT_count_by_atom = mp.Value(c_int) equal_CONECT_count_by_atom = mp.Value(c_int) more_than_CONECT_count_by_file = mp.Value(c_int) less_than_CONECT_count_by_file = mp.Value(c_int) equal_CONECT_count_by_file = mp.Value(c_int) pool = mp.Pool(processes = ppn) pool.map(ligand_scanner, glob.iglob(pdbpath)) # Write results
def run(self, genomic_files, tln_tables): """Run Prodigal across a set of genomes. Parameters ---------- genomic_files : dict Dictionary indicating the genomic and gene file for each genome. tln_tables : Dict[str, int] Mapping of genome id to user-specified translation table. """ # populate worker queue with data to process worker_queue = mp.Queue() writer_queue = mp.Queue() for genome_id, file_path in genomic_files.items(): worker_queue.put((genome_id, file_path, tln_tables.get(genome_id))) for _ in range(self.threads): worker_queue.put(None) try: manager = mp.Manager() out_dict = manager.dict() n_skipped = mp.Value('i', 0) worker_proc = [ mp.Process(target=self._worker, args=(out_dict, worker_queue, writer_queue, n_skipped)) for _ in range(self.threads) ] writer_proc = mp.Process(target=self._writer, args=(len(genomic_files), writer_queue)) writer_proc.start() for p in worker_proc: p.start() for p in worker_proc: p.join() # Gracefully terminate the program. if p.exitcode != 0: raise ProdigalException( 'Prodigal returned a non-zero exit code.') writer_queue.put(None) writer_proc.join() except Exception: for p in worker_proc: p.terminate() writer_proc.terminate() raise ProdigalException( 'An exception was caught while running Prodigal.') # Report if any genomes were skipped due to having already been processed. if n_skipped.value > 0: genome_s = 'genome' if n_skipped.value == 1 else 'genomes' self.logger.warning( f'Prodigal skipped {n_skipped.value} {genome_s} ' f'due to pre-existing data, see warnings.log') # Report on any genomes which failed to have any genes called result_dict = dict() lq_gids = list() for gid, gid_dict in out_dict.items(): if os.path.getsize(gid_dict['aa_gene_path']) <= 1: lq_gids.append(gid) else: result_dict[gid] = gid_dict if len(lq_gids) > 0: self.logger.warning( f'Skipping {len(lq_gids)} of {len(genomic_files)} ' f'genomes as no genes were called by Prodigal. ' f'Check the genome quality (see gtdb.warnings.log).') self.warnings.warning( f'The following {len(lq_gids)} genomes have ' f'been excluded from analysis due to Prodigal ' f'failing to call any genes:') # If there are few low-quality genomes just output to console. if len(lq_gids) > 10: for lq_gid in lq_gids: self.warnings.info(lq_gid) else: for lq_gid in lq_gids: self.logger.warning(f'Skipping: {lq_gid}') self.warnings.info(lq_gid) return result_dict
def __init__(self, train_fn, args=None, resource=None, checkpoint='./exp/checkpoint.ag', resume=False, num_trials=None, time_attr='epoch', reward_attr='accuracy', visualizer='none', controller_lr=1e-3, ema_baseline_decay=0.95, controller_resource={ 'num_cpus': 0, 'num_gpus': 0 }, controller_batch_size=1, dist_ip_addrs=[], sync=True, **kwargs): assert isinstance(train_fn, _autogluon_method), 'Please use @ag.args ' + \ 'to decorate your training script.' self.ema_baseline_decay = ema_baseline_decay self.sync = sync # create RL searcher/controller searcher = RLSearcher(train_fn.kwspaces) super(RLScheduler, self).__init__(train_fn, train_fn.args, resource, searcher, checkpoint=checkpoint, resume=False, num_trials=num_trials, time_attr=time_attr, reward_attr=reward_attr, visualizer=visualizer, dist_ip_addrs=dist_ip_addrs, **kwargs) # reserve controller computation resource on master node master_node = self.REMOTE_MANAGER.get_master_node() self.controller_resource = DistributedResource(**controller_resource) assert self.RESOURCE_MANAGER.reserve_resource( master_node, self.controller_resource), 'Not Enough Resource on Master Node' + \ ' for Training Controller' self.controller_ctx = [mx.gpu(i) for i in self.controller_resource.gpu_ids] if \ controller_resource['num_gpus'] > 0 else [mx.cpu()] # controller setup self.controller = searcher.controller self.controller.collect_params().reset_ctx(self.controller_ctx) self.controller_optimizer = mx.gluon.Trainer( self.controller.collect_params(), 'adam', optimizer_params={ 'learning_rate': controller_lr * controller_batch_size }) self.controller_batch_size = controller_batch_size self.baseline = None self.lock = mp.Lock() # async buffers if not sync: self.mp_count = mp.Value('i', 0) self.mp_seed = mp.Value('i', 0) self.mp_fail = mp.Value('i', 0) if resume: if os.path.isfile(checkpoint): self.load_state_dict(load(checkpoint)) else: msg = 'checkpoint path {} is not available for resume.'.format( checkpoint) logger.exception(msg)
import multiprocessing as mp def job(n, w, l): l.acquire() w.value += n print(w.value) l.release() if __name__=='__main__': w = mp.Value('d', 0.0) l = mp.Lock() p1 = mp.Process(target=job, args=(1, w, l)) p2 = mp.Process(target=job, args=(2, w, l)) p1.start() p2.start() p1.join() p2.join()
import multiprocessing import ctypes import time def func(num): num.value = 0 while 1: print(num.value) time.sleep(0.1) if __name__ == "__main__": num = multiprocessing.Value('b', 1) print(num.value) p = multiprocessing.Process(target=func, args=(num, )) p.start() # p.join() print(num.value) while 1: num.value += 1 time.sleep(0.05)
def runCapture(config, duration=None, video_file=None, nodetect=False, detect_end=False, \ upload_manager=None, resume_capture=False): """ Run capture and compression for the given time.given Arguments: config: [config object] Configuration read from the .config file. Keyword arguments: duration: [float] Time in seconds to capture. None by default. video_file: [str] Path to the video file, if it was given as the video source. None by default. nodetect: [bool] If True, detection will not be performed. False by defualt. detect_end: [bool] If True, detection will be performed at the end of the night, when capture finishes. False by default. upload_manager: [UploadManager object] A handle to the UploadManager, which handles uploading files to the central server. None by default. resume_capture: [bool] Resume capture in the last data directory in CapturedFiles. Return: night_archive_dir: [str] Path to the archive folder of the processed night. """ global STOP_CAPTURE # Check if resuming capture to the last capture directory night_data_dir_name = None if resume_capture: log.info("Resuming capture in the last capture directory...") # Find the latest capture directory capturedfiles_path = os.path.join(os.path.abspath(config.data_dir), config.captured_dir) most_recent_dir_time = 0 for dir_name in sorted(os.listdir(capturedfiles_path)): dir_path_check = os.path.join(capturedfiles_path, dir_name) # Check it's a directory if os.path.isdir(dir_path_check): # Check if it starts with the correct station code if dir_name.startswith(str(config.stationID)): dir_mod_time = os.path.getmtime(dir_path_check) # Check that it is the most recent directory if (night_data_dir_name is None) or (dir_mod_time > most_recent_dir_time): night_data_dir_name = dir_name night_data_dir = dir_path_check most_recent_dir_time = dir_mod_time if night_data_dir_name is None: log.info("Previous capture directory could not be found! Creating a new one...") else: log.info("Previous capture directory found: {:s}".format(night_data_dir)) # Resume run is finished now, reset resume flag cml_args.resume = False # Make a name for the capture data directory if night_data_dir_name is None: # Create a directory for captured files night_data_dir_name = str(config.stationID) + '_' \ + datetime.datetime.utcnow().strftime('%Y%m%d_%H%M%S_%f') # Full path to the data directory night_data_dir = os.path.join(os.path.abspath(config.data_dir), config.captured_dir, \ night_data_dir_name) # Make a directory for the night mkdirP(night_data_dir) log.info('Data directory: ' + night_data_dir) # Get the platepar file platepar, platepar_path, platepar_fmt = getPlatepar(config, night_data_dir) log.info('Initializing frame buffers...') ### For some reason, the RPi 3 does not like memory chunks which size is the multipier of its L2 ### cache size (512 kB). When such a memory chunk is provided, the compression becomes 10x slower ### then usual. We are applying a dirty fix here where we just add an extra image row and column ### if such a memory chunk will be created. The compression is performed, and the image is cropped ### back to its original dimensions. array_pad = 0 # Check if the image dimensions are divisible by RPi3 L2 cache size and add padding if (256*config.width*config.height)%(512*1024) == 0: array_pad = 1 # Init arrays for parallel compression on 2 cores sharedArrayBase = multiprocessing.Array(ctypes.c_uint8, 256*(config.width + array_pad)*(config.height + array_pad)) sharedArray = np.ctypeslib.as_array(sharedArrayBase.get_obj()) sharedArray = sharedArray.reshape(256, (config.height + array_pad), (config.width + array_pad)) startTime = multiprocessing.Value('d', 0.0) sharedArrayBase2 = multiprocessing.Array(ctypes.c_uint8, 256*(config.width + array_pad)*(config.height + array_pad)) sharedArray2 = np.ctypeslib.as_array(sharedArrayBase2.get_obj()) sharedArray2 = sharedArray2.reshape(256, (config.height + array_pad), (config.width + array_pad)) startTime2 = multiprocessing.Value('d', 0.0) log.info('Initializing frame buffers done!') # Check if the detection should be performed or not if nodetect: detector = None else: if detect_end: # Delay detection until the end of the night delay_detection = duration else: # Delay the detection for 2 minutes after capture start delay_detection = 120 # Initialize the detector detector = QueuedPool(detectStarsAndMeteors, cores=1, log=log, delay_start=delay_detection, \ backup_dir=night_data_dir) detector.startPool() # If the capture is being resumed into the directory, load all previously saved FF files if resume_capture: for ff_name in sorted(os.listdir(night_data_dir)): # Check if the file is a valid FF files ff_path = os.path.join(night_data_dir, ff_name) if os.path.isfile(ff_path) and (str(config.stationID) in ff_name) and validFFName(ff_name): # Add the FF file to the detector detector.addJob([night_data_dir, ff_name, config]) log.info("Added existing FF files for detection: {:s}".format(ff_name)) # Initialize buffered capture bc = BufferedCapture(sharedArray, startTime, sharedArray2, startTime2, config, video_file=video_file) # Initialize the live image viewer if config.live_maxpixel_enable: live_view = LiveViewer(night_data_dir, slideshow=False, banner_text="Live") live_view.start() else: live_view = None # Initialize compression compressor = Compressor(night_data_dir, sharedArray, startTime, sharedArray2, startTime2, config, detector=detector) # Start buffered capture bc.startCapture() # Init and start the compression compressor.start() # Capture until Ctrl+C is pressed wait(duration, compressor) # If capture was manually stopped, end capture if STOP_CAPTURE: log.info('Ending capture...') # Stop the capture log.debug('Stopping capture...') bc.stopCapture() log.debug('Capture stopped') dropped_frames = bc.dropped_frames log.info('Total number of late or dropped frames: ' + str(dropped_frames)) # Stop the compressor log.debug('Stopping compression...') detector = compressor.stop() # Free shared memory after the compressor is done try: log.debug('Freeing frame buffers...') del sharedArrayBase del sharedArray del sharedArrayBase2 del sharedArray2 except Exception as e: log.debug('Freeing frame buffers failed with error:' + repr(e)) log.debug(repr(traceback.format_exception(*sys.exc_info()))) log.debug('Compression stopped') if live_view is not None: # Stop the live viewer log.debug('Stopping live viewer...') live_view.stop() live_view.join() del live_view live_view = None log.debug('Live view stopped') # If detection should be performed if not nodetect: try: log.info('Finishing up the detection, ' + str(detector.input_queue.qsize()) \ + ' files to process...') except: print('Finishing up the detection... error when getting input queue size!') # Reset the Ctrl+C to KeyboardInterrupt resetSIGINT() try: # If there are some more files to process, process them on more cores if detector.input_queue.qsize() > 0: # Let the detector use all cores, but leave 2 free available_cores = multiprocessing.cpu_count() - 2 if available_cores > 1: log.info('Running the detection on {:d} cores...'.format(available_cores)) # Start the detector detector.updateCoreNumber(cores=available_cores) log.info('Waiting for the detection to finish...') # Wait for the detector to finish and close it detector.closePool() log.info('Detection finished!') except KeyboardInterrupt: log.info('Ctrl + C pressed, exiting...') if upload_manager is not None: # Stop the upload manager if upload_manager.is_alive(): log.debug('Closing upload manager...') upload_manager.stop() del upload_manager # Terminate the detector if detector is not None: del detector sys.exit() # Set the Ctrl+C back to 'soft' program kill setSIGINT() ### SAVE DETECTIONS TO FILE log.info('Collecting results...') # Get the detection results from the queue detection_results = detector.getResults() else: detection_results = [] # Save detection to disk and archive detection night_archive_dir, archive_name, _ = processNight(night_data_dir, config, \ detection_results=detection_results, nodetect=nodetect) # Put the archive up for upload if upload_manager is not None: log.info('Adding file to upload list: ' + archive_name) upload_manager.addFiles([archive_name]) log.info('File added...') # Delete detector backup files if detector is not None: detector.deleteBackupFiles() # If the capture was run for a limited time, run the upload right away if (duration is not None) and (upload_manager is not None): log.info('Uploading data before exiting...') upload_manager.uploadData() # Run the external script runExternalScript(night_data_dir, night_archive_dir, config) # If capture was manually stopped, end program if STOP_CAPTURE: log.info('Ending program') # Stop the upload manager if upload_manager is not None: if upload_manager.is_alive(): upload_manager.stop() log.info('Closing upload manager...') sys.exit() return night_archive_dir
def create_app( agent: Optional["Agent"] = None, cors_origins: Union[Text, List[Text], None] = "*", auth_token: Optional[Text] = None, jwt_secret: Optional[Text] = None, jwt_method: Text = "HS256", endpoints: Optional[AvailableEndpoints] = None, ): """Class representing a Rasa HTTP server.""" app = Sanic(__name__) app.config.RESPONSE_TIMEOUT = 60 * 60 configure_cors(app, cors_origins) # Setup the Sanic-JWT extension if jwt_secret and jwt_method: # since we only want to check signatures, we don't actually care # about the JWT method and set the passed secret as either symmetric # or asymmetric key. jwt lib will choose the right one based on method app.config["USE_JWT"] = True Initialize( app, secret=jwt_secret, authenticate=authenticate, algorithm=jwt_method, user_id="username", ) app.agent = agent # Initialize shared object of type unsigned int for tracking # the number of active training processes app.active_training_processes = multiprocessing.Value("I", 0) @app.exception(ErrorResponse) async def handle_error_response(request: Request, exception: ErrorResponse): return response.json(exception.error_info, status=exception.status) add_root_route(app) @app.get("/version") async def version(request: Request): """Respond with the version number of the installed Rasa.""" return response.json({ "version": rasa.__version__, "minimum_compatible_version": MINIMUM_COMPATIBLE_VERSION, }) @app.get("/status") @requires_auth(app, auth_token) @ensure_loaded_agent(app) async def status(request: Request): """Respond with the model name and the fingerprint of that model.""" return response.json({ "model_file": app.agent.path_to_model_archive or app.agent.model_directory, "fingerprint": model.fingerprint_from_path(app.agent.model_directory), "num_active_training_jobs": app.active_training_processes.value, }) @app.get("/conversations/<conversation_id>/tracker") @requires_auth(app, auth_token) @ensure_loaded_agent(app) async def retrieve_tracker(request: Request, conversation_id: Text): """Get a dump of a conversation's tracker including its events.""" verbosity = event_verbosity_parameter(request, EventVerbosity.AFTER_RESTART) until_time = rasa.utils.endpoints.float_arg(request, "until") tracker = await get_tracker(app.agent.create_processor(), conversation_id) try: if until_time is not None: tracker = tracker.travel_back_in_time(until_time) state = tracker.current_state(verbosity) return response.json(state) except Exception as e: logger.debug(traceback.format_exc()) raise ErrorResponse(500, "ConversationError", f"An unexpected error occurred. Error: {e}") @app.post("/conversations/<conversation_id>/tracker/events") @requires_auth(app, auth_token) @ensure_loaded_agent(app) async def append_events(request: Request, conversation_id: Text): """Append a list of events to the state of a conversation""" validate_request_body( request, "You must provide events in the request body in order to append them" "to the state of a conversation.", ) verbosity = event_verbosity_parameter(request, EventVerbosity.AFTER_RESTART) try: async with app.agent.lock_store.lock(conversation_id): tracker = await get_tracker(app.agent.create_processor(), conversation_id) # Get events after tracker initialization to ensure that generated # timestamps are after potential session events. events = _get_events_from_request_body(request) for event in events: tracker.update(event, app.agent.domain) app.agent.tracker_store.save(tracker) return response.json(tracker.current_state(verbosity)) except Exception as e: logger.debug(traceback.format_exc()) raise ErrorResponse(500, "ConversationError", f"An unexpected error occurred. Error: {e}") def _get_events_from_request_body(request: Request) -> List[Event]: events = request.json if not isinstance(events, list): events = [events] events = [Event.from_parameters(event) for event in events] events = [event for event in events if event] if not events: raise_warning( f"Append event called, but could not extract a valid event. " f"Request JSON: {request.json}") raise ErrorResponse( 400, "BadRequest", "Couldn't extract a proper event from the request body.", { "parameter": "", "in": "body" }, ) return events @app.put("/conversations/<conversation_id>/tracker/events") @requires_auth(app, auth_token) @ensure_loaded_agent(app) async def replace_events(request: Request, conversation_id: Text): """Use a list of events to set a conversations tracker to a state.""" validate_request_body( request, "You must provide events in the request body to set the sate of the " "conversation tracker.", ) verbosity = event_verbosity_parameter(request, EventVerbosity.AFTER_RESTART) try: async with app.agent.lock_store.lock(conversation_id): tracker = DialogueStateTracker.from_dict( conversation_id, request.json, app.agent.domain.slots) # will override an existing tracker with the same id! app.agent.tracker_store.save(tracker) return response.json(tracker.current_state(verbosity)) except Exception as e: logger.debug(traceback.format_exc()) raise ErrorResponse(500, "ConversationError", f"An unexpected error occurred. Error: {e}") @app.get("/conversations/<conversation_id>/story") @requires_auth(app, auth_token) @ensure_loaded_agent(app) async def retrieve_story(request: Request, conversation_id: Text): """Get an end-to-end story corresponding to this conversation.""" # retrieve tracker and set to requested state tracker = await get_tracker(app.agent.create_processor(), conversation_id) until_time = rasa.utils.endpoints.float_arg(request, "until") try: if until_time is not None: tracker = tracker.travel_back_in_time(until_time) # dump and return tracker state = tracker.export_stories(e2e=True) return response.text(state) except Exception as e: logger.debug(traceback.format_exc()) raise ErrorResponse(500, "ConversationError", f"An unexpected error occurred. Error: {e}") @app.post("/conversations/<conversation_id>/execute") @requires_auth(app, auth_token) @ensure_loaded_agent(app) async def execute_action(request: Request, conversation_id: Text): request_params = request.json action_to_execute = request_params.get("name", None) if not action_to_execute: raise ErrorResponse( 400, "BadRequest", "Name of the action not provided in request body.", { "parameter": "name", "in": "body" }, ) # Deprecation warning raise_warning( "Triggering actions via the execute endpoint is deprecated. " "Trigger an intent via the " "`/conversations/<conversation_id>/trigger_intent` " "endpoint instead.", FutureWarning, ) policy = request_params.get("policy", None) confidence = request_params.get("confidence", None) verbosity = event_verbosity_parameter(request, EventVerbosity.AFTER_RESTART) try: async with app.agent.lock_store.lock(conversation_id): tracker = await get_tracker(app.agent.create_processor(), conversation_id) output_channel = _get_output_channel(request, tracker) await app.agent.execute_action( conversation_id, action_to_execute, output_channel, policy, confidence, ) except Exception as e: logger.debug(traceback.format_exc()) raise ErrorResponse(500, "ConversationError", f"An unexpected error occurred. Error: {e}") tracker = await get_tracker(app.agent.create_processor(), conversation_id) state = tracker.current_state(verbosity) response_body = {"tracker": state} if isinstance(output_channel, CollectingOutputChannel): response_body["messages"] = output_channel.messages return response.json(response_body) @app.post("/conversations/<conversation_id>/trigger_intent") @requires_auth(app, auth_token) @ensure_loaded_agent(app) async def trigger_intent(request: Request, conversation_id: Text) -> HTTPResponse: request_params = request.json intent_to_trigger = request_params.get("name") entities = request_params.get("entities", []) if not intent_to_trigger: raise ErrorResponse( 400, "BadRequest", "Name of the intent not provided in request body.", { "parameter": "name", "in": "body" }, ) verbosity = event_verbosity_parameter(request, EventVerbosity.AFTER_RESTART) try: async with app.agent.lock_store.lock(conversation_id): tracker = await get_tracker(app.agent.create_processor(), conversation_id) output_channel = _get_output_channel(request, tracker) if intent_to_trigger not in app.agent.domain.intents: raise ErrorResponse( 404, "NotFound", f"The intent {trigger_intent} does not exist in the domain.", ) await app.agent.trigger_intent( intent_name=intent_to_trigger, entities=entities, output_channel=output_channel, tracker=tracker, ) except ErrorResponse: raise except Exception as e: logger.debug(traceback.format_exc()) raise ErrorResponse(500, "ConversationError", f"An unexpected error occurred. Error: {e}") state = tracker.current_state(verbosity) response_body = {"tracker": state} if isinstance(output_channel, CollectingOutputChannel): response_body["messages"] = output_channel.messages return response.json(response_body) @app.post("/conversations/<conversation_id>/predict") @requires_auth(app, auth_token) @ensure_loaded_agent(app) async def predict(request: Request, conversation_id: Text): try: # Fetches the appropriate bot response in a json format responses = await app.agent.predict_next(conversation_id) responses["scores"] = sorted(responses["scores"], key=lambda k: (-k["score"], k["action"])) return response.json(responses) except Exception as e: logger.debug(traceback.format_exc()) raise ErrorResponse(500, "ConversationError", f"An unexpected error occurred. Error: {e}") @app.post("/conversations/<conversation_id>/messages") @requires_auth(app, auth_token) @ensure_loaded_agent(app) async def add_message(request: Request, conversation_id: Text): validate_request_body( request, "No message defined in request body. Add a message to the request body in " "order to add it to the tracker.", ) request_params = request.json message = request_params.get("text") sender = request_params.get("sender") parse_data = request_params.get("parse_data") verbosity = event_verbosity_parameter(request, EventVerbosity.AFTER_RESTART) # TODO: implement for agent / bot if sender != "user": raise ErrorResponse( 400, "BadRequest", "Currently, only user messages can be passed to this endpoint. " "Messages of sender '{}' cannot be handled.".format(sender), { "parameter": "sender", "in": "body" }, ) user_message = UserMessage(message, None, conversation_id, parse_data) try: async with app.agent.lock_store.lock(conversation_id): tracker = await app.agent.log_message(user_message) return response.json(tracker.current_state(verbosity)) except Exception as e: logger.debug(traceback.format_exc()) raise ErrorResponse(500, "ConversationError", f"An unexpected error occurred. Error: {e}") @app.post("/model/train") @requires_auth(app, auth_token) async def train(request: Request) -> HTTPResponse: """Train a Rasa Model.""" validate_request_body( request, "You must provide training data in the request body in order to " "train your model.", ) rjs = request.json validate_request(rjs) # create a temporary directory to store config, domain and # training data temp_dir = tempfile.mkdtemp() config_path = os.path.join(temp_dir, "config.yml") rasa.utils.io.write_text_file(rjs["config"], config_path) if "nlu" in rjs: nlu_path = os.path.join(temp_dir, "nlu.md") rasa.utils.io.write_text_file(rjs["nlu"], nlu_path) if "stories" in rjs: stories_path = os.path.join(temp_dir, "stories.md") rasa.utils.io.write_text_file(rjs["stories"], stories_path) domain_path = DEFAULT_DOMAIN_PATH if "domain" in rjs: domain_path = os.path.join(temp_dir, "domain.yml") rasa.utils.io.write_text_file(rjs["domain"], domain_path) if rjs.get("save_to_default_model_directory", True) is True: model_output_directory = DEFAULT_MODELS_PATH else: model_output_directory = tempfile.gettempdir() try: with app.active_training_processes.get_lock(): app.active_training_processes.value += 1 info = dict( domain=domain_path, config=config_path, training_files=temp_dir, output=model_output_directory, force_training=rjs.get("force", False), ) loop = asyncio.get_event_loop() from rasa import train as train_model # Declare `model_path` upfront to avoid pytype `name-error` model_path: Optional[Text] = None # pass `None` to run in default executor model_path = await loop.run_in_executor( None, functools.partial(train_model, **info)) filename = os.path.basename(model_path) if model_path else None return await response.file(model_path, filename=filename, headers={"filename": filename}) except InvalidDomain as e: raise ErrorResponse( 400, "InvalidDomainError", f"Provided domain file is invalid. Error: {e}", ) except Exception as e: logger.debug(traceback.format_exc()) raise ErrorResponse( 500, "TrainingError", f"An unexpected error occurred during training. Error: {e}", ) finally: with app.active_training_processes.get_lock(): app.active_training_processes.value -= 1 def validate_request(rjs): if "config" not in rjs: raise ErrorResponse( 400, "BadRequest", "The training request is missing the required key `config`.", { "parameter": "config", "in": "body" }, ) if "nlu" not in rjs and "stories" not in rjs: raise ErrorResponse( 400, "BadRequest", "To train a Rasa model you need to specify at least one type of " "training data. Add `nlu` and/or `stories` to the request.", { "parameters": ["nlu", "stories"], "in": "body" }, ) if "stories" in rjs and "domain" not in rjs: raise ErrorResponse( 400, "BadRequest", "To train a Rasa model with story training data, you also need to " "specify the `domain`.", { "parameter": "domain", "in": "body" }, ) @app.post("/model/test/stories") @requires_auth(app, auth_token) @ensure_loaded_agent(app, require_core_is_ready=True) async def evaluate_stories(request: Request): """Evaluate stories against the currently loaded model.""" validate_request_body( request, "You must provide some stories in the request body in order to " "evaluate your model.", ) stories = rasa.utils.io.create_temporary_file(request.body, mode="w+b") use_e2e = rasa.utils.endpoints.bool_arg(request, "e2e", default=False) try: evaluation = await test(stories, app.agent, e2e=use_e2e) return response.json(evaluation) except Exception as e: logger.debug(traceback.format_exc()) raise ErrorResponse( 500, "TestingError", f"An unexpected error occurred during evaluation. Error: {e}", ) @app.post("/model/test/intents") @requires_auth(app, auth_token) async def evaluate_intents(request: Request): """Evaluate intents against a Rasa model.""" validate_request_body( request, "You must provide some nlu data in the request body in order to " "evaluate your model.", ) eval_agent = app.agent model_path = request.args.get("model", None) if model_path: model_server = app.agent.model_server if model_server is not None: model_server.url = model_path eval_agent = await _load_agent(model_path, model_server, app.agent.remote_storage) nlu_data = rasa.utils.io.create_temporary_file(request.body, mode="w+b") data_path = os.path.abspath(nlu_data) if not os.path.exists(eval_agent.model_directory): raise ErrorResponse(409, "Conflict", "Loaded model file not found.") model_directory = eval_agent.model_directory _, nlu_model = model.get_model_subdirectories(model_directory) try: evaluation = run_evaluation(data_path, nlu_model) return response.json(evaluation) except Exception as e: logger.debug(traceback.format_exc()) raise ErrorResponse( 500, "TestingError", f"An unexpected error occurred during evaluation. Error: {e}", ) @app.post("/model/predict") @requires_auth(app, auth_token) @ensure_loaded_agent(app, require_core_is_ready=True) async def tracker_predict(request: Request): """ Given a list of events, predicts the next action""" validate_request_body( request, "No events defined in request_body. Add events to request body in order to " "predict the next action.", ) sender_id = UserMessage.DEFAULT_SENDER_ID verbosity = event_verbosity_parameter(request, EventVerbosity.AFTER_RESTART) request_params = request.json try: tracker = DialogueStateTracker.from_dict(sender_id, request_params, app.agent.domain.slots) except Exception as e: logger.debug(traceback.format_exc()) raise ErrorResponse( 400, "BadRequest", f"Supplied events are not valid. {e}", { "parameter": "", "in": "body" }, ) try: policy_ensemble = app.agent.policy_ensemble probabilities, policy = policy_ensemble.probabilities_using_best_policy( tracker, app.agent.domain) scores = [{ "action": a, "score": p } for a, p in zip(app.agent.domain.action_names, probabilities)] return response.json({ "scores": scores, "policy": policy, "tracker": tracker.current_state(verbosity), }) except Exception as e: logger.debug(traceback.format_exc()) raise ErrorResponse(500, "PredictionError", f"An unexpected error occurred. Error: {e}") @app.post("/model/parse") @requires_auth(app, auth_token) @ensure_loaded_agent(app) async def parse(request: Request): validate_request_body( request, "No text message defined in request_body. Add text message to request body " "in order to obtain the intent and extracted entities.", ) emulation_mode = request.args.get("emulation_mode") emulator = _create_emulator(emulation_mode) try: data = emulator.normalise_request_json(request.json) try: parsed_data = await app.agent.parse_message_using_nlu_interpreter( data.get("text")) except Exception as e: logger.debug(traceback.format_exc()) raise ErrorResponse( 400, "ParsingError", f"An unexpected error occurred. Error: {e}") response_data = emulator.normalise_response_json(parsed_data) return response.json(response_data) except Exception as e: logger.debug(traceback.format_exc()) raise ErrorResponse(500, "ParsingError", f"An unexpected error occurred. Error: {e}") @app.put("/model") @requires_auth(app, auth_token) async def load_model(request: Request): validate_request_body( request, "No path to model file defined in request_body.") model_path = request.json.get("model_file", None) model_server = request.json.get("model_server", None) remote_storage = request.json.get("remote_storage", None) if model_server: try: model_server = EndpointConfig.from_dict(model_server) except TypeError as e: logger.debug(traceback.format_exc()) raise ErrorResponse( 400, "BadRequest", f"Supplied 'model_server' is not valid. Error: {e}", { "parameter": "model_server", "in": "body" }, ) app.agent = await _load_agent(model_path, model_server, remote_storage, endpoints, app.agent.lock_store) logger.debug(f"Successfully loaded model '{model_path}'.") return response.json(None, status=204) @app.delete("/model") @requires_auth(app, auth_token) async def unload_model(request: Request): model_file = app.agent.model_directory app.agent = Agent(lock_store=app.agent.lock_store) logger.debug(f"Successfully unloaded model '{model_file}'.") return response.json(None, status=204) @app.get("/domain") @requires_auth(app, auth_token) @ensure_loaded_agent(app) async def get_domain(request: Request): """Get current domain in yaml or json format.""" accepts = request.headers.get("Accept", default="application/json") if accepts.endswith("json"): domain = app.agent.domain.as_dict() return response.json(domain) elif accepts.endswith("yml") or accepts.endswith("yaml"): domain_yaml = app.agent.domain.as_yaml() return response.text(domain_yaml, status=200, content_type="application/x-yml") else: raise ErrorResponse( 406, "NotAcceptable", "Invalid Accept header. Domain can be " "provided as " 'json ("Accept: application/json") or' 'yml ("Accept: application/x-yml"). ' "Make sure you've set the appropriate Accept " "header.", ) return app
root_copy_room_id = config.get('root-copy', 'room_id') ftp_ip = config.get('ftp', 'ip') ftp_port = config.getint('ftp', 'port') ftp_user = config.get('ftp', 'user') ftp_pwd = config.get('ftp', 'pwd') ftp_dir = config.get('ftp', 'dir') server_type = config.get('server', 'server_type') zone_room_id = config.get('server', 'zone_room_id') node_name = config.get('server', 'node_name') server_id = config.get('server', 'server_id') dns_id = config.get('server', 'dns_id') loop_count = 0 share_delay = multiprocessing.Value('d', 86400) network = {} network['ip'] = config.get('local-net', 'ip') network['port'] = config.getint('local-net', 'port') network['crt'] = config.get('local-net', 'crt') network['key'] = config.get('local-net', 'key') waj_conf['net'] = network upload = {} upload['url'] = config.get('upload', 'url') upload['org_id'] = config.get('upload', 'org_id') waj_conf['upload'] = upload security = {} security['user_pwd'] = config.get('waj-security', 'user_pwd')
def __init__(self): self.val = multiprocessing.Value('i', 0)
def music_load(path): music = [] for filename in os.listdir(path): if filename.endswith('.ogg'): music.append(os.path.join(path, filename)) return music music = music_load(path) choice_music = random.randrange(0, len(music)) pygame.mixer.music.load(music[choice_music]) autoplay = False filename = "none" choice = multiprocessing.Value('i', 0) # Executed on Ctrl-C def signal_handler(sig, frame): p.terminate() print('Ctrl+C detected: Exiting') os._exit(0) signal.signal(signal.SIGINT, signal_handler) def loop(cho): while (True): char = getch.getch()
def deposite(balance, lock): for i in range(100): time.sleep(0.01) lock.acquire() # critical section balance.value = balance.value + 1 lock.release() def withdraw(balance, lock): for i in range(100): time.sleep(0.01) lock.acquire() # critical section balance.value = balance.value - 1 lock.release() if __name__ == "__main__": balance = multiprocessing.Value("i", 200) lock = multiprocessing.Lock() d = multiprocessing.Process(target=deposite, args=(balance, lock)) w = multiprocessing.Process(target=withdraw, args=(balance, lock)) d.start() w.start() d.join() w.join() print(balance.value)
def spawn_import_clients(options, files_info): # Spawn one reader process for each db.table, as well as many client processes task_queue = SimpleQueue() error_queue = SimpleQueue() exit_event = multiprocessing.Event() interrupt_event = multiprocessing.Event() errors = [] reader_procs = [] client_procs = [] parent_pid = os.getpid() signal.signal( signal.SIGINT, lambda a, b: abort_import(a, b, parent_pid, exit_event, task_queue, client_procs, interrupt_event)) try: progress_info = [] rows_written = multiprocessing.Value(ctypes.c_longlong, 0) for i in xrange(options["clients"]): client_procs.append( multiprocessing.Process(target=client_process, args=(options["host"], options["port"], options["auth_key"], task_queue, error_queue, rows_written, options["force"], options["durability"]))) client_procs[-1].start() for file_info in files_info: progress_info.append(( multiprocessing.Value(ctypes.c_longlong, -1), # Current lines/bytes processed multiprocessing.Value(ctypes.c_longlong, 0))) # Total lines/bytes to process reader_procs.append( multiprocessing.Process(target=table_reader, args=(options, file_info, task_queue, error_queue, progress_info[-1], exit_event))) reader_procs[-1].start() # Wait for all reader processes to finish - hooray, polling while len(reader_procs) > 0: time.sleep(0.1) # If an error has occurred, exit out early while not error_queue.empty(): exit_event.set() errors.append(error_queue.get()) reader_procs = [proc for proc in reader_procs if proc.is_alive()] update_progress(progress_info) # Wait for all clients to finish alive_clients = sum([client.is_alive() for client in client_procs]) for i in xrange(alive_clients): task_queue.put("exit") while len(client_procs) > 0: time.sleep(0.1) client_procs = [ client for client in client_procs if client.is_alive() ] # If we were successful, make sure 100% progress is reported if len(errors) == 0 and not interrupt_event.is_set(): print_progress(1.0) def plural(num, text): return "%d %s%s" % (num, text, "" if num == 1 else "s") # Continue past the progress output line print("") print("%s imported in %s" % (plural( rows_written.value, "row"), plural(len(files_info), "table"))) finally: signal.signal(signal.SIGINT, signal.SIG_DFL) if interrupt_event.is_set(): raise RuntimeError("Interrupted") if not task_queue.empty(): errors.append( (RuntimeError, RuntimeError("Error: Items remaining in the task queue"), None)) if len(errors) != 0: # multiprocessing queues don't handling tracebacks, so they've already been stringified in the queue for error in errors: print("%s" % error[1], file=sys.stderr) if options["debug"]: print("%s traceback: %s" % (error[0].__name__, error[2]), file=sys.stderr) if len(error) == 4: print("In file: %s" % error[3], file=sys.stderr) raise RuntimeError("Errors occurred during import")
with pathos.multiprocessing.ProcessingPool(MAX_CONS) as pool: pool.map(attack.start_attack, range(MAX_CONS)) print("SLOW "+args.type.upper()+" ATTACK has been executed successfully on " +str(args.connection)+" connections.") print("Total execution time is {}s.".format(time.time() - start_time)) if __name__ == '__main__': database = CSVHandler() initial_row = ["Time", "Connected", "Pending", "Closed", "Service_Available"] database.write(initial_row) timer = Timer(0.5) var = multiprocessing.Value('i', int(MAX_CONS/10)) p1 = multiprocessing.Process(target=thread1) p1.start() p2 = multiprocessing.Process(target=thread2) p2.start() p1.join() p2.join() df = pd.read_csv(args.out+".csv") fig = go.Figure() fig.update_layout(title="SLOW "+args.type.upper()+" ATTACK against "+ "http://"+str(args.target)+":"+str(args.port), xaxis_title="Time [s]", yaxis_title="Number of connections", font=dict(