def test_pipespeed(): c, d = processing.Pipe() cond = processing.Condition() elapsed = 0 iterations = 1 while elapsed < delta: iterations *= 2 p = processing.Process(target=pipe_func, args=(d, cond, iterations)) cond.acquire() p.start() cond.wait() cond.release() result = None t = _timer() while result != 'STOP': result = c.recv() elapsed = _timer() - t p.join() print iterations, 'objects passed through connection in', elapsed, 'seconds' print 'average number/sec:', iterations / elapsed
def create(cls, operation_spec, result_directory, debug=False, input_paths=[]): """ A factory method that creates an WEKA operation based on the information given in the operation specification operation_spec """ assert(operation_spec["type"] == "weka_classification") # Determine all parameter combinations that should be tested parameter_settings = cls._get_parameter_space(operation_spec) # Read the command template from a file template_file = open(os.path.join(pySPACE.configuration.spec_dir, "operations", "weka_templates", operation_spec["template"]), 'r') command_template = template_file.read() template_file.close() # number of processes if "runs" in operation_spec: number_processes = len(input_paths) * len(parameter_settings) * \ operation_spec["runs"] else: # approximate the number of processes runs = [] for dataset_dir in input_paths: collection = BaseDataset.load(dataset_dir) runs.append(collection.meta_data["runs"]) runs = max(runs) number_processes = len(input_paths) * len(parameter_settings) * \ runs if debug == True: # To better debug creation of processes we don't limit the queue # and create all processes before executing them processes = processing.Queue() cls._createProcesses(processes, result_directory, operation_spec, parameter_settings, input_paths, command_template) # create and return the weka operation object return cls(processes, operation_spec, result_directory, number_processes) else: # Create all processes by calling a recursive helper method in # another thread so that already created processes can be executed in # parallel. Therefore a queue is used which size is maximized to # guarantee that not to much objects are created (because this costs # memory). However, the actual number of 100 is arbitrary and might # be reviewed. processes = processing.Queue(100) create_process = processing.Process(target=cls._createProcesses, args=( processes, result_directory, operation_spec, parameter_settings, input_paths, command_template)) create_process.start() # create and return the weka operation object return cls(processes, operation_spec, result_directory, number_processes, create_process)
def test_join_timeout(): p = processing.Process(target=join_timeout_func) p.start() print 'waiting for process to finish' while 1: p.join(timeout=1) if not p.isAlive(): break print '.', sys.stdout.flush()
def create(cls, operation_spec, result_directory, debug=False, input_paths=[]): """ A factory method that creates an Analysis operation based on the information given in the operation specification operation_spec """ assert (operation_spec["type"] == "analysis") input_path = operation_spec["input_path"] summary = BaseDataset.load( os.path.join(pySPACE.configuration.storage, input_path)) data_dict = summary.data # Determine the parameters that should be analyzed parameters = operation_spec["parameters"] # Determine the metrics that should be plotted metrics = operation_spec["metrics"] # Determine how many processes will be created number_parameter_values = [ len(set(data_dict[param])) for param in parameters ] number_processes = cls._numberOfProcesses(0, number_parameter_values) + 1 if debug == True: # To better debug creation of processes we don't limit the queue # and create all processes before executing them processes = processing.Queue() cls._createProcesses(processes, result_directory, data_dict, parameters, metrics, True) return cls(processes, operation_spec, result_directory, number_processes) else: # Create all plot processes by calling a recursive helper method in # another thread so that already created processes can be executed # although creation of processes is not finished yet. Therefore a queue # is used which size is limited to guarantee that not to much objects # are created (since this costs memory). However, the actual number # of 100 is arbitrary and might be changed according to the system at hand. processes = processing.Queue(100) create_process = processing.Process( target=cls._createProcesses, args=(processes, result_directory, data_dict, parameters, metrics, True)) create_process.start() # create and return the operation object return cls(processes, operation_spec, result_directory, number_processes, create_process)
def test_sharedvalues(): values = [('i', 10), ('h', -2), ('d', 1.25)] arrays = [('i', range(100)), ('d', [0.25 * i for i in range(100)]), ('H', range(1000))] shared_values = [processing.Value(id, v) for id, v in values] shared_arrays = [processing.Array(id, a) for id, a in arrays] p = processing.Process(target=sharedvalues_func, args=(values, arrays, shared_values, shared_arrays)) p.start() p.join() assert p.getExitCode() == 0
def test_semaphore(): sema = processing.Semaphore(3) mutex = processing.RLock() running = processing.Value('i', 0) processes = [ processing.Process(target=semaphore_func, args=(sema, mutex, running)) for i in range(10) ] for p in processes: p.start() for p in processes: p.join()
def test_queue(): q = processing.Queue() p = processing.Process(target=queue_func, args=(q, )) p.start() o = None while o != 'STOP': try: o = q.get(timeout=0.3) print o, sys.stdout.flush() except Empty: print 'TIMEOUT' print
def test_value(): TASKS = 10 running = processing.Value('i', TASKS) mutex = processing.Lock() for i in range(TASKS): processing.Process(target=value_func, args=(running, mutex)).start() while running.value > 0: time.sleep(0.08) mutex.acquire() print running.value, sys.stdout.flush() mutex.release() print print 'No more running processes'
def test_event(): event = processing.Event() processes = [ processing.Process(target=event_func, args=(event, )) for i in range(5) ] for p in processes: p.start() print 'main is sleeping' time.sleep(2) print 'main is setting event' event.set() for p in processes: p.join()
def submit(self, plan): """ Starts a subprocess to manage the submission, and returns quickly. """ tmp_dir = tempfile.mkdtemp(dir="/tmp", prefix="jobsub-") state = ServerSubmissionState(submission_state=State.SUBMITTED, tmp_dir=tmp_dir) state.save() # Early save to get an "id" process = processing.Process(target=run_plan, args=(state.id, plan, tmp_dir), name=plan.name) process.setDaemon(True) process.start() state.pid = process.getPid() state.submission_state = State.RUNNING state.save() return SubmissionHandle(id=state.id)
def test_condition(): cond = processing.Condition() p = processing.Process(target=condition_func, args=(cond, )) print cond cond.acquire() print cond cond.acquire() print cond p.start() print 'main is waiting' cond.wait() print 'main has woken up' print cond cond.release() print cond cond.release() p.join() print cond
def create(cls, operation_spec, result_directory, debug=False, input_paths=[]): """ A factory method that creates an MMLF operation based on the information given in the operation specification operation_spec """ assert (operation_spec["type"] == "mmlf") # The generic world configuration YAML file world_conf = """ worldPackage : %s environment: %s agent: %s monitor: policyLogFrequency : 100000 %s """ # Create directory for the experiment world_name = operation_spec['world_name'] world_path = "%s/config/%s" % (result_directory, world_name) if not os.path.exists(world_path): os.makedirs(world_path) # Compute all possible parameter combinations # Determine all parameter combinations that should be tested parameter_settings = cls._get_parameter_space(operation_spec) # If the operation spec defines parameters for a generalized domain: if "generalized_domain" in operation_spec: # We have to test each parameter setting in each instantiation of # the generalized domain. This can be achieved by computing the # crossproduct of parameter settings and domain settings augmented_parameter_settings = [] for parameter_setting in parameter_settings: for domain_parameter_setting in operation_spec[ "generalized_domain"]: instantiation = dict(parameter_setting) instantiation.update(domain_parameter_setting) augmented_parameter_settings.append(instantiation) parameter_settings = augmented_parameter_settings # Create and remember all worlds for the given parameter_settings world_pathes = [] # for all parameter setting for parameter_setting in parameter_settings: # Add 4 blanks to all lines in templates environment_template = operation_spec['environment_template'] environment_template = \ "\n".join(" " + line for line in environment_template.split("\n")) agent_template = operation_spec['agent_template'] agent_template = \ "\n".join(" " + line for line in agent_template.split("\n")) monitor_conf = \ operation_spec['monitor_conf'] if 'monitor_conf' in operation_spec else "" monitor_conf = \ "\n".join(" " + line for line in monitor_conf.split("\n")) # Instantiate the templates environment_conf = environment_template agent_conf = agent_template for parameter, value in parameter_setting.iteritems(): environment_conf = environment_conf.replace( parameter, str(value)) agent_conf = agent_conf.replace(parameter, str(value)) def get_parameter_str(parameter_name): return "".join(subpart[:4] for subpart in parameter_name.split("_")) configuration_str = "{" + "}{".join([ "%s:%s" % (get_parameter_str(parameter), str(value)[:6]) for parameter, value in parameter_setting.iteritems() ]) + "}" configuration_str = configuration_str.replace('_', '') world_file_name = "world_%s.yaml" % configuration_str open(os.path.join(world_path, world_file_name), 'w').write( world_conf % (world_name, environment_conf, agent_conf, monitor_conf)) world_pathes.append(os.path.join(world_path, world_file_name)) number_processes = len(world_pathes) * int(operation_spec["runs"]) if debug is True: # To better debug creation of processes we don't limit the queue # and create all processes before executing them processes = processing.Queue() cls._createProcesses(processes, world_pathes, operation_spec, result_directory) return cls(processes, operation_spec, result_directory, world_name, number_processes) else: # Create all processes by calling a recursive helper method # in another thread so that already created processes can be # executed although creation of processes is not finished yet. # Therefor a queue is used which size is limited to guarantee that # not to much objects are created (since this costs memory). # However, the actual number of 100 is arbitrary and might be # changed according to the system at hand. processes = processing.Queue(100) create_process = processing.Process(target=cls._createProcesses, args=(processes, world_pathes, operation_spec, result_directory)) create_process.start() # create and return the operation object return cls(processes, operation_spec, result_directory, world_name, number_processes, create_process)
models = create_model(flux) # Processing support? if has_processing: # Wait until one thread has finished while len(processing.activeChildren()) >= max_threads: time.sleep(60) # Set arguments args = (loge, emin, emax, models) kwargs = {} # Generate pull distribution p = processing.Process(target=create_pull, args=args, kwargs=kwargs) p.start() print("Process emin=%.4f emax=%.4f started." % (emin, emax)) # Wait some time time.sleep(1) # ... no else: create_pull(loge, emin, emax, models) # Processing support if has_processing: # Wait until all threads finished
def create(cls, operation_spec, result_directory, debug=False, input_paths=[]): """ A factory method that creates an Analysis operation based on the information given in the operation specification operation_spec. If debug is TRUE the creation of the Analysis Processes will not be in a separated thread. """ assert (operation_spec["type"] == "comp_analysis") input_path = operation_spec["input_path"] summary = BaseDataset.load( os.path.join(pySPACE.configuration.storage, input_path)) data_dict = summary.data ## Done # Determine the parameters that should be analyzed parameters = operation_spec["parameters"] # Determine dependent parameters, which don't get extra resolution try: dep_par = operation_spec["dep_par"] except KeyError: dep_par = [] # Determine the metrics that should be plotted spec_metrics = operation_spec["metrics"] metrics = [] for metric in spec_metrics: if data_dict.has_key(metric): metrics.append(metric) else: import warnings warnings.warn('The metric "' + metric + '" is not contained in the results csv file.') if len(metrics) == 0: warnings.warn( 'No metric available from spec file, default to first dict entry.' ) metrics.append(data_dict.keys()[0]) # Determine how many processes will be created number_parameter_values = [ len(set(data_dict[param])) for param in parameters ] number_processes = cls._numberOfProcesses(0, number_parameter_values) + 1 logscale = False if operation_spec.has_key('logscale'): logscale = operation_spec['logscale'] markertype = 'x' if operation_spec.has_key('markertype'): markertype = operation_spec['markertype'] if debug == True: # To better debug creation of processes we don't limit the queue # and create all processes before executing them processes = processing.Queue() cls._createProcesses(processes, result_directory, data_dict, parameters, dep_par, metrics, logscale, markertype, True) return cls(processes, operation_spec, result_directory, number_processes) else: # Create all plot processes by calling a recursive helper method in # another thread so that already created processes can be executed # although creation of processes is not finished yet. Therefore a queue # is used which size is limited to guarantee that not to much objects # are created (since this costs memory). However, the actual number # of 100 is arbitrary and might be reviewed. processes = processing.Queue(100) create_process = processing.Process( target=cls._createProcesses, args=(processes, result_directory, data_dict, parameters, dep_par, metrics, logscale, markertype, True)) create_process.start() # create and return the comp_analysis operation object return cls(processes, operation_spec, result_directory, number_processes, create_process)
from PIL import Image import processing import utils import sys import numpy as np sys.setrecursionlimit(10000) filename = '54.png' img = Image.open(filename).convert('L') w, h = img.size pixels = img.load() pixels = np.reshape([pixels[i, j] for j in range(h) for i in range(w)], (w, h)) process = processing.Process(w, h, pixels) img = process.make()
def __startWorker(self): #- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - self.t = pc.Process(target=self._worker) self.t.setDaemon(True) self.t.start()
def startProcess(self, target, **kwargs): process = processing.Process(target=target, **kwargs) self.processes.append(process) process.start()
def create(cls, operation_spec, result_directory, debug=False, input_paths=[]): """ A factory method that creates the processes which form an operation based on the information given in the operation specification, *operation_spec*. In debug mode this is done in serial. In the other default mode, at the moment 4 processes are created in parallel and can be immediately executed. So generation of processes and execution are made in parallel. This kind of process creation is done independently from the backend. For huge parameter spaces this is necessary! Otherwise numerous processes are created and corresponding data is loaded but the concept of spreading the computation to different processors can not really be used, because process creation is blocking only one processor and memory space, but nothing more is done, till the processes are all created. .. todo:: Use :class:`~pySPACE.resources.dataset_defs.dummy.DummyDataset` for empty data, when no input_path is given. """ assert (operation_spec["type"] == "node_chain") # Determine all parameter combinations that should be tested parameter_settings = cls._get_parameter_space(operation_spec) ## Use node_chain parameter if no templates are given ## if not operation_spec.has_key("templates"): if operation_spec.has_key("node_chain"): operation_spec["templates"] = [ operation_spec.pop("node_chain") ] # extract_key_str(operation_spec["base_file"], # keyword="node_chain")] # operation_spec.pop("node_chain") else: warnings.warn( "Specify parameter 'templates' or 'node_chain' in your operation spec!" ) elif operation_spec.has_key("node_chain"): operation_spec.pop("node_chain") warnings.warn( "node_chain parameter is ignored. Templates are used.") # load files in templates as dictionaries elif type(operation_spec["templates"][0]) == str: operation_spec["template_files"] = \ copy.deepcopy(operation_spec["templates"]) for i in range(len(operation_spec["templates"])): rel_node_chain_file = operation_spec["templates"][i] abs_node_chain_file_name = os.sep.join([ pySPACE.configuration.spec_dir, "node_chains", rel_node_chain_file ]) with open(abs_node_chain_file_name, "r") as read_file: node_chain = read_file.read() #node_chain = yaml.load(read_file) operation_spec["templates"][i] = node_chain storage = pySPACE.configuration.storage if not input_paths: raise Exception("No input datasets found in input_path %s in %s!" % (operation_spec["input_path"], storage)) # Get relative path rel_input_paths = [name[len(storage):] for name in input_paths] # Determine approximate number of runs if "runs" in operation_spec: runs = operation_spec["runs"] else: runs = [] for dataset_dir in rel_input_paths: abs_collection_path = \ pySPACE.configuration.storage + os.sep \ + dataset_dir collection_runs = \ BaseDataset.load_meta_data(abs_collection_path).get('runs',1) runs.append(collection_runs) runs = max(runs) # Determine splits dataset_dir = rel_input_paths[0] abs_collection_path = \ pySPACE.configuration.storage + os.sep + dataset_dir splits = BaseDataset.load_meta_data(abs_collection_path).get( 'splits', 1) # Determine how many processes will be created number_processes = len(operation_spec["templates"]) * \ len(parameter_settings) * len(rel_input_paths) * \ runs * splits if debug: # To better debug creation of processes we don't limit the queue # and create all processes before executing them processes = processing.Queue() cls._createProcesses(processes, result_directory, operation_spec, parameter_settings, rel_input_paths) # create and return the operation object return cls(processes, operation_spec, result_directory, number_processes) else: # Create all processes by calling a recursive helper method in # another thread so that already created processes can be executed in # parallel. Therefore a queue is used which size is maximized to # guarantee that not to much objects are created (because this costs # memory). However, the actual number of 4 is arbitrary and might # be changed according to the system at hand. processes = processing.Queue(4) create_process = \ processing.Process(target=cls._createProcesses, args=(processes, result_directory, operation_spec, parameter_settings, rel_input_paths)) create_process.start() # create and return the operation object return cls(processes, operation_spec, result_directory, number_processes, create_process)