def __init__(self, number_of_threads=10): self.log = logger.get("infra") self.log.info("Initiating TaskManager with %d threads" % number_of_threads) self.number_of_threads = number_of_threads self.pool = Executors.newFixedThreadPool(self.number_of_threads) self.futures = dict()
def __init__(self, sentiworddicname, amplifiersname, decrementersname): self.sentiworddic = importsentixcomplex(sentiworddicname) self.amplifiers = loadlines(amplifiersname) self.decrementers = loadlines(decrementersname) # start thread pool self.k = available_cpu_count() self.pool = Executors.newFixedThreadPool(self.k)
def main(opts): # set up our channel conn_factory = ConnectionFactory() conn_factory.setUri(config['RABBITMQ_URI']) conn = conn_factory.newConnection() channel = conn.createChannel() channel.queueDeclare(opts.queue_name, False, False, False, None) channel.basicQos(1); # tells the channel we're only going to deliver one response before req acknowledgement workers = [PdfExtractor(channel, opts) for i in xrange(opts.workers)] log.info("creating pool with %d threads" % opts.workers) tpool = Executors.newFixedThreadPool(opts.workers) log.info("executing threads") futures = tpool.invokeAll(workers) log.info("shutting down thread pool") tpool.shutdown() try: if not tpool.awaitTermination(50, TimeUnit.SECONDS): log.info("thread pool not shutting down; trying again") tpool.shutdownNow() if not tpool.awaitTermination(50, TimeUnit.SECONDS): log.error("Pool did not terminate") except InterruptedException: log.info("exception during thread pool shutdown; trying again") tpool.shutdownNow() Thread.currentThread().interrupt()
def __init__(self, sentiworddicname, amplifiersname, decrementersname): self.sentiworddic = importsentixcomplex(sentiworddicname) self.amplifiers = loadlines(amplifiersname) self.decrementers = loadlines(decrementersname) # start thread pool self.k = available_cpu_count() self.pool = Executors.newFixedThreadPool(self.k)
def load_data(self): executors = [] num_executors = 5 doc_executors = 5 pool = Executors.newFixedThreadPool(5) self.num_items = self.total_num_items / doc_executors for i in xrange(doc_executors): executors.append( GleambookUser_Docloader(self.bucket, self.num_items, self.items_start_from + i * self.num_items, batch_size=2000)) executors.append( GleambookMessages_Docloader(self.msg_bucket, self.num_items, self.items_start_from + i * self.num_items, batch_size=2000)) futures = pool.invokeAll(executors) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) self.updates_from = self.items_start_from self.deletes_from = self.items_start_from + self.total_num_items / 10 self.items_start_from += self.total_num_items
def main(opts): # set up our channel conn_factory = ConnectionFactory() conn_factory.setUri(config['RABBITMQ_URI']) conn = conn_factory.newConnection() channel = conn.createChannel() channel.queueDeclare(opts.queue_name, False, False, False, None) channel.basicQos(1) # tells the channel we're only going to deliver one response before req acknowledgement workers = [PdfExtractor(channel, opts) for i in xrange(opts.workers)] log.info("creating pool with %d threads" % opts.workers) tpool = Executors.newFixedThreadPool(opts.workers) log.info("executing threads") futures = tpool.invokeAll(workers) log.info("shutting down thread pool") tpool.shutdown() try: if not tpool.awaitTermination(50, TimeUnit.SECONDS): log.info("thread pool not shutting down; trying again") tpool.shutdownNow() if not tpool.awaitTermination(50, TimeUnit.SECONDS): log.error("Pool did not terminate") except InterruptedException: log.info("exception during thread pool shutdown; trying again") tpool.shutdownNow() Thread.currentThread().interrupt()
def __init__(self, number_of_threads=10): self.log = logging.getLogger("infra") self.log.debug("Initiating TaskManager with {0} threads" .format(number_of_threads)) self.number_of_threads = number_of_threads self.pool = Executors.newFixedThreadPool(self.number_of_threads) self.futures = {} self.tasks = []
def setUp(self): super(SuperIndexSearcherTest, self).setUp() self.executor = Executors.newFixedThreadPool(5) indexDirectory = SimpleFSDirectory(File(self.tempdir)) conf = IndexWriterConfig(Version.LUCENE_4_10_0, MerescoStandardAnalyzer()) self.writer = IndexWriter(indexDirectory, conf) self.reader = DirectoryReader.open(self.writer, True) self.sis = SuperIndexSearcher(self.reader)
def __init__(self, sentiworddicname = None, amplifiersname = None, decrementersname = None, serializedDictsName = None): if serializedDictsName is not None: (self.sentiworddic,self.amplifiers,self.decrementers) = loaddictionariesfromfile(serializedDictsName) else: self.sentiworddic = importsentixcomplex(sentiworddicname) self.amplifiers = loadlines(amplifiersname) self.decrementers = loadlines(decrementersname) # start thread pool self.k = available_cpu_count() self.pool = Executors.newFixedThreadPool(self.k)
def play(self): num_threads = Runtime.getRuntime().availableProcessors() executor = Executors.newFixedThreadPool(num_threads) callables = [_Worker(start_pos) for start_pos in self.positions] futures = executor.invokeAll(callables) # calculate stats for future in futures: worker = future.get() self.process_scores(worker) executor.shutdown()
def testParallelMultiSort(self): """ test a variety of sorts using a parallel multisearcher """ threadPool = Executors.newFixedThreadPool(self.getRandomNumber(2, 8), NamedThreadFactory("testParallelMultiSort")) searcher = IndexSearcher(MultiReader([self.searchX.getIndexReader(), self.searchY.getIndexReader()]), threadPool) self._runMultiSorts(searcher, False) threadPool.shutdown(); threadPool.awaitTermination(1000L, TimeUnit.MILLISECONDS);
def getDataWriter(self, meta): """ generated source for method getDataWriter """ if self.writer == None: with lock_for_object(self): if self.writer == None: self.writerExecutor = Executors.newFixedThreadPool( Runtime.getRuntime().availableProcessors()) self.writer = DefaultTableStoreWriter( self.asyncClient, config.getDataTableName(), WriterConfig(), None, self.writerExecutor) return TableStoreDataWriter(self.writer, self.config.getDataTableName(), meta)
def newFixedThreadPool(n_threads=0, name="jython-worker"): """ Return an ExecutorService whose Thread instances belong to the same group as the caller's Thread, and therefore will be interrupted when the caller is. n_threads: number of threads to use. If zero, use as many as available CPUs. If negative, use as many as available CPUs minus that number, but at least one. """ if n_threads <= 0: n_threads = max(1, Runtime.getRuntime().availableProcessors() + n_threads) return Executors.newFixedThreadPool(n_threads, ThreadFactorySameGroup(name))
def run_queries(self): num_executors = 1 executors = [] pool = Executors.newFixedThreadPool(num_executors) for i in xrange(num_executors): executors.append( QueryRunner(random.choice(self.queries), self.num_query, self.cbas_util)) futures = pool.invokeAll(executors) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors)
def runBlockMatching(params_list, image_pairs): MAX_CONCURRENT = 20 block_matching_inputs = zip(image_pairs, params_list) pool = Executors.newFixedThreadPool(MAX_CONCURRENT) block_matchers = [BlockMatcher(pair[0], pair[1], params) for (pair, params) in block_matching_inputs] futures = pool.invokeAll(block_matchers) for future in futures: print future.get(5, TimeUnit.SECONDS) shutdownAndAwaitTermination(pool, 5)
def __init__(self, state, burpCallbacks): """ Main constructor. Creates an instance of a FixedThreadPool for threading operations, such as issuing multiple HTTP requests. All calls to this class to methods that end in "Clicked" are made in an independent thread to avoid locking up the Burp UI. Args: state: the state object. burpCallbacks: the burp callbacks object. """ self.state = state self.burpCallbacks = burpCallbacks self.lock = Lock() self.extensions = [] self.maxConcurrentRequests = 8 # Avoid instantiating during unit test as it is not needed. if not utility.INSIDE_UNIT_TEST: self.state.executorService = Executors.newFixedThreadPool(16) self.state.fuzzExecutorService = Executors.newFixedThreadPool(16) # Beware: if the second argument to two of these importBurpExtension calls is the same, the same extension will be loaded twice. The solution is to recompile the JARs so that the classes do not have the same name. log("[+] Loading Backslash Powered Scanner") self.extensions.append( ("bps", utility.importBurpExtension( "lib/backslash-powered-scanner-fork.jar", 'burp.BackslashBurpExtender', burpCallbacks))) # log("[+] Loading SHELLING") # self.extensions.append(("shelling", utility.importBurpExtension("lib/shelling.jar", 'burp.BurpExtender', burpCallbacks))) log("[+] Loading ParamMiner") self.extensions.append( ("paramminer", utility.importBurpExtension("lib/param-miner-fork.jar", 'paramminer.BurpExtender', burpCallbacks)))
def searcher(self): if not self._reopenSearcher: return self._searcher if self._settings.multithreaded: if self._executor: self._executor.shutdown(); self._executor = Executors.newFixedThreadPool(self._numberOfConcurrentTasks); self._searcher = SuperIndexSearcher(self._reader, self._executor, self._numberOfConcurrentTasks) else: self._searcher = IndexSearcher(self._reader) self._searcher.setSimilarity(self._similarity) self._reopenSearcher = False return self._searcher
def __init__(self, sentiworddicname=None, amplifiersname=None, decrementersname=None, serializedDictsName=None): if serializedDictsName is not None: (self.sentiworddic, self.amplifiers, self.decrementers) = loaddictionariesfromfile(serializedDictsName) else: self.sentiworddic = importsentixcomplex(sentiworddicname) self.amplifiers = loadlines(amplifiersname) self.decrementers = loadlines(decrementersname) # start thread pool self.k = available_cpu_count() self.pool = Executors.newFixedThreadPool(self.k)
def start(cls): try: inetSocketAddress = InetSocketAddress(cls.host, cls.port) cls.httpServer = HttpServer.create(inetSocketAddress, cls.socketBackLog) cls.httpServer.createContext("/callback", CallbackHandler()) cls.httpServer.createContext("/start", StartHandler()) cls.httpServer.setExecutor( Executors.newFixedThreadPool(cls.poolsize)) cls.httpServer.start() logger.info("HTTPServerCallback is listening on %s %s" % (cls.host, cls.port)) except IOException, e: logger.error('(start) %s : stacktrace=%s' % (cls.__name__, e)) raise UnboundLocalError('(start) %s : stacktrace=%s' % (cls.__name__, e))
def getShiftFromViews(v1, v2): # Thread pool exe = Executors.newFixedThreadPool( Runtime.getRuntime().availableProcessors()) try: # PCM: phase correlation matrix pcm = PhaseCorrelation2.calculatePCM( v1, v2, ArrayImgFactory(FloatType()), FloatType(), ArrayImgFactory(ComplexFloatType()), ComplexFloatType(), exe) # Minimum image overlap to consider, in pixels minOverlap = v1.dimension(0) / 10 # Returns an instance of PhaseCorrelationPeak2 peak = PhaseCorrelation2.getShift(pcm, v1, v2, nHighestPeaks, minOverlap, True, True, exe) except Exception, e: print e
def searcher(self): if not self._reopenSearcher: return self._searcher if self._settings.multithreaded: if self._executor: self._executor.shutdown() self._executor = Executors.newFixedThreadPool( self._numberOfConcurrentTasks) self._searcher = SuperIndexSearcher(self._reader, self._executor, self._numberOfConcurrentTasks) else: self._searcher = IndexSearcher(self._reader) self._searcher.setSimilarity(self._similarity) self._reopenSearcher = False return self._searcher
def runBlockMatchingAll(wafer_title): MAX_CONCURRENT = 40 params = BlockMatcherParameters(wafer_title=wafer_title) start = 0 finish = len(os.listdir(params.input_folder)) neighbors = 2 image_pairs = make_image_pairs(start, finish, neighbors) # neighbors = 2 # 1200, 2400, 3600, 4500, .. # neighbors = 4 # 1210 - 1213 # 1267 - 1270 # neighbors = 3 # 1334 - 1336 # Log file t = time.localtime() ts = str(t[0]) + str(t[1]) + str(t[2]) + str(t[3]) + str(t[4]) + str(t[5]) writefile = params.output_folder + ts + "_block_matching_loop_log.txt" wf = open(writefile, 'w') wf.write(time.asctime() + "\n") wf.write(writefile + "\n") param_values = vars(params) for key in param_values: wf.write(key + "\t" + str(param_values[key]) + "\n") wf.write("B_idx\t" + "A_idx\t" + "B_file\t" + "A_file\t" + "matches\t" + "smooth_removed\t" + "max_displacement\t" + "eff_dist\t" + "eff_sigma\t" + "mesh\n") pool = Executors.newFixedThreadPool(MAX_CONCURRENT) block_matchers = [BlockMatcher(pair[0], pair[1], params, wf) for pair in image_pairs] futures = pool.invokeAll(block_matchers) for future in futures: print future.get(5, TimeUnit.SECONDS) wf.write(time.asctime() + "\n") wf.close() shutdownAndAwaitTermination(pool, 5)
def update_data(self): pool = Executors.newFixedThreadPool(5) executors = [] num_executors = 5 doc_executors = 4 executors.append( GleambookUser_Docloader(self.bucket, 2 * self.num_items / 10, self.updates_from, "update")) # executors.append(GleambookUser_Docloader(bucket, num_items/10, deletes_from,"delete")) executors.append( GleambookMessages_Docloader(self.msg_bucket, 2 * self.num_items / 10, self.updates_from, "update")) # executors.append(GleambookMessages_Docloader(msg_bucket, num_items/10, deletes_from,"delete")) futures = pool.invokeAll(executors) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors)
def register(view_index, filepaths, modelclass, csv_dir, params): n_threads = Runtime.getRuntime().availableProcessors() exe = Executors.newFixedThreadPool(n_threads) try: name = "matrices-view-%i" % view_index matrices = loadMatrices(name, csvdir) if not matrices: matrices = computeForwardTransforms(filepaths[view_index], klb_loader, getCalibration, csv_dir, exe, modelclass, params) saveMatrices(name, csv_dir) finally: exe.shutdown() transforms = asBackwardConcatTransforms(matrices, transformclass=Translation3D) path_transforms = dict(izip(filepaths[view_index], transforms)) registered_loader = RegisteredLoader(klb_loader, path_transforms) return Load.lazyStack(filepaths[view_index], registered_loader)
def load(self, k, v, docs=10000, server="localhost", bucket="default"): cluster = CouchbaseCluster.create(server) cluster.authenticate("Administrator", "password") bucket = cluster.openBucket(bucket) pool = Executors.newFixedThreadPool(5) docloaders = [] num_executors = 5 total_num_executors = 5 num_docs = docs / total_num_executors for i in xrange(total_num_executors): docloaders.append( DocloaderTask(bucket, num_docs, i * num_docs, k, v)) futures = pool.invokeAll(docloaders) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, 5) if bucket.close() and cluster.disconnect(): pass
def execute(self, frees, threads): """ Begins multithreaded execution Parameters ---------- frees: list(Freeway) list of freeways to begin multithreaded execution threads: number of threads """ from java.util.concurrent import Executors, ExecutorCompletionService pool = Executors.newFixedThreadPool(threads) ecs = ExecutorCompletionService(pool) for f in frees: ecs.submit(f) submitted = len(frees) while submitted > 0: result = ecs.take().get() print str(result) submitted -= 1
# Extract red channel: alpha:0, red:1, green:2, blue:3 red = Converters.argbChannel(img, 1) # Cut out two overlapping ROIs r1 = Rectangle(1708, 680, 1792, 1760) r2 = Rectangle(520, 248, 1660, 1652) cut1 = Views.zeroMin( Views.interval(red, [r1.x, r1.y], [r1.x + r1.width - 1, r1.y + r1.height - 1])) cut2 = Views.zeroMin( Views.interval(red, [r2.x, r2.y], [r2.x + r2.width - 1, r2.y + r2.height - 1])) # Thread pool exe = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()) try: # PCM: phase correlation matrix pcm = PhaseCorrelation2.calculatePCM(cut1, cut2, ArrayImgFactory(FloatType()), FloatType(), ArrayImgFactory(ComplexFloatType()), ComplexFloatType(), exe) # Number of phase correlation peaks to check with cross-correlation nHighestPeaks = 10 # Minimum image overlap to consider, in pixels minOverlap = cut1.dimension(0) / 10
def test_ups_volume(self): nodes_in_cluster = [self.servers[0]] print "Start Time: %s" % str( time.strftime("%H:%M:%S", time.gmtime(time.time()))) ######################################################################################################################## self.log.info("Add a KV nodes - 2") self.query_node = self.servers[1] rest = RestConnection(self.servers[1]) rest.set_data_path(data_path=self.servers[1].data_path, index_path=self.servers[1].index_path, cbas_path=self.servers[1].cbas_path) result = self.add_node(self.servers[1], rebalance=False) self.assertTrue(result, msg="Failed to add N1QL/Index node.") self.log.info("Add a KV nodes - 3") rest = RestConnection(self.servers[2]) rest.set_data_path(data_path=self.kv_servers[1].data_path, index_path=self.kv_servers[1].index_path, cbas_path=self.kv_servers[1].cbas_path) result = self.add_node(self.kv_servers[1], services=["kv"], rebalance=False) self.assertTrue(result, msg="Failed to add KV node.") self.log.info("Add one more KV node") rest = RestConnection(self.servers[3]) rest.set_data_path(data_path=self.kv_servers[3].data_path, index_path=self.kv_servers[3].index_path, cbas_path=self.kv_servers[3].cbas_path) result = self.add_node(self.kv_servers[3], services=["kv"], rebalance=False) self.assertTrue(result, msg="Failed to add KV node.") self.log.info("Add one more KV node") rest = RestConnection(self.servers[4]) rest.set_data_path(data_path=self.kv_servers[4].data_path, index_path=self.kv_servers[4].index_path, cbas_path=self.kv_servers[4].cbas_path) result = self.add_node(self.kv_servers[4], services=["kv"], rebalance=False) self.assertTrue(result, msg="Failed to add KV node.") nodes_in_cluster = nodes_in_cluster + [ self.servers[1], self.servers[2], self.servers[3], self.servers[4] ] ######################################################################################################################## self.log.info("Step 2: Create Couchbase buckets.") self.create_required_buckets() ######################################################################################################################## self.log.info( "Step 3: Create 10M docs average of 1k docs for 8 couchbase buckets." ) env = DefaultCouchbaseEnvironment.builder().mutationTokensEnabled( True).computationPoolSize(5).socketConnectTimeout( 100000).connectTimeout(100000).maxRequestLifetime( TimeUnit.SECONDS.toMillis(300)).build() cluster = CouchbaseCluster.create(env, self.master.ip) cluster.authenticate("Administrator", "password") bucket = cluster.openBucket("GleambookUsers") msg_bucket = cluster.openBucket("GleambookMessages") pool = Executors.newFixedThreadPool(5) items_start_from = 0 total_num_items = self.input.param("num_items", 5000) executors = [] num_executors = 5 doc_executors = 5 num_items = total_num_items / num_executors for i in xrange(doc_executors): executors.append( GleambookUser_Docloader(bucket, num_items, items_start_from + i * num_items)) executors.append( GleambookMessages_Docloader(msg_bucket, num_items, items_start_from + i * num_items)) futures = pool.invokeAll(executors) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) updates_from = items_start_from deletes_from = items_start_from + total_num_items / 10 items_start_from += total_num_items ######################################################################################################################## self.log.info("Step 6: Verify the items count.") self.validate_items_count() ######################################################################################################################## self.log.info("Step 8: Delete 1M docs. Update 1M docs.") pool = Executors.newFixedThreadPool(5) num_items = self.input.param("num_items", 5000) executors = [] num_executors = 5 doc_executors = 4 executors.append( GleambookUser_Docloader(bucket, num_items / 10, updates_from, "update")) executors.append( GleambookUser_Docloader(bucket, num_items / 10, deletes_from, "delete")) executors.append( GleambookMessages_Docloader(msg_bucket, num_items / 10, updates_from, "update")) executors.append( GleambookMessages_Docloader(msg_bucket, num_items / 10, deletes_from, "delete")) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) ######################################################################################################################## self.log.info("Step 9: Connect cbas buckets.") self.connect_cbas_buckets() self.sleep(10, "Wait for the ingestion to complete") ######################################################################################################################## self.log.info("Step 10: Verify the items count.") self.validate_items_count() ######################################################################################################################## self.log.info( "Step 12: When 11 is in progress do a KV Rebalance in of 1 nodes.") rest = RestConnection(self.servers[5]) rest.set_data_path(data_path=self.servers[5].data_path, index_path=self.servers[5].index_path, cbas_path=self.servers[5].cbas_path) rebalance = self.cluster.async_rebalance(nodes_in_cluster, [self.servers[5]], []) nodes_in_cluster += [self.servers[2]] ######################################################################################################################## self.log.info("Step 11: Create 10M docs.") pool = Executors.newFixedThreadPool(5) total_num_items = self.input.param("num_items", 5000) executors = [] num_executors = 5 doc_executors = 4 num_items = total_num_items / doc_executors for i in xrange(doc_executors): executors.append( GleambookUser_Docloader(bucket, num_items, items_start_from + i * num_items)) executors.append( GleambookMessages_Docloader(msg_bucket, num_items, items_start_from + i * num_items)) futures = pool.invokeAll(executors) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) updates_from = items_start_from deletes_from = items_start_from + total_num_items / 10 items_start_from += total_num_items ######################################################################################################################## self.log.info("Step 13: Wait for rebalance to complete.") rebalance.get_result() reached = RestHelper(self.rest).rebalance_reached(wait_step=120) self.assertTrue(reached, "rebalance failed, stuck or did not complete") self.sleep(20) ######################################################################################################################## self.log.info("Step 14: Verify the items count.") self.validate_items_count() ######################################################################################################################## self.log.info("Step 15: Delete 1M docs. Update 1M docs.") pool = Executors.newFixedThreadPool(5) num_items = self.input.param("num_items", 5000) executors = [] num_executors = 5 doc_executors = 4 executors.append( GleambookUser_Docloader(bucket, num_items / 10, updates_from, "update")) executors.append( GleambookUser_Docloader(bucket, num_items / 10, deletes_from, "delete")) executors.append( GleambookMessages_Docloader(msg_bucket, num_items / 10, updates_from, "update")) executors.append( GleambookMessages_Docloader(msg_bucket, num_items / 10, deletes_from, "delete")) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) ######################################################################################################################## self.log.info( "Step 16: Verify Results that 1M docs gets deleted from analytics datasets." ) self.validate_items_count() ######################################################################################################################## self.log.info("Step 17: Disconnect CBAS buckets.") self.disconnect_cbas_buckets() ######################################################################################################################## self.log.info("Step 18: Create 10M docs.") pool = Executors.newFixedThreadPool(5) total_num_items = self.input.param("num_items", 5000) executors = [] num_executors = 5 doc_executors = 4 num_items = total_num_items / doc_executors for i in xrange(doc_executors): executors.append( GleambookUser_Docloader(bucket, num_items, items_start_from + i * num_items)) executors.append( GleambookMessages_Docloader(msg_bucket, num_items, items_start_from + i * num_items)) futures = pool.invokeAll(executors) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) updates_from = items_start_from deletes_from = items_start_from + total_num_items / 10 items_start_from += total_num_items ######################################################################################################################## self.log.info("Step 20: Verify the docs count.") self.validate_items_count() ######################################################################################################################## pool = Executors.newFixedThreadPool(5) executors = [] num_executors = 5 self.log.info( "Step 22: When 21 is in progress do a KV Rebalance out of 2 nodes." ) rebalance = self.cluster.async_rebalance(nodes_in_cluster, [], self.servers[1:3]) nodes_in_cluster = [ node for node in nodes_in_cluster if node not in self.servers[1:3] ] futures = pool.invokeAll(executors) self.log.info("Step 23: Wait for rebalance.") rebalance.get_result() reached = RestHelper(self.rest).rebalance_reached(wait_step=120) self.assertTrue(reached, "rebalance failed, stuck or did not complete") self.sleep(20) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) ######################################################################################################################## self.log.info("Step 24: Create 10M docs.") pool = Executors.newFixedThreadPool(5) total_num_items = self.input.param("num_items", 5000) executors = [] num_executors = 6 doc_executors = 4 num_items = total_num_items / doc_executors for i in xrange(doc_executors): executors.append( GleambookUser_Docloader(bucket, num_items, items_start_from + i * num_items)) executors.append( GleambookMessages_Docloader(msg_bucket, num_items, items_start_from + i * num_items)) ##################################################### NEED TO BE UPDATED ################################################################## self.log.info( "Step 25: When 24 is in progress do a KV Rebalance in of 2 nodes.") for node in self.servers[1:3]: rest = RestConnection(node) rest.set_data_path(data_path=node.data_path, index_path=node.index_path, cbas_path=node.cbas_path) rebalance = self.cluster.async_rebalance(nodes_in_cluster, self.servers[1:3], []) nodes_in_cluster = nodes_in_cluster + self.servers[1:3] futures = pool.invokeAll(executors) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) updates_from = items_start_from deletes_from = items_start_from + total_num_items / 10 items_start_from += total_num_items self.log.info("Step 27: Wait for rebalance to complete.") rebalance.get_result() reached = RestHelper(self.rest).rebalance_reached(wait_step=120) self.assertTrue(reached, "rebalance failed, stuck or did not complete") ######################################################################################################################## self.log.info("Step 28: Verify the docs count.") self.validate_items_count() ######################################################################################################################## self.log.info("Step 29: Delete 1M docs. Update 1M docs.") pool = Executors.newFixedThreadPool(5) num_items = self.input.param("num_items", 5000) executors = [] num_executors = 5 doc_executors = 4 executors.append( GleambookUser_Docloader(bucket, num_items / 10, updates_from, "update")) executors.append( GleambookUser_Docloader(bucket, num_items / 10, deletes_from, "delete")) executors.append( GleambookMessages_Docloader(msg_bucket, num_items / 10, updates_from, "update")) executors.append( GleambookMessages_Docloader(msg_bucket, num_items / 10, deletes_from, "delete")) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) ######################################################################################################################## self.log.info("Step 30: Verify the docs count.") self.validate_items_count() ######################################################################################################################## self.log.info("Step 31: Create 10M docs.") pool = Executors.newFixedThreadPool(5) total_num_items = self.input.param("num_items", 5000) executors = [] num_executors = 5 doc_executors = 4 num_items = total_num_items / doc_executors for i in xrange(doc_executors): executors.append( GleambookUser_Docloader(bucket, num_items, items_start_from + i * num_items)) executors.append( GleambookMessages_Docloader(msg_bucket, num_items, items_start_from + i * num_items)) ###################################################### NEED TO BE UPDATED ################################################################## self.log.info( "Step 32: When 31 is in progress do a KV Rebalance out of 2 nodes." ) rebalance = self.cluster.async_rebalance(nodes_in_cluster, [], self.servers[1:3]) nodes_in_cluster = [ node for node in nodes_in_cluster if node not in self.servers[1:3] ] futures = pool.invokeAll(executors) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) updates_from = items_start_from deletes_from = items_start_from + total_num_items / 10 items_start_from += total_num_items ######################################################################################################################## self.log.info("Step 33: Wait for rebalance to complete.") rebalance.get_result() reached = RestHelper(self.rest).rebalance_reached(wait_step=120) self.assertTrue(reached, "rebalance failed, stuck or did not complete") self.sleep(20) ######################################################################################################################## self.log.info("Step 34: Verify the docs count.") self.validate_items_count() ######################################################################################################################## self.log.info("Step 35: Delete 1M docs. Update 1M docs.") pool = Executors.newFixedThreadPool(5) num_items = self.input.param("num_items", 5000) executors = [] num_executors = 5 doc_executors = 4 executors.append( GleambookUser_Docloader(bucket, num_items / 10, updates_from, "update")) executors.append( GleambookUser_Docloader(bucket, num_items / 10, deletes_from, "delete")) executors.append( GleambookMessages_Docloader(msg_bucket, num_items / 10, updates_from, "update")) executors.append( GleambookMessages_Docloader(msg_bucket, num_items / 10, deletes_from, "delete")) futures = pool.invokeAll(executors) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) ######################################################################################################################## self.log.info("Step 36: Verify the docs count.") self.validate_items_count() ######################################################################################################################## self.log.info("Step 37: Create 10M docs.") pool = Executors.newFixedThreadPool(5) total_num_items = self.input.param("num_items", 5000) executors = [] num_executors = 5 doc_executors = 4 num_items = total_num_items / doc_executors for i in xrange(doc_executors): executors.append( GleambookUser_Docloader(bucket, num_items, items_start_from + i * num_items)) executors.append( GleambookMessages_Docloader(msg_bucket, num_items, items_start_from + i * num_items)) ###################################################### NEED TO BE UPDATED ################################################################## self.log.info( "Step 38: When 37 is in progress do a CBAS SWAP Rebalance of 2 nodes." ) for node in self.cbas_servers[-1:]: rest = RestConnection(node) rest.set_data_path(data_path=node.data_path, index_path=node.index_path, cbas_path=node.cbas_path) rebalance = self.cluster.async_rebalance(nodes_in_cluster, self.servers[6], [self.servers[5]], services=["kv"], check_vbucket_shuffling=False) nodes_in_cluster += self.servers[6] nodes_in_cluster.remove(self.servers[5]) futures = pool.invokeAll(executors) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) ######################################################################################################################## self.log.info("Step 39: Wait for rebalance to complete.") rebalance.get_result() reached = RestHelper(self.rest).rebalance_reached(wait_step=120) self.assertTrue(reached, "rebalance failed, stuck or did not complete") self.sleep(20) updates_from = items_start_from deletes_from = items_start_from + total_num_items / 10 items_start_from += total_num_items ######################################################################################################################## self.log.info("Step 40: Verify the docs count.") self.validate_items_count() ######################################################################################################################## self.log.info("Step 41: Delete 1M docs. Update 1M docs.") pool = Executors.newFixedThreadPool(5) num_items = self.input.param("num_items", 5000) executors = [] num_executors = 5 doc_executors = 4 executors.append( GleambookUser_Docloader(bucket, num_items / 10, updates_from, "update")) executors.append( GleambookUser_Docloader(bucket, num_items / 10, deletes_from, "delete")) executors.append( GleambookMessages_Docloader(msg_bucket, num_items / 10, updates_from, "update")) executors.append( GleambookMessages_Docloader(msg_bucket, num_items / 10, deletes_from, "delete")) futures = pool.invokeAll(executors) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) ######################################################################################################################## self.log.info("Step 42: Verify the docs count.") self.validate_items_count() ######################################################################################################################## self.log.info("Step 43: Create 10M docs.") pool = Executors.newFixedThreadPool(5) total_num_items = self.input.param("num_items", 5000) executors = [] num_executors = 5 doc_executors = 4 num_items = total_num_items / doc_executors for i in xrange(doc_executors): executors.append( GleambookUser_Docloader(bucket, num_items, items_start_from + i * num_items)) executors.append( GleambookMessages_Docloader(msg_bucket, num_items, items_start_from + i * num_items)) ###################################################### NEED TO BE UPDATED ################################################################## self.log.info("Step 44: When 43 is in progress do a KV Rebalance IN.") rest = RestConnection(self.servers[5]) rest.set_data_path(data_path=self.servers[5].data_path, index_path=self.servers[5].index_path, cbas_path=self.servers[5].cbas_path) rebalance = self.cluster.async_rebalance(nodes_in_cluster, [self.servers[5]], [], services=["kv"]) nodes_in_cluster += [self.servers[5]] self.assertTrue(reached, "rebalance failed, stuck or did not complete") futures = pool.invokeAll(executors) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) ######################################################################################################################## self.log.info("Step 45: Wait for rebalance to complete.") rebalance.get_result() reached = RestHelper(self.rest).rebalance_reached(wait_step=120) self.assertTrue(reached, "rebalance failed, stuck or did not complete") self.sleep(20) updates_from = items_start_from deletes_from = items_start_from + total_num_items / 10 items_start_from += total_num_items ######################################################################################################################## self.log.info("Step 46: Verify the docs count.") self.validate_items_count() ######################################################################################################################## self.log.info("Step 47: Delete 1M docs. Update 1M docs.") pool = Executors.newFixedThreadPool(5) num_items = self.input.param("num_items", 5000) executors = [] num_executors = 5 doc_executors = 4 executors.append( GleambookUser_Docloader(bucket, num_items / 10, updates_from, "update")) executors.append( GleambookUser_Docloader(bucket, num_items / 10, deletes_from, "delete")) executors.append( GleambookMessages_Docloader(msg_bucket, num_items / 10, updates_from, "update")) executors.append( GleambookMessages_Docloader(msg_bucket, num_items / 10, deletes_from, "delete")) futures = pool.invokeAll(executors) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) ######################################################################################################################## self.log.info("Step 48: Verify the docs count.") self.validate_items_count() ######################################################################################################################## self.log.info("Step 49: Create 10M docs.") pool = Executors.newFixedThreadPool(5) total_num_items = self.input.param("num_items", 5000) executors = [] num_executors = 5 doc_executors = 4 num_items = total_num_items / doc_executors for i in xrange(doc_executors): executors.append( GleambookUser_Docloader(bucket, num_items, items_start_from + i * num_items)) executors.append( GleambookMessages_Docloader(msg_bucket, num_items, items_start_from + i * num_items)) ######################################################################################################################## self.log.info( "Step 50: When 49 is in progress do a KV+CBAS Rebalance OUT.") rest = RestConnection(self.servers[6]) rest.set_data_path(data_path=self.servers[6].data_path, index_path=self.servers[6].index_path, cbas_path=self.kv_servers[6].cbas_path) rebalance = self.cluster.async_rebalance(nodes_in_cluster, [], [self.servers[6]]) nodes_in_cluster.remove(self.servers[6]) futures = pool.invokeAll(executors) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) ######################################################################################################################## self.log.info("Step 51: Wait for rebalance to complete.") rebalance.get_result() reached = RestHelper(self.rest).rebalance_reached(wait_step=120) self.assertTrue(reached, "rebalance failed, stuck or did not complete") self.sleep(20) updates_from = items_start_from deletes_from = items_start_from + total_num_items / 10 items_start_from += total_num_items ######################################################################################################################## self.log.info("Step 52: Verify the docs count.") self.validate_items_count() ######################################################################################################################## self.log.info("Step 53: Delete 1M docs. Update 1M docs.") pool = Executors.newFixedThreadPool(5) num_items = self.input.param("num_items", 5000) executors = [] num_executors = 5 doc_executors = 4 executors.append( GleambookUser_Docloader(bucket, num_items / 10, updates_from, "update")) executors.append( GleambookUser_Docloader(bucket, num_items / 10, deletes_from, "delete")) executors.append( GleambookMessages_Docloader(msg_bucket, num_items / 10, updates_from, "update")) executors.append( GleambookMessages_Docloader(msg_bucket, num_items / 10, deletes_from, "delete")) futures = pool.invokeAll(executors) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) ######################################################################################################################## self.log.info("Step 54: Verify the docs count.") self.validate_items_count() ######################################################################################################################## self.log.info("Step 55: Create 10M docs.") pool = Executors.newFixedThreadPool(5) total_num_items = self.input.param("num_items", 5000) executors = [] num_executors = 5 doc_executors = 4 num_items = total_num_items / doc_executors for i in xrange(doc_executors): executors.append( GleambookUser_Docloader(bucket, num_items, items_start_from + i * num_items)) executors.append( GleambookMessages_Docloader(msg_bucket, num_items, items_start_from + i * num_items)) ######################################################################################################################## self.log.info( "Step 56: When 55 is in progress do a KV+CBAS SWAP Rebalance .") rest = RestConnection(self.servers[7]) rest.set_data_path(data_path=self.servers[7].data_path, index_path=self.servers[7].index_path, cbas_path=self.servers[7].cbas_path) rebalance = self.cluster.async_rebalance(nodes_in_cluster, [self.servers[7]], [self.servers[6]]) # rebalance.get_result() nodes_in_cluster.remove(self.servers[6]) nodes_in_cluster += [self.servers[7]] futures = pool.invokeAll(executors) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) ######################################################################################################################## self.log.info("Step 57: Wait for rebalance to complete.") rebalance.get_result() reached = RestHelper(self.rest).rebalance_reached(wait_step=240) self.assertTrue(reached, "rebalance failed, stuck or did not complete") self.sleep(20) updates_from = items_start_from deletes_from = items_start_from + total_num_items / 10 items_start_from += total_num_items ######################################################################################################################## self.log.info("Step 58: Verify the docs count.") self.validate_items_count() ######################################################################################################################## self.log.info("Step 59: Delete 1M docs. Update 1M docs.") pool = Executors.newFixedThreadPool(5) num_items = self.input.param("num_items", 5000) executors = [] num_executors = 5 doc_executors = 4 executors.append( GleambookUser_Docloader(bucket, num_items / 10, updates_from, "update")) executors.append( GleambookUser_Docloader(bucket, num_items / 10, deletes_from, "delete")) executors.append( GleambookMessages_Docloader(msg_bucket, num_items / 10, updates_from, "update")) executors.append( GleambookMessages_Docloader(msg_bucket, num_items / 10, deletes_from, "delete")) futures = pool.invokeAll(executors) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) ######################################################################################################################## self.log.info("Step 60: Verify the docs count.") self.validate_items_count() bucket.close() msg_bucket.close() cluster.disconnect() print "End Time: %s" % str( time.strftime("%H:%M:%S", time.gmtime(time.time())))
# read full input data f = open('texts.csv','r') cr = csv.reader(f) texts = [] for r in cr: if len(r) > 0: texts.append(r[0]) n = len(texts) # get CPU count k = available_cpu_count() tdatas = [] # split the work (optimal static splitting) # implicit assumption: atomic units of work of equal weight (not true in this case) for i in range(k): tdatas.append(texts[(n*i)/k : (n*(i+1))/k]) ####################################### pool = Executors.newFixedThreadPool(k) workers = [Worker(sentiworddic,amplifiers,decrementers,i,tdatas[i]) for i in range(k)] futures = pool.invokeAll(workers) for future in futures: f = future.get(5, TimeUnit.SECONDS) resd[f.tid] = f.result # shutdown_and_await_termination(pool, 5)
if j < len(server.protocols) - 1: sys.stdout.write(', ') sys.stdout.write(str(protocol)) print '.' # # Scheduler # scheduler.start() # # Tasks # fixed_executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2 + 1) if len(tasks) > 0: futures = [] start_time = System.currentTimeMillis() print 'Executing %s startup tasks...' % len(tasks) for task in tasks: futures.append(fixed_executor.submit(task)) for future in futures: try: future.get() except: pass print 'Finished all startup tasks in %s seconds.' % ((System.currentTimeMillis() - start_time) / 1000.0) for application in applications: applicationService = ApplicationService(application)
def process(self, dataSource, progressBar): def addArtifact(file, message): art = file.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT) att = BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, VadCheckModuleFactory.moduleName, message) art.addAttribute(att) start = time.clock() # get or create "Transcribed" tag tagsManager = Case.getCurrentCase().getServices().getTagsManager() tagTranscribed = getOrAddTag(tagsManager, "Transcribed") self.log(Level.INFO, "Starting vad_check_ingest with settings minPercVoiced " + str(self.local_settings.minPercVoiced) + " minTotalVoiced " + str(self.local_settings.minTotalVoiced)) fileManager = Case.getCurrentCase().getServices().getFileManager() #get all files files = fileManager.findFiles(dataSource, "%") numFiles = len(files) self.log(Level.INFO, "found " + str(numFiles) + " files") progressBar.switchToDeterminate(4) ffmpeg_clock_start = time.clock() pool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()) futures = pool.invokeAll(map(lambda file: RunProcessAVFile(file, self.local_settings.minTotalVoiced, self), files)) pool.shutdownNow() filesForVoiceClassification = filter(lambda x: x is not None, map(lambda future: future.get(), futures)) ffmpeg_clock_end = time.clock() self.log(Level.INFO, "ffmpeg conversions completed in " + str(ffmpeg_clock_end - ffmpeg_clock_start) + "s") if len(filesForVoiceClassification) == 0: self.log(Level.INFO, "No files contain speech.") return IngestModule.ProcessResult.OK tmpFiles = map(lambda x: x[1], filesForVoiceClassification) fileListTxt = writeListToFile(tmpFiles, "filelist.txt") self.log(Level.INFO, "Files to classify speech/not speech:\n" + "\n".join(tmpFiles)) progressBar.progress("Running voice activity detection on " + str(len(tmpFiles)) + " files. Be patient, this may take a while.", 1) #now run all files of interest through ina_speech_segmenter to detect voice activity try: ina_run_time = runInaSpeechSegmener(fileListTxt, self) self.log(Level.INFO, "ina_speech_segmenter completed in " + str(ina_run_time) + "s") except SubprocessError: self.log(Level.INFO, "inaSpeechSegmenter failed") return IngestModule.ProcessResult.ERROR progressBar.progress("Importing " + str(len(filesForVoiceClassification)) + " csv files", 2) filesForDeepspeech = [] for file, tmpFile, duration in filesForVoiceClassification: tmpFileBase, _ = os.path.splitext(tmpFile) csvFile = tmpFileBase + ".csv" total_voiced, total_female, total_male = processInaSpeechSegmenterCSV(csvFile, self) perc_voiced_frames = total_voiced / duration * 100 if ((perc_voiced_frames > self.local_settings.minPercVoiced ) and (total_voiced > self.local_settings.minTotalVoiced)): self.log(Level.INFO, "Found an audio file with speech: " + file.getName()) addArtifact(file, "Audio file with speech") if total_male > 0: addArtifact(file, "Audio file with speech - male") if total_female > 0: addArtifact(file, "Audio file with speech - female") filesForDeepspeech.append((file, tmpFile)) else: self.log(Level.INFO, "Audio file " + file.getName() + "doesn't match conditions. perc_voiced_frames = " + str(perc_voiced_frames)+ "total_voiced = " + str(total_voiced)) # Fire an event to notify the UI and others that there is a new artifact IngestServices.getInstance().fireModuleDataEvent( ModuleDataEvent(VadCheckModuleFactory.moduleName, BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, None)) progressBar.progress("Transcribing " + str(len(filesForDeepspeech)) + " files. Be patient, this may take a while.", 3) if self.local_settings.runVadTranscriber and len(filesForDeepspeech) > 0: tmpFiles = map(lambda x: x[1], filesForDeepspeech) try: deepspeech_clock_start = time.clock() #transcribe all files in one go transcribeFiles(fileListTxt, self.local_settings.vadTranscriberLanguage, self.local_settings.showTextSegmentStartTime, self) importTranscribedTextFiles(filesForDeepspeech, self, VadCheckModuleFactory, tagsManager, tagTranscribed) deepspeech_clock_end = time.clock() self.log(Level.INFO, "deepspeech completed in " + str(deepspeech_clock_end - deepspeech_clock_start) + "s") except SubprocessError: self.log(Level.INFO, "deepspeech failed") return IngestModule.ProcessResult.ERROR end = time.clock() self.log(Level.INFO, "Vad_check_ingest completed in " + str(end-start) + "s") return IngestModule.ProcessResult.OK
def __init__(self, number_of_threads=10): self.number_of_threads = number_of_threads self.pool = Executors.newFixedThreadPool(self.number_of_threads) self.futures = {} self.tasks = []
with open(configurationFile) as data_file: json_string = data_file.read() except EnvironmentError, err: print str(err) usage() sys.exit(3) try: config = json.loads(json_string.decode('utf-8')) except: print "JSON from file '" + configurationFile + "' is malformed." e = sys.exc_info()[0] print str(e) sys.exit(4) pool = Executors.newFixedThreadPool(len(config["input"])) ecs = ExecutorCompletionService(pool) def scheduler(roots): for inputConfig in roots: yield inputConfig def getClassByName(module, className): if not module: if className.startswith("services."): className = className.split("services.")[1] l = className.split(".") m = __services__[l[0]] return getClassByName(m, ".".join(l[1:])) elif "." in className: l = className.split(".")
"angle_epsilon": 0.02, # in radians. 0.05 is 2.8 degrees, 0.02 is 1.1 degrees "len_epsilon_sq": pow(somaDiameter, 2), # in calibrated units, squared } # RANSAC parameters: reduce list of pointmatches to a spatially coherent subset paramsModel = { "maxEpsilon": somaDiameter, # max allowed alignment error in calibrated units (a distance) "minInlierRatio": 0.0000001, # ratio inliers/candidates "minNumInliers": 5, # minimum number of good matches to accept the result "n_iterations": 2000, # for estimating the model "maxTrust": 4, # for rejecting candidates } # Joint dictionary of parameters params = {} params.update(paramsDoG) params.update(paramsFeatures) params.update(paramsModel) # The model type to fit. Could also be: # TranslationModel3D, SimilarityModel3D, AffineModel3D modelclass = RigidModel3D n_threads = Runtime.getRuntime().availableProcessors() exe = Executors.newFixedThreadPool(n_threads) csv_dir = "/tmp/" registered = registeredView(img_filenames, img_loader, getCalibration, modelclass, csv_dir, exe, params) IL.wrap(registered, "registered").show()
def testEmptyFieldSort(self): """ test sorting when the sort field is empty(undefined) for some of the documents """ sort = self.sort sort.setSort(SortField("string", SortField.Type.STRING)) self._assertMatches(self.full, self.queryF, sort, "ZJI") sort.setSort(SortField("string", SortField.Type.STRING, True)) self._assertMatches(self.full, self.queryF, sort, "IJZ") sort.setSort(SortField("int", SortField.Type.INT)) self._assertMatches(self.full, self.queryF, sort, "IZJ") sort.setSort(SortField("int", SortField.Type.INT, True)) self._assertMatches(self.full, self.queryF, sort, "JZI") sort.setSort(SortField("float", SortField.Type.FLOAT)) self._assertMatches(self.full, self.queryF, sort, "ZJI") # using a nonexisting field as first sort key shouldn't make a # difference: sort.setSort([SortField("nosuchfield", SortField.Type.STRING), SortField("float", SortField.Type.FLOAT)]) self._assertMatches(self.full, self.queryF, sort, "ZJI") sort.setSort(SortField("float", SortField.Type.FLOAT, True)) self._assertMatches(self.full, self.queryF, sort, "IJZ") # When a field is None for both documents, the next SortField should # be used. # Works for sort.setSort([SortField("int", SortField.Type.INT), SortField("string", SortField.Type.STRING), SortField("float", SortField.Type.FLOAT)]) self._assertMatches(self.full, self.queryG, sort, "ZWXY") # Reverse the last criterium to make sure the test didn't pass by # chance sort.setSort([SortField("int", SortField.Type.INT), SortField("string", SortField.Type.STRING), SortField("float", SortField.Type.FLOAT, True)]) self._assertMatches(self.full, self.queryG, sort, "ZYXW") # Do the same for a ParallelMultiSearcher threadPool = Executors.newFixedThreadPool(self.getRandomNumber(2, 8), NamedThreadFactory("testEmptyFieldSort")) parallelSearcher=IndexSearcher(self.full.getIndexReader(), threadPool) sort.setSort([SortField("int", SortField.Type.INT), SortField("string", SortField.Type.STRING), SortField("float", SortField.Type.FLOAT)]) self._assertMatches(parallelSearcher, self.queryG, sort, "ZWXY") sort.setSort([SortField("int", SortField.Type.INT), SortField("string", SortField.Type.STRING), SortField("float", SortField.Type.FLOAT, True)]) self._assertMatches(parallelSearcher, self.queryG, sort, "ZYXW") threadPool.shutdown() threadPool.awaitTermination(1000L, TimeUnit.MILLISECONDS)
DEFAULT_ACCOUNTING = ('idv', '0') DEFAULT_SIZE = (480, 640) CoordinateSystems = enum('AREA', 'LATLON', 'IMAGE') AREA = CoordinateSystems.AREA LATLON = CoordinateSystems.LATLON IMAGE = CoordinateSystems.IMAGE Places = enum(ULEFT='Upper Left', CENTER='Center') ULEFT = Places.ULEFT CENTER = Places.CENTER MAX_CONCURRENT = 5 pool = Executors.newFixedThreadPool(MAX_CONCURRENT) ecs = ExecutorCompletionService(pool) def _satBandUrl(**kwargs): # needs at least server, port, debug, user, and proj # follow AddeImageChooser.appendMiscKeyValues in determining which extra keys to add satbandUrlFormat = "adde://%(server)s/text?&FILE=SATBAND&COMPRESS=gzip&PORT=%(port)s&DEBUG=%(debug)s&VERSION=1&USER=%(user)s&PROJ=%(proj)s" return satbandUrlFormat % kwargs # NOTE: remember that Callable means that the "task" returns some kind of # result from CallableObj.get()! # RunnableObj.get() just returns null. class _SatBandReq(Callable): def __init__(self, url): self.url = url
def run(title): gd = GenericDialog("Record Window") gd.addMessage("Maximum number of frames to record.\nZero means infinite, interrupt with ESC key.") gd.addNumericField("Max. frames:", 50, 0) gd.addNumericField("Milisecond interval:", 300, 0) gd.addSlider("Start in (seconds):", 0, 20, 5) frames = [] titles = [] for f in Frame.getFrames(): if f.isEnabled() and f.isVisible(): frames.append(f) titles.append(f.getTitle()) gd.addChoice("Window:", titles, titles[0]) gd.addCheckbox("To file", False) gd.showDialog() if gd.wasCanceled(): return n_frames = int(gd.getNextNumber()) interval = gd.getNextNumber() / 1000.0 # in seconds frame = frames[gd.getNextChoiceIndex()] delay = int(gd.getNextNumber()) tofile = gd.getNextBoolean() dir = None if tofile: dc = DirectoryChooser("Directory to store image frames") dir = dc.getDirectory() if dir is None: return # dialog canceled snaps = [] borders = None executors = Executors.newFixedThreadPool(1) try: while delay > 0: IJ.showStatus("Starting in " + str(delay) + "s.") time.sleep(1) # one second delay -= 1 IJ.showStatus("Capturing frame borders...") bounds = frame.getBounds() robot = Robot() frame.toFront() time.sleep(0.5) # half a second borders = robot.createScreenCapture(bounds) IJ.showStatus("Recording " + frame.getTitle()) # Set box to the inside borders of the frame insets = frame.getInsets() box = bounds.clone() box.x = insets.left box.y = insets.top box.width -= insets.left + insets.right box.height -= insets.top + insets.bottom start = System.currentTimeMillis() / 1000.0 # in seconds last = start intervals = [] real_interval = 0 i = 1 fus = None if tofile: fus = [] # 0 n_frames means continuous acquisition while 0 == n_frames or (len(snaps) < n_frames and last - start < n_frames * interval): now = System.currentTimeMillis() / 1000.0 # in seconds real_interval = now - last if real_interval >= interval: last = now img = snapshot(frame, box) if tofile: fus.append(executors.submit(Saver(i, dir, bounds, borders, img, insets))) # will flush img i += 1 else: snaps.append(img) intervals.append(real_interval) else: time.sleep(interval / 5) # interrupt capturing: if IJ.escapePressed(): IJ.showStatus("Recording user-interrupted") break # debug: # print "insets:", insets # print "bounds:", bounds # print "box:", box # print "snap dimensions:", snaps[0].getWidth(), snaps[0].getHeight() # Create stack stack = None if tofile: for fu in snaps: fu.get() # wait on all stack = VirtualStack(bounds.width, bounds.height, None, dir) files = File(dir).list(TifFilter()) Arrays.sort(files) for f in files: stack.addSlice(f) else: stack = ImageStack(bounds.width, bounds.height, None) t = 0 for snap, real_interval in zip(snaps, intervals): bi = BufferedImage(bounds.width, bounds.height, BufferedImage.TYPE_INT_RGB) g = bi.createGraphics() g.drawImage(borders, 0, 0, None) g.drawImage(snap, insets.left, insets.top, None) stack.addSlice(str(IJ.d2s(t, 3)), ImagePlus("", bi).getProcessor()) t += real_interval snap.flush() bi.flush() borders.flush() ImagePlus(frame.getTitle() + " recording", stack).show() IJ.showStatus("Done recording " + frame.getTitle()) except Exception, e: print "Some error ocurred:" print e.printStackTrace() IJ.showStatus("") if borders is not None: borders.flush() for snap in snaps: snap.flush()
from downloader import Downloader from shutdown import shutdown_and_await_termination from java.util.concurrent import Executors, TimeUnit MAX_CONCURRENT = 3 SITES = [ "http://www.cnn.com/", "http://www.nytimes.com/", "http://www.washingtonpost.com/", "http://www.dailycamera.com/", "http://www.timescall.com/", ] pool = Executors.newFixedThreadPool(MAX_CONCURRENT) downloaders = [Downloader(url) for url in SITES] futures = pool.invokeAll(downloaders) for future in futures: print future.get(5, TimeUnit.SECONDS) shutdown_and_await_termination(pool, 5)
def __del__(self): del self.result from java.util.concurrent import Executors, ExecutorCompletionService import java.lang.Runtime as JavaRunTime rt = JavaRunTime.getRuntime() # over 2 because most intel machines report double the number of real cores, # returned by this function call, -1 because we want to leave something for WAT/ResSim/the OS. MAX_THREADS = max(1, rt.availableProcessors()/2 - 1) ... pool = Executors.newFixedThreadPool(numThreads) ecs = ExecutorCompletionService(pool) jobs = list() def runKnnAndStorageAreas(tableLabel, modelFPart, ...): # function to do compute # call kNN compute functions and post-processors for interpolations ... return listOfTimeSeriesContainers for task in listOfTasksCsvFile: ... # get the parameters from task ... jobs.append(ThreadWrapper(runKnnAndStorageAreas, tableLabel, modelFPart, ..., jobName=tableLabel))
# Propagate before or propagate after # TO DO # Apply transforms to patches progress = 0 for mesh, layer in zip(meshes, layers): Utils.log("Applying transforms to patches...") IJ.showProgress(0, len(layers)) mlt = MovingLeastSquaresTransform2() mlt.setModel(AffineModel2D) mlt.setAlpha(2.0) mlt.setMatches(mesh.getVA().keySet()) # ElasticLayerAlignment uses newer concurrent methods for this pool = Executors.newFixedThreadPool(MAX_NUM_THREADS) patch_transforms = [] patches = layer.getDisplayables(Patch) for patch in patches: pt = PatchTransform(patch, mlt.copy()) patch_transforms.append(pt) futures = pool.invokeAll(patch_transforms) for future in futures: print future.get(5, TimeUnit.SECONDS) shutdown_and_await_termination(pool, 5) for vd in vector_data: vd.apply(layer, inf_area, mlt) progress += 1 IJ.showProgress(progress, len(layers))
indices = range(1, search.numNeighbors()) if furthest: indices.reverse() # Make as many constellations as possible, up to n_max count = 0 for i, k in combinations(indices, 2): p1, d1 = search.getPosition(i), search.getSquareDistance(i) p2, d2 = search.getPosition(k), search.getSquareDistance(k) cons = Constellation(peak, p1, d1, p2, d2) if cons.angle > 0.25 and count < n_max: count += 1 yield cons """ exe = Executors.newFixedThreadPool(4) try: # A map of image indices and collections of DoG peaks in calibrated 3D coordinates # (Must be calibrated, or the KDTree radius search wouldn't work as intended.) futures = [ exe.submit(Task(getDoGPeaks, img, calibration)) for img in [img1, img2] ] soma_detections = {ti: f.get() for ti, f in enumerate(futures)} for ti, peaks in soma_detections.iteritems(): print "Found %i peaks in %i" % (len(peaks), ti) # Extract features from the detected soma: # Each feature is a constellation of a soma position and two other nearby somas.
with open(configurationFile) as data_file: json_string = data_file.read() except EnvironmentError, err: print str(err) usage() sys.exit(3) try: config = json.loads(json_string.decode('utf-8')) except: print "JSON from file '" + configurationFile + "' is malformed." e = sys.exc_info()[0] print str(e) sys.exit(4) pool = Executors.newFixedThreadPool(len(config["input"])) ecs = ExecutorCompletionService(pool) def scheduler(roots): for inputConfig in roots: yield inputConfig def getClassByName(module, className): if not module: if className.startswith("services."): className = className.split("services.")[1] l = className.split(".") m = __services__[l[0]] return getClassByName(m, ".".join(l[1:]))
cr = csv.reader(f) texts = [] for r in cr: if len(r) > 0: texts.append(r[0]) n = len(texts) # get CPU count k = available_cpu_count() tdatas = [] # split the work (optimal static splitting) # implicit assumption: atomic units of work of equal weight (not true in this case) for i in range(k): tdatas.append(texts[(n * i) / k:(n * (i + 1)) / k]) ####################################### pool = Executors.newFixedThreadPool(k) workers = [ Worker(sentiworddic, amplifiers, decrementers, i, tdatas[i]) for i in range(k) ] futures = pool.invokeAll(workers) for future in futures: f = future.get(5, TimeUnit.SECONDS) resd[f.tid] = f.result # shutdown_and_await_termination(pool, 5)
def run(title): gd = GenericDialog('Record Window') gd.addMessage( "Maximum number of frames to record.\nZero means infinite, interrupt with ESC key." ) gd.addNumericField('Max. frames:', 50, 0) gd.addNumericField('Milisecond interval:', 300, 0) gd.addSlider('Start in (seconds):', 0, 20, 5) frames = [] titles = [] for f in Frame.getFrames(): if f.isEnabled() and f.isVisible(): frames.append(f) titles.append(f.getTitle()) gd.addChoice('Window:', titles, titles[0]) gd.addCheckbox("To file", False) gd.showDialog() if gd.wasCanceled(): return n_frames = int(gd.getNextNumber()) interval = gd.getNextNumber() / 1000.0 # in seconds frame = frames[gd.getNextChoiceIndex()] delay = int(gd.getNextNumber()) tofile = gd.getNextBoolean() dir = None if tofile: dc = DirectoryChooser("Directory to store image frames") dir = dc.getDirectory() if dir is None: return # dialog canceled snaps = [] borders = None executors = Executors.newFixedThreadPool(1) try: while delay > 0: IJ.showStatus('Starting in ' + str(delay) + 's.') time.sleep(1) # one second delay -= 1 IJ.showStatus('Capturing frame borders...') bounds = frame.getBounds() robot = Robot() frame.toFront() time.sleep(0.5) # half a second borders = robot.createScreenCapture(bounds) IJ.showStatus("Recording " + frame.getTitle()) # Set box to the inside borders of the frame insets = frame.getInsets() box = bounds.clone() box.x = insets.left box.y = insets.top box.width -= insets.left + insets.right box.height -= insets.top + insets.bottom start = System.currentTimeMillis() / 1000.0 # in seconds last = start intervals = [] real_interval = 0 i = 1 fus = None if tofile: fus = [] # 0 n_frames means continuous acquisition while 0 == n_frames or (len(snaps) < n_frames and last - start < n_frames * interval): now = System.currentTimeMillis() / 1000.0 # in seconds real_interval = now - last if real_interval >= interval: last = now img = snapshot(frame, box) if tofile: fus.append( executors.submit( Saver(i, dir, bounds, borders, img, insets))) # will flush img i += 1 else: snaps.append(img) intervals.append(real_interval) else: time.sleep(interval / 5) # interrupt capturing: if IJ.escapePressed(): IJ.showStatus("Recording user-interrupted") break # debug: #print "insets:", insets #print "bounds:", bounds #print "box:", box #print "snap dimensions:", snaps[0].getWidth(), snaps[0].getHeight() # Create stack stack = None if tofile: for fu in snaps: fu.get() # wait on all stack = VirtualStack(bounds.width, bounds.height, None, dir) files = File(dir).list(TifFilter()) Arrays.sort(files) for f in files: stack.addSlice(f) else: stack = ImageStack(bounds.width, bounds.height, None) t = 0 for snap, real_interval in zip(snaps, intervals): bi = BufferedImage(bounds.width, bounds.height, BufferedImage.TYPE_INT_RGB) g = bi.createGraphics() g.drawImage(borders, 0, 0, None) g.drawImage(snap, insets.left, insets.top, None) stack.addSlice(str(IJ.d2s(t, 3)), ImagePlus('', bi).getProcessor()) t += real_interval snap.flush() bi.flush() borders.flush() ImagePlus(frame.getTitle() + " recording", stack).show() IJ.showStatus('Done recording ' + frame.getTitle()) except Exception, e: print "Some error ocurred:" print e.printStackTrace() IJ.showStatus('') if borders is not None: borders.flush() for snap in snaps: snap.flush()
def test_volume(self): nodes_in_cluster = [self.servers[0]] print "Start Time: %s" % str( time.strftime("%H:%M:%S", time.gmtime(time.time()))) ######################################################################################################################## self.log.info("Add a N1QL/Index nodes") self.query_node = self.servers[1] rest = RestConnection(self.query_node) rest.set_data_path(data_path=self.query_node.data_path, index_path=self.query_node.index_path, cbas_path=self.query_node.cbas_path) result = self.add_node(self.query_node, rebalance=False) self.assertTrue(result, msg="Failed to add N1QL/Index node.") self.log.info("Add a KV nodes") result = self.add_node(self.servers[2], services=["kv"], rebalance=True) self.assertTrue(result, msg="Failed to add KV node.") nodes_in_cluster = nodes_in_cluster + [ self.servers[1], self.servers[2] ] ######################################################################################################################## self.log.info("Step 2: Create Couchbase buckets.") self.create_required_buckets() for node in nodes_in_cluster: NodeHelper.do_a_warm_up(node) NodeHelper.wait_service_started(node) ######################################################################################################################## self.log.info( "Step 3: Create 10M docs average of 1k docs for 8 couchbase buckets." ) env = DefaultCouchbaseEnvironment.builder().mutationTokensEnabled( True).computationPoolSize(5).socketConnectTimeout( 100000).connectTimeout(100000).maxRequestLifetime( TimeUnit.SECONDS.toMillis(300)).build() cluster = CouchbaseCluster.create(env, self.master.ip) cluster.authenticate("Administrator", "password") bucket = cluster.openBucket("GleambookUsers") pool = Executors.newFixedThreadPool(5) items_start_from = 0 total_num_items = self.input.param("num_items", 5000) executors = [] num_executors = 5 doc_executors = 5 num_items = total_num_items / num_executors for i in xrange(doc_executors): executors.append( GleambookUser_Docloader(bucket, num_items, items_start_from + i * num_items, batch_size=2000)) futures = pool.invokeAll(executors) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) updates_from = items_start_from deletes_from = items_start_from + total_num_items / 10 items_start_from += total_num_items ######################################################################################################################## self.sleep(120, "Sleeping after 1st cycle.") self.log.info("Step 8: Delete 1M docs. Update 1M docs.") pool = Executors.newFixedThreadPool(5) num_items = self.input.param("num_items", 5000) executors = [] num_executors = 5 doc_executors = 4 executors.append( GleambookUser_Docloader(bucket, num_items / 10, updates_from, "update")) executors.append( GleambookUser_Docloader(bucket, num_items / 10, deletes_from, "delete")) futures = pool.invokeAll(executors) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) ######################################################################################################################## self.sleep(120, "Sleeping after 2nd cycle.") pool = Executors.newFixedThreadPool(5) num_items = self.input.param("num_items", 5000) executors = [] num_executors = 5 doc_executors = 5 num_items = total_num_items / doc_executors for i in xrange(doc_executors): executors.append( GleambookUser_Docloader(bucket, num_items, items_start_from + i * num_items, batch_size=2000)) rebalance = self.cluster.async_rebalance(nodes_in_cluster, [self.servers[3]], []) futures = pool.invokeAll(executors) for future in futures: print future.get(num_executors, TimeUnit.SECONDS) print "Executors completed!!" shutdown_and_await_termination(pool, num_executors) rebalance.get_result() reached = RestHelper(self.rest).rebalance_reached(wait_step=120) self.assertTrue(reached, "rebalance failed, stuck or did not complete") bucket.close() cluster.disconnect() print "End Time: %s" % str( time.strftime("%H:%M:%S", time.gmtime(time.time())))
def recompress(run_id, conf): """Proceed to recompression of a run. Arguments: run_id: The run id conf: configuration dictionary """ common.log('INFO', 'Recompress step: Starting', conf) # Check if input root fastq root data exists if not common.is_dir_exists(FASTQ_DATA_PATH_KEY, conf): error("FASTQ data directory does not exist", "FASTQ data directory does not exist: " + conf[FASTQ_DATA_PATH_KEY], conf) return False start_time = time.time() fastq_input_dir = conf[FASTQ_DATA_PATH_KEY] + '/' + run_id # initial du for comparing with ending disk usage previous_du_in_bytes = common.du(fastq_input_dir) # get information about compression type compression_type = conf[RECOMPRESS_COMPRESSION_KEY] compression_level = conf[RECOMPRESS_COMPRESSION_LEVEL_KEY] compression_info_tuple = get_info_from_file_type(compression_type, compression_level) if compression_info_tuple is None: error("Unknown compression type", "Unknown compression type: " + compression_type, conf) return False (compression_type_result, output_file_extension, output_compression_command, output_decompression_command, compression_level_argument) = compression_info_tuple # The following list contains the processed type of files to recompress types_to_recompress = ["fastq.gz", "fastq"] # list of program to check if exists in path before execution program_set = {"bash", "tee", "touch", "chmod", "md5sum", output_compression_command, output_decompression_command} # get list of file to process input_files = [] for extension in types_to_recompress: input_files.extend(list_files(fastq_input_dir, extension)) simple_extension = os.path.splitext(extension)[-1][1:] extension_info_tuple = get_info_from_file_type(simple_extension) if extension_info_tuple is None: error("Unknown extension type", "Unknown extension type: " + extension, conf) return False program_set.add(extension_info_tuple[3]) # actual program list check for program in program_set: if not common.exists_in_path(program): error("Can't find all needed commands in PATH env var", "Can't find all needed commands in PATH env var. Unable to find: " + program + " command.", conf) return False # Create executor and for parallelization of processus executor = Executors.newFixedThreadPool(int(conf[RECOMPRESS_THREADS_KEY])) workers = [] # process each fastq and fastq.gz recursively in each fastq directory for input_file in input_files: simple_extension = os.path.splitext(input_file)[-1][1:] # get info about the type of input file extension_info_tuple = get_info_from_file_type(simple_extension) if extension_info_tuple is None: error("Unknown extension type", "Unknown extension type: " + simple_extension, conf) return False input_decompression_command = extension_info_tuple[3] # get file base name and create output_file name, if file is already .fastq its ready to be base_input_file base_input_file = input_file[0: input_file.index(".fastq") + 6] output_file = base_input_file + "." + output_file_extension # Skip if the output_file already exists if not os.path.exists(output_file): # Create worker then execute in thread worker = Worker(input_file, output_file, input_decompression_command, output_compression_command, output_decompression_command, compression_level_argument, common.is_conf_value_equals_true(RECOMPRESS_DELETE_ORIGINAL_FASTQ_KEY, conf)) workers.append(worker) executor.execute(worker) else: common.log("WARNING", "Recompress step: Omitting processing file " + input_file + ". The associated output file " + output_file + " already exists.", conf) # Wait for all thread to finish executor.shutdown() while not executor.isTerminated(): time.sleep(1) # Check if any worker is in error for worker in workers: if not worker.is_successful(): error(worker.get_error_message(), worker.get_long_error_message(), conf) return False # check new disk usage df_in_bytes = common.df(fastq_input_dir) du_in_bytes = common.du(fastq_input_dir) previous_du = previous_du_in_bytes / (1024 * 1024) df = df_in_bytes / (1024 * 1024 * 1024) du = du_in_bytes / (1024 * 1024) common.log("WARNING", "Recompress step: output disk free after step: " + str(df_in_bytes), conf) common.log("WARNING", "Recompress step: space previously used: " + str(previous_du_in_bytes), conf) common.log("WARNING", "Recompress step: space now used by step: " + str(du_in_bytes), conf) duration = time.time() - start_time msg = 'Ending recompression for run ' + run_id + '.' + \ '\nJob finished at ' + common.time_to_human_readable(time.time()) + \ ' without error in ' + common.duration_to_human_readable(duration) + '. ' msg += '\n\nAfter recompress step FASTQ folder is now %.2f MB (previously %.2f MB) and %.2f GB still free.' % ( du, previous_du, df) common.send_msg('[Aozan] Ending recompress for run ' + run_id + ' on ' + common.get_instrument_name(run_id, conf), msg, False, conf) common.log('INFO', 'Recompress step: successful in ' + common.duration_to_human_readable(duration), conf) return True