def train(training_dbs, validation_db, start_iter=0): learning_rate = system_configs.learning_rate max_iteration = system_configs.max_iter pretrained_model = system_configs.pretrain snapshot = system_configs.snapshot val_iter = system_configs.val_iter display = system_configs.display decay_rate = system_configs.decay_rate stepsize = system_configs.stepsize val_ind = 0 print("building model...") nnet = NetworkFactory(training_dbs[0]) # getting the size of each database training_size = len(training_dbs[0].db_inds) validation_size = len(validation_db.db_inds) # queues storing data for training training_queue = Queue(32) # queues storing pinned data for training pinned_training_queue = queue.Queue(32) # load data sampling function data_file = "sample.{}".format(training_dbs[0].data) sample_data = importlib.import_module(data_file).sample_data # allocating resources for parallel reading training_tasks = init_parallel_jobs(training_dbs, training_queue, sample_data, True) training_pin_semaphore = threading.Semaphore() training_pin_semaphore.acquire() training_pin_args = (training_queue, pinned_training_queue, training_pin_semaphore) training_pin_thread = threading.Thread(target=pin_memory, args=training_pin_args) training_pin_thread.daemon = True training_pin_thread.start() run = Run.get_context() if pretrained_model is not None: if not os.path.exists(pretrained_model): raise ValueError("pretrained model does not exist") print("loading from pretrained model") nnet.load_pretrained_params(pretrained_model) if start_iter: if start_iter == -1: print("training starts from the latest iteration") save_list = os.listdir(system_configs.snapshot_dir) save_list.sort(reverse=True) if len(save_list) > 0: target_save = save_list[0] start_iter = int(re.findall(r'\d+', target_save)[0]) learning_rate /= (decay_rate ** (start_iter // stepsize)) nnet.load_params(start_iter) else: start_iter = 0 nnet.set_lr(learning_rate) print("training starts from iteration {} with learning_rate {}".format(start_iter + 1, learning_rate)) else: nnet.set_lr(learning_rate) print("training start...") nnet.cuda() nnet.train_mode() if not os.path.exists('./outputs'): os.makedirs('./outputs') print('outputs file created') else: print(os.listdir('./outputs')) error_count = 0 for iteration in tqdm(range(start_iter + 1, max_iteration + 1)): try: training = pinned_training_queue.get(block=True) except: print('Error when extracting data') error_count += 1 if error_count > 10: print('failed') time.sleep(1) break continue training_loss = nnet.train(**training) if display and iteration % display == 0: print("training loss at iteration {}: {}".format(iteration, training_loss.item())) run.log('train_loss', training_loss.item()) if val_iter and validation_db.db_inds.size and iteration % val_iter == 0: nnet.eval_mode() validation, val_ind = sample_data(validation_db, val_ind, data_aug=False) validation_loss = nnet.validate(**validation) print("validation loss at iteration {}: {}".format(iteration, validation_loss.item())) run.log('val_loss', validation_loss.item()) nnet.train_mode() if iteration % snapshot == 0: nnet.save_params(iteration) if iteration % stepsize == 0: learning_rate /= decay_rate nnet.set_lr(learning_rate) # sending signal to kill the thread training_pin_semaphore.release() # terminating data fetching processes for training_task in training_tasks: training_task.terminate()
# Tomo una impresora de la lista. # (Esta línea va a fallar si no quedan impresoras, agregar sincronización para que no pase) sema.acquire() try: impresora = impresorasDisponibles.pop() # La utilizo. impresora.imprimir(self.texto) # La vuelvo a dejar en la lista para que la use otro. impresorasDisponibles.append(impresora) finally: sema.release() # o la versión con with # with sema: # impresora = impresorasDisponibles.pop() # impresora.imprimir(self.texto) # impresorasDisponibles.append(impresora) sema = threading.Semaphore(5) impresorasDisponibles = [] for i in range(3): # Creo tres impresoras y las meto en la lista. Se puede cambiar el 3 por otro número para hacer pruebas. impresorasDisponibles.append(Impresora(i)) Computadora('hola').start() Computadora('qué tal').start() Computadora('todo bien').start() Computadora('esta explota').start() Computadora('esta también').start()
raise ServerError(name, args) class Notifier(Proxy): """Proxy methods of server handler, asynchronously. Call Notifier(connection).foo(*args, **kwargs) to invoke method handler.foo(*args, **kwargs) of server handler. """ def __call__(self, *args, **kwargs): """Call method on server, don't wait for response.""" data = dumps((NOTIFY, self._name, args, kwargs)) self._conn.write(data) g_threads_semaphore = threading.Semaphore(MAX_THREADS) def run_in_thread(foo): """Decorate to run foo using bounded number of threads.""" def wrapper1(*args, **kwargs): try: foo(*args, **kwargs) finally: g_threads_semaphore.release() def wrapper2(*args, **kwargs): g_threads_semaphore.acquire() thread.start_new_thread(wrapper1, args, kwargs) return wrapper2
@return: None @rtype : None """ thread_id = threading.current_thread().ident filename = build_logfile_path(GARBAGE_COLLECTOR_LOGS, LOG_TYPE_GC, str(thread_id)) if not os.path.exists(os.path.dirname(filename)): try: os.makedirs(os.path.dirname(filename)) except OSError: pass with open(filename, "a+") as log_file: print(msg, file=log_file) main_lock = threading.Semaphore(1) def write_to_main(data: str, print_to_console: bool=False): """ Writes to the main log @param data: The data to write @param print_to_console: If true, print to console as well as log @return: None """ try: main_lock.acquire() with open(MAIN_LOGS, "a+") as f: print(data, file=f) f.flush() except Exception as err:
def wait(self): self.sem = threading.Semaphore(0) self.sem.acquire()
import os import random import time import threading inicioPuente = 10 largoPuente = 20 semaforoPuente = threading.Semaphore(1) class Vaca(threading.Thread): def __init__(self): super().__init__() self.posicion = 0 self.velocidad = random.uniform(0.1, 0.5) def avanzar(self): if (self.posicion == inicioPuente - 1): semaforoPuente.acquire( ) #esto congela a las vacas en una posicion anterior al puente time.sleep(self.velocidad) self.posicion += 1 if (self.posicion == inicioPuente + largoPuente): semaforoPuente.release() def dibujar(self): print(' ' * self.posicion + "v") def run(self):
from BaseThread import BaseThread import threading import time # 多執行緒的前工作 def my_thread_job(): with sem: print("{} runing".format("hi")) time.sleep(1) # 多執行緒的後工作 def cb(argv1, argv2): with sem: print("{} {}".format(argv1, argv2)) sem=threading.Semaphore(4) for i in range(5): BaseThread( name = 'test', target=my_thread_job, callback=cb, callback_args=("hello","word") ).start()
def _start(self, process): pythics.libproxy.PartialAutoProxy._start(self, process) self._semaphore = threading.Semaphore(1) self._thread = threading.Thread(target=self._thread_loop)
""" Core class for the NG/AMS DB interface. """ import importlib import logging import random import tempfile import threading import time from ngamsCore import TRACE, toiso8601, fromiso8601 from DBUtils.PooledDB import PooledDB # Global DB Semaphore to protect critical, global DB interaction. _globalDbSem = threading.Semaphore(1) logger = logging.getLogger(__name__) # Define lay-out of ngas_disks table _ngasDisksDef = [["nd.disk_id", "NGAS_DISKS_DISK_ID"], ["nd.archive", "NGAS_DISKS_ARCHIVE"], ["nd.logical_name", "NGAS_DISKS_LOG_NAME"], ["nd.host_id", "NGAS_DISKS_HOST_ID"], ["nd.slot_id", "NGAS_DISKS_SLOT_ID"], ["nd.mounted", "NGAS_DISKS_MOUNTED"], ["nd.mount_point", "NGAS_DISKS_MT_PT"], ["nd.number_of_files", "NGAS_DISKS_NO_OF_FILES"], ["nd.available_mb", "NGAS_DISKS_AVAIL_MB"], ["nd.bytes_stored", "NGAS_DISKS_BYTES_STORED"], ["nd.type", "NGAS_DISKS_TYPE"],
''' ideal usage: python ideal.py <trace> 0 0 10 ''' def run_ideal(trace): outputname = 'ideal_' + '_'.join(trace.split('_')[1:]) with open(outputname, 'w') as output: output = subprocess.call(['python', '../ideal.py', trace, '0', '0', '10'], stdout = output) def maketrace_and_run(result, semaphore): sempahore.acquire() tracename = maketrace(result) print result run_ideal(tracename) semaphore.release() if __name__ == '__main__': if (sys.argv < 2): print 'not enough args!' else: results = sys.argv[1:] sempahore = threading.Semaphore(32) threads = [threading.Thread(target = maketrace_and_run, args = (r, sempahore)) for r in results] [t.start() for t in threads] [t.join() for t in threads]
time_slot = 1000 min_confidence_value = (1, ) max_confidence_value = (1, ) number_of_queues = 0 total_buffers_length = 0 priority_buffer = {} beta = 2 alpha = 0.9 packet_in_counters_list = {} total_count_per_timeslot = 0 threshold_user = {} threshold_malicious_user = 0.1 max_threshold = 5 service_rate = 0 sem_incoming_packetin_list = threading.Semaphore() sem_priority_buffer = threading.Semaphore() sem_packet_in_counters_list = threading.Semaphore() sem_total_count_per_timeslot = threading.Semaphore() sem_threshold_user = threading.Semaphore() sem_confidence_list = threading.Semaphore() sem_max_min_confidence_value = threading.Semaphore() class SimpleSwitch14(app_manager.RyuApp): OFP_VERSIONS = [ofproto_v1_4.OFP_VERSION] # OFP_VERSIONS = [ofproto_v1_3.OFP_VERSION] def __init__(self, *args, **kwargs): #do not delete or comment this print
def __init__(self, max_value: int, exc_class: type) -> None: self.max_value = max_value self.exc_class = exc_class self._semaphore = threading.Semaphore(max_value)
def Train(self, display_interval=100): ''' User function: Start training Args: display_interval (int): Post every specified iteration the training losses and accuracies will be printed Returns: None ''' # reading arguments from command start_iter = self.system_dict["training"]["params"]["start_iter"] distributed = self.system_dict["model"]["params"]["distributed"] world_size = self.system_dict["model"]["params"]["world_size"] initialize = self.system_dict["model"]["params"]["initialize"] gpu = None rank = self.system_dict["model"]["params"]["rank"] # reading arguments from json file batch_size = self.system_dict["dataset"]["params"]["batch_size"] learning_rate = self.system_dict["training"]["params"]["lr"] max_iteration = self.system_dict["training"]["params"][ "total_iterations"] pretrained_model = None stepsize = int( self.system_dict["training"]["params"]["total_iterations"] * 0.8) snapshot = int( self.system_dict["training"]["params"]["total_iterations"] * 0.5) val_iter = self.system_dict["training"]["params"]["val_interval"] display = display_interval decay_rate = self.system_dict["local"]["system_config"].decay_rate print("start_iter = {}".format(start_iter)) print("distributed = {}".format(distributed)) print("world_size = {}".format(world_size)) print("initialize = {}".format(initialize)) print("batch_size = {}".format(batch_size)) print("learning_rate = {}".format(learning_rate)) print("max_iteration = {}".format(max_iteration)) print("stepsize = {}".format(stepsize)) print("snapshot = {}".format(snapshot)) print("val_iter = {}".format(val_iter)) print("display = {}".format(display)) print("decay_rate = {}".format(decay_rate)) print("Process {}: building model...".format(rank)) self.system_dict["local"]["nnet"] = NetworkFactory( self.system_dict["local"]["system_config"], self.system_dict["local"]["model"], distributed=distributed, gpu=gpu) # queues storing data for training training_queue = Queue( self.system_dict["local"]["system_config"].prefetch_size) validation_queue = Queue(5) # queues storing pinned data for training pinned_training_queue = queue.Queue( self.system_dict["local"]["system_config"].prefetch_size) pinned_validation_queue = queue.Queue(5) # allocating resources for parallel reading training_tasks = init_parallel_jobs( self.system_dict["local"]["system_config"], self.system_dict["local"]["training_dbs"], training_queue, data_sampling_func, True) if self.system_dict["dataset"]["val"]["status"]: validation_tasks = init_parallel_jobs( self.system_dict["local"]["system_config"], [self.system_dict["local"]["validation_db"]], validation_queue, data_sampling_func, False) training_pin_semaphore = threading.Semaphore() validation_pin_semaphore = threading.Semaphore() training_pin_semaphore.acquire() validation_pin_semaphore.acquire() training_pin_args = (training_queue, pinned_training_queue, training_pin_semaphore) training_pin_thread = threading.Thread(target=pin_memory, args=training_pin_args) training_pin_thread.daemon = True training_pin_thread.start() validation_pin_args = (validation_queue, pinned_validation_queue, validation_pin_semaphore) validation_pin_thread = threading.Thread(target=pin_memory, args=validation_pin_args) validation_pin_thread.daemon = True validation_pin_thread.start() if pretrained_model is not None: if not os.path.exists(pretrained_model): raise ValueError("pretrained model does not exist") print("Process {}: loading from pretrained model".format(rank)) self.system_dict["local"]["nnet"].load_pretrained_params( pretrained_model) if start_iter: self.system_dict["local"]["nnet"].load_params(start_iter) learning_rate /= (decay_rate**(start_iter // stepsize)) self.system_dict["local"]["nnet"].set_lr(learning_rate) print( "Process {}: training starts from iteration {} with learning_rate {}" .format(rank, start_iter + 1, learning_rate)) else: self.system_dict["local"]["nnet"].set_lr(learning_rate) if rank == 0: print("training start...") self.system_dict["local"]["nnet"].cuda() self.system_dict["local"]["nnet"].train_mode() if (self.system_dict["dataset"]["val"]["status"]): old_val_loss = 100000.0 with stdout_to_tqdm() as save_stdout: for iteration in tqdm(range(start_iter + 1, max_iteration + 1), file=save_stdout, ncols=80): training = pinned_training_queue.get(block=True) training_loss = self.system_dict["local"]["nnet"].train( **training) if display and iteration % display == 0: print("Process {}: training loss at iteration {}: {}". format(rank, iteration, training_loss.item())) del training_loss if val_iter and self.system_dict["local"][ "validation_db"].db_inds.size and iteration % val_iter == 0: self.system_dict["local"]["nnet"].eval_mode() validation = pinned_validation_queue.get(block=True) validation_loss = self.system_dict["local"][ "nnet"].validate(**validation) print( "Process {}: validation loss at iteration {}: {}". format(rank, iteration, validation_loss.item())) if (validation_loss < old_val_loss): print("Loss Reduced from {} to {}".format( old_val_loss, validation_loss)) self.system_dict["local"]["nnet"].save_params( "best") old_val_loss = validation_loss else: print( "validation loss did not go below {}, current loss - {}" .format(old_val_loss, validation_loss)) self.system_dict["local"]["nnet"].train_mode() if iteration % stepsize == 0: learning_rate /= decay_rate self.system_dict["local"]["nnet"].set_lr(learning_rate) self.system_dict["local"]["nnet"].save_params("final") # sending signal to kill the thread training_pin_semaphore.release() validation_pin_semaphore.release() # terminating data fetching processes terminate_tasks(training_tasks) terminate_tasks(validation_tasks) else: with stdout_to_tqdm() as save_stdout: for iteration in tqdm(range(start_iter + 1, max_iteration + 1), file=save_stdout, ncols=80): training = pinned_training_queue.get(block=True) training_loss = self.system_dict["local"]["nnet"].train( **training) if display and iteration % display == 0: print("Process {}: training loss at iteration {}: {}". format(rank, iteration, training_loss.item())) del training_loss if (iteration % val_iter == 0): self.system_dict["local"]["nnet"].save_params( "intermediate") if iteration % stepsize == 0: learning_rate /= decay_rate self.system_dict["local"]["nnet"].set_lr(learning_rate) self.system_dict["local"]["nnet"].save_params("final") # sending signal to kill the thread training_pin_semaphore.release() # terminating data fetching processes terminate_tasks(training_tasks)
def download(__url__): url = __url__ self.destination = str(self.destination) stopButton['command'] = lambda: terminate(self.downloadObject) stopButton['state'] = NORMAL pauseButton['command'] = lambda: pauseResume(self.downloadObject) pauseButton['state'] = NORMAL def doDownload(sem): with sem: try: if self.downloadObject: self.downloadObject.start() except Exception as e: print(f"------> {e}") print( f"object error ---> {self.downloadObject.get_errors()}" ) self.statusMessage.set(f" Status: {e}") root.update_idletasks() def showProgress(sem): with sem: time.sleep(1) startTime = time.perf_counter() if self.downloadObject: while not self.downloadObject.isFinished() and len( self.downloadObject.get_errors()) == 0: self.statusMessage.set( f" Status: {self.downloadObject.get_status().capitalize()}" ) self.speedMessage.set( f" Speed: {self.downloadObject.get_speed(human=True)}" ) self.destinationMessage.set( f" Working directory: {self.destination}") self.sizeMessage.set( f" Downloaded so far: {self.downloadObject.get_dl_size(human=True)}" ) self.timeMessage.set( f" Elapsed Time: {round(time.perf_counter() - startTime, 1)}" if self.downloadObject.get_status() != 'paused' else ' Elapsed Time: . . . ') progress[ 'value'] = 100 * self.downloadObject.get_progress( ) time.sleep(0.2) root.update_idletasks() if len(self.downloadObject.get_errors()) == 0: startPoint = time.perf_counter() while time.perf_counter() - startPoint < 2: self.statusMessage.set( f" Status: {self.downloadObject.get_status().capitalize()}" ) self.speedMessage.set( f" Speed: {self.downloadObject.get_speed(human=True)}" ) self.destinationMessage.set( f" Saved at: {self.downloadObject.get_dest()}" ) self.sizeMessage.set( f" Total File Size: {self.downloadObject.get_final_filesize(human=True)}" ) self.timeMessage.set( f" Total Time: {str(self.downloadObject.get_dl_time(human=True))}" ) progress[ 'value'] = 100 * self.downloadObject.get_progress( ) time.sleep(0.2) root.update_idletasks() if progress['value'] == 100: speak('File Downloaded') else: self.statusMessage.set( f" Status: Download Failed") self.speedMessage.set( f" Reason: {self.downloadObject.get_errors()[0]}" ) root.update_idletasks() speak('Download Failed') if len(url) == 0: downloadButton.flash() else: try: self.downloadObject = SmartDL(url, self.destination) except Exception as e: print(f"Error in {e}") self.statusMessage.set(f" Status: {e}") root.update_idletasks() semaphore = threading.Semaphore(2) threading.Thread(target=doDownload, args=(semaphore, )).start() threading.Thread(target=showProgress, args=(semaphore, )).start()
class ActivePool(object): def __init__(self): super(ActivePool, self).__init__() self.active = [] self.lock = threading.Lock() def makeActive(self, name): with self.lock: self.active.append(name) logging.debug('Running: %s', self.active) def makeInactive(self, name): with self.lock: self.active.remove(name) logging.debug('Running: %s', self.active) def worker(s, pool): logging.debug('Waiting to join the poll') with s: name = threading.currentThread().getName() pool.makeActive(name) pool.makeInactive(name) pool = ActivePool() s = threading.Semaphore() for i in range(10): t = threading.Thread(target = worker, name = str(i), args=(s, pool)) t.start()
import psycopg2 import openerp from openerp import SUPERUSER_ID import openerp.release import openerp.sql_db import openerp.tools import security _logger = logging.getLogger(__name__) self_actions = {} self_id = 0 self_id_protect = threading.Semaphore() # This should be moved to openerp.modules.db, along side initialize(). def _initialize_db(id, db_name, demo, lang, user_password): try: self_actions[id]['progress'] = 0 db = openerp.sql_db.db_connect(db_name) with closing(db.cursor()) as cr: # TODO this should be removed as it is done by RegistryManager.new(). openerp.modules.db.initialize(cr) openerp.tools.config['lang'] = lang cr.commit() registry = openerp.modules.registry.RegistryManager.new( db_name, demo, self_actions[id], update_module=True)
def __init__(self): pdcom.Subscriber.__init__(self) self.sem = threading.Semaphore(0) self.__subscription = []
res = res['data'] else: print("get proxy err in data") count = res['count'] if 'proxy_list' in res: proxys = res['proxy_list'] else: print("get proxy err in proxy_list") for proxy in proxys: proxiesList.append(proxy) return proxiesList sm = threading.Semaphore(500) if __name__ == "__main__": db = dbSettings.db_connect() cursor = db.cursor() sql = ''' SELECT `listingId` FROM `listing_location_us` ''' cursor.execute(sql) results = cursor.fetchall() existLocation = [row['listingId'] for row in results] existLocation = set(existLocation) print(len(existLocation)) sql = '''
def __init__(self): threading.Thread.__init__(self) self.daemon = True self.sem = threading.Semaphore() self.s = set() self.q = gs.queue.Queue()
def __init__(self, server): threading.Thread.__init__(self) self.server = server self.go = True self.queue = [] self.queue_sem = threading.Semaphore(value=0)
def __init__(self, mp, name, value): self.mp = mp self.name = name self.sema = threading.Semaphore(value)
def main(args=None): """Run the tests.""" parsed_args = _PARSER.parse_args(args=args) # Make sure that third-party libraries are up-to-date before running tests, # otherwise import errors may result. install_third_party_libs.main() for directory in DIRS_TO_ADD_TO_SYS_PATH: if not os.path.exists(os.path.dirname(directory)): raise Exception('Directory %s does not exist.' % directory) # The directories should only be inserted starting at index 1. See # https://stackoverflow.com/a/10095099 and # https://stackoverflow.com/q/10095037 for more details. sys.path.insert(1, directory) import dev_appserver dev_appserver.fix_sys_path() if parsed_args.generate_coverage_report: python_utils.PRINT('Checking whether coverage is installed in %s' % common.OPPIA_TOOLS_DIR) if not os.path.exists( os.path.join(common.OPPIA_TOOLS_DIR, 'coverage-%s' % common.COVERAGE_VERSION)): raise Exception( 'Coverage is not installed, please run the start script.') pythonpath_components = [COVERAGE_DIR] if os.environ.get('PYTHONPATH'): pythonpath_components.append(os.environ.get('PYTHONPATH')) os.environ['PYTHONPATH'] = os.pathsep.join(pythonpath_components) if parsed_args.test_target and parsed_args.test_path: raise Exception( 'At most one of test_path and test_target should be specified.') if parsed_args.test_path and '.' in parsed_args.test_path: raise Exception('The delimiter in test_path should be a slash (/)') if parsed_args.test_target and '/' in parsed_args.test_target: raise Exception('The delimiter in test_target should be a dot (.)') if parsed_args.test_target: if '_test' in parsed_args.test_target: all_test_targets = [parsed_args.test_target] else: python_utils.PRINT('') python_utils.PRINT( '---------------------------------------------------------') python_utils.PRINT( 'WARNING : test_target flag should point to the test file.') python_utils.PRINT( '---------------------------------------------------------') python_utils.PRINT('') time.sleep(3) python_utils.PRINT('Redirecting to its corresponding test file...') all_test_targets = [parsed_args.test_target + '_test'] else: include_load_tests = not parsed_args.exclude_load_tests all_test_targets = _get_all_test_targets( test_path=parsed_args.test_path, include_load_tests=include_load_tests) # Prepare tasks. max_concurrent_runs = 25 concurrent_count = min(multiprocessing.cpu_count(), max_concurrent_runs) semaphore = threading.Semaphore(concurrent_count) task_to_taskspec = {} tasks = [] for test_target in all_test_targets: test = TestingTaskSpec(test_target, parsed_args.generate_coverage_report) task = concurrent_task_utils.create_task(test.run, parsed_args.verbose, semaphore, name=test_target) task_to_taskspec[task] = test tasks.append(task) task_execution_failed = False try: concurrent_task_utils.execute_tasks(tasks, semaphore) except Exception: task_execution_failed = True for task in tasks: if task.exception: concurrent_task_utils.log( python_utils.convert_to_bytes(task.exception.args[0])) python_utils.PRINT('') python_utils.PRINT('+------------------+') python_utils.PRINT('| SUMMARY OF TESTS |') python_utils.PRINT('+------------------+') python_utils.PRINT('') # Check we ran all tests as expected. total_count = 0 total_errors = 0 total_failures = 0 for task in tasks: spec = task_to_taskspec[task] if not task.finished: python_utils.PRINT('CANCELED %s' % spec.test_target) test_count = 0 elif (task.exception and 'No tests were run' in python_utils.convert_to_bytes(task.exception.args[0])): python_utils.PRINT('ERROR %s: No tests found.' % spec.test_target) test_count = 0 elif task.exception: exc_str = python_utils.convert_to_bytes(task.exception.args[0]) python_utils.PRINT(exc_str[exc_str.find('='):exc_str.rfind('-')]) tests_failed_regex_match = re.search( r'Test suite failed: ([0-9]+) tests run, ([0-9]+) errors, ' '([0-9]+) failures', python_utils.convert_to_bytes(task.exception.args[0])) try: test_count = int(tests_failed_regex_match.group(1)) errors = int(tests_failed_regex_match.group(2)) failures = int(tests_failed_regex_match.group(3)) total_errors += errors total_failures += failures python_utils.PRINT('FAILED %s: %s errors, %s failures' % (spec.test_target, errors, failures)) except AttributeError: # There was an internal error, and the tests did not run (The # error message did not match `tests_failed_regex_match`). test_count = 0 total_errors += 1 python_utils.PRINT('') python_utils.PRINT( '------------------------------------------------------') python_utils.PRINT(' WARNING: FAILED TO RUN %s' % spec.test_target) python_utils.PRINT('') python_utils.PRINT( ' This is most likely due to an import error.') python_utils.PRINT( '------------------------------------------------------') else: try: tests_run_regex_match = re.search( r'Ran ([0-9]+) tests? in ([0-9\.]+)s', task.output) test_count = int(tests_run_regex_match.group(1)) test_time = float(tests_run_regex_match.group(2)) python_utils.PRINT('SUCCESS %s: %d tests (%.1f secs)' % (spec.test_target, test_count, test_time)) except Exception: python_utils.PRINT('An unexpected error occurred. ' 'Task output:\n%s' % task.output) total_count += test_count python_utils.PRINT('') if total_count == 0: raise Exception('WARNING: No tests were run.') python_utils.PRINT('Ran %s test%s in %s test class%s.' % (total_count, '' if total_count == 1 else 's', len(tasks), '' if len(tasks) == 1 else 'es')) if total_errors or total_failures: python_utils.PRINT('(%s ERRORS, %s FAILURES)' % (total_errors, total_failures)) else: python_utils.PRINT('All tests passed.') if task_execution_failed: raise Exception('Task execution failed.') elif total_errors or total_failures: raise Exception('%s errors, %s failures' % (total_errors, total_failures)) if parsed_args.generate_coverage_report: subprocess.check_call( [sys.executable, COVERAGE_MODULE_PATH, 'combine']) process = subprocess.Popen([ sys.executable, COVERAGE_MODULE_PATH, 'report', '--omit="%s*","third_party/*","/usr/share/*"' % common.OPPIA_TOOLS_DIR, '--show-missing' ], stdout=subprocess.PIPE) report_stdout, _ = process.communicate() python_utils.PRINT(report_stdout) coverage_result = re.search( r'TOTAL\s+(\d+)\s+(\d+)\s+(?P<total>\d+)%\s+', report_stdout) if coverage_result.group('total') != '100': raise Exception('Backend test coverage is not 100%') python_utils.PRINT('') python_utils.PRINT('Done!')
from Raspberry_Producer.ProducerThread import ProducerThread from Raspberry_Producer.Button import Button import threading print("Here we go! Press CTRL+Z to exit") semaphore = threading.Semaphore(1) producer = ProducerThread(semaphore) button = Button(semaphore) # The start method bellow will call the the run methods inside of ProducerThread and Button classes. producer.start() button.start() producer.join() button.join() print ("Exiting Main Thread")
self.lock.acquire() retRun = self.evp.run() _logger.debug("%s : %s" % (retRun, self.fileName)) self.pool.remove(self) self.lock.release() # get files _logger.debug("EVP session") timeNow = datetime.datetime.utcnow() timeInt = datetime.datetime.utcnow() fileList = glob.glob(evpFilePatt) fileList.sort() # create thread pool and semaphore adderLock = threading.Semaphore(1) adderThreadPool = ThreadPool() # add while len(fileList) != 0: # time limit to aviod too many copyArchve running at the sametime if (datetime.datetime.utcnow() - timeNow) > datetime.timedelta(minutes=overallTimeout): _logger.debug("time over in EVP session") break # try to get Semaphore adderLock.acquire() # get fileList if (datetime.datetime.utcnow() - timeInt) > datetime.timedelta(minutes=15): timeInt = datetime.datetime.utcnow() # get file
# Defaults for retrying commands. POLL_INTERVAL = 30 TIMEOUT = 1200 FUZZ = .5 MAX_RETRIES = -1 WINDOWS = 'nt' DARWIN = 'Darwin' PASSWORD_LENGTH = 15 OUTPUT_STDOUT = 0 OUTPUT_STDERR = 1 OUTPUT_EXIT_CODE = 2 _SIMULATE_MAINTENANCE_SEMAPHORE = threading.Semaphore(0) flags.DEFINE_integer('default_timeout', TIMEOUT, 'The default timeout for ' 'retryable commands in seconds.') flags.DEFINE_integer('burn_cpu_seconds', 0, 'Amount of time in seconds to burn cpu on vm before ' 'starting benchmark') flags.DEFINE_integer('burn_cpu_threads', 1, 'Number of threads to use to ' 'burn cpu before starting benchmark.') flags.DEFINE_integer('background_cpu_threads', None, 'Number of threads of background cpu usage while ' 'running a benchmark') flags.DEFINE_integer('background_network_mbits_per_sec', None, 'Number of megabits per second of background ' 'network traffic to generate during the run phase ' 'of the benchmark')
coords_parana = [(-54.3603516, -24.5371294), (-54.3933105, -24.7468313), (-54.4757080, -24.9711199), (-54.4702148, -25.1800878), (-54.5251465, -25.3340967), (-54.5910645, -25.4035850), (-54.6350098, -25.5275717), (-54.5526123, -25.6365741), (-54.4592285, -25.7058875), (-54.3438721, -25.5870398), (-54.2065430, -25.5870398), (-54.1241455, -25.5176574), (-54.1241455, -25.5870398), (-54.0032959, -25.5969483), (-53.8989258, -25.6811373), (-53.8494873, -25.8888786), (-53.8385010, -26.0074242), (-53.7231445, -26.1357136), (-53.6682129, -26.1726940), (-53.6517334, -26.2343020), (-53.5528564, -26.2687883), (-53.4704590, -26.2737140), (-53.3770752, -26.2540097), (-53.2836914, -26.2835649), (-53.1738281, -26.3524979), (-53.1024170, -26.3820280), (-52.9815674, -26.3721854), (-52.8167725, -26.3623421), (-52.6684570, -26.3820280), (-52.5695801, -26.4263090), (-52.4047852, -26.4410656), (-52.1520996, -26.4656556), (-52.0916748, -26.4951568), (-51.9818115, -26.5639634), (-51.8170166, -26.5836148), (-51.6412354, -26.5787023), (-51.5039063, -26.5934393), (-51.4160156, -26.7063599), (-51.3720703, -26.6720045), (-51.2951660, -26.6523683), (-51.2127686, -26.5983512), (-51.2402344, -26.5099045), (-51.2786865, -26.4410656), (-51.2786865, -26.4017105), (-51.2347412, -26.3180365), (-51.1029053, -26.2343020), (-50.9985352, -26.2343020), (-50.8502197, -26.2737140), (-50.7348633, -26.2244469), (-50.6140137, -26.0863881), (-50.5480957, -26.0419774), (-50.3723145, -26.0715865), (-50.3393555, -26.1357136), (-50.2514648, -26.0617176), (-50.0482178, -26.0419774), (-49.9053955, -26.0469128), (-49.5922852, -26.2244469), (-49.4989014, -26.2244469), (-49.4110107, -26.1702290), (-49.1857910, -26.0172976), (-48.8122559, -25.9777990), (-48.6254883, -25.9827370), (-48.4606934, -25.6563820), (-48.3453369, -25.5672204), (-48.2849121, -25.5325285), (-48.0706787, -25.2446960), (-48.1915283, -25.1850589), (-48.1530762, -25.1403119), (-48.2244873, -25.0507688), (-48.2244873, -24.9960157), (-48.2684326, -24.9711199), (-48.3013916, -25.0433039), (-48.3233643, -25.0358386), (-48.4030151, -24.9910370), (-48.4167480, -24.9686301), (-48.4552002, -24.9860580), (-48.4991455, -25.0532570), (-48.5403442, -25.0855989), (-48.5705566, -25.0433039), (-48.6007690, -24.9960157), (-48.5705566, -24.9511996), (-48.5540771, -24.8640106), (-48.5760498, -24.8490577), (-48.5595703, -24.7942150), (-48.5073853, -24.7617965), (-48.5046387, -24.7293696), (-48.5952759, -24.6644904), (-48.6584473, -24.6769698), (-48.6776733, -24.6919434), (-48.6996460, -24.6570022), (-48.7600708, -24.6869524), (-48.8095093, -24.6769698), (-48.8534546, -24.6395279), (-48.9523315, -24.6669864), (-49.0045166, -24.6470172), (-49.0127563, -24.6220511), (-49.0567017, -24.6270447), (-49.0676880, -24.6619944), (-49.1638184, -24.6694823), (-49.2297363, -24.6919434), (-49.3011475, -24.6744740), (-49.3176270, -24.5471232), (-49.2901611, -24.5371294), (-49.2901611, -24.4996456), (-49.2462158, -24.4721504), (-49.1967773, -24.3395895), (-49.2846680, -24.2970405), (-49.3588257, -24.2093947), (-49.3286133, -24.1292086), (-49.4302368, -24.0790667), (-49.4604492, -24.0213793), (-49.4879150, -23.9912713), (-49.5263672, -23.9034159), (-49.6032715, -23.8506740), (-49.5620728, -23.8104753), (-49.5593262, -23.6847742), (-49.6170044, -23.6243946), (-49.6005249, -23.5337730), (-49.6115112, -23.4556877), (-49.5730591, -23.4204082), (-49.6636963, -23.1807636), (-49.7351074, -23.1201536), (-49.7460938, -23.0898384), (-49.9053955, -23.0443527), (-49.9438477, -22.9786240), (-50.0152588, -22.8723793), (-50.2130127, -22.9280417), (-50.3393555, -22.9280417), (-50.3942871, -22.8875622), (-50.4492188, -22.9229824), (-50.5371094, -22.9229824), (-50.6414795, -22.8825014), (-50.7183838, -22.8825014), (-50.7403564, -22.9280417), (-50.7952881, -22.9331007), (-50.8227539, -22.8571947), (-50.9106445, -22.7711166), (-51.0534668, -22.7508550), (-51.2622070, -22.7001879), (-51.3500977, -22.6495021), (-51.5093994, -22.6596408), (-51.6302490, -22.6545715), (-51.8060303, -22.6190816), (-51.9763184, -22.5632932), (-52.0477295, -22.5277798), (-52.1850586, -22.5024075), (-52.1850586, -22.6140109), (-52.2949219, -22.6140109), (-52.4597168, -22.6140109), (-52.5366211, -22.6038688), (-52.6025391, -22.5632932), (-52.7014160, -22.6140109), (-52.8552246, -22.5835825), (-53.0310059, -22.5632932), (-53.1518555, -22.6444325), (-53.3166504, -22.7559207), (-53.4924316, -22.8470707), (-53.6022949, -22.9381596), (-53.6572266, -23.0392977), (-53.6682129, -23.1908626), (-53.7561035, -23.3220800), (-53.8989258, -23.4330091), (-54.0087891, -23.4632463), (-54.0856934, -23.7752912), (-54.1186523, -23.9360549), (-54.3273926, -24.0865893), (-54.3713379, -24.1467536), (-54.3273926, -24.2669973), (-54.2834473, -24.3771208), (-54.3603516, -24.4771500), (-54.3603516, -24.5371294)] parana = Polygon(coords_parana) remote_start = 0 local_pronto = False remoto_pronto = False itens_enviados = {} indexes_enviados = [] indexes_recebidos = [] utilizar_garantia_recebimento = True semaforo = threading.Semaphore() class ClientProtocol(DatagramProtocol): def esta_no_poligono(self, lon, lat): global poligono # Verifica se esta no poligono ponto = Point(lon, lat) # Se estiver, escrever resultado em arquivo if ponto.within(parana): self.escrever_resultado(lon, lat) def escrever_resultado(self, lon, lat): # # Abrir arquivo .csv resultado
def initSemaphore(value): global chapterThreadSemaphore chapterThreadSemaphore = threading.Semaphore(value)
def __init__(self, id, function, instances, parameters = [], loopSuspension = None, loopPeriod = None): """ Constructor method initializing the object and starting the threads id: ID of this group of threads (string). function: The function containing the code that will be executed as the N threads within the context of the class (Python function reference). instances: Number of instances (threads) to run internally (integer). parameters: Parameters to hand over to the thread call-back function (list). loopSuspension: Loop suspension in seconds to carry out during each iteration (float). loopPeriod: The minimum time each iteration should take in seconds. If an iteration takes shorter time to execute than the loopPeriod specified, the execution is suspended for a period of time to make the total execution time of the iteration equal to the specified loopPeriod. Specifying this parameter, makes it possible to achieve a certain 'real-time behavior' of the execution of each loop, although this is only the case when the loopPeriod is larger than the time it takes to execute the business logical of the thread (float). """ T = TRACE() self.__id = id self.__function = function self.__instances = int(instances) self.__parameters = parameters self.__loopSuspension = loopSuspension self.__loopPeriod = loopPeriod self.__execute = True self.__pauseEvent = threading.Event() self.__generalMux = threading.Semaphore(1) self.__threadHandles = [] self.__threadTiming = {} # Create the thread handles as requested. for n in range(1, (self.__instances + 1)): args = (self, None) thrId = "%s-%d" % (id, n) thrObj = threading.Thread(None, self.__threadEncapsulator, thrId, args) thrObj.setDaemon(0) thrObj.start() self.__threadHandles.append(thrObj)
import optparse import socket import threading screenLock = threading.Semaphore(value=1) def connScan(tgtHost, tgtPort): try: connSkt = socket.socket(socket.AF_INET, socket.SOCK_STREAM) connSkt.connect((tgtHost, tgtPort)) connSkt.send('SomeText\r\n') results = connSkt.recv(100) screenLock.acquire() print '[+]%d/tcp open' % tgtPort print ' %s' % str(results).split('\n')[0] except socket.timeout: screenLock.acquire() print '[-]%d/tcp closed' % tgtPort finally: screenLock.release() connSkt.close() def portScan(tgtHost, tgtPorts): try: tgtIP = socket.gethostbyname(tgtHost) except socket.gaierror: print "[-] Cannot resolve '%s': Unknown host" % tgtHost return try:
from localstack.utils import bootstrap from localstack.config import DEFAULT_ENCODING from localstack.constants import ENV_DEV from localstack.utils.bootstrap import FuncThread # arrays for temporary files and resources TMP_FILES = [] TMP_THREADS = [] TMP_PROCESSES = [] # cache clean variables CACHE_CLEAN_TIMEOUT = 60 * 5 CACHE_MAX_AGE = 60 * 60 CACHE_FILE_PATTERN = os.path.join(tempfile.gettempdir(), '_random_dir_', 'cache.*.json') last_cache_clean_time = {'time': 0} MUTEX_CLEAN = threading.Semaphore(1) # misc. constants TIMESTAMP_FORMAT = '%Y-%m-%dT%H:%M:%S' TIMESTAMP_FORMAT_MICROS = '%Y-%m-%dT%H:%M:%S.%fZ' CODEC_HANDLER_UNDERSCORE = 'underscore' # chunk size for file downloads DOWNLOAD_CHUNK_SIZE = 1024 * 1024 # set up logger LOG = logging.getLogger(__name__) # flag to indicate whether we've received and processed the stop signal INFRA_STOPPED = False