def lauchCMAESForListOfPoints(target_size, rs, save, points): p = ThreadPool(processes=len(points)) posIni = np.loadtxt(pathDataFolder + rs.experimentFilePosIni) p.map( partial(launchCMAESForSpecificTargetSizeAndSpecificPointMulti, target_size, rs, save), [[i, posIni[i]] for i in points]) p.close() p.join()
def launchCMAESForAllTargetSizesMulti(rs): ''' Launch in parallel (on differents processor) the cmaes optimization for each target size ''' #initializes setup variables #initializes a pool of worker, ie multiprocessing p = ThreadPool(processes=4) #run cmaes on each targets size on separate processor p.map(partial(launchCMAESForSpecificTargetSize, rs=rs, save=False), rs.target_size) p.close() p.join()
def launch(rs): all_points = [] for target_size in [0.005, 0.01, 0.02, 0.04]: for i in range(15): if not check_if_theta_file_exists(target_size, i): all_points.append([i, target_size]) p = ThreadPool(processes=len(all_points)) posIni = np.loadtxt(pathDataFolder + rs.experimentFilePosIni) p.map(partial(launchCMAESMissing, rs, True, 6), [[point[0], posIni[point[0]], point[1]] for point in all_points]) p.close() p.join()
def launchCMAESForAllPoint(rs, target_size, save, noise=None): """ Launch in parallel (on differents processor) the cmaes optimization for each point input: rs: setup file target_size: size of the target save: for save experience log noise: noise on muscle, if None, defalt noise from muscle setup """ p = ThreadPool(processes=15) #run cmaes on each targets size on separate processor posIni = np.loadtxt(pathDataFolder + rs.experimentFilePosIni) p.map(partial(launchCMAESForSpecificTargetSizeAndSpecificPoint, target_size, rs, save, noise=noise), enumerate(posIni)) p.close() p.join()
def launchCMAESForAllPoint(rs, target_size, save, noise=None): """ Launch in parallel (on differents processor) the cmaes optimization for each point input: rs: setup file target_size: size of the target save: for save experience log noise: noise on muscle, if None, defalt noise from muscle setup """ p = ThreadPool(processes=15) #run cmaes on each targets size on separate processor posIni = np.loadtxt(pathDataFolder + rs.experimentFilePosIni) p.map( partial(launchCMAESForSpecificTargetSizeAndSpecificPoint, target_size, rs, save, noise=noise), enumerate(posIni)) p.close() p.join()
def lauchCMAESForListOfPoints(target_size, rs, save, points): p = ThreadPool(processes=len(points)) posIni = np.loadtxt(pathDataFolder + rs.experimentFilePosIni) p.map(partial(launchCMAESForSpecificTargetSizeAndSpecificPointMulti, target_size, rs, save), [[i, posIni[i]] for i in points]) p.close() p.join()
'./data/raw/data-sample_data-nyctaxi-trips-2012-json_corrigido.json' ] def send_register(registers): global total_sent length = len(registers) total_sent += length records = [{ 'Data': data.encode(), 'PartitionKey': '1' } for data in registers] print(f"Total: {total_sent}/4MI") client.put_records( Records=records, StreamName=STREAM_NAME, ) def process_file(file): with open(file, 'r') as buffer: registers = buffer.readlines(max_bytes) while len(registers): send_register(registers) registers = buffer.readlines(max_bytes) pool = ThreadPool(4) pool.map(process_file, files)