def run_clustering_for_rank(rank, distances_input=None, distance_ranks_input=None, isolates=None, previous_seeds=None): """ Clusters isolates into lineages based on their relative distances using a single R to enable parallelisation. Args: rank (int) Integer specifying the maximum rank of neighbour used for clustering. Should be changed to int list for hierarchical clustering. qlist (list) List of query sequences being added to an existing clustering. Should be included within rlist. use_existing (bool) Whether to extend a previously generated analysis or not. Returns: lineage_assignation (dict) Assignment of each isolate to a cluster. lineage_seed (dict) Seed isolate used to initiate each cluster. connections (set of tuples) Edges to add to network describing lineages. """ # load shared memory objects distances_shm = shared_memory.SharedMemory(name=distances_input.name) distances = np.ndarray(distances_input.shape, dtype=distances_input.dtype, buffer=distances_shm.buf) distance_ranks_shm = shared_memory.SharedMemory( name=distance_ranks_input.name) distance_ranks = np.ndarray(distance_ranks_input.shape, dtype=distance_ranks_input.dtype, buffer=distance_ranks_shm.buf) isolate_list = isolates isolate_indices = range(0, len(isolate_list)) # load previous scheme seeds = {} if previous_seeds is not None: seeds = previous_seeds[rank] # identify nearest neighbours nn = get_nearest_neighbours(rank, ranks=distance_ranks_input, isolates=isolate_list) # iteratively identify lineages lineage_index = 1 connections = set() lineage_assignation = {isolate: None for isolate in isolate_list} while None in lineage_assignation.values(): if lineage_index in seeds.keys(): seed_isolate = seeds[lineage_index] else: seed_isolate = pick_seed_isolate(lineage_assignation, distances=distances_input) # skip over previously-defined seeds if amalgamated into different lineage now if lineage_assignation[seed_isolate] is None: seeds[lineage_index] = seed_isolate lineage_assignation, added_connections = get_lineage( lineage_assignation, nn, seed_isolate, lineage_index) connections.update(added_connections) lineage_index = lineage_index + 1 # return clustering return lineage_assignation, seeds, nn, connections
#x = 0 frames = 0 text_to_read = "¿Qué objeto desea encontrar?" reading_from_string(text_to_read, object_filename) original_text = input(text_to_read) valid_object, object_to_find = find_word.find_word(original_text) object_found = False coords = [] if valid_object: while (not (object_found)): t = None frame = pepper.getCameraFrame(handle) if frames == 0: print('Sending initial informantion') shm = shared_memory.SharedMemory( create=True, size=frame.nbytes) # name='image_random' data = json.dumps({ "shape": frame.shape, "name": shm.name, "type": frame.dtype.name, "object": object_to_find }) s.send(data.encode()) #data = s.recv(1024) #s.close() #print('Received', data) # Now create a NumPy array backed by shared memory b = np.ndarray(frame.shape, dtype=frame.dtype, buffer=shm.buf) b[:] = frame[:] # Copy the original data into shared memory #print('Sending')
def destroy(self): shm = shared_memory.SharedMemory(self.shmname) shm.unlink()
import time import sys from multiprocessing import shared_memory SHM_NAME = "xxx" SHM_SIZE = 1024 DATA = b"x" * SHM_SIZE N = 100000 argv = sys.argv is_server = False if len(argv) > 1: if argv[1] == "-s": is_server = True if is_server: shm = shared_memory.SharedMemory(create=True, size=SHM_SIZE, name=SHM_NAME) else: shm = shared_memory.SharedMemory(size=SHM_SIZE, name=SHM_NAME) try: shm.buf[0] = 0 for i in range(int(N) + 1): if is_server: while shm.buf[0] == 0: time.sleep(0) shm.buf[0] = 0 for x in range(1, len(shm.buf)): shm.buf[x] = DATA[x] # print(shm.buf) else: while shm.buf[0] == 1:
red_rects.append(rect) # keep rotating the rectangle until running is set to False joy_steer = 0 joy_throttle = 0 joy_strafe = 0 vel_cmd = 0 steer_cmd = 0 last_vel_cmd = 0 tick_time = time.time() strafe = 0 myFont = py.font.SysFont("Times New Roman", int(40 * WINDOW_SCALING)) existing_shm = shared_memory.SharedMemory(name='acorn_steering_debug') # Untrack the resource so it does not get destroyed. This allows the # steering debug window to stay open. resource_tracker.unregister(existing_shm._name, 'shared_memory') calc = np.ndarray((8, ), dtype=np.float64, buffer=existing_shm.buf) running = True while running: # set FPS clock.tick(FPS) # clear the screen every time before drawing new objects screen.fill(BLACK) # check for the exit for event in py.event.get(): if event.type == py.QUIT: running = False
def perform_fft(input_info, plot=False): print("\nspawn FFT process nr : ", input_info["process_nr"]) existing_shm = shared_memory.SharedMemory(name=input_info["shm"].name) # check if poly calculation is on to be executed, because then there must be an additional key in the dictionary if "poly" in input_info.keys(): poly = True reference_to_data_block = numpy.ndarray( input_info["dim"], dtype=numpy.float64, buffer=existing_shm.buf)[:, input_info["from"]:input_info["to"], :] else: poly = False reference_to_data_block = numpy.ndarray( input_info["dim"], dtype=numpy.float64, buffer=existing_shm.buf)[:, input_info["from"]:input_info["to"], :] # get data to process out of buffer print("ref data dtype: ", reference_to_data_block.dtype) print("\n") # reshape data from buffer to 2d matrix with the time as y coords and x as the values data_mat = reference_to_data_block.reshape( reference_to_data_block.shape[0], reference_to_data_block.shape[1] * reference_to_data_block.shape[2]) # strore orig time, cols and row information - needed for reshaping orig_time = reference_to_data_block.shape[0] orig_rows = reference_to_data_block.shape[1] orig_cols = reference_to_data_block.shape[2] # if plots are whished if plot: existing_shm_qual = shared_memory.SharedMemory( name=input_info["shm_qual"].name) if poly: reference_to_qual_block = numpy.ndarray( input_info["dim"], dtype=numpy.float64, buffer=existing_shm_qual.buf )[:, input_info["from"]:input_info["to"], :] else: reference_to_qual_block = numpy.ndarray( input_info["dim"], dtype=numpy.float64, buffer=existing_shm_qual.buf )[:, input_info["from"]:input_info["to"], :] qual_weights = input_info["weights"] qual_factor = 1 qual_mat = reference_to_qual_block.reshape( reference_to_qual_block.shape[0], reference_to_qual_block.shape[1] * reference_to_qual_block.shape[2]) qual_mat[qual_mat == 255] = numpy.nan print("Data Mat Shape", data_mat.shape) # setting int Nan value to numpy.nan --> transforms dataytpe to floa64!!! data_mat = numpy.where(data_mat == 32767, numpy.nan, data_mat) n = data_mat.shape[0] t = numpy.arange(0, n, 1) # iter through for i in range(0, data_mat.shape[1], 1): data_mat_v_nan = numpy.isfinite(data_mat[:, i]) data_mat_v_t = numpy.arange(0, len(data_mat_v_nan), 1) if False in data_mat_v_nan: try: # interpolate on that spots data_mat_v_interp = numpy.round( numpy.interp(data_mat_v_t, data_mat_v_t[data_mat_v_nan], data_mat[:, i][data_mat_v_nan])) # calculate the fft f_hat = numpy.fft.fft(data_mat_v_interp, n) # and the power spectrum - which frequencies are dominant power_spectrum = f_hat * numpy.conj(f_hat) / n # get the max power value max_fft_spectr_value = numpy.max(power_spectrum) # set it to zeros so one can find those frequencies that are far lower and important but still no noise power_spec_no_max = numpy.where( power_spectrum == max_fft_spectr_value, 0, power_spectrum) threshold_remaining_values = numpy.nanmax( power_spec_no_max) / 2 indices = power_spectrum > threshold_remaining_values f_hat = indices * f_hat ffilt = numpy.fft.ifft(f_hat) if plot: if i <= 3: print("proces nr %d i == %d" % (input_info["process_nr"], i)) print("data mat: ", data_mat[:, i]) print("data_mat_v_interp", data_mat_v_interp) # print("data mat:", data_mat[:, i]) # print("data_mat.dtype: ", data_mat.dtype) # print("data_mat_interp.dtype: ", data_mat_v_interp.dtype) ffilt = numpy.round(ffilt).astype(numpy.int16) # print("\ntransfrom to int16: ", data_mat_v_interp) # print("\ndata_mat_interp.dtype: ", data_mat_v_interp.dtype) fig, axs = plt.subplots(3, 1) good_qual = numpy.where( qual_mat[:, i] == qual_weights[0], qual_weights[0], numpy.nan) * qual_factor okay_qual = numpy.where( qual_mat[:, i] == qual_weights[1], qual_weights[1], numpy.nan) * qual_factor bad_qual = numpy.where( qual_mat[:, i] == qual_weights[2], qual_weights[2], numpy.nan) * qual_factor really_bad_qual = numpy.where( qual_mat[:, i] == qual_weights[3], qual_weights[3], numpy.nan) * qual_factor nan_values = numpy.where(qual_mat[:, i] == 255, qual_weights[3], numpy.nan) plt.sca(axs[0]) plt.plot(t, data_mat[:, i], color='c', LineWidth=3, label="raw data") plt.plot(t, data_mat_v_interp, color='k', LineWidth=1, linestyle='--', label='lin interp') plt.plot(t, ffilt, color="k", LineWidth=2, label='FFT Filtered') plt.plot(t, good_qual, 'go', label="Good Quality") plt.plot(t, okay_qual, 'yo', label="Okay Quality") plt.plot(t, bad_qual, 'o', color='orange', label="Bad Quality") plt.plot(t, really_bad_qual, 'ro', label="Really Bad Quality") plt.plot(t, nan_values, 'ko', label="NaN Values") plt.xlim(t[0], t[-1]) plt.ylabel("Intensity [%]") plt.xlabel("Time [days]") plt.legend() plt.sca(axs[1]) plt.plot(t, power_spectrum, color="c", LineWidth=2, label="Noisy") plt.plot(t, power_spectrum, 'b*', LineWidth=2, label="Noisy") plt.plot(t[0], t[-1]) plt.xlabel("Power Spectrum [Hz]") plt.ylabel("Power") plt.title( "Power Spectrum Analyses - Max: {} - Threshold: {}" .format(max_fft_spectr_value, numpy.nanmean(power_spectrum))) plt.sca(axs[2]) plt.plot(t, power_spec_no_max, color="c", LineWidth=2, label="Noisy") plt.plot(t, power_spec_no_max, 'b*', LineWidth=2, label="Noisy") plt.plot(t[0], t[-1]) plt.xlabel("Power Spectrum no max [Hz]") plt.ylabel("Power") plt.title( "Power Spectrum Analysis - removed big max {} - Max: {} - Threshold: {}" .format(max_fft_spectr_value, numpy.nanmax(power_spec_no_max), threshold_remaining_values)) # plot data plt.show() data_mat[:, i] = ffilt except: # gets triggered most if there are only nans in the array continue else: # calculate the fft f_hat = numpy.fft.fft(data_mat[:, i], n) # and the power spectrum - which frequencies are dominant power_spectrum = f_hat * numpy.conj(f_hat) / n # get the max power value max_fft_spectr_value = numpy.max(power_spectrum) # set it to zeros so one can find those frequencies that are far lower and important but still no noise power_spec_no_max = numpy.where( power_spectrum == max_fft_spectr_value, 0, power_spectrum) threshold_remaining_values = numpy.nanmax(power_spec_no_max) / 2 indices = power_spectrum > threshold_remaining_values f_hat = indices * f_hat ffilt = numpy.fft.ifft(f_hat) data_mat[:, i] = ffilt # transorm float64 back to INT16!! # save interpolation results on the shared memory object if poly: reference_to_data_block[:] = numpy.round( data_mat.reshape(orig_time, orig_rows, orig_cols)) else: reference_to_data_block[:] = numpy.round( data_mat.reshape(orig_time, orig_rows, orig_cols))
def perform_dft(input_info, plot=False): print("\nspawn DFT process nr : ", input_info["process_nr"]) existing_shm = shared_memory.SharedMemory(name=input_info["shm"].name) existing_shm_qual = shared_memory.SharedMemory( name=input_info["shm_qual"].name) # check if poly calculation is on to be executed, because then there must be an additional key in the dictionary poly = False reference_to_data_block = numpy.ndarray( input_info["dim"], dtype=numpy.float64, buffer=existing_shm.buf)[:, input_info["from"]:input_info["to"], :] reference_to_qual_block = numpy.ndarray( input_info["dim"], dtype=numpy.float64, buffer=existing_shm_qual.buf)[:, input_info["from"]:input_info["to"], :] # get data to process out of buffer A = input_info["A"] print("ref data dtype: ", reference_to_data_block.dtype) print("Shape A: ", A.shape) print("\n") # reshape data from buffer to 2d matrix with the time as y coords and x as the values data_mat = reference_to_data_block.reshape( reference_to_data_block.shape[0], reference_to_data_block.shape[1] * reference_to_data_block.shape[2]) qual_mat = reference_to_qual_block.reshape( reference_to_qual_block.shape[0], reference_to_qual_block.shape[1] * reference_to_qual_block.shape[2]) print("Start DFT in process nr: ", input_info["process_nr"]) sing_cou = 0 for i in range(0, data_mat.shape[1], 1): # check if nan fields are in a vector - if yes interpolate linear on that spots # at the qual vector the indizes of where data is nan should allready be set to zero ( see in main file ) data_mat_v_nan = numpy.isfinite(data_mat[:, i]) data_mat_v_t = numpy.arange(0, len(data_mat_v_nan), 1) if False in data_mat_v_nan: # interpolate on that spots try: data_mat_v = numpy.round( numpy.interp(data_mat_v_t, data_mat_v_t[data_mat_v_nan], data_mat[:, i][data_mat_v_nan])) except: #print("crash ") #print(data_mat[:, i]) continue else: data_mat_v = data_mat[:, i] qual_mat_v = qual_mat[:, i].reshape(len(data_mat_v), 1) data_mat_v = data_mat_v.reshape(len(data_mat_v), 1) # ATPA try: P = numpy.eye(len(data_mat_v)) * qual_mat_v ATPA = numpy.linalg.inv(numpy.dot(numpy.dot(A.T, P), A)) ATPL = numpy.dot(numpy.dot(A.T, P), data_mat_v) x_hat = numpy.dot(ATPA, ATPL) l_hat = numpy.dot(A, x_hat) data_mat[:, i] = l_hat.reshape(len(l_hat)) except: # print("singular matrix") sing_cou += 1 # for e in range(0, len(data_mat_v), 1): # print("data: ", data_mat_v[e], " - qual:", qual_mat_v[e]) continue # ------------------------------------------ # optimize to using vektors and not matrizes # ------------------------------------------ # APv = numpy.multiply(A,qual_mat_v) # # check singular matrix --> maybe try/except implement # ATPA = numpy.linalg.inv(numpy.dot(APv.T, A)) # # ATPL # ATPL = numpy.dot(APv.T, data_mat_v) # x_hat = numpy.dot(ATPA, ATPL) # l_hat = numpy.dot(A, x_hat) # qual_mat[:, i] = l_hat.reshape(len(l_hat)) # print("APv: ", APv) # print("ATPA.shape", ATPA.shape) # print("data shape: ", data_mat_v.shape) # print("qual shape: ", qual_mat_v.shape) # #print("data: ", data_mat_v) # #print("qual: ", qual_mat_v) # print("A.shape : ", A.shape) # print("A.T shape : ", A.T.shape) # print("DFT x_HAT: ", x_hat) # print("lhat:", l_hat) # strore orig time, cols and row information - needed for reshaping orig_time = reference_to_data_block.shape[0] orig_rows = reference_to_data_block.shape[1] orig_cols = reference_to_data_block.shape[2] print( "- finished process %d - singular matrix count " % input_info["process_nr"], sing_cou) # save interpolation results on the shared memory object reference_to_data_block[:] = numpy.round( data_mat.reshape(orig_time, orig_rows, orig_cols))
# In the first Python interactive shell import numpy as np a = np.array([1, 1, 2, 3, 5, 8]) # Start with an existing NumPy array from multiprocessing import shared_memory shm = shared_memory.SharedMemory(create=True, size=a.nbytes) # Now create a NumPy array backed by shared memory b = np.ndarray(a.shape, dtype=a.dtype, buffer=shm.buf) b[:] = a[:] # Copy the original data into shared memory print(b) print(shm.name) # In either the same shell or a new Python shell on the same machine import numpy as np from multiprocessing import shared_memory # Attach to the existing shared memory block existing_shm = shared_memory.SharedMemory(name=shm.name) # Note that a.shape is (6,) and a.dtype is np.int64 in this example c = np.ndarray((6, ), dtype=np.int64, buffer=existing_shm.buf) print(c) c[-1] = 888 print(c) # Back in the first Python interactive shell, b reflects this change print(b) # Clean up from within the second Python shell del c # Unnecessary; merely emphasizing the array is no longer used existing_shm.close() # Clean up from within the first Python shell del b # Unnecessary; merely emphasizing the array is no longer used
def gen_time_results(mat_size, core_list): mat_shape = (mat_size, mat_size) data_A = np.random.rand(*mat_shape).astype(np.float32) data_B = np.random.rand(*mat_shape).astype(np.float32) data_C = np.empty((mat_size, mat_size), dtype=np.float32) shm_A = shared_memory.SharedMemory(create=True, size=data_A.nbytes) shm_B = shared_memory.SharedMemory(create=True, size=data_B.nbytes) shm_C = shared_memory.SharedMemory(create=True, size=data_C.nbytes) mat_A = np.ndarray(data_A.shape, dtype=data_A.dtype, buffer=shm_A.buf) mat_A[:] = data_A[:] mat_B = np.ndarray(data_B.shape, dtype=data_B.dtype, buffer=shm_B.buf) mat_B[:] = data_B[:] mat_C = np.ndarray(data_C.shape, dtype=data_C.dtype, buffer=shm_C.buf) mat_C[:] = data_C[:] name_A = shm_A.name name_B = shm_B.name name_C = shm_C.name total_times = [] send_times = [] calc_times = [] recv_times = [] for no_cores in core_list: print(no_cores) #Assuming the matrix is of size 2^n for int N, we take log2 to find the value of n power = np.log2(no_cores) / 2 #Represents the number of partitons that must be calculated in the result matrix C pars_i = int(2**(np.ceil(power))) pars_j = int(2**(np.floor(power))) #Represents the size of each partiton in the i and j axis len_i = int(mat_size / pars_i) len_j = int(mat_size / pars_j) send_list = [] total_start = time.time() send_list = [[i, len_i, j, len_j, mat_size, name_A, name_B, name_C] for j in range(pars_j) for i in range(pars_i)] p = Pool(processes=no_cores) res_list = p.starmap(matrix_mult, send_list) p.close() total_finish = time.time() calc_start_list = [] calc_finish_list = [] res_list = list(res_list) for i in range(len(res_list)): time_difference = res_list[0][0] - res_list[i][0] calc_start_list.append(res_list[i][1] + time_difference) calc_finish_list.append(res_list[i][2] + time_difference) calc_start = min(calc_start_list) calc_finish = max(calc_finish_list) send_time = calc_start - total_start calc_time = calc_finish - calc_start gather_time = total_finish - calc_finish total_time = total_finish - total_start assert send_time + calc_time + gather_time == total_time send_times.append(round(send_time, 10)) calc_times.append(round(calc_time, 10)) recv_times.append(round(gather_time, 10)) total_times.append(round(total_time, 10)) shm_A.close() shm_B.close() shm_C.close() shm_A.unlink() shm_B.unlink() shm_C.unlink() return tuple(send_times), tuple(calc_times), tuple(recv_times), tuple( total_times)
import numpy as np a = np.array([1, 1, 2, 3, 5, 8]) # Start with an existing NumPy array from multiprocessing import shared_memory shm = shared_memory.SharedMemory(create=True, size=a.nbytes, name='psm_957212c1') # Now create a NumPy array backed by shared memory b = np.ndarray(a.shape, dtype=a.dtype, buffer=shm.buf) b[:] = a[:] # Copy the original data into shared memory print(b) type(b) type(a) print( shm.name ) # If we did not specify a name one would be chosen for us and we would have to paste it in the client wait = input('Launch client then press enter') print(b) # Clean up from within the first Python shell del b # Unnecessary; merely emphasizing the array is no longer used shm.close() shm.unlink() # Free and release the shared memory block at the very end
import psutil import numpy as np from humanize import naturalsize # # Make a big array # E.g., analagously, this could be a large ML model we want to serve array_n = 10**4 big_matrix = np.random.rand(array_n, array_n).astype(np.float) print("array size: ", naturalsize(big_matrix.nbytes)) # # Create dedicated shared memory object shared_block = shared_memory.SharedMemory(create=True, size=big_matrix.nbytes) shared_matrix = np.ndarray(big_matrix.shape, dtype=np.float, buffer=shared_block.buf) shared_matrix[:] = big_matrix[:] shared_block_id = shared_block.name # # Worker func. Run on forked process def worker_func(worker_number, matrix_block_id, matrix_shape): block = shared_memory.SharedMemory(name=shared_block_id) matrix = np.ndarray(matrix_shape, dtype=np.float, buffer=block.buf) for _ in range(100): print(
# Test of how the virtual environment process might pass data to and from # the simbox parent process. **Don't run this file as-is**; run it in two # separate Python processes (simplest: two shells), entering code in each # process in order, going down the page. # This seems to work great! # -=-=- Process 1 -=-=- import numpy as np import multiprocessing.shared_memory as sm # Set up and initialize shared array # Note: mem size and array shape and dtype should be determined for our # specific purposes. Here it's an array of 6 bools (1 byte each) mem = sm.SharedMemory(create=True, name="valve_states", size=6) valves = np.ndarray((6, ), dtype=np.bool, buffer=mem.buf) valves[:] = [False, False, True, False, False, True] # valves is now useable as an array in Process 1. # -=-=- Process 2 -=-=- import numpy as np import multiprocessing.shared_memory as sm # Create array using the same shared memory # Note that the shape and dtype shoudl eb the same as defined above mem = sm.SharedMemory(name="valve_states") valves = np.ndarray((6, ), dtype=np.bool, buffer=mem.buf) # valves is now useable as an array in Process 2. It shares the same data # as the array in Process 1
if t_end-window_start_time<0: xgrid = int(gridnumber) else: xgrid = int(gridnumber + c*(t_end-window_start_time)/delta_x) #################### x_interval=const.x_interval #10 t_total=1e15*x_end/c #fs t_size=int(t_total/dt)+1+1 ######allay define SHAPE = ((int(xgrid/x_interval)+1,t_size)) #xt = Array('f',SHAPE) #global a a=np.zeros((int(xgrid/x_interval)+1,t_size)) shm = shared_memory.SharedMemory(create=True, size=a.nbytes) #xt = np.ndarray(a.shape, dtype=a.dtype, buffer=shm.buf) #xt[:,:]=a[:,:] #import multiprocessing #import time #import numpy as np #global_arr_shared = None #def init_pool(arr_shared): # global global_arr_shared # global_arr_shared = arr_shared #def worker(i): # arr = np.frombuffer(global_arr_shared, np.double).reshape(SHAPE)
def midi_worker_main(mem1, mem2, cfg, q): # noinspection PyBroadException try: shm1 = shared_memory.SharedMemory(mem1) shm2 = shared_memory.SharedMemory(mem2) def update_status(txt, val): write_shm_text(shm1, txt) write_shm_long(shm2, val) update_status("<b>准备:</b>加载插件。", 0) plugins = [] def _load_module(type_): _names = cfg['midi'][f'{type_}s'] for name in _names: _pkg_name, klass_name = name.split(".") _pkg = importlib.import_module(f"mid.{type_}s.{_pkg_name}") klass = getattr(_pkg, klass_name) args = cfg['midi'][f'{type_}s'][name] real_args = {} for arg, value in args.items(): arg = arg.replace( f"#{type_}-cfg-{_pkg_name}-{klass_name}-", "") if not (isinstance(value, str) and value.strip()): continue real_args[arg] = j if (j := eval_or_null(value, vars(_pkg))) \ is not NotImplemented else value try: instance = klass(**real_args) except TypeError: q.put( f"猜测:无法初始化“{name}”,因为参数类型不正确或缺少。\n{traceback.format_exc()}" ) return None except AttributeError: q.put( f"猜测:无法初始化“{name}”,因为找不到指定的类或模块。\n{traceback.format_exc()}" ) return None yield instance plugins.extend(_load_module("plugin")) fp = cfg["midi"]["path"] ig = InGameGenerator if cfg["midi"]["generate_type"] == "rt": update_status("<b>准备:</b>创建生成器实例。", 0) try: gen = RealTimeGenerator(fp=fp, plugins=plugins, namespace=cfg["func_namespace"]) except FileNotFoundError: q.put( f"猜测:无法创建生成器实例,因为无法找到MIDI文件。\n{traceback.format_exc()}" ) return None except PermissionError: q.put( f"猜测:无法创建生成器实例,因为无法读取MIDI文件。\n{traceback.format_exc()}" ) return None except (OSError, IOError): q.put( f"猜测:无法创建生成器实例,因为MIDI文件无效。\n{traceback.format_exc()}") return None elif cfg["midi"]["generate_type"] == "ig": update_status("<b>准备:</b>加载前端。", 0) names = cfg['midi']['frontend'] pkg_name = names.split(".")[0] frn_name = names.split(".")[1] pkg = importlib.import_module(f"mid.frontends.{pkg_name}") try: frontend = getattr(pkg, frn_name) except AttributeError: q.put( f"猜测:无法初始化前端“{frn_name}”,因为找不到指定的前端。\n{traceback.format_exc()}" ) return None update_status("<b>准备:</b>加载中间件。", 0) middles = [] middles.extend(_load_module("middle")) update_status("<b>准备:</b>创建生成器实例。", 0) try: gen = InGameGenerator(fp=fp, frontend=frontend, plugins=plugins, middles=middles, namespace=cfg["func_namespace"]) except FileNotFoundError: q.put( f"猜测:无法创建生成器实例,因为无法找到MIDI文件。\n{traceback.format_exc()}" ) return None except PermissionError: q.put( f"猜测:无法创建生成器实例,因为无法读取MIDI文件。\n{traceback.format_exc()}" ) return None except (OSError, IOError): q.put( f"猜测:无法创建生成器实例,因为MIDI文件无效。\n{traceback.format_exc()}") return None try: gen.gvol_enabled = not cfg["midi"]["overrides"]["gvol"] gen.prog_enabled = not cfg["midi"]["overrides"]["program"] gen.phase_enabled = not cfg["midi"]["overrides"]["phase"] gen.pitch_enabled = not cfg["midi"]["overrides"]["pitch"] gen.pitch_factor = float(cfg["midi"]["pitch_factor"]) gen.volume_factor = float(cfg["midi"]["volume_factor"]) except ValueError: q.put(f"猜测:无法配置生成器实例,因为无法解析浮点数。\n{traceback.format_exc()}") return None else: raise RuntimeError("不合法的生成类型。") try: gen.wrap_length = round(float(cfg["midi"]["wrap_length"])) gen._use_function_array = cfg["midi"]["use_func"] gen._auto_function_array = cfg["midi"]["auto_func"] gen.blank_ticks = round(float(cfg["midi"]["blank_ticks"])) gen.tick_rate = float(cfg["midi"]["tick_rate"]) gen.tick_scale = float(cfg["midi"]["tick_scale"]) except ValueError: q.put(f"猜测:无法配置生成器实例,因为无法解析浮点数。\n{traceback.format_exc()}") return None if (i := cfg["midi"]["auto_adjust"])["is_enabled"]: update_status("<b>生成:</b>自动调整参数。", 20) try: gen.auto_tick_rate(base=float(i["base"]), step=float(i["step"]), tolerance=float(i["tolerance"])) except ValueError: q.put(f"猜测:无法自动调整参数,因为无法解析浮点数。\n{traceback.format_exc()}") return None
def cluster_proc(q_lett, digs, acc, path_data, shared_list, bordes, std_ini, cluster_report_dicc, report_dicc_circula): q_db = collections.deque() extendidos = {} existing_shm = shared_memory.SharedMemory(name='memo04_wf1460') existing_shm_bin = shared_memory.SharedMemory(name='memo04_wf1460_bin') c = np.ndarray((1000, end_bin), dtype=np.float64, buffer=existing_shm.buf) c_bin = np.ndarray((1000, end_bin), dtype=np.float64, buffer=existing_shm_bin.buf) # q.put('a dta es') print('salgo de process 2 to sleep:', c[-20:, -1]) # du=collections.deque() # da=collections.deque() # std_ini=0 bins = bins_constant try: bins = 22201 big_wf = load_data(std_ini, bins, path_data) except: pass try: bins = 22500 big_wf = load_data(std_ini, bins, path_data) except: pass try: bins = 14261 big_wf = load_data(std_ini, bins, path_data) except: pass bin_filt_1 = 25 bin_filt_2 = 15 bin_filt_3 = 8 posicion1 = 5500 posicion2 = 8700 desplazo = 10 Niter = int(big_wf.shape[0] / desplazo) # pi=len(du) print('termine de cargar la data') os.chdir(desvios_path) mad_data = np.load("desvios.npy") median_data = np.load("medias.npy") base_fin = median_data.size c[:, :] = np.eye(N=1000, M=end_bin) * 0.2 wf_actual = c.copy() wf_actual_bin = c_bin.copy() print('antes de entrar al loop', wf_actual.shape) tiempo = 0 medianM = np.zeros((desplazo, end_bin)) madM = np.zeros((desplazo, end_bin)) for jj in range(desplazo): medianM[jj, :] = median_data[:end_bin] madM[jj, :] = mad_data[:end_bin] flag_forward = True flag_backward = False flag_next = False flag_previous = False jkl = -1 while jkl < Niter: # jkl+=1 # time.sleep(0.4) if q_lett.qsize() > 0: print('entro a la cola') letter = q_lett.get_nowait() if letter == 'f': flag_forward = True flag_backward = False flag_next = False flag_previous = False if letter == 'b': flag_forward = False flag_backward = True flag_next = False flag_previous = False # hacer adelantar la data if letter == 'n': flag_forward = False flag_backward = False flag_next = True flag_previous = False # hacer adelantar la data if letter == 'a': flag_forward = True flag_backward = False flag_next = False flag_previous = True # hacer adelantar la data if letter == 'p': flag_forward = False flag_backward = False flag_next = False flag_previous = False # hacer adelantar la data if not flag_forward and not flag_backward and not flag_next and not flag_previous: time.sleep(0.5) if flag_forward: jkl += 1 tiempo = jkl wf_actual = np.roll(wf_actual, -desplazo, axis=0) wf_actual_bin = np.roll(wf_actual_bin, -desplazo, axis=0) # st3=time.time() wf_actual[-desplazo:, :] = big_wf[tiempo * desplazo:(tiempo + 1) * desplazo, :] MAD = np.abs(big_wf[tiempo * desplazo:(tiempo + 1) * desplazo, :] - medianM) / madM wf_actual_bin[ -desplazo:, :posicion1] = MAD[:, :posicion1] > bin_filt_1 wf_actual_bin[ -desplazo:, posicion1:posicion2] = MAD[:, posicion1:posicion2] > bin_filt_2 wf_actual_bin[-desplazo:, posicion2:] = MAD[:, posicion2:] > bin_filt_3 if flag_backward: jkl -= 1 tiempo = jkl wf_actual = np.roll(wf_actual, desplazo, axis=0) wf_actual_bin = np.roll(wf_actual_bin, desplazo, axis=0) # st3=time.time() wf_actual[:desplazo, :] = big_wf[tiempo * desplazo:(tiempo + 1) * desplazo, :] MAD = np.abs(big_wf[tiempo * desplazo:(tiempo + 1) * desplazo, :] - medianM) / madM wf_actual_bin[:desplazo, : posicion1] = MAD[:, :posicion1] > bin_filt_1 wf_actual_bin[:desplazo, posicion1: posicion2] = MAD[:, posicion1:posicion2] > bin_filt_2 wf_actual_bin[:desplazo, posicion2:] = MAD[:, posicion2:] > bin_filt_3 if flag_next: jkl += 1 tiempo = jkl flag_next = False if flag_previous: jkl -= 1 tiempo = jkl flag_previous = False ### AVANZO TEMPORALMENTE SEA CUAL SEA EL FLAG DE EVOLUCION ... ### aca entro a calcular clusters y propiedades porque modifique el tiempo if flag_forward or flag_backward or flag_next or flag_previous: keyboard.press_and_release( 'k' ) ## with k letter, the other process executes plot_binary to update the mpl figure # k for key, group in groupby( sorted(list(extendidos.keys()), key=lambda x: x[0]), lambda x: x[0]): for thing in group: try: clave = max([item for item in group], key=itemgetter(1)) print("grouped: %s : %s " % (key, clave)) print(clave) if min(extendidos[clave][0][:, 1]) < desplazo: cluster,cs,not_cs,fit,ll,w,h,good_fit, edgecolor, \ txt_c, fontsize,velocidad, txt_data,\ delta_posicion, intensidad_posicion, delta_tiempo, intensidad_tiempo= extendidos[clave] cluster[:, 1] -= desplazo extendidos[clave]= cluster,cs,not_cs,fit,ll,w,h,good_fit, edgecolor, \ txt_c, fontsize,velocidad, txt_data,\ delta_posicion, intensidad_posicion, delta_tiempo, intensidad_tiempo except: pass time_da = tiempo ctr = 0 threshold = 0.03 #parametro cluster primera filtrada eps = 4 #15 min_samples = 21 #250 filas_imagen_binaria = 1000 size_wf = (filas_imagen_binaria, median_data.size) startu = time.time() pts_cluster = np.transpose( np.array(((np.nonzero(wf_actual_bin)[1], np.nonzero(wf_actual_bin)[0])))) if len(pts_cluster.shape) > 1: db = DBSCAN(eps=eps, min_samples=min_samples).fit(pts_cluster) else: time.sleep(0.01) finu = time.time() rr, noise_pts = calculo_cluster_properties(db, pts_cluster, wf_actual) gnu = time.time() # shared_list=[] ### extiendo clusters flag_extend = True while flag_extend: rr, noise_pts, flag_extend = extiendo_clusters( rr, noise_pts, db, pts_cluster, wf_actual, cluster_report_dicc) print(flag_extend) ### chequeo clusters reportados extendidos y no cortados ###tiempo es el tiempo de abajo ### tiempo - 1000 es el tiempo de arriba lista_posiciones = [ int(item[4][0]) if item[11] > 0 else int(item[4][0] + item[5]) for item in rr ] for p, n in list(cluster_report_dicc.keys())[::]: # extendidos[p,n,] if p in lista_posiciones: # print (p) print(p) idx_cluster_to_modify = np.where( np.array(lista_posiciones) == p)[0][0] tup = rr[idx_cluster_to_modify] cluster,cs,not_cs,fit,ll,w,h,good_fit, edgecolor, \ txt_c, fontsize,velocidad, txt_data,\ delta_posicion, intensidad_posicion, delta_tiempo, intensidad_tiempo=tup # print (np.min(cluster[:,1]) , 'jjkl el cluster es;,', cluster[:,1]) # min_tiem= keys = (p, n, (tiempo + 1) * desplazo - 1000 + np.min(cluster[:, 1])) if good_fit: extendidos[keys] = tup # extendidos[keys]= cluster,cs,not_cs,fit,ll,w,h,good_fit, edgecolor, \ # txt_c, fontsize,velocidad, txt_data,\ # delta_posicion, intensidad_posicion, delta_tiempo, intensidad_tiempo=tup # for tup in rr: # shared_list.append(tup) print('') print('') print('tiempo:', tiempo * desplazo) # for things in extendidos.keys(): print('kelsssssss', extendidos.keys()) for key, group in groupby( sorted(list(extendidos.keys()), key=lambda x: x[0]), lambda x: x[0]): for thing in group: try: clave = max([item for item in group], key=itemgetter(1)) print("grouped: %s : %s " % (key, clave)) print(clave) if min(extendidos[clave][0][:, 1]) < desplazo: cluster,cs,not_cs,fit,ll,w,h,good_fit, edgecolor, \ txt_c, fontsize,velocidad, txt_data,\ delta_posicion, intensidad_posicion, delta_tiempo, intensidad_tiempo= extendidos[clave] print('limites time clust', min(extendidos[clave][0][:, 1]), max(extendidos[clave][0][:, 1])) shared_list.append(extendidos[clave]) except: pass print('') print('') # time.sleep(1) print('updating shm c, ___ iteracion n:', jkl) ###(jkl* desplazo is the time executed... ) c[:, :] = wf_actual.copy() c_bin[:, :] = wf_actual_bin.copy() print('DBSCAN time is:', finu - startu) print('cluster_ properties time is:', gnu - finu) ## rr, noise_pts= extiendo_clusters(rr, noise_pts, db, pts_cluster,wf_actual, cluster_report_dicc,ax) # ### aca itero sobre todos los clusters, esto es el lugar donde tengo que extenderlos. # ### en cluster fit.... ## for cluster,cs,not_cs,fit,ll,w,h,good_fit,edgecolor,txt_c,fontsize,velocidad, txt_data in rr: ##%% # bordea=[] ### aca solo clasifico segun las caracteristicas ,,,, for tup in rr: # ### unpacking tup cluster,cs,not_cs,fit,ll,w,h,good_fit, edgecolor, \ txt_c, fontsize,velocidad, txt_data,\ delta_posicion, intensidad_posicion, delta_tiempo, intensidad_tiempo=tup # jom+=1 # rect_clust=ptc.Rectangle(ll,w,h,facecolor='none',edgecolor=edgecolor) # ax.add_patch(rect_clust) # rect_clust_bin=ptc.Rectangle(ll,w,h,facecolor='none',edgecolor=edgecolor) ## ax_bin.add_patch(rect_clust_bin) # bordes.append(rect_clust) ## ax.plot(cluster[:,0],cluster[:,1],'o') # tr=(ll[0]+w,ll[1] + h) # ax.annotate(delta_tiempo,tr) # if h > 49 and 2000 > w > 10: # rect_clust=ptc.Rectangle(ll,w,h,facecolor='none',edgecolor=edgecolor) # ax.add_patch(rect_clust) # rect_clust_bin=ptc.Rectangle(ll,w,h,facecolor='none',edgecolor=edgecolor) # ax_bin.add_patch(rect_clust_bin) # bordes.append(rect_clust) tr = (ll[0] + w, ll[1] + h) # ax.annotate(delta_posicion,tr) pts_cant = cluster.size # ax.plot() ### por ahora vamos a extender solo los clusters grandes ### capaz que hay que extenderlos en calculo_cluster_properties # yoffset=[0, 10,20,30,40] # if good_fit: # for off in yoffset: # w_clus= max(cluster[:,0]) - min(cluster[:,0]) # if velocidad<0: xfit= np.arange(min(cluster[:,0])- w ,max(cluster[:,0])) # if velocidad>0: xfit= np.arange(min(cluster[:,0]), max(cluster[:,0]) + w) # yfit= fit(xfit) # ax.plot(xfit,yfit+off) # ax_bin.plot(xfit,yfit+off) # if velocidad<0: # ax.add_patch(ptc.Rectangle(ll-(5,-h),10,off,facecolor='none', # edgecolor=edgecolor)) ## ax_bin.add_patch(ptc.Rectangle(ll-(5,-h),10,off,facecolor='none',edgecolor=edgecolor)) # if velocidad>0: # ax.add_patch(ptc.Rectangle(tr+(5,0)),10,off,facecolor='none', # edgecolor=edgecolor) # ax_bin.add_patch(ptc.Rectangle(tr+(5,0)),10,off,facecolor='none',edgecolor=edgecolor) ## reporte db da el vehiculo ## cluster_report ve si es actualizacion o finalizacion ## st tipo de actualizacion, st_cant cantidad de actualizacion if velocidad > 0: st, st_cant = cluster_report(velocidad, pts_cant, ll, h, cluster_report_dicc) vh = reporte_db(st, st_cant, velocidad, good_fit, ll, h, w, q_db, report_dicc_circula, time_da) if velocidad < 0: st, st_cant = cluster_report(velocidad, pts_cant, (ll[0] + w, ll[1]), h, cluster_report_dicc) vh = reporte_db(st, st_cant, velocidad, good_fit, (ll[0] + w, ll[1]), h, w, q_db, report_dicc_circula, time_da) if velocidad == 0: vh = 'nny' st = 'none' st_cant = '-3' if (edgecolor == 'r' or edgecolor == 'g') and velocidad != 0: ann = (vh, st, st_cant, txt_data, tr, txt_c, fontsize) bordes.append(ann) texto_dic = str(vh + '\n' + st + ':' + str(st_cant) + '\n' + str(txt_data)) texto_dic = { 'vehiculo': str(vh), 'st': st, 'st_cant': st_cant, 'w': str(int(w)), 'h': int(h), 'lower': ll[0], 'left': ll[1], 'velocidad': str(round(velocidad, 3)), 'ajuste': str(round((h / w) / fit[1], 3)) }
def _open(self): if self.shm_name and not self.memory: self.memory = shm.SharedMemory(name=self.shm_name) return self.memory
def init_data_block_sg_fft(sg_window, band, in_dir_qs, in_dir_tf, tile, list_qual, list_data, num_ob_buf_bytes, fit_nr, name_weights_addition, master_raster_info, poly=False): """ Parameters ---------- sg_window band in_dir_qs in_dir_tf tile list_qual list_data num_ob_buf_bytes fit_nr name_weights_addition master_raster_info Returns ------- """ #data_block = numpy.zeros([sg_window, master_raster_info[2], master_raster_info[3]]) shm = shared_memory.SharedMemory(create=True, size=num_ob_buf_bytes) shm_qual = shared_memory.SharedMemory(create=True, size=num_ob_buf_bytes) if poly: data_block = numpy.ndarray( (sg_window, master_raster_info[2], master_raster_info[3]), dtype=numpy.float64, buffer=shm.buf) qual_block = numpy.ndarray( (sg_window, master_raster_info[2], master_raster_info[3]), dtype=numpy.float64, buffer=shm_qual.buf) else: data_block = numpy.ndarray( (sg_window, master_raster_info[2], master_raster_info[3]), dtype=numpy.int16, buffer=shm.buf) qual_block = numpy.ndarray( (sg_window, master_raster_info[2], master_raster_info[3]), dtype=numpy.int16, buffer=shm_qual.buf) print("\n# START READING SATDATA for BAND {}".format(band)) for i in range(0, sg_window, 1): # load qual file try: qual_ras = gdal.Open(os.path.join(in_dir_qs, tile, list_qual[i]), gdal.GA_ReadOnly) print("# load qual data for band %d: %s" % (band, list_qual[i])) qual_block[i, :, :] = qual_ras.ReadAsArray() del qual_ras except Exception as ErrorQualRasReading: print("### ERROR while reading quality raster:\n {}".format( ErrorQualRasReading)) # load satellite data try: data_ras = gdal.Open(os.path.join(in_dir_tf, tile, list_data[i]), gdal.GA_ReadOnly) print("# load sat data for band %s: %s" % (str(band), list_data[i])) #data_band = data_ras.GetRasterBand(1) data_block[i, :, :] = data_ras.ReadAsArray() # collect epochs raster name if fit_nr == i: print("\n# Name of fitted tile will be: {} \n".format( os.path.join(tile, list_data[i]))) fitted_raster_band_name = list_data[ i][:-4] + name_weights_addition % str(sg_window) del data_ras except Exception as ErrorRasterDataReading: print("### ERROR while reading satellite raster:\n {}".format( ErrorRasterDataReading)) print("data_block from readout: ", data_block[:, 2000, 100]) return data_block, qual_block, shm, shm_qual, fitted_raster_band_name
def create_shm(self): self.shm = shared_memory.SharedMemory(name=self.name, create=True, size=self.capacity) self.holder = True
def init_data_block_dft(sg_window, band, in_dir_qs, in_dir_tf, tile, list_qual, list_data, num_ob_buf_bytes, master_raster_info): """ Creates a initial datablock for the modis data and returns a numpy ndim array Parameters ---------- sg_window band in_dir_qs in_dir_tf tile list_qual list_data device master_raster_info fit_nr name_weights_addition Returns ------- """ #data_block = numpy.zeros([sg_window, master_raster_info[2], master_raster_info[3]]) shm = shared_memory.SharedMemory(create=True, size=num_ob_buf_bytes) data_block = numpy.ndarray( (sg_window, master_raster_info[2], master_raster_info[3]), dtype=numpy.float64, buffer=shm.buf) shm_qual = shared_memory.SharedMemory(create=True, size=num_ob_buf_bytes) qual_block = numpy.ndarray( (sg_window, master_raster_info[2], master_raster_info[3]), dtype=numpy.float64, buffer=shm_qual.buf) print("\n# START READING SATDATA for BAND {}".format(band)) for i in range(0, len(list_data), 1): # load qual file try: qual_ras = gdal.Open(os.path.join(in_dir_qs, tile, list_qual[i]), gdal.GA_ReadOnly) #print("load qual data for band %d: %s" % (band, list_qual[i])) #qual_band = qual_ras.GetRasterBand(1) qual_block[i, :, :] = qual_ras.ReadAsArray() del qual_ras except Exception as ErrorQualRasReading: print("### ERROR while reading quality raster:\n {}".format( ErrorQualRasReading)) # load satellite data try: data_ras = gdal.Open(os.path.join(in_dir_tf, tile, list_data[i]), gdal.GA_ReadOnly) print("# load sat data for band %s: %s" % (str(band), list_data[i])) #data_band = data_ras.GetRasterBand(1) data_block[i, :, :] = data_ras.ReadAsArray() del data_ras except Exception as ErrorRasterDataReading: print("### ERROR while reading satellite raster:\n {}".format( ErrorRasterDataReading)) print("data_block from readout: ", data_block[:, 2000, 100]) return data_block, qual_block, shm, shm_qual
def load_shm(self): self.shm = shared_memory.SharedMemory(name=self.name, create=False) self.holder = False
from multiprocessing import shared_memory import time import sys shm_a = shared_memory.SharedMemory(name="memtest", create=True, size=1) print(shm_a) a = shm_a.buf[0] print(a) print(shm_a.name) a = 10 shm_a.buf[0] = 10 try: for i in range(40): print("i = " + str(i) + ", a = " + str(shm_a.buf[0])) time.sleep(1) except KeyboardInterrupt: shm_a.close() shm_a.unlink() sys.exit() shm_a.close() shm_a.unlink()
async def check_for_file_transfer_requests(self): # 2. `Y -> S`: every one second, Y asks server for any requests await self.write(bytes(FileTransferRequestResponsePackets())) # 3. `S -> X/F -> Y`: server responds with active requests file_transfer_requests = FileTransferCheckRequestsPackets( data=(await self.read())[4:]).requests if not file_transfer_requests: return print("Incoming file transfer request(s):") index_to_email = dict() index_to_file_info = dict() i = 1 for email, file_info in file_transfer_requests.items(): print("\t{}. {}".format(i, email)) print("\t\tname: ", file_info["name"]) print("\t\tsize: ", sizeof_fmt(int(file_info["size"]))) print("\t\tSHA256: ", file_info["SHA256"]) index_to_email[i] = email index_to_file_info[i] = file_info i += 1 try: selection = input( "\nEnter the number for which request you'd like to accept, or 0 to deny all: " ) accept = True selection_num = int(selection) if selection_num <= 0 or selection_num >= i: raise ValueError packets = FileTransferAcceptRequestPackets( index_to_email[selection_num]) except ValueError or KeyboardInterrupt: packets = FileTransferAcceptRequestPackets("") accept = False if accept: while True: out_directory = input("Enter the output directory: ") file_path = os.path.join( out_directory, index_to_file_info[selection_num]["name"]) if not os.path.isdir(out_directory): print("The path {} is not a directory".format( os.path.abspath(out_directory))) elif os.path.exists(file_path): print("The file {} already exists".format(file_path)) elif not os.access(out_directory, os.X_OK | os.W_OK): print( "Cannot write file path {} permission denied.".format( file_path)) else: break # 4. `Y -> Yes/No -> S`: Y accepts or denies transfer request await self.write(bytes(packets)) if not accept: return False # 5. `S -> Token -> Y`: if Y accepted, server sends a unique token Y token = FileTransferSendTokenPackets( data=(await self.read())[4:]).token lock = Lock() progress = shared_memory.SharedMemory(create=True, size=8) server_sentinel = shared_memory.SharedMemory(create=True, size=1) status_sentinel = shared_memory.SharedMemory(create=True, size=1) listen_port = shared_memory.SharedMemory(create=True, size=4) # 6. `Y -> Port -> S`: Y binds to 0 (OS chooses) and sends the port it's listening on to S p2p_server = P2PServer(token, os.path.abspath(out_directory), progress.name, lock, listen_port.name, status_sentinel.name) p2p_server_process = Process(target=p2p_server.run, args=(0, server_sentinel.name)) p2p_server_process.start() print("Started P2P server. Waiting for listen...") # wait for listen port = 0 while port == 0: with lock: port = int.from_bytes(listen_port.buf, byteorder='little') await self.write(bytes(FileTransferSendPortPackets(port))) # Wait until file received time_start = time.time() chunk_size = FILE_TRANSFER_P2P_CHUNK_SIZE def unguarded_print_received_progress(final=False): utils.print_status( *utils.get_progress( int.from_bytes(progress.buf[0:4], byteorder='little'), int.from_bytes(progress.buf[4:8], byteorder='little'), chunk_size), "received", final) def print_received_progress(): while True: with lock: if status_sentinel.buf[0] == 1: break unguarded_print_received_progress() time.sleep(0.03) status_thread = Thread(target=print_received_progress) status_thread.start() try: p2p_server_process.join() except KeyboardInterrupt: raise RuntimeError("User requested abort") finally: if p2p_server_process.is_alive(): p2p_server_process.terminate() with lock: status_sentinel.buf[0] = 1 status_thread.join() unguarded_print_received_progress(final=True) progress.close() progress.unlink() server_sentinel.close() server_sentinel.unlink() status_sentinel.close() status_sentinel.unlink() listen_port.close() listen_port.unlink() time_end = time.time() print("File transfer completed successfully in {} seconds.".format( time_end - time_start)) return True
sys.path.insert(1, '/home/solcanma/darknet') import darknet from multiprocessing import Process, Manager from multiprocessing import shared_memory manager = Manager() manager_detections = manager.list() from pyimagesearch.centroidtracker import * from distance import * from navigate import * from map import * #from pyimagesearch.centroidtracker import * shm = shared_memory.SharedMemory(create=True, size=6520800, name='psm_c013ddb5') shm_image = np.ndarray((Yresolution, Xresolution, 3), dtype=np.uint8, buffer=shm.buf) logging.basicConfig( level=logging.DEBUG, format='[%(levelname)s] (%(threadName)-10s) %(message)s', ) # import the necessary packages import numpy as np def convertBack(x, y, w, h): xmin = int(round(x - (w / 2)))
async def send_file(self): msg = None try: # 1. `X -> Y/F -> S`: X wants to send F to Y recipient_email = input("Enter the recipient's email address: ") file_path = os.path.abspath(input("Enter the file path: ")) valid_email = validate_and_normalize_email(recipient_email) if valid_email is None: raise RuntimeError("Invalid Email Address.") if not file_path: raise RuntimeError("Empty file path.") if not os.path.exists(file_path): raise RuntimeError("Cannot find file: {}".format(file_path)) if not os.path.isfile(file_path): raise RuntimeError("Not a file: {}".format(file_path)) file_base = os.path.basename(file_path) file_size = os.path.getsize(file_path) file_sha256 = sha256_file(file_path) file_info = { "name": file_base, "size": file_size, "SHA256": file_sha256, } # send request await self.write( bytes(FileTransferRequestPackets(valid_email, file_info))) # this only checks if the request is valid # this does not check if the recipient accepted or denied the request msg = StatusPackets(data=(await self.read())[4:]).message if msg != "": raise RuntimeError(msg) # 7. `S -> Token/Port -> X`: S sends the same token and port to X # denied request is indicated by empty token and port port_and_token = FileTransferSendPortTokenPackets( data=(await self.read())[4:]) port, token = port_and_token.port, port_and_token.token if token and port: print( "User {} accepted the file transfer Connecting to recipient on port " .format(valid_email, port)) else: raise RuntimeError( "User {} declined the file transfer request".format( valid_email)) progress = shared_memory.SharedMemory(create=True, size=8) progress_lock = Lock() p2p_client = P2PClient(port, token, file_path, file_size, file_sha256, progress.name, progress_lock) time_start = time.time() sentinel = False chunk_size = FILE_TRANSFER_P2P_CHUNK_SIZE def unguarded_print_sent_progress(final=False): utils.print_status( *utils.get_progress( int.from_bytes(progress.buf[0:4], byteorder='little'), int.from_bytes(progress.buf[4:8], byteorder='little'), chunk_size), "sent", final) def print_sent_progress(): while not sentinel: with progress_lock: unguarded_print_sent_progress() time.sleep(0.03) # i was having trouble with asyncio.gather, so just run status printer in a new thread status_thread = Thread(target=print_sent_progress) status_thread.start() # wait until p2p transfer completes, unless keyboard interrupt try: await p2p_client.main() except KeyboardInterrupt: raise RuntimeError("User requested abort") finally: sentinel = True status_thread.join() unguarded_print_sent_progress(final=True) progress.close() progress.unlink() time_end = time.time() print( "\nFile transfer completed in {} seconds.".format(time_end - time_start)) except RuntimeError as e: msg = str(e) if msg != "": print("Failed to send file: ", msg)
def __init__(self): self.shm = shared_memory.SharedMemory(create=True, size=4) self.server = Server(21234) self.server.config()
def __init__(self): self._Shm = shared_memory.SharedMemory("_SharedMem_Modbus")
import random from multiprocessing import shared_memory from array import array from util import wait, signal mem = shared_memory.SharedMemory(name='prod_con_buffer') buff = mem.buf.cast('i') print(mem.name) try: while True: print("{0:-^50}".format("PRODUCER")) seed = int(input("Enter a seed: ")) wait(buff, 1) random.seed = seed buff[0] = random.randrange(-2**31, 2**31 - 1) print("shared variable write: %d" % buff[0]) signal(buff, 2) signal(buff, 1) except KeyboardInterrupt: pass del buff mem.close()
def create_shared_memory(self): self.shared_memory_instance = shared_memory.SharedMemory(create=True, size=2) self.shared_memory_name = self.shared_memory_instance.name self.shared_memory_buffer = self.shared_memory_instance.buf
shm = shared_memory.SharedMemory(shared_memory_tag) AC_in_bytes = shm.buf[0:ac_size] ac_in_process = AC.from_buff(AC_in_bytes, copy=False) string_to_search = "asdpythonasdasdruby" print("Executing search in {}".format(processname)) for id, start, end in ac_in_process.match(string_to_search): print(id, string_to_search[start:end]) #time.sleep(100) ac_in_process = None AC_in_bytes.release() # MUST release memory beforing closing shm insance, otherwise error is raised. shm.close() if __name__ == "__main__": shared_memory_tag = "shared_ac" # Put mm in shared memory of py3.8 shm = shared_memory.SharedMemory(name=shared_memory_tag, create=True, size=ac_size) shm.buf[0:ac_size] = mm mm.close() processes_list = list() for x in range(0, 6): p = Process( target=get_shared_memory_find_matches, args=( "process_" + str(x), shared_memory_tag, ac_size ) ) p.start() processes_list.append(p) for p in processes_list:
def create(self, name, size) -> AnyStr: shm = shared_memory.SharedMemory(name=name, create=True, size=size) self.shm_store[name] = shm return shm.buf