def unpack(self): if os.path.isdir(self.build_dir): utils.output("SCIPY source already unpacked, not redoing.") return utils.output("Unpacking SCIPY source.") if os.name == 'posix': utils.unpack_build(self.tbfilename) else: os.mkdir(self.build_dir) os.chdir(self.build_dir) utils.unpack(self.tbfilename)
def unpack(self): if os.path.isdir(self.build_dir): utils.output("MATPLOTLIB source already unpacked, not redoing.") else: if os.name == 'posix': utils.output("Unpacking MATPLOTLIB source.") utils.unpack_build(self.tbfilename) else: utils.output("Unpacking MATPLOTLIB binaries.") os.mkdir(self.build_dir) os.chdir(self.build_dir) utils.unpack(self.tbfilename)
def install_nt(self): config.CMAKE_BINPATH = os.path.join( self.inst_dir, 'bin', 'cmake.exe') utils.goto_inst() if os.path.exists('cmake'): utils.output('CMAKE already installed, Skipping.') return # this will unpack into inst/cmake-VER-win32-x86/bin etc utils.unpack(self.afilename) # so we rename it to plain 'cmake' os.rename(CMAKE_DIRBASE, 'cmake')
def process(mj, since, till, path_incoming, path_to, force_unpack=False): if force_unpack or not os.path.isdir('tmp/%s' % mj): logging.debug('will unpack') utils.unpack(mj, since, till, path_incoming, path_to) production_times, unmatched_files = get_production_times(path_to, mj) logging.debug('production times count: %s' % len(production_times)) base_ini_fn = '%s/%s/atx300/set/base/base.ini' % (path_incoming, mj) base_to_ior, ior_to_base = atxutils.build_signal_maps(base_ini_fn) signals_fns = utils.get_signals_files('%s/%s' % (path_incoming, mj), since, till) ''' for fn in signals_fns: header = atxsignals.read_header(fn) assert(header['is_deaggregated']) ''' logging.debug('reading signals') deaggregated = atxsignals.read_deaggregated(signals_fns, since, till) illegal_cement_openings = [] cem_bin_closed = None cem_scale_value = None for rec in deaggregated: if rec['type'] == 'header': continue k = rec['k'] v = rec['v'] if 'v' in rec else rec['avg'] #v_recalc = rec['v'] * coeffs[rec['k']] - offs[rec['k']] if ior_to_base.get(k) == 'Cement_Scale1': cem_scale_value = v elif ior_to_base.get(k) == 'I_Bin1CEMclosed': if v == 0 and cem_bin_closed != 0: is_illegal = True for prod_since, prod_till in production_times: if prod_since <= rec['t'] <= prod_till: is_illegal = False break if is_illegal: #logging.debug('%s %s' % (dt, cem_scale_value)) illegal_cement_openings.append((rec['t'], cem_scale_value)) cem_bin_closed = v logging.debug('found %d illegal cement openings' % len(illegal_cement_openings)) return unmatched_files, illegal_cement_openings
def parse_option(self, option_data): p = 0 while p<len(option_data): kind = ord(option_data[p]) if kind == 0: # end break elif kind == 1: # NOP p+=1 continue p+=1 length = ord(option_data[p]) p+=1 if kind == 2: # MAX segment_size self.option_max_segment_size = unpack('!H', option_data[p:p+2]) elif kind == 3: # Window scale factor self.option_window_scale_factor = ord(option_data[p]) elif kind == 4: # TCP SACK Permitted Option self.option_SACK_Permit = True elif kind == 5: # SACK pass elif kind == 8: # time stamp self.option_timestamp = struct.unpack("!II", option_data[p:p+8]) # timestamp value / timestamp echo reply else: print "Warning, unknown TCP options:", kind p+=length-2
def update(self, uid, properties): # lookup edge self._cursor.set_key(uid) if self._cursor.search() == WT_NOT_FOUND: raise KeyError('Edge not found, identifier: %s' % uid) else: # update properties start, label, end, data = self._cursor.get_value() other = unpack(data) # remove old indices if any for key in other.keys(): if key in self._indices: self._keys.set_key(key, other[key], uid) self._cursor.remove() # update data = pack(properties) self._cursor.set_value(start, label, end, data) # index properties for key in properties.keys(): if key in self._indices: self._keys.set_key(key, properties[key], uid) self._keys.set_value('') self._keys.insert()
def collect_data(self, num_steps: int = 1000, deterministic: Optional[Dict[str, bool]] = None, disable_tqdm: bool = True, train_mode: bool = True, gamma: float = 0.99, tau: float = 0.95) -> DataBatch: """ Performs a rollout of the agents in the environment, for an indicated number of steps or episodes. Args: num_steps: number of steps to take; either this or num_episodes has to be passed (not both) deterministic: whether each agent should use the greedy policy; False by default disable_tqdm: whether a live progress bar should be (not) displayed train_mode: Returns: dictionary with the gathered data """ if deterministic is None: deterministic = defaultdict(lambda: False) self.reset() obs, _, _, _ = unpack(self.env.reset(train_mode=train_mode)[self.brain_name], self.agent_ids) for step in trange(num_steps + 1, disable=disable_tqdm): # Compute the action for each agent action_info = { # action, logprob, entropy agent_id: self.agents[agent_id].compute_single_action(obs[agent_id], deterministic[agent_id]) for agent_id in self.agent_ids } action = {agent_id: action_info[agent_id][0] for agent_id in self.agent_ids} logprob = {agent_id: action_info[agent_id][1] for agent_id in self.agent_ids} entropy = {agent_id: action_info[agent_id][2] for agent_id in self.agent_ids} value = {agent_id: action_info[agent_id][3] for agent_id in self.agent_ids} # Actual step in the environment next_obs, reward, done, info = unpack(self.env.step(pack(action))[self.brain_name], self.agent_ids) # Saving to memory self.memory.store(obs, action, reward, logprob, entropy, value, done) obs = next_obs return self.memory.get_torch_data(gamma=gamma, tau=tau)
def decompress_into(self, params, scalar_weight, messages, metadata): """ Adds scaled, decompressed, parameters from `messages` to `params`. This modifies `params`. """ shapes = [p.shape for p in params] for param, values in zip(params, unpack(messages[0], shapes)): param[:].add_(scalar_weight, values)
def word_at_offset(filename, offset): # filename = ensure_src_file_path(filename) with open(filename, "rb") as f: result = "" f.seek(offset) data = f.read(2) for b in data: result += "%02x" % ord(b) return unpack(result[0:2], result[2:])
def word_at_offset(filename, offset): #filename = ensure_src_file_path(filename) with open(filename, 'rb') as f: result = "" f.seek(offset) data = f.read(2) for b in data: result += "%02x" % ord(b) return unpack(result[0:2], result[2:])
def read(name): try: with open(name, "r") as f: s = f.readline() s = unpack(s) return s except Exception as e: tf.logging.warning(e) return None
def init_state(self, params, seed=0): my_rank = torch.distributed.get_rank() ps = {} qs = {} rngs = {} for neighbor in self.topology.neighbor_ranks(my_rank): rngs[neighbor] = self._rng_for_neighbor(seed, neighbor) # Ensure that the p's and q's are consequtive in memory so we can quickly send them p_buffer, shapes = pack([self._init_p(param) for param in params]) self.fill_with_random_values(p_buffer, rngs[neighbor]) ps[neighbor] = {"list": unpack(p_buffer, shapes), "buffer": p_buffer} q_buffer, shapes = pack([self._init_q(param) for param in params]) self.fill_with_random_values(q_buffer, rngs[neighbor]) qs[neighbor] = {"list": unpack(q_buffer, shapes), "buffer": q_buffer} return self.State(ps=ps, qs=qs, iteration_number=0, bits_sent=0, messages_sent=0, rngs=rngs)
def plot_trajectories(goal_object, runs_dir, img_dir, filename, n_runs=10): """ :param runs_dir: :param img_dir: :param filename: :param goal_object :param n_runs: """ dataset_states = load_dataset(runs_dir) dataset_states = dataset_states[[ 'goal_position', 'goal_angle', 'position', 'initial_position', 'initial_angle' ]] runs = np.arange(n_runs) run_states = dataset_states.where(dataset_states.run.isin(runs), drop=True) fig, ax = plt.subplots(figsize=(7.8, 4.8), constrained_layout=True) ax.set_xlabel('x axis', fontsize=11) ax.set_ylabel('y axis', fontsize=11) ax.grid() for run_id, run in run_states.groupby('run'): init_position = run.initial_position[run_id] init_angle = run.initial_angle[run_id] x_position, y_position = unpack(run.position, 'axis') if run_id == 0: goal_position = run_states.goal_position[run_id] goal_angle = run_states.goal_angle[run_id] draw_docking_station(ax, goal_object) draw_marxbot(ax, goal_position, goal_angle, label='goal position') draw_marxbot(ax, init_position, init_angle, label='initial positions') plt.plot(x_position, y_position, color='black', label='trajectories', linewidth=1) else: draw_marxbot(ax, init_position, init_angle) plt.plot(x_position, y_position, color='black', linewidth=1) ax.set_ylim(-220, 220) ax.set_xlim(-250, 250) ax.set_aspect('equal') plt.legend() save_visualisation(filename, img_dir)
def surface(symbol, qdate, qtime=None, errors=False, n=False, loc=0, scale=0): df = query.vols(["*"], ["Date", "Time", "Tenor", "Strike"], symbol, qdate, qtime) st = utils.pack(df) del df #---------------------------------------------------------------------------------- f = lambda LST: pricer.norm_weights(LST, loc, scale) wgt = utils.apply(f, 1, st, ["Time", "Tenor"], ["LogStrike"]) #retrive weights vector sv = pricer.SSVI(errors) f = lambda LST, VAR, TNR, VOL: sv.calibrate(LST, VAR, TNR, VOL, wgt) if errors == True: prm = utils.apply(f, 6, st, ["Time"], ["LogStrike", "TotAtmfVar", "Tenor", "SmtVol"], diff=True, fill=False) eps = prm[-1] prm = np.asarray(prm[:-1]).T else: prm = utils.apply(f, 5, st, ["Time"], ["LogStrike", "TotAtmfVar", "Tenor", "SmtVol"], fill=False) #---------------------------------------------------------------------------------- reduced = [ "Time", "Group", "Tenor", "SpotMid", "Forward", "CumDivDays", "Div", "ImpBor", "Rate", "CallH0", "CallH1", "CallKh", "PutH0", "PutH1", "PutKh", "TotAtmfVar" ] st = utils.unique_nosort(st[reduced]) prm = utils.restack(prm, st["TotAtmfVar"], tile=False) rho = sv.correlation(st["TotAtmfVar"], prm[:, 0], prm[:, 1], prm[:, 2]) phi = sv.power_law(st["TotAtmfVar"], prm[:, 3], prm[:, 4]) atm, skw, krt = sv.raw2jw(st["Tenor"] / base, st["TotAtmfVar"], phi, rho) arr = unstruct(st) arr = np.column_stack([arr, atm, skw, krt, phi, rho]) df = utils.unpack(arr, qdate, raw=False, symbol=symbol) if errors == True and n == False: return df, eps elif errors == False and n == True: return df, n elif errors == True and n == True: return df, eps, n else: return df
def get(self, uid): """Look for a vertice with the given identifier `uid`""" # lookup the uid self._cursor.set_key(uid) if self._cursor.search() == WT_NOT_FOUND: raise KeyError('Vertex not found, identifier: %s' % uid) else: # uid found, return label and properties label, data = self._cursor.get_value() return label, unpack(data)
def decompress_into(self, params, scalar_weight, messages, metadata): """ Adds scaled, decompressed, parameters from `messages` to `params`. This modifies `params`. """ parts = unpack(messages[0], metadata) us = parts[: len(parts) // 2] vs = parts[len(parts) // 2 :] for param, u, v in zip(params, us, vs): param[:].addmm_(1, scalar_weight, u, v.t())
def parse(self, incoming_data): self.hardware_type = unpack("!H", incoming_data[:2]) # should be 1 assert self.hardware_type == 1 self.protocol = unpack("!H", incoming_data[2:4]) # should be ETHERTYPE_IP assert self.protocol == ETHERTYPE_IP self.hardware_size = ord(incoming_data[4]) # should be 6 assert self.hardware_size == 6 self.protocol_size = ord(incoming_data[5]) # should be 4 assert self.protocol_size == 4 self.opcode = unpack("!H", incoming_data[6:8]) self.sender_mac = incoming_data[8:14] self.sender_ip = tuple([ord(x) for x in incoming_data[14:18]]) self.target_mac = incoming_data[18:24] self.target_ip = tuple([ord(x) for x in incoming_data[24:28]]) self.headlen = 28
def get(self, uid): """Retrieve the edge with the given `uid`""" # lookup `uid` self._cursor.set_key(uid) if self._cursor.search() == WT_NOT_FOUND: raise KeyError('Edge not found, identifier: %s' % uid) else: # return edge start, label, end, data = self._cursor.get_value() return start, label, end, unpack(data)
def parse(self, incoming_data): data = [ord(x) for x in incoming_data[:20]] self.version = data[0] >> 4 self.headlen = (data[0] & 0xf) * 4 self.identification = unpack("!H", incoming_data[4:6]) self.flags = (data[6] >> 5) self.segment_id = ((data[6] & 0x1f) << 8) + data[7] self.TTL = data[8] self.protocol = data[9] self.src_ip = tuple(data[12:16]) self.dst_ip = tuple(data[16:20])
def forward(self, sentences:list, to_str:bool=False) -> PackedSequence: packed = self.extractor(sentences) packed = PackedSequence(self.dropout(packed.data), packed.batch_sizes) packed, _ = self.rnn(packed) data = self.linear(packed.data) if to_str: max_idx = data.argmax(-1) word_pack = [self.extractor.vocab[i] for i in max_idx.cpu()] return unpack(word_pack, packed.batch_sizes) else: return PackedSequence(data, packed.batch_sizes)
def runSequence(sequence, intcode, endCondition, startSignal=0): runners = [Runner(iter([phase])) for phase in sequence] runnerIterators = [runner.output_iterator(intcode) for runner in runners] # pipe last signal to current runner's input and add its next output signal to signals def step(signals, index): runners[index].feed(signals[-1]) signal = next(runnerIterators[index], None) return signals + [signal], (index + 1) % len(runners) return reduceUntil(endCondition, unpack(step), ([startSignal], 0))[0]
def find_pointers(): p = re.compile(pointer_regex) for file in files: file_path = os.path.join(abs_file_path, file) in_file = open(file_path, "rb") print file bytes = in_file.read() # text.decode('shift_jis', errors='ignore').encode('utf-8') only_hex = "" for c in bytes: # print ord(c) only_hex += "\\x%02x" % ord(c) # print only_hex # print bytes.encode('hex') tables = p.finditer(only_hex) for table in tables: last_part = table.group(4).split("\\x") # print last_part if last_part[1] == last_part[2] == last_part[3] == last_part[4]: # ignore FFFFFFFFFF sections pass elif "\\x00\\x00\\x00\\x00" in table.group(0): # sometimes they sneak by. catch them here pass else: # print table.group(0) start = table.start() / 4 # divide by four, since 4 characters per byte in our dump) stop = table.end() / 4 count = (stop - start) / 4 # div by 4 again, since 4 bytes per pointer delimiter = table.group(2) + table.group(3) # print table.group(0) # print delimiter values = [] # Can't just do this - sometimes part of the delimiter shows up in the pointer itself! (10-00-00-00) # values = table.group(0).split(delimiter) # So just slice the string into the first two bytes. for x in range(0, len(table.group(0)) - 15, 16): pointer_string = table.group(0)[x : x + 8] pointer_tuple = pointer_string.split("\\x")[1], pointer_string.split("\\x")[2] values.append(pointer_tuple) pointers = [] for (first, second) in values: pointers.append(hex(unpack(first, second))) print str(count) + " pointers at " + hex(start) + ", delimiter: " + delimiter pointers.sort() print pointers pointers_filename = "pointers_" + file out_file = open(pointers_filename, "w") for ptr in pointers: out_file.write(ptr + "\n") # next, calculate the diffs. and figure out if it's just 4 over and over diffs = [] for pointer in range(0, len(pointers) - 1): diffs.append(int(pointers[pointer + 1], 16) - (int(pointers[pointer], 16))) print diffs
def key_set(): if request.method == "POST": try: data = loads(request.data.decode('utf-8')) app.sk = unpack(data["sk"]) return dumps({"status": "OK"}) except KeyError as e: return dumps({"status": "ERROR", "message": e.args}) except Exception as e: return dumps({"status": "ERROR", "message": e.args}) else: return dumps({"status": "ERROR", "message":"Use POST method."})
def read(name): try: start_time = time.time() with pa.OSFile(name) as f: s = f.read_buffer() readtime = time.time() - start_time start_time = time.time() s = unpack(s) untime = time.time() - start_time return s, readtime, untime except Exception as e: logging.warning(e) return None
def laserize(asteroid): # sorter of asteroid tracings depending on the laser movement def laserSort(slope, positive): prioritize_if = lambda value: 0 if value else 1 startPointingUp = prioritize_if(slope == infinity and not positive) clockwise = (prioritize_if(positive), slope) return startPointingUp, clockwise rotation = cycle(sorted(asteroid.traces.keys(), key=unpack(laserSort))) while any(asteroid.traces.values()): aligned = asteroid.traces[next(rotation)] if aligned: yield aligned.pop(0)
def capture_mic(): tscale = 1.0 p = pyaudio.PyAudio() in_stream = p.open(format=pyaudio.paInt16, channels=2, rate=44100, input = True) out_stream = p.open(format=pyaudio.paInt16, channels=2, rate=int(44100*tscale), output=True) data_to_file = [] chunk = 512 prev_data = [] data = [] after_data = in_stream.read(chunk) for i in range(300): after_data = utils.unpack(after_data) # after_data = reduce_noise(after_data) if prev_data != []: concat = np.concatenate( (( np.concatenate(((prev_data,data))),after_data )) ) shifted_data = shift_pv(concat, tscale) # data_to_file.extend(shift_data) data_out = utils.pack(shifted_data) data_out = ''.join(data_out) out_stream.write(data_out) prev_data = data data = after_data after_data = in_stream.read(chunk) in_stream.stop_stream() in_stream.close() out_stream.stop_stream() out_stream.close() wav.write('test.wav',44100*2, array(data_to_file, dtype='int16')) p.terminate()
def decompress_into(self, params, scalar_weight, messages, metadata): """ Adds scaled, decompressed, parameters from `messages` to `params`. This modifies `params`. """ sign_size = metadata signs = bit2byte.unpacking(messages[0], sign_size) norms = messages[1] shapes = [p.shape for p in params] for param, signs, norm in zip(params, unpack(signs, shapes), norms): param[:].add_(scalar_weight * norm / signs.nelement(), signs)
def collect_episodes(self, num_episodes: int = 100, deterministic: Optional[Dict[str, bool]] = None, disable_tqdm: bool = True, train_mode: bool = True, gamma: float = 0.99, tau: float = 0.95) -> DataBatch: if deterministic is None: deterministic = defaultdict(lambda: False) self.reset() for episode in trange(num_episodes, disable=disable_tqdm): obs, _, _, _ = unpack(self.env.reset(train_mode=train_mode)[self.brain_name], self.agent_ids) done_ = False while not done_: # Compute the action for each agent action_info = { # action, logprob, entropy agent_id: self.agents[agent_id].compute_single_action(obs[agent_id], deterministic[agent_id]) for agent_id in self.agent_ids } action = {agent_id: action_info[agent_id][0] for agent_id in self.agent_ids} logprob = {agent_id: action_info[agent_id][1] for agent_id in self.agent_ids} entropy = {agent_id: action_info[agent_id][2] for agent_id in self.agent_ids} value = {agent_id: action_info[agent_id][3] for agent_id in self.agent_ids} # Actual step in the environment next_obs, reward, done, info = unpack(self.env.step(pack(action))[self.brain_name], self.agent_ids) # Saving to memory self.memory.store(obs, action, reward, logprob, entropy, value, done) obs = next_obs done_ = done['Agent0'] return self.memory.get_torch_data(gamma=gamma, tau=tau)
def step(self, params, state): bits_sent = state.bits_sent messages_sent = state.messages_sent # Send our values to the neighbors buffer, shapes = pack(params) torch.distributed.all_reduce(buffer) bits_sent += num_bits(buffer) messages_sent += 1 buffer /= torch.distributed.get_world_size() params = unpack(buffer, shapes) return params, self.State(bits_sent, messages_sent)
def plot_trajectory(goal_object, runs_dir, img_dir, filename, run_id=0): """ :param goal_object :param runs_dir: :param img_dir: :param filename: :param run_id: """ dataset_states = load_dataset(runs_dir) dataset_states = dataset_states[[ 'goal_position', 'goal_angle', 'position', 'initial_position', 'initial_angle' ]] run_states = dataset_states.where(dataset_states.run == run_id, drop=True) init_position = run_states.initial_position[run_id] init_angle = run_states.initial_angle[run_id] goal_position = run_states.goal_position[run_id] goal_angle = run_states.goal_angle[run_id] x_position, y_position = unpack(run_states.position, 'axis') fig, ax = plt.subplots(figsize=(7.8, 4.8), constrained_layout=True) ax.set_xlabel('x axis', fontsize=11) ax.set_ylabel('y axis', fontsize=11) ax.grid() draw_docking_station(ax, goal_object) draw_marxbot(ax, init_position, init_angle, label='initial position') draw_marxbot(ax, goal_position, goal_angle, label='goal position') plt.plot(x_position, y_position, color='black', label='trajectory', linewidth=1) ax.set_ylim(-220, 220) ax.set_xlim(-250, 250) ax.set_aspect('equal') plt.legend() plt.title('Run %d' % run_id, fontsize=14, weight='bold') save_visualisation(filename, img_dir)
def update(self): include('jquery-plugins') request = self.request if 'form.upload' in request: dataField = IPhoto['data'] context = self.context files = [] data = request.form.get('uploadFile', None) if data is not None: if type(data) is list: files.extend(data) else: files.append(data) updated = False for file in files: if not file: continue for file in unpack(file): name = file.filename if name in context and \ not IPhoto.providedBy(context[name]): photo = context[name] field = dataField.bind(photo) field.set(photo, file) transaction.commit() continue elif name in context: del context[name] photo = Photo(name) event.notify(ObjectCreatedEvent(photo)) context[name] = photo field = dataField.bind(photo) field.set(photo, file) transaction.commit() updated = True if updated: IStatusMessage(request).add(_(u'Files have been uploaded.'))
def delete(self, uid): """Remove the edge with the given identifier `uid`""" # lookup the uid self._cursor.set_key(uid) if self._cursor.search() == WT_NOT_FOUND: raise KeyError('Edge not found, identifier: %s' % uid) else: # remove primary row _, _, _, data = self._cursor.get_value() properties = unpack(data) self._cursor.remove() # remove indices if any for key in properties.keys(): if key in self._indices: self._keys.set_key(key, properties[key], uid) self._cursor.remove()
def plot_positions_heatmap(goal_object, runs_dir, img_dir, filename): """ :param goal_object: :param runs_dir: :param img_dir: :param filename: """ dataset_states = load_dataset(runs_dir) x, y = unpack(dataset_states.position, 'axis') n_bins = 100 grid_x, bins_x = pd.cut(x.data, n_bins, retbins=True) grid_y, bins_y = pd.cut(y.data, n_bins, retbins=True) grid = np.stack([grid_y.codes, grid_x.codes]) unique, counts = np.unique(grid, axis=-1, return_counts=True) mesh = np.zeros([n_bins, n_bins]) mesh[unique[0], unique[1]] = counts plt.figure() cmap = plt.get_cmap('viridis') cmap.set_over('w') plt.pcolormesh(bins_x, bins_y, mesh, cmap=cmap, norm=colors.PowerNorm(0.5), vmax=200) cbar = plt.colorbar() cbar.set_label('samples per grid cell (clipped)', labelpad=15) plt.axis('image') plt.xlabel('x axis', fontsize=11) plt.ylabel('y axis', fontsize=11) draw_docking_station(plt.gca(), goal_object) save_visualisation(filename, img_dir)
def generator_read_next_data_block_int8(self, chan=-1, nchan=1): """ Read the next block of data and its header Returns: (header, data) header (dict): dictionary of header metadata data (np.array): Numpy array of data, converted into to complex64. """ header, head_idx = self.read_header() #print(head_idx) n_chan = int(header['OBSNCHAN']) n_pol = int(header['NPOL']) #should be 4 if n_pol == 2: n_pol = 4 n_bit = int(header['NBITS']) blocsize = int(header['BLOCSIZE']) if chan > 0: data_idx = head_idx + chan * n_pol * int(n_bit / 8) blocsize //= (n_chan // nchan) n_chan = nchan else: data_idx = head_idx self.file_obj.seek(data_idx, 0) n_samples = int(blocsize / (n_chan * n_pol * (float(n_bit) / 8))) d = np.fromfile(self.file_obj, count=blocsize, dtype='int8') # Handle 2-bit and 4-bit data if n_bit != 8: d = unpack(d, n_bit) d = d.reshape((n_chan, n_samples, n_pol)) # Real, imag if self._d_x.shape != d[..., 0:2].shape: self._d_x = np.ascontiguousarray( np.zeros(d[..., 0:2].shape, dtype='int8')) self._d_y = np.ascontiguousarray( np.zeros(d[..., 2:4].shape, dtype='int8')) self._d_x[:] = d[..., 0:2] self._d_y[:] = d[..., 2:4] if chan > 0: data_idx = head_idx + header['BLOCSIZE'] self.file_obj.seek(data_idx, 0) return header, self._d_x, self._d_y
def parse(self, data): self.src_port = unpack("!H", data[:2]) self.dst_port = unpack("!H", data[2:4]) self.seq_num = unpack("!I", data[4:8]) self.ack_num = unpack("!I", data[8:12]) self.headlen = (ord(data[12]) >> 4) * 4 # head len is counted as 4 byte words self.flags = ((ord(data[12]) & 1) << 8) + (ord(data[13])) self.window = unpack("!H", data[14:16]) self.urgent = unpack("!H", data[18:20]) if self.headlen>20: # have option self.parse_option(data[20:self.headlen])
def post_userCon_setupFirstUser(): jdata = bu.get_jdata(ensure="email fname lname pw") email, fname, lname, pw = utils.unpack(jdata, "email fname lname pw") print(email, fname, lname, pw) userCount = userMod.getUserCount() if userCount >= 1: return bu.abort( "Setup already completed. Please visite /login to log in.") user = userMod.buildUser( email=email, fname=fname, lname=lname, pw=pw, isRootAdmin=True, isVerified=True, accessLevel=auth.alm.getMaxLevel(), ) userMod.insertUser(user) return auth.sendAuthSuccessResponse(user)
def plot_initial_positions(goal_object, runs_dir, img_dir, filename): """ :param goal_object :param runs_dir: :param img_dir: :param filename: """ dataset_states, splits = load_dataset(runs_dir, load_splits=True) step_states = dataset_states.where(dataset_states.step == 0, drop=True) plt.figure(figsize=(7.8, 4.8), constrained_layout=True) radius = 8.5 for i, name in enumerate(splits.split_names): split_states = step_states.where(splits == i) x, y = unpack(split_states.initial_position, 'axis') plt.plot(x, y, 'o', label=name, alpha=0.1, markersize=(radius * np.pi) / 2, markeredgecolor='none') ax = plt.gca() draw_docking_station(ax, goal_object) goal_position = step_states.goal_position[0] goal_angle = step_states.goal_angle[0] draw_marxbot(ax, goal_position, goal_angle, label='goal position') ax.set_ylim(-220, 220) ax.set_xlim(-250, 250) ax.set_aspect('equal') plt.legend() plt.xlabel('x axis', fontsize=11) plt.ylabel('y axis', fontsize=11) save_visualisation(filename, img_dir)
def recv_packet(self, payme=False, timeout=None): buff = [] packet_len = -1 t = time.time() while len(buff) != packet_len: if timeout and time.time() >= t + timeout: print('Timeout') return None #print(packet_len, print(len(buff))) c = self.ser.read() c = ord(c) if not buff: if c in self.packet_starts: buff.append(c) elif len(buff) == 1: buff.append(c) packet_len = 4 + (buff[0] & 0x07) + buff[1] else: buff.append(c) payload = bytes(buff[4:]) if buff[0] == 0x80: try: c, attr, typ = unpack('BHB', payload[:4]) if (attr in self.emg_char) and not self.terminate: #==0x27: if self.emg_count == 0: self.start_time = time.time() self.emg_count += 1 vals = tuple(int(b) - (b // 128) * 256 for b in buff[9:]) #unpack('8HB', payload[5:] self.on_emg_data(vals) #self.on_emg_data(vals[8:]) except Exception: pass if time.time() >= (t + time.time()): print('Timeout.') return None elif payme: return buff[:4] + [payload]
def step(self, params, state): bits_sent, messages_sent = state # Pack all parameters in one flat vector `p` to speed up the computation p, original_shapes = pack(params) seed = self.rng.randint(1_000_000_000) noise = torch.rand_like(p) q = self.stochastic_rounding(self.modulo(p / self.Btheta, 1) + 0.5, noise) - 0.5 xhat = q * self.Btheta - self.modulo(p, self.Btheta) # left out + x_{k_i} # Send compressed messages to the neighbors my_rank = torch.distributed.get_rank() send_handles = [] for neighbor_rank in self.topology.neighbor_ranks(my_rank): handle = isend(q, neighbor_rank) bits_sent += q.nelement() * 2 messages_sent += 1 send_handles.append(handle) # Receive messages and update the parameter p recv_buffer = torch.empty_like(q) for neighbor_rank in self.topology.neighbor_ranks(my_rank): recv(recv_buffer, neighbor_rank) weight = self.topology.weight(my_rank, neighbor_rank) p.add_( self.diffusion_rate * weight, self.modulo(recv_buffer * self.Btheta - p, self.Btheta) - xhat, ) # Make sure all sends are finished for handle in send_handles: handle.wait() # Unpack the parameters back into a list form params = unpack(p, original_shapes) return params, self.State(bits_sent, messages_sent)
def fromUnicode(self, u): uri, title, description = utils.unpack(u) m = utils.uri.match(uri) if m is not None and m.group('protocol'): return ExternalLink(uri, title, description) else: # resolve object path = str(uri) try: portal = getSite() context = getattr(self.context, '__parent__', portal) if path.startswith('/'): item = portal.restrictedTraverse(path[1:]) else: item = context.restrictedTraverse(path) reference = item.UID() except KeyError: reference = None return InternalLink(reference, title, description)
def parse(self, incoming_data): self.src_port = unpack("!H", incoming_data[:2]) self.dst_port = unpack("!H", incoming_data[2:4]) self.headlen = unpack("!H", incoming_data[4:6])
def parse(self, incoming_data): self.dst_mac = incoming_data[:6] self.src_mac = incoming_data[6:12] self.protocol = unpack("!H", incoming_data[12:14]) self.headlen = 14
def fromtype(self, stuff): if(G.SEEINTERNAL): #print "The type sees: " + repr(stuff) #print "Function sees: " + repr(unpack(stuff)) pass return unpack(stuff)
install, and also for correct apt-get invocation to install them all on for example Debian / Ubuntu. """ return if not posix_test_cc(): utils.output('c++ compiler not found.') return utils.goto_build() tbfn = os.path.join(config.archive_dir, python_fname) pybasename = 'Python-%s' % (PYVER_STR,) build_dir = os.path.join(config.build_dir, pybasename) if not os.path.exists(build_dir): utils.unpack(tbfn) os.chdir(build_dir) ret = os.system( './configure --enable-shared --prefix=%s/python' % (config.inst_dir,)) if ret != 0: utils.error('Python configure error.') # config.MAKE contains -j setting # I've had this break with Python 2.6.2, so I'm using straight make here... ret = os.system('%s install' % ('make',)) if ret != 0: utils.error('Python build error.')
def read_chunk(host, port, chunkid, version, size, offset=0): if offset + size > CHUNKSIZE: raise ValueError("size too large %s > %s" % (size, CHUNKSIZE-offset)) from dpark.accumulator import RemoteReadBytes conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) conn.settimeout(10) conn.connect((host, port)) msg = pack(CUTOCS_READ, uint64(chunkid), version, offset, size) n = conn.send(msg) while n < len(msg): if not n: raise IOError("write failed") msg = msg[n:] n = conn.send(msg) def recv(n): d = conn.recv(n) while len(d) < n: nd = conn.recv(n-len(d)) if not nd: raise IOError("not enough data") d += nd return d while size > 0: cmd, l = unpack("II", recv(8)) if cmd == CSTOCU_READ_STATUS: if l != 9: raise Exception("readblock: READ_STATUS incorrect message size") cid, code = unpack("QB", recv(l)) if cid != chunkid: raise Exception("readblock; READ_STATUS incorrect chunkid") conn.close() return elif cmd == CSTOCU_READ_DATA: if l < 20 : raise Exception("readblock; READ_DATA incorrect message size") cid, bid, boff, bsize, crc = unpack("QHHII", recv(20)) if cid != chunkid: raise Exception("readblock; READ_STATUS incorrect chunkid") if l != 20 + bsize: raise Exception("readblock; READ_DATA incorrect message size ") if bsize == 0 : # FIXME raise Exception("readblock; empty block") #yield "" #continue if bid != offset >> 16: raise Exception("readblock; READ_DATA incorrect block number") if boff != offset & 0xFFFF: raise Exception("readblock; READ_DATA incorrect block offset") breq = 65536 - boff if size < breq: breq = size if bsize != breq: raise Exception("readblock; READ_DATA incorrect block size") while breq > 0: data = conn.recv(breq) if not data: #print chunkid, version, offset, size, bsize, breq raise IOError("unexpected ending: need %d" % breq) RemoteReadBytes.add(len(data)) yield data breq -= len(data) offset += bsize size -= bsize else: raise Exception("readblock; unknown message: %s" % cmd) conn.close()
def fromtype(self, stuff): result = unpack(stuff[1:-1:2]) if(stuff[0]=="\x02"): result *= -1 return result
def get(self, key): self._cursor.set_key(key) if self._cursor.search() == WT_NOT_FOUND: raise KeyError(key) return unpack(self._cursor.get_value())
def real_time_modify(): tscale = 1.0 p = pyaudio.PyAudio() in_stream = p.open(format=pyaudio.paInt16, channels=2, rate=44100, input = True) out_stream = p.open(format=pyaudio.paInt16, channels=2, rate=int(44100*tscale), output=True) chunk = 1024 data = in_stream.read(chunk) data = utils.unpack(data) print 'data', len(data) next_data = in_stream.read(chunk) next_data = utils.unpack(next_data) amp = 0 out_data = np.concatenate(([], zeros(chunk))) L = len(data) N = L/2 H = N/2 phi = zeros(N) out = zeros(N, dtype=complex) win = hanning(N) p = 0 pp = 0 for i in range(0,300): concat = np.concatenate(((data, next_data))) amp = max(amp, max(concat)) out_data = np.concatenate((out_data, zeros(N/tscale))) p = 0 for i in range(2): # take the spectra of two consecutive windows p1 = int(p) spec1 = fft(win*concat[p1:p1+N]) spec2 = fft(win*concat[p1+H:p1+N+H]) # take their phase difference and integrate phi += (angle(spec2) - angle(spec1)) out.real, out.imag = cos(phi), sin(phi) # inverse FFT and overlap-add print 'pp:pp+N', pp, pp+N out_data[pp:pp+N] += win*ifft(abs(spec2)*out) pp += H p += H*tscale out_data = amp*out_data/max(out_data) out_formatted = utils.pack(out_data[i*chunk:(i+1)*chunk]) out_formatted = ''.join(out_formatted) out_stream.write(out_formatted) data = next_data next_data = in_stream.read(chunk) next_data = utils.unpack(next_data) in_stream.stop_stream() in_stream.close() out_stream.stop_stream() out_stream.close() # wav.write('test.wav',44100*2, array(data_to_file, dtype='int16')) p.terminate()