def disconnect(self, h=None): if not h is None: print('h:', h) return self.send_command(3, 0, pack('B', h)) elif not self.conn is None: self.write_attr(0x19, b'\x09\x01\x00') return self.send_command(3, 0, pack('B', self.conn))
def build_rowpanel(self, parent): "build right hand panel" panel = wx.Panel(parent) sizer = wx.GridBagSizer(5, 2) row = -1 for attr, dtype, extra in self.attrs: row += 1 label = None if dtype is None: wid = wx.TextCtrl(panel, value="", size=(TWID, -1)) elif dtype == 'float': prec, xmin, xmax = extra wid = FloatCtrl(panel, value="", precision=prec, minval=xmin, maxval=xmax, size=(TWID, -1)) elif dtype == 'multi': height = extra wid = wx.TextCtrl(panel, value="", size=(TWID, height), style=wx.TE_MULTILINE) elif dtype == 'foreign': choiceclass, manager, llabel = extra _wid = HyperChoice(panel, choiceclass, label='%s:' % llabel.title(), manager=manager, db=self.db) wid = _wid.choice label = _wid.link setattr(self, attr, wid) if label is None: label = wx.StaticText(panel, label="%s:" % attr.title()) sizer.Add(label, (row, 0), (1, 1), wx.ALIGN_LEFT|wx.ALL, 2) sizer.Add(wid, (row, 1), (1, 1), wx.ALIGN_LEFT|wx.ALL, 2) pack(panel, sizer) return panel
def munge_jag_results(jag_run): """ Return a properly formatted result to be serialized. """ result = {} for out in jag_run.data: result[out] = jag_run.data[out] try: pack(result[out]) except BaseException: print("WARNING: Un-Pickleable Data: " + str(result[out])) # Conduit doesn't like unicode, thus the excessive ensurance of use # of str type instead. result[out] = str(str('Un-Pickleable!: ') + str(result[out])) if out == 'outputs': if 'images' in result['outputs']: result['outputs']['images'] = format_jag_images( result['outputs']['images']) if 'scalars' in result['outputs']: result['outputs']['scalars'] = format_jag_scalars( result['outputs']['scalars']) if 'timeseries' in result['outputs']: result['outputs']['timeseries'] = format_jag_timeseries( result['outputs']['timeseries']) return result
def build_datapanel(self): self.data_panel.SetBackgroundColour(wx.Colour(250, 250, 250)) dpan = self.data_panel dlabel = wx.StaticText(dpan, label='No Data Selected') sizer = wx.BoxSizer(wx.VERTICAL) sizer.Add(dlabel, 0, wx.ALIGN_LEFT|wx.ALIGN_TOP, 5) pack(dpan, sizer)
def build_datapanel(self): self.data_panel.SetBackgroundColour(wx.Colour(250, 250, 250)) dpan = self.data_panel dlabel = wx.StaticText(dpan, label='No Data Selected') sizer = wx.BoxSizer(wx.VERTICAL) sizer.Add(dlabel, 0, wx.ALIGN_LEFT | wx.ALIGN_TOP, 5) pack(dpan, sizer)
def request_blind_sign(): (_, pub) = elgamal_keygen(params) (cm, c, proof_s) = prepare_blind_sign(params, ATTRIBUTE, pub) json = { "cm": pack(cm), "c": pack(c), "proof_s": pack(proof_s), "pub": pack(pub) } async_request(ROUTE_SIGN_PRIVATE, json)
def load_packets(filename, packet_size): packets = [pack(0, 0, 0, 0, 0, 0, b'')] f = open(filename, 'rb') seqNum = 1 while True: rawPacket = f.read(packet_size) if rawPacket: packets.append(pack(len(rawPacket), seqNum, seqNum, 0, 0, 0, rawPacket)) else: packets.append(pack(0, 0, 0, 1, 0, 0, b'')) break seqNum += 1 return packets
def pack(self, payload, peek=None): s1 = chr((self.version << 4) + (self.headlen / 4)) s2 = chr(0) # different service field s3 = pack("!H", 20 + len(payload)) s4 = pack("!H", self.identification) s5 = chr((self.flags << 5) + (self.segment_id >> 8)) s6 = chr(self.segment_id & 0xff) s7 = chr(self.TTL) s8 = chr(self.protocol) s10 = "".join([chr(x) for x in self.src_ip]) s11 = "".join([chr(x) for x in self.dst_ip]) s9 = pack("!H", checksum(s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s10 + s11)) return s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10 + s11 + payload
def compress(self, params): """Create a flat-packed message that can be sent to another worker""" values = [] indices = [] for tensor in params: sample_size = self.num_entries_for_param(tensor) positions = np.random.choice(tensor.nelement(), sample_size, replace=False) indices.append(torch.from_numpy(positions)) values.append(tensor.view(-1)[positions].contiguous()) # Pack compressed parameters together to reduce the number of communications value_buffer, shapes = pack(values) indices_buffer, _ = pack(indices) return [value_buffer, indices_buffer], shapes
def update(self, uid, properties): # lookup edge self._cursor.set_key(uid) if self._cursor.search() == WT_NOT_FOUND: raise KeyError('Edge not found, identifier: %s' % uid) else: # update properties start, label, end, data = self._cursor.get_value() other = unpack(data) # remove old indices if any for key in other.keys(): if key in self._indices: self._keys.set_key(key, other[key], uid) self._cursor.remove() # update data = pack(properties) self._cursor.set_value(start, label, end, data) # index properties for key in properties.keys(): if key in self._indices: self._keys.set_key(key, properties[key], uid) self._keys.set_value('') self._keys.insert()
def train_discriminator(self, real_data, current_batch_size, real_label, fake_label): # Generate with noise latent_noise = torch.randn(current_batch_size, self.latent_input, 1, 1, device=self.device) generated_batch = self.generator(latent_noise) fake_data = utils.pack(generated_batch, self.packing) ### Train discriminator self.discriminator.zero_grad() # Train on real data real_prediction = self.discriminator(real_data).squeeze() loss_discriminator_real = self.discriminator.loss( real_prediction, real_label) #loss_discriminator_real.backward() # Train on fake data fake_prediction = self.discriminator(fake_data.detach()).squeeze() loss_discriminator_fake = self.discriminator.loss( fake_prediction, fake_label) #loss_discriminator_fake.backward() # Add losses loss_discriminator_total = loss_discriminator_real + loss_discriminator_fake loss_discriminator_total.backward() self.D_optimiser.step() return loss_discriminator_total
def step(e=None): _s_t_batch = env.get_state()[None, None, :, :, :] _a_t_batch = [[env.get_act()]] _state_in_batch = [env.get_state_in()] _a_t_new, _a_t_logits, _v_cur, _state_out_batch = sess.run( [model.get_current_act(), model.get_current_act_logits(), model.current_value, model.state_out], feed_dict={model.s_t: _s_t_batch, model.previous_actions: _a_t_batch, model.state_in: _state_in_batch}) # _a_t_new = np.argmax(_a_t_logits, axis=-1) if e is not None: _a_t_new[0][0] = e env.step( _a_t_new[0][0], _a_t_logits[0][0], _state_out_batch[0] ) env.update_v(_v_cur[0][0]) seg = env.get_history() if seg is not None: if not os.path.exists("Demos"): os.mkdir("Demos") dicseg = {k: v for k, v in seg._asdict().items()} with pa.OSFile(os.path.join("Demos", game + "_%d.demo" % count), "wb") as f: f.write(pack(dicseg)) env.reset()
def compress(self, params): """Create a flat-packed message that can be sent to another worker""" us = [] vs = [] for param in params: matrix = param.view(param.shape[0], -1) m, n = matrix.shape rnk = min(self.rank, m, n) if not matrix.is_cuda: try: u, s, vT = np.linalg.svd(matrix) v = vT.T u, s, v = u[:, :rnk], s[:rnk], v[:, :rnk] except np.linalg.LinAlgError as e: print("SVD failed ... using zeros") u, s, v = ( np.zeros([matrix.shape[0], rnk], dtype=np.float32), np.zeros([rnk], dtype=np.float32), np.zeros([matrix.shape[1], rnk], dtype=np.float32), ) u = torch.from_numpy(u) s = torch.from_numpy(s) v = torch.from_numpy(v) else: u, s, v = torch.svd(matrix) u, s, v = u[:, :rnk], s[:rnk], v[:, :rnk] us.append((s * u).contiguous()) vs.append(v.contiguous()) buffer, shapes = pack(us + vs) return [buffer], shapes
def set_key(): for i in range(N): r = requests.post( "http://"+SERVER_ADDR[i]+":"+str(SERVER_PORT[i])+ROUTE_KEY_SET, data = dumps({"sk": pack(sk[i])}) ) assert loads(r.text)["status"] == "OK"
def step(self, params, state): bits_sent = state.bits_sent messages_sent = state.messages_sent my_rank = torch.distributed.get_rank() # Send our values to the neighbors buffer, shapes = pack(params) send_request_handles = [] for neighbor_rank in self.topology.neighbor_ranks(my_rank): handle = isend(buffer, neighbor_rank) bits_sent += num_bits(buffer) messages_sent += 1 send_request_handles.append(handle) # Average with the neighbors own_weight = self.topology.weight(my_rank, my_rank) own_weight = 1.0 - (1.0 - own_weight) * self.diffusion_rate params = [param.data * own_weight for param in params] buffer = torch.empty_like(buffer) for neighbor_rank in self.topology.neighbor_ranks(my_rank): weight = self.topology.weight(my_rank, neighbor_rank) recv(buffer, neighbor_rank) for param, neighbor_param in zip(params, unpack(buffer, shapes)): param.data.add_(weight * self.diffusion_rate, neighbor_param) # Make sure all send requests finished for handle in send_request_handles: handle.wait() return params, self.State(bits_sent, messages_sent)
def blind_sign_wrapper(data): cm = unpack(data["cm"]) c = unpack(data["c"]) proof_s = unpack(data["proof_s"]) pub = unpack(data["pub"]) blind_sig = blind_sign(params, app.sk, cm, c, pub, proof_s) return format(pack(blind_sig))
def send_command(self, cls, cmd, payload=b'', wait_resp=True): s = pack('4B', 0, len(payload), cls, cmd) + payload self.ser.write(s) while True: packet = self.recv_packet(payme=True) if packet[0] == 0: return packet
def lui(rd, imm): rd = check_and_trans_reg(rd) imm = check_and_trans_imm(imm, 20) return pack([ (0b0110111, 7), (rd, 5), (imm, 20) ])
def __unicode__(self): brain = self._brain() if brain is None: return u"" path = "/" + brain.getPath().lstrip('/') return utils.pack(path, self.title, self.description)
def __init__(self, in_size, mid_size, out_size, bn=True, mode='nearest'): super().__init__() self.block = nn.Sequential(*pack([ Interpolate(scale_factor=2, mode=mode), ConvRelu(in_size, mid_size), nn.BatchNorm2d(mid_size) if bn else None, ConvRelu(mid_size, out_size), nn.BatchNorm2d(out_size) if bn else None]))
def auipc(rd, imm): rd = check_and_trans_reg(rd) v = check_and_trans_imm(imm, 20) return pack([ (0b0010111, 7), (rd, 5), (v, 20) ])
def totype(self, num): num = int(num) assert num >= 0 result = pack(num) len_result = len(result) assert len_result <= self.size result = "\x00" * (self.size - len_result) + result return result
def __init__(self, in_size, mid_size, out_size, kernel_size=4, bn=True): super().__init__() self.block = nn.Sequential(*pack([ ConvRelu(in_size, mid_size), nn.BatchNorm2d(mid_size) if bn else None, nn.ConvTranspose2d(mid_size, out_size, stride=2, kernel_size=kernel_size, padding=kernel_size//2-1), nn.ReLU(inplace=True), nn.BatchNorm2d(out_size) if bn else None, ]))
def jal(rd, imm): rd = check_and_trans_reg(rd) imm = check_and_trans_imm(imm, 20) # check_alignment(imm, 2) return pack([ (0b1101111, 7), (rd, 5), (bit_reorder(imm, [19, *range(9, -1, -1), 10, *range(18, 10, -1)]), 20) ])
def totype(self, num): num = int(num) result = '\x01'.join(i for i in pack(num)) if num >= 0: header = "\x01" else: header = "\x02" result = header + result + "\x00" return result
def init_state(self, params, seed=0): my_rank = torch.distributed.get_rank() ps = {} qs = {} rngs = {} for neighbor in self.topology.neighbor_ranks(my_rank): rngs[neighbor] = self._rng_for_neighbor(seed, neighbor) # Ensure that the p's and q's are consequtive in memory so we can quickly send them p_buffer, shapes = pack([self._init_p(param) for param in params]) self.fill_with_random_values(p_buffer, rngs[neighbor]) ps[neighbor] = {"list": unpack(p_buffer, shapes), "buffer": p_buffer} q_buffer, shapes = pack([self._init_q(param) for param in params]) self.fill_with_random_values(q_buffer, rngs[neighbor]) qs[neighbor] = {"list": unpack(q_buffer, shapes), "buffer": q_buffer} return self.State(ps=ps, qs=qs, iteration_number=0, bits_sent=0, messages_sent=0, rngs=rngs)
def compress(self, params): """Create a flat-packed message that can be sent to another worker""" norms = torch.tensor([p.norm(p=1) for p in params], device=params[0].device) flat, _ = pack(params) bits, sign_size = bit2byte.packing(flat) return [bits, norms], sign_size
def train(self, nb_epoch = NB_EPOCH): print("Start training.") for epoch in range(nb_epoch): print("Epoch : " + str(epoch)) g_loss = [] d_loss = [] for batch_id, (x, target) in enumerate(self.train_loader): real_batch_data = x.to(self.device) current_batch_size = x.shape[0] one_hot_label = torch.zeros(current_batch_size, self.num_labels).scatter_(1, target.unsqueeze(1), 1).to(self.device) packed_real_data = utils.pack(real_batch_data, self.packing) packed_batch_size = packed_real_data.shape[0] # labels label_real = torch.full((packed_batch_size,), 1, device=self.device) label_fake = torch.full((packed_batch_size,), 0, device=self.device) # smoothed real labels between 0.7 and 1, and fake between 0 and 0.3 label_real_smooth = torch.rand((packed_batch_size,)).to(self.device) * 0.3 + 0.7 label_fake_smooth = torch.rand((packed_batch_size,)).to(self.device) * 0.3 temp_discriminator_loss = [] temp_generator_loss = [] ### Train discriminator multiple times for i in range(self.nb_discriminator_step): loss_discriminator_total = self.train_discriminator(packed_real_data, current_batch_size, one_hot_label, label_real_smooth if self.real_label_smoothing else label_real, label_fake_smooth if self.fake_label_smoothing else label_fake) temp_discriminator_loss.append(loss_discriminator_total.item()) ### Train generator for i in range(self.nb_generator_step): loss_generator_total = self.train_generator(current_batch_size, one_hot_label, label_real) temp_generator_loss.append(loss_generator_total.item()) ### Keep track of losses d_loss.append(torch.mean(torch.tensor(temp_discriminator_loss))) g_loss.append(torch.mean(torch.tensor(temp_generator_loss))) self.discriminator_losses.append(torch.mean(torch.tensor(d_loss))) self.generator_losses.append(torch.mean(torch.tensor(g_loss))) utils.save_images(self.generator(self.saved_latent_input, self.saved_label), self.save_path + "gen_", self.image_size, self.image_channels, self.nb_image_to_gen, epoch) utils.write_loss_plot(self.generator_losses, "G loss", self.save_path, clear_plot=False) utils.write_loss_plot(self.discriminator_losses, "D loss", self.save_path, clear_plot=True) print("Training finished.")
def surface(symbol, qdate, qtime=None, errors=False, n=False, loc=0, scale=0): df = query.vols(["*"], ["Date", "Time", "Tenor", "Strike"], symbol, qdate, qtime) st = utils.pack(df) del df #---------------------------------------------------------------------------------- f = lambda LST: pricer.norm_weights(LST, loc, scale) wgt = utils.apply(f, 1, st, ["Time", "Tenor"], ["LogStrike"]) #retrive weights vector sv = pricer.SSVI(errors) f = lambda LST, VAR, TNR, VOL: sv.calibrate(LST, VAR, TNR, VOL, wgt) if errors == True: prm = utils.apply(f, 6, st, ["Time"], ["LogStrike", "TotAtmfVar", "Tenor", "SmtVol"], diff=True, fill=False) eps = prm[-1] prm = np.asarray(prm[:-1]).T else: prm = utils.apply(f, 5, st, ["Time"], ["LogStrike", "TotAtmfVar", "Tenor", "SmtVol"], fill=False) #---------------------------------------------------------------------------------- reduced = [ "Time", "Group", "Tenor", "SpotMid", "Forward", "CumDivDays", "Div", "ImpBor", "Rate", "CallH0", "CallH1", "CallKh", "PutH0", "PutH1", "PutKh", "TotAtmfVar" ] st = utils.unique_nosort(st[reduced]) prm = utils.restack(prm, st["TotAtmfVar"], tile=False) rho = sv.correlation(st["TotAtmfVar"], prm[:, 0], prm[:, 1], prm[:, 2]) phi = sv.power_law(st["TotAtmfVar"], prm[:, 3], prm[:, 4]) atm, skw, krt = sv.raw2jw(st["Tenor"] / base, st["TotAtmfVar"], phi, rho) arr = unstruct(st) arr = np.column_stack([arr, atm, skw, krt, phi, rho]) df = utils.unpack(arr, qdate, raw=False, symbol=symbol) if errors == True and n == False: return df, eps elif errors == False and n == True: return df, n elif errors == True and n == True: return df, eps, n else: return df
def pack_alu(opcode, rd, funct3, rs1, rs2, funct7): l = [ (opcode, 7), (rd, 5), (funct3, 3), (rs1, 5), (rs2, 5), (funct7, 7) ] return pack(l)
def main(): comm = MPI.COMM_WORLD intra_rank = utils.get_intra_rank(comm) chainer.cuda.get_device(intra_rank).use() mpi_print = common.create_mpi_print(comm) nelems_list = [2, 4, 8, 16, 32, 64, 128, 256] nelems_max = nelems_list[-1] * pow(2, 20) sendarr = cupy.random.rand(nelems_max, dtype=cupy.float32) recvarr = cupy.zeros((nelems_max, ), dtype=cupy.float32) if comm.rank == 0: print('array initialized...') sendbuf_gpu = _memory_utility.DeviceMemory() sendbuf_gpu.assign(nelems_max * 4) recvbuf_gpu = _memory_utility.DeviceMemory() recvbuf_gpu.assign(nelems_max * 4) if comm.rank == 0: print('GPU buffer initialized...') utils.pack([sendarr], sendbuf_gpu) if comm.rank == 0: print('packed...') for nelems in nelems_list: nelems *= pow(2, 20) sendbuf = [sendbuf_gpu.buffer(nelems * 4), MPI.FLOAT] recvbuf = [recvbuf_gpu.buffer(nelems * 4), MPI.FLOAT ] if comm.rank == 0 else None if comm.rank == 0: s_time = time.time() # WE MUST SYNC BEFORE COMMUNICATION !!! chainer.cuda.Stream.null.synchronize() comm.Reduce(sendbuf, recvbuf, root=0) if comm.rank == 0: print('COUNT {} MiBytes, TIME {} sec'.format( (nelems * 4) / pow(2, 20), time.time() - s_time))
def jalr(rd, rs, imm): rd = check_and_trans_reg(rd) rs = check_and_trans_reg(rs) v = check_and_trans_imm(imm, 12) return pack([ (0b1100111, 7), (rd, 5), (0, 3), (rs, 5), (v, 12), ])
def flw(rd, rs1, imm): imm = check_and_trans_imm(imm, 12) rd = check_and_trans_reg(rd) rs1 = check_and_trans_reg(rs1) return pack([ (0b0000111, 7), (rd, 5), (0b010, 3), (rs1, 5), (imm, 12) ])
def alui(rd, funct3, rs1, imm): imm = check_and_trans_imm(imm, 12) rd = check_and_trans_reg(rd) rs1 = check_and_trans_reg(rs1) return pack([ (0b0010011, 7), (rd, 5), (funct3, 3), (rs1, 5), (imm, 12) ])
def __init__(self, parent): wx.Panel.__init__(self, parent, -1) labstyle = wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.ALL|wx.GROW toprow = wx.Panel(self) self.label = wx.StaticText(toprow, label='Spectra:') self.element = wx.StaticText(toprow, label='Element:') self.edge = wx.StaticText(toprow, label='Edge: ') tsizer = wx.BoxSizer(wx.HORIZONTAL) tsizer.Add(self.label, 1, labstyle, 4) tsizer.Add(self.element, 0, labstyle, 4) tsizer.Add(self.edge, 0, labstyle, 4) pack(toprow, tsizer) nb = self.notebook = wx.Notebook(self) self.data_panel = wx.Panel(nb) self.sample_panel = wx.Panel(nb) self.beamline_panel = wx.Panel(nb) self.info_panel = wx.Panel(nb) self.data_panel.SetBackgroundColour(wx.Colour(255, 255, 250)) self.info_panel.SetBackgroundColour(wx.Colour(215, 215, 250)) self.sample_panel.SetBackgroundColour(wx.Colour(255, 235, 250)) nb.AddPage(self.data_panel, 'Data') nb.AddPage(self.sample_panel, 'Sample') nb.AddPage(self.beamline_panel, 'Beamline/Mono') nb.AddPage(self.info_panel, 'Info') nb.Bind(wx.EVT_NOTEBOOK_PAGE_CHANGED, self.onPageChange) sizer = wx.BoxSizer(wx.VERTICAL) sizer.Add(toprow, 0, wx.ALIGN_LEFT|wx.ALIGN_TOP, 5) sizer.Add(nb, 1, wx.ALIGN_LEFT|wx.ALIGN_TOP|wx.GROW, 5) pack(self, sizer) self.build_datapanel()
def capture_mic(): tscale = 1.0 p = pyaudio.PyAudio() in_stream = p.open(format=pyaudio.paInt16, channels=2, rate=44100, input = True) out_stream = p.open(format=pyaudio.paInt16, channels=2, rate=int(44100*tscale), output=True) data_to_file = [] chunk = 512 prev_data = [] data = [] after_data = in_stream.read(chunk) for i in range(300): after_data = utils.unpack(after_data) # after_data = reduce_noise(after_data) if prev_data != []: concat = np.concatenate( (( np.concatenate(((prev_data,data))),after_data )) ) shifted_data = shift_pv(concat, tscale) # data_to_file.extend(shift_data) data_out = utils.pack(shifted_data) data_out = ''.join(data_out) out_stream.write(data_out) prev_data = data data = after_data after_data = in_stream.read(chunk) in_stream.stop_stream() in_stream.close() out_stream.stop_stream() out_stream.close() wav.write('test.wav',44100*2, array(data_to_file, dtype='int16')) p.terminate()
def buildFrame(self): # Now create the Panel to put the other controls on. splitter = wx.SplitterWindow(self, style=wx.SP_LIVE_UPDATE) splitter.SetMinimumPaneSize(200) self.rowlist = wx.ListBox(splitter) self.rowlist.SetBackgroundColour(wx.Colour(255, 255, 255)) self.rowlist.Bind(wx.EVT_LISTBOX, self.onShow) self.rightpanel = wx.Panel(splitter) self.set_rowlist() rpanel = self.rightpanel rsizer = wx.BoxSizer(wx.VERTICAL) self.toplabel = wx.StaticText(rpanel, label='') rsizer.Add(self.toplabel, 0, wx.ALIGN_CENTRE|wx.ALL, 2) rowpanel = self.build_rowpanel(rpanel) rsizer.Add(rowpanel, 1, wx.ALIGN_CENTRE|wx.ALL, 2) btn_save = add_btn(rpanel, "Save Changes", action=self.onSave) btn_remove = add_btn(rpanel, "Remove This %s" % self.tablename, action=self.onRemove) brow = wx.BoxSizer(wx.HORIZONTAL) brow.Add(btn_save, 0, wx.ALIGN_LEFT|wx.ALL, 10) brow.Add(btn_remove, 0, wx.ALIGN_LEFT|wx.ALL, 10) rsizer.Add(brow, 0, wx.ALIGN_LEFT|wx.ALL, 4) pack(rpanel, rsizer) splitter.SplitVertically(self.rowlist, self.rightpanel, 1) sizer = wx.BoxSizer(wx.VERTICAL) sizer.Add(splitter, 1, wx.GROW|wx.ALL, 5) pack(self, sizer) self.show_row() self.Show() self.Raise()
def add(self, start, label, end, properties=None): """Add edge with and return its unique identifier `start`, and `end` must be respectively vertices identifiers. `label` must be a string and `properties` a dictionary""" # append edge and return identifier properties = properties if properties else dict() self._append.set_value(start, label, end, pack(properties)) self._append.insert() uid = self._append.get_key() # index properties if any for key in properties.keys(): if key in self._indices: self._keys.set_key(key, properties[key], uid) self._keys.set_value('') self._keys.insert() return uid
def add(self, label, properties=None): """Add vertex with the given `label` and `properties` `properties` are packed using msgpack""" properties = properties if properties else dict() # append vertex to the list of vertices # and retrieve its assigned identifier self._append.set_value(label, pack(properties)) self._append.insert() uid = self._append.get_key() # index properties if any for key in properties.keys(): if key in self._indices: self._keys.set_key(key, properties[key], uid) self._keys.set_value('') self._keys.insert() return uid
def pack(self, payload, peek=None): s1 = pack("!H", self.src_port) s2 = pack("!H", self.dst_port) s3 = pack("!I", self.seq_num) s4 = pack("!I", self.ack_num) s6 = chr(self.flags & 0xff) s7 = pack("!H", self.window) s9 = pack("!H", self.urgent) s10 = self.pack_options() self.headlen = len(s1+s2+s3+s4+' '+s6+s7+s9+s10+' ') s5 = chr(((self.headlen /4) <<4) + (self.flags >> 8)) # construct fake IP header: src_ip, dst_ip, reserved(0x00), protocol(tcp is 0x06), length(header+payload) fake_header = "".join([chr(x) for x in peek.src_ip]) + "".join([chr(x) for x in peek.dst_ip]) + "\x00\x06"+pack("!H", self.headlen+len(payload)) csum = checksum(fake_header + s1+s2+s3+s4+s5+s6+s7+s9+s10+payload) s8 = pack("!H", csum) return s1+s2+s3+s4+s5+s6+s7+s8+s9+s10+payload
def pack_options(self): opt_str = [] for option in [x for x in dir(self) if x.startswith("option_")]: option_value = getattr(self, option) if option == 'option_max_segment_size': opt_str.append("\x02\x04"+pack("!H", option_value)) elif option == 'option_window_scale_factor': opt_str.append("\x03\x03"+chr(option_value)) elif option == 'option_SACK_Permit': opt_str.append("\x04\02") elif option == 'option_timestamp': opt_str.append("\x08\x0a"+struct.pack("!II", *self.option_timestamp)) else: print "Warning, unsupported tcp option:", option opt = "".join(opt_str) # import ipdb; ipdb.set_trace() if len(opt) % 4 >0: # we have remaining opt+="\x01" * (4-(len(opt) % 4)) # use NOP to fill, make it 4 bytes aligned return opt
def buildSamplePanel(self): "build Sample panel" # temperature # sample crystal structure data, format # ref sample formula # ref sample material source # ref sample crystal structure data, format panel = self.sample_panel = wx.Panel(self.nb) self.nb.AddPage(panel, 'Sample') sizer = wx.GridBagSizer(10, 6) def stext(label): return wx.StaticText(panel, label=label) # self.sample_formula = wx.TextCtrl(panel, value='', size=(220, -1)) # self.refer_formula = wx.TextCtrl(panel, value='', size=(220, -1)) # self.sample_xtal = add_btn(panel, "Add Crystal Structure Data", # action=self.onSampleXTAL) # self.sample_source = wx.TextCtrl(panel, value='', size=(220, 75), # style=wx.TE_MULTILINE) self.user_notes = wx.TextCtrl(panel, value='', size=(450, 90), style=wx.TE_MULTILINE) irow = 1 sizer.Add(Title(panel, 'User Notes from\nData Collection:'), (0, 0), (1, 1), titlestyle, 1) sizer.Add(self.user_notes, (0, 1), (1, 3), namestyle, 1) sizer.Add(wx.StaticLine(panel, size=(150, 1), style=wx.LI_HORIZONTAL), (1, 0), (1, 5), wx.ALIGN_CENTER|wx.GROW|wx.ALL, 0) _sample = HyperChoice(panel, SampleChoice, label='Sample:', manager=SampleManager, db=self.db) self.sample = _sample.choice _refsample = HyperChoice(panel, SampleChoice, label='Reference Sample:', manager=SampleManager, db=self.db) self.refsample = _refsample.choice sizer.Add(_sample.link, (2, 0), (1, 1), labstyle, 1) sizer.Add(self.sample, (2, 1), (1, 1), namestyle, 1) irow = 3 tpanel = wx.Panel(panel) tsizer = wx.BoxSizer(wx.HORIZONTAL) self.sample_temperature = FloatCtrl(tpanel, precision=3, size=(100, -1), value=300, minval=-500) self.temperature_units = TypeChoice(tpanel, choices=('K','C','F'), size=(50, -1)) self.temperature_units.SetSelection(0) tsizer.Add(self.sample_temperature, 0, wx.ALIGN_CENTER|wx.ALL, 1) tsizer.Add(self.temperature_units, 0, wx.ALIGN_CENTER|wx.ALL, 1) pack(tpanel, tsizer) sizer.Add(stext('Temperature:'), (irow, 0), (1, 1), tlabstyle, 1) sizer.Add(tpanel, (irow, 1), (1, 2), tnamestyle, 1) irow += 1 sizer.Add(wx.StaticLine(panel, size=(150, 1), style=wx.LI_HORIZONTAL), (irow, 0), (1, 5), wx.ALIGN_CENTER|wx.GROW|wx.ALL, 0) irow += 1 sizer.Add(_refsample.link, (irow, 0), (1, 1), labstyle, 1) sizer.Add(self.refsample, (irow, 1), (1, 1), namestyle, 1) pack(panel, sizer)
def create_frame(self): "create top level frame" # Create the menubar menuBar = wx.MenuBar() fmenu = wx.Menu() add_menu(self, fmenu, "Open Library", "Open Existing XAS Data Library", action = self.onReadXDLFile) add_menu(self, fmenu, "New Library", "Create New XAS Data Library", action=self.OpenNewXDLFile) fmenu.AppendSeparator() add_menu(self, fmenu, "Import Spectra", "Read Spectra from ASCII File", action=self.ImportSpectra) add_menu(self, fmenu, "Export Spectra", "Write Spectra to ASCII (XDI) File", action=self.ExportSpectra) fmenu.AppendSeparator() add_menu(self, fmenu, "E&xit\tAlt-X", "Exit this Program", action=self.onClose) omenu = wx.Menu() add_menu(self, omenu, "Suites of Spectra", "Manage Suites of Spectra", action=self.onSuiteManager) add_menu(self, omenu, "Samples", "Manage Samples", action=self.onSampleManager) add_menu(self, omenu, "People", "Manage People Adding to Library", action=self.onPersonManager) add_menu(self, omenu, "Beamlines", "Manage Beamline", action=self.onBeamlineManager) # and put the menu on the menubar menuBar.Append(fmenu, "&File") menuBar.Append(omenu, "&Tables") self.SetMenuBar(menuBar) self.CreateStatusBar(1, wx.CAPTION|wx.THICK_FRAME) # Now create the Main Panel to put the other controls on. # SelectFilter | CurrentFilter | SpectraList # ------------------------------------------ # Spectra details # ------------------------------------------ topsection = wx.Panel(self) self.filterchoice = wx.Choice(topsection, size=(-1,-1), choices = self.filters) self.filterchoice.SetStringSelection(self.current_filter) self.filterchoice.Bind(wx.EVT_CHOICE, self.onFilterChoice) sizer = wx.BoxSizer(wx.HORIZONTAL) sizer.Add(wx.StaticText(topsection, label='Filter Spectra by:'), 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.GROW|wx.ALL, 4) sizer.Add(self.filterchoice, 0, wx.ALIGN_LEFT|wx.GROW|wx.ALL, 0) pack(topsection, sizer) splitter1 = wx.SplitterWindow(self, style=wx.SP_LIVE_UPDATE) splitter1.SetMinimumPaneSize(200) self.top_panel = wx.Panel(splitter1) # top self.bot_panel = SpectraPanel(splitter1) # bottom splitter2 = wx.SplitterWindow(self.top_panel, style=wx.SP_LIVE_UPDATE) splitter2.SetMinimumPaneSize(200) # left hand side -- filter self.left_panel = wx.Panel(splitter2) self.selection_list = wx.ListBox(self.left_panel) self.selection_label = wx.StaticText(self.left_panel, label=self.current_filter) sizer = wx.BoxSizer(wx.VERTICAL) sizer.Add(self.selection_label, 0, wx.ALIGN_LEFT|wx.ALL, 1) sizer.Add(self.selection_list, 1, wx.ALIGN_LEFT|wx.GROW|wx.ALL, 1) pack(self.left_panel, sizer) # right hand side -- filtered spectra self.right_panel = wx.Panel(splitter2) self.spectra_list = wx.ListBox(self.right_panel) self.spectra_label = wx.StaticText(self.right_panel, label='Spectra') sizer = wx.BoxSizer(wx.VERTICAL) sizer.Add(self.spectra_label, 0, wx.ALIGN_LEFT|wx.ALL, 1) sizer.Add(self.spectra_list, 1, wx.ALIGN_LEFT|wx.GROW|wx.ALL, 1) pack(self.right_panel, sizer) self.selection_list.SetBackgroundColour(wx.Colour(255, 250, 250)) self.spectra_list.SetBackgroundColour(wx.Colour(250, 250, 240)) self.selection_list.Clear() # for name in ('Fe Compounds, GSECARS', 'As standards'): # self.selection_list.Append(name) self.spectra_list.Clear() # 'FeCO3', 'Fe metal', 'maghemite'): for name in ('Fe metal', 'Fe2O3', 'FeO'): self.spectra_list.Append(name) self.selection_list.Bind(wx.EVT_LISTBOX, self.onSelectionSelect) self.spectra_list.Bind(wx.EVT_LISTBOX, self.onSpectraSelect) splitter2.SplitVertically(self.left_panel, self.right_panel, 0) sizer2 = wx.BoxSizer(wx.HORIZONTAL) sizer2.Add(splitter2, 1, wx.GROW|wx.ALL, 5) pack(self.top_panel, sizer2) splitter1.SplitHorizontally(self.top_panel, self.bot_panel, 1) sizer = wx.BoxSizer(wx.VERTICAL) sizer.Add(topsection, 0, wx.ALL|wx.GROW, 1) sizer.Add(splitter1, 1, wx.GROW|wx.ALL, 1) pack(self, sizer)
def pack(self, payload, peek=None): return self.dst_mac + self.src_mac + pack("!H", self.protocol) + payload
def buildBeamlinePanel(self): "build Beamline info panel" # beamline # facility # monochromator # energy units panel = self.beamline_panel = wx.Panel(self.nb) self.nb.AddPage(panel, 'Data Collection') sizer = wx.GridBagSizer(10, 6) def stext(label): return wx.StaticText(panel, label=label) _person = HyperChoice(panel, PersonChoice, label='Person:', manager=PersonManager, db=self.db) self.person = _person.choice _beamline = HyperChoice(panel, BeamlineChoice, label='Beamline:', manager=BeamlineManager, db=self.db) self.beamline = _beamline.choice _cite = HyperChoice(panel, CitationChoice, label='Citation:', db=self.db) self.citation = _cite.choice self.collection_datetime = DateTimeCtrl(panel, name="collection_time") self.submission_datetime = DateTimeCtrl(panel, name="collection_time", use_now=True) sizer.Add(Title(panel, 'Data Collection Information:'), (0, 0), (1, 3), titlestyle, 1) irow = 1 sizer.Add(_person.link, (irow, 0), (1, 1), labstyle, 1) sizer.Add(self.person, (irow, 1), (1, 1), namestyle, 1) irow += 1 sizer.Add(stext('Collection Date:'), (irow, 0), (1, 1), labstyle, 1) sizer.Add(self.collection_datetime[0], (irow, 1), (1, 1), namestyle, 1) irow += 1 sizer.Add(stext('Submission Date:'), (irow, 0), (1, 1), labstyle, 1) sizer.Add(self.submission_datetime[0], (irow, 1), (1, 1), namestyle, 1) irow += 1 sizer.Add(_beamline.link, (irow, 0), (1, 1), labstyle, 1) sizer.Add(self.beamline, (irow, 1), (1, 1), namestyle, 1) irow += 1 sizer.Add(_cite.link, (irow, 0), (1, 1), labstyle, 1) sizer.Add(self.citation, (irow, 1), (1, 1), namestyle, 1) ## RHS of Beamline Panel sizer.Add(wx.StaticLine(panel, size=(1, 160), style=wx.LI_VERTICAL), (1, 2), (6, 1), wx.ALIGN_CENTER|wx.GROW|wx.ALL, 0) irow = 1 self.ring_current = FloatCtrl(panel, precision=2, value=0, minval=0, maxval=1e6) self.ring_energy = FloatCtrl(panel, precision=3, value=0, minval=0, maxval=1e3) self.xray_source = wx.TextCtrl(panel, value='', size=(180, -1)) self.focusing = wx.TextCtrl(panel, value='', size=(180, -1)) self.collimation = wx.TextCtrl(panel, value='', size=(180, -1)) self.harmonic_rej = wx.TextCtrl(panel, value='', size=(180, -1)) sizer.Add(stext('Ring Current:'), (irow, 4), (1, 1), labstyle, 1) sizer.Add(self.ring_current, (irow, 5), (1, 1), namestyle, 1) sizer.Add(stext('mA'), (irow, 6), (1, 1), labstyle, 1) irow += 1 sizer.Add(stext('Ring Energy:'), (irow, 4), (1, 1), labstyle, 1) sizer.Add(self.ring_energy, (irow, 5), (1, 1), namestyle, 1) sizer.Add(stext('GeV'), (irow, 6), (1, 1), labstyle, 1) irow += 1 sizer.Add(stext('X-ray Source:'), (irow, 4), (1, 1), labstyle, 1) sizer.Add(self.xray_source, (irow, 5), (1, 2), namestyle, 1) irow += 1 sizer.Add(stext('Focusing:'), (irow, 4), (1, 1), labstyle, 1) sizer.Add(self.focusing, (irow, 5), (1, 2), namestyle, 1) irow += 1 sizer.Add(stext('Collimation:'), (irow, 4), (1, 1), labstyle, 1) sizer.Add(self.collimation, (irow, 5), (1, 2), namestyle, 1) irow += 1 sizer.Add(stext('Harmonic\nRejection:'), (irow, 4), (1, 1), labstyle, 1) sizer.Add(self.harmonic_rej, (irow, 5), (1, 2), namestyle, 1) pack(panel, sizer)
def set(self, key, value): self._cursor.set_key(key) self._cursor.set_value(pack(value)) self._cursor.insert()
def buildDataPanel(self): "build Data / Column selection panel" panel = self.columns_panel = wx.Panel(self.nb) self.nb.AddPage(panel, 'Data') sizer = wx.GridBagSizer(10, 6) def stext(label): return wx.StaticText(panel, label=label) self.elem = ElementChoice(panel, db=self.db) self.edge = EdgeChoice(panel, db=self.db) self.name = wx.TextCtrl(panel, value='', size=(220,-1)) self.name_ok = stext('') self.column_en = ColumnChoice(panel) self.column_i0 = ColumnChoice(panel) self.column_it = ColumnChoice(panel) self.column_if = ColumnChoice(panel) self.column_ir = ColumnChoice(panel) self.type_en = TypeChoice(panel, self.energy_choices) self.type_i0 = TypeChoice(panel, ('not collected', 'intensity',)) self.type_it = TypeChoice(panel, ('not collected', 'intensity','mu')) self.type_if = TypeChoice(panel, ('not collected', 'intensity','mu')) self.type_ir = TypeChoice(panel, ('not collected', 'intensity','mu')) self.column_en.SetSelection(0) self.type_en.SetSelection(0) self.type_i0.SetSelection(0) self.type_it.SetSelection(0) self.type_ir.SetSelection(0) self.type_if.SetSelection(0) _mono = HyperChoice(panel, MonochromatorChoice, label='Monochromator:', manager=MonochromatorManager, db=self.db) self.mono_choice = _mono.choice self.mono_dspacing = FloatCtrl(panel, precision=6, value=0, minval=0) self.mono_steps = FloatCtrl(panel, precision=2, value=0, minval=0) self.type_en.Bind(wx.EVT_CHOICE, self.onEnergyChoice) sizer.Add(stext('Spectra Name:'), (0, 0), (1, 1), labstyle, 1) sizer.Add(self.name, (0, 1), (1, 4), namestyle, 1) sizer.Add(self.name_ok, (0, 5), (1, 1), clabstyle, 1) sizer.Add(stext('Element:'), (1, 0), (1, 1), labstyle, 1) sizer.Add(self.elem, (1, 1), (1, 1), choicestyle, 1) sizer.Add(stext('Edge:'), (1, 2), (1, 1), rlabstyle, 1) sizer.Add(self.edge, (1, 3), (1, 1), choicestyle, 1) sizer.Add(wx.StaticLine(panel, size=(350, 2), style=wx.LI_HORIZONTAL), (2, 0), (1, 6), wx.ALIGN_CENTER|wx.GROW|wx.ALL, 0) irow = 3 sizer.Add(stext('Data:'), (irow, 0), (1, 1), labstyle, 1) sizer.Add(stext(' Type:'), (irow, 1), (1, 1), labstyle, 1) sizer.Add(stext('Column #:'), (irow, 2), (1, 1), labstyle, 1) irow += 1 sizer.Add(stext('Energy:'), (irow, 0), (1, 1), labstyle, 1) sizer.Add(self.type_en, (irow, 1), (1, 1), choicestyle, 1) sizer.Add(self.column_en, (irow, 2), (1, 1), choicestyle, 1) irow += 1 sizer.Add(stext('I0:'), (irow, 0), (1, 1), labstyle, 1) sizer.Add(self.type_i0, (irow, 1), (1, 1), choicestyle, 1) sizer.Add(self.column_i0, (irow, 2), (1, 1), choicestyle, 1) irow += 1 sizer.Add(stext('Transmission:'),(irow, 0), (1, 1), labstyle, 1) sizer.Add(self.type_it, (irow, 1), (1, 1), choicestyle, 1) sizer.Add(self.column_it, (irow, 2), (1, 1), choicestyle, 1) irow += 1 sizer.Add(stext('Fluorescence:'),(irow, 0), (1, 1), labstyle, 1) sizer.Add(self.type_if, (irow, 1), (1, 1), choicestyle, 1) sizer.Add(self.column_if, (irow, 2), (1, 1), choicestyle, 1) irow += 1 sizer.Add(stext('Reference:'), (irow, 0), (1, 1), labstyle, 1) sizer.Add(self.type_ir, (irow, 1), (1, 1), choicestyle, 1) sizer.Add(self.column_ir, (irow, 2), (1, 1), choicestyle, 1) irow = 4 sizer.Add(_mono.link, (irow, 3), (1, 1), labstyle, 1) sizer.Add(self.mono_choice, (irow, 4), (1, 2), choicestyle, 1) irow += 1 sizer.Add(stext(' d spacing ='), (irow, 3), (1, 1), rlabstyle, 1) sizer.Add(self.mono_dspacing , (irow, 4), (1, 1), namestyle, 1) sizer.Add(stext('Angtroms'), (irow, 5), (1, 1), labstyle, 1) irow += 1 sizer.Add(stext(' resolution ='), (irow, 3), (1, 1), rlabstyle, 1) sizer.Add(self.mono_steps , (irow, 4), (1, 1), namestyle, 1) sizer.Add(stext('steps/degree'), (irow, 5), (1, 1), labstyle, 1) #self.mono_dspacing.DisableEntry() #self.mono_steps.DisableEntry() pack(panel, sizer)
def __init__(self, fname=None, db=None): wx.Frame.__init__(self, parent=None, title=self.title, size=(650, 650)) self.fname = fname self.db = db menuBar = wx.MenuBar() fmenu = wx.Menu() add_menu(self, fmenu, "E&xit\tAlt-X", "Exit this Program", action=self.onClose) menuBar.Append(fmenu, "&File") self.SetMenuBar(menuBar) self.CreateStatusBar(1, wx.CAPTION|wx.THICK_FRAME) splitter = wx.SplitterWindow(self, style=wx.SP_LIVE_UPDATE) splitter.SetMinimumPaneSize(150) self.top_panel = wx.Panel(splitter) # top self.bot_panel = wx.Panel(splitter) # bottom self.filename = wx.StaticText(self.bot_panel, -1, label='') self.fileview = wx.TextCtrl(self.bot_panel, size=(175, 175), style=wx.TE_READONLY|wx.TE_MULTILINE|wx.HSCROLL) self.fileview.SetBackgroundColour(wx.Colour(250, 250, 250)) bot_sizer = wx.BoxSizer(wx.VERTICAL) bot_sizer.Add(self.filename, 0, wx.ALIGN_TOP|wx.ALIGN_CENTER_VERTICAL, 5) bot_sizer.Add(self.fileview, 1, wx.ALIGN_TOP|wx.GROW|wx.ALL, 5) pack(self.bot_panel, bot_sizer) top_sizer = wx.BoxSizer(wx.VERTICAL) title_row = wx.Panel(self.top_panel) title_sizer = wx.BoxSizer(wx.HORIZONTAL) self.top_label = Title(title_row, 'Data from File:', colour=(200,20,20)) self.save_btn = add_btn(title_row, "Import Data", action=self.onImport) title_sizer.Add(self.top_label, 1, wx.ALIGN_LEFT|wx.ALL, 3) title_sizer.Add(self.save_btn, 0, wx.ALIGN_RIGHT|wx.GROW|wx.ALL, 3) pack(title_row, title_sizer) self.nb = wx.Notebook(self.top_panel, size=(600, 275)) self.buildDataPanel() self.buildBeamlinePanel() self.buildSamplePanel() top_sizer.Add(title_row, 0, wx.ALIGN_LEFT|wx.GROW|wx.ALL, 1) top_sizer.Add(self.nb, 1, wx.ALIGN_CENTER|wx.GROW|wx.ALL, 1) pack(self.top_panel, top_sizer) splitter.SplitHorizontally(self.top_panel, self.bot_panel, -1) framesizer = wx.BoxSizer(wx.VERTICAL) framesizer.Add(splitter, 1, wx.GROW|wx.ALL, 5) self.SetSizer(framesizer) if fname is not None: self.ShowFile(fname) self.Show() self.Raise()
def real_time_modify(): tscale = 1.0 p = pyaudio.PyAudio() in_stream = p.open(format=pyaudio.paInt16, channels=2, rate=44100, input = True) out_stream = p.open(format=pyaudio.paInt16, channels=2, rate=int(44100*tscale), output=True) chunk = 1024 data = in_stream.read(chunk) data = utils.unpack(data) print 'data', len(data) next_data = in_stream.read(chunk) next_data = utils.unpack(next_data) amp = 0 out_data = np.concatenate(([], zeros(chunk))) L = len(data) N = L/2 H = N/2 phi = zeros(N) out = zeros(N, dtype=complex) win = hanning(N) p = 0 pp = 0 for i in range(0,300): concat = np.concatenate(((data, next_data))) amp = max(amp, max(concat)) out_data = np.concatenate((out_data, zeros(N/tscale))) p = 0 for i in range(2): # take the spectra of two consecutive windows p1 = int(p) spec1 = fft(win*concat[p1:p1+N]) spec2 = fft(win*concat[p1+H:p1+N+H]) # take their phase difference and integrate phi += (angle(spec2) - angle(spec1)) out.real, out.imag = cos(phi), sin(phi) # inverse FFT and overlap-add print 'pp:pp+N', pp, pp+N out_data[pp:pp+N] += win*ifft(abs(spec2)*out) pp += H p += H*tscale out_data = amp*out_data/max(out_data) out_formatted = utils.pack(out_data[i*chunk:(i+1)*chunk]) out_formatted = ''.join(out_formatted) out_stream.write(out_formatted) data = next_data next_data = in_stream.read(chunk) next_data = utils.unpack(next_data) in_stream.stop_stream() in_stream.close() out_stream.stop_stream() out_stream.close() # wav.write('test.wav',44100*2, array(data_to_file, dtype='int16')) p.terminate()
def __unicode__(self): return utils.pack(self.uri, self.title, self.description)
def read_chunk(host, port, chunkid, version, size, offset=0): if offset + size > CHUNKSIZE: raise ValueError("size too large %s > %s" % (size, CHUNKSIZE-offset)) from dpark.accumulator import RemoteReadBytes conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) conn.settimeout(10) conn.connect((host, port)) msg = pack(CUTOCS_READ, uint64(chunkid), version, offset, size) n = conn.send(msg) while n < len(msg): if not n: raise IOError("write failed") msg = msg[n:] n = conn.send(msg) def recv(n): d = conn.recv(n) while len(d) < n: nd = conn.recv(n-len(d)) if not nd: raise IOError("not enough data") d += nd return d while size > 0: cmd, l = unpack("II", recv(8)) if cmd == CSTOCU_READ_STATUS: if l != 9: raise Exception("readblock: READ_STATUS incorrect message size") cid, code = unpack("QB", recv(l)) if cid != chunkid: raise Exception("readblock; READ_STATUS incorrect chunkid") conn.close() return elif cmd == CSTOCU_READ_DATA: if l < 20 : raise Exception("readblock; READ_DATA incorrect message size") cid, bid, boff, bsize, crc = unpack("QHHII", recv(20)) if cid != chunkid: raise Exception("readblock; READ_STATUS incorrect chunkid") if l != 20 + bsize: raise Exception("readblock; READ_DATA incorrect message size ") if bsize == 0 : # FIXME raise Exception("readblock; empty block") #yield "" #continue if bid != offset >> 16: raise Exception("readblock; READ_DATA incorrect block number") if boff != offset & 0xFFFF: raise Exception("readblock; READ_DATA incorrect block offset") breq = 65536 - boff if size < breq: breq = size if bsize != breq: raise Exception("readblock; READ_DATA incorrect block size") while breq > 0: data = conn.recv(breq) if not data: #print chunkid, version, offset, size, bsize, breq raise IOError("unexpected ending: need %d" % breq) RemoteReadBytes.add(len(data)) yield data breq -= len(data) offset += bsize size -= bsize else: raise Exception("readblock; unknown message: %s" % cmd) conn.close()