def decode(cls, data): '''Deserialize the data and return an object.''' with _convert_exceptions(): xdr = Unpacker(data) ret = cls.decode_xdr(xdr) xdr.done() return ret
def gmetric_read(header_msg, data_msg): header = Unpacker(header_msg) data = Unpacker(data_msg) values = dict() header.unpack_int() values['HOSTNAME'] = str(header.unpack_string().decode('ascii')) values['NAME'] = str(header.unpack_string().decode('ascii')) values['SPOOFENABLED'] = header.unpack_int() values['TYPE'] = str(header.unpack_string().decode('ascii')) values['NAME'] = str(header.unpack_string().decode('ascii')) values['UNITS'] = str(header.unpack_string().decode('ascii')) values['SLOPE'] = slope_int2str[header.unpack_int()] values['TMAX'] = header.unpack_uint() values['DMAX'] = header.unpack_uint() if header.unpack_int() == 1: header.unpack_string() values['GROUP'] = str(header.unpack_string().decode('ascii')) # Actual data in the second packet data.unpack_int() values['HOSTNAME'] = str(data.unpack_string().decode('ascii')) values['NAME'] = str(data.unpack_string().decode('ascii')) values['SPOOFENABLED'] = data.unpack_int() data.unpack_string() values['VAL'] = str(data.unpack_string().decode('ascii')) header.done() data.done() return values
def gmetric_read(msg): unpacker = Unpacker(msg) values = dict() unpacker.unpack_int() values['TYPE'] = unpacker.unpack_string() values['NAME'] = unpacker.unpack_string() values['VAL'] = unpacker.unpack_string() values['UNITS'] = unpacker.unpack_string() values['SLOPE'] = slope_int2str[unpacker.unpack_int()] values['TMAX'] = unpacker.unpack_uint() values['DMAX'] = unpacker.unpack_uint() unpacker.done() return values
def gmetric_read(msg): unpacker = Unpacker(msg) values = dict() unpacker.unpack_int() values["TYPE"] = unpacker.unpack_string() values["NAME"] = unpacker.unpack_string() values["VAL"] = unpacker.unpack_string() values["UNITS"] = unpacker.unpack_string() values["SLOPE"] = slope_int2str[unpacker.unpack_int()] values["TMAX"] = unpacker.unpack_uint() values["DMAX"] = unpacker.unpack_uint() unpacker.done() return values
def _length_from_bytes(four_bytes): """ The RPC standard calls for the length of a message to be sent as the least significant 31 bits of an XDR encoded unsigned integer. The most significant bit encodes a True/False bit which indicates that this message will be the last. """ from xdrlib import Unpacker unpacker = Unpacker(four_bytes) val = unpacker.unpack_uint() unpacker.done() if val < 2**31: return (val, False) return (val-2**31, True)
def handle(self): data = self.request[0] unpacker = Unpacker(data) type = unpacker.unpack_int() if type not in GANGLIA_DECODE: return host = unpacker.unpack_string() name = unpacker.unpack_string() unpacker.unpack_int() # spoof boolean unpacker.unpack_string() # format string value = GANGLIA_DECODE[type](unpacker) unpacker.done() graphite.record_stat(name, value)
def read_flow_record(up, sample): """Reads a 'struct flow_record' (p. 29)""" flow_format = up.unpack_uint() flow_data = up.unpack_opaque() up_flow_data = Unpacker(flow_data) if flow_format == FLOW_DATA_RAW_HEADER: res = FlowRecord(sample, read_sampled_header(up_flow_data)) elif flow_format == FLOW_DATA_ETHERNET_HEADER: res = FlowRecord(sample, read_sampled_ethernet(up_flow_data)) elif flow_format == FLOW_DATA_IPV4_HEADER: res = FlowRecord(sample, read_sampled_ipv4(up_flow_data)) else: res = 'read_flow_record:Unknown data_format (%d)' % flow_format up_flow_data.done() return res
def decode_data(self, data): ''' Decodes 1 UDP Ganglia packet, returns a dict of the decoded data ''' ret = {} xdr = Unpacker(data) message_type = xdr.unpack_int() if message_type == GANGLIA_METADATA_MESSAGE: ret = { 'METRIC': 'METADATA', 'hostname': xdr.unpack_string(), 'name': xdr.unpack_string(), 'spoof': bool(xdr.unpack_int()), 'metric_type': xdr.unpack_string(), 'metric_name': xdr.unpack_string(), 'units': xdr.unpack_string(), 'slope': xdr.unpack_int(), 'time_max': xdr.unpack_int(), 'data_max': xdr.unpack_int() } assert xdr.unpack_int() == 1 assert xdr.unpack_string() == 'GROUP' ret['group'] = xdr.unpack_string() elif message_type == GANGLIA_METRIC_MESSAGE: ret = { 'type': 'METRIC', 'hostname': xdr.unpack_string(), 'metric': xdr.unpack_string(), 'spoof': bool(xdr.unpack_int()), 'format': xdr.unpack_string(), 'value': xdr.unpack_string() } xdr.done() return ret
def read_sample_record(up, sample_datagram): # Unpack sample_record structure # data_format sample_type; # Specifies the type of sample data # opaque sample_data<>; # A structure corresponding to the sample_type sample_type = up.unpack_uint() sample_data = up.unpack_opaque() up_sample_data = Unpacker(sample_data) if sample_type == SAMPLE_DATA_FLOW_RECORD: return read_flow_sample(up_sample_data, sample_datagram) elif sample_type == SAMPLE_DATA_COUNTER_RECORD: return read_counter_sample(up_sample_data, sample_datagram) else: raise Exception() # Check if whole data block was unpacked up_sample_data.done()
def parse(self, wired_string): """ @returns (counterparty_pub_key_sexp, cleartext,) @raises SessionInvalidated if the incoming message was an "invalidate session" message \000\000\000\002. @raises UnknownSession error if the incoming message did not identify a known session key. @precondition `wired_string' must be a string.: type(wired_string) == types.StringType: "wired_string: %s :: %s" % (hr(wired_string), hr(type(wired_string))) @postcondition `counterparty_pub_key_sexp' is a public key.: MojoKey.publicRSAKeyForCommunicationSecurityIsWellFormed(counterparty_pub_key_sexp): "counterparty_pub_key_sexp: %s" % hr(counterparty_pub_key_sexp) """ assert type(wired_string) == types.StringType, "precondition: `wired_string' must be a string." + " -- " + "wired_string: %s :: %s" % (hr(wired_string), hr(type(wired_string))) session = None try: u = Unpacker(wired_string) mtype = u.unpack_fstring(4) if mtype == '\000\000\000\000': # a message with a full PK header header = u.unpack_string() iv = u.unpack_fstring(8) prefix = wired_string[:u.get_position()] encrypted = u.unpack_string() u.done() counterparty_pub_key_sexp, symmetric_key = self._session_keeper.parse_header(header) decrypted = tripledescbc.new(symmetric_key).decrypt(iv, encrypted) u = Unpacker(decrypted) message = u.unpack_string() mac = u.unpack_fstring(SIZE_OF_UNIQS) u.done() # debugprint("------ ------ ------ ------ hmachish(key=%s, message=%s)\n" % (`symmetric_key`, `message`)) maccomp = cryptutil.hmacish(key=symmetric_key, message=message) if mac != maccomp: raise Error, 'incorrect MAC' return (counterparty_pub_key_sexp, message) elif mtype == '\000\000\000\001': # a message using an already established session id session = u.unpack_fstring(SIZE_OF_UNIQS) iv = u.unpack_fstring(8) prefix = wired_string[:u.get_position()] encrypted = u.unpack_string() u.done() counterparty_pub_key_sexp, symmetric_key, want_ack = self._session_keeper.get_session_info(session) decrypted = tripledescbc.new(symmetric_key).decrypt(iv, encrypted) u = Unpacker(decrypted) message = u.unpack_string() mac = u.unpack_fstring(SIZE_OF_UNIQS) u.done() counterparty_id = idlib.make_id(counterparty_pub_key_sexp, 'broker') # debugprint("------ ------ ------ ------ hmachish(key=%s, message=%s)\n" % (`symmetric_key`, `message`)) maccomp = cryptutil.hmacish(key=symmetric_key, message=message) if mac != maccomp: raise Error, 'incorrect MAC' if want_ack: self._session_keeper.got_ack(counterparty_id) return (counterparty_pub_key_sexp, message) elif mtype == '\000\000\002\002': # a short "message" invalidating an outgoing session id bad_session_id_out = u.unpack_fstring(SIZE_OF_UNIQS) unverified_counterparty_id = u.unpack_fstring(SIZE_OF_UNIQS) self._session_keeper.invalidate_session(bad_session_id_out, unverified_counterparty_id) raise SessionInvalidated, 'session_id %s with %s invalidated' % (`bad_session_id_out`, idlib.to_ascii(unverified_counterparty_id)) else: raise Error, 'unsupported message type' except (modval.Error, tripledescbc.Error, xdrlib.Error, EOFError), le: debugprint("got error in mesgen.parse(): %s", args=(le,), v=4, vs="debug") if session is not None: raise UnknownSession(session, self.get_id()) else: raise Error, le
def __parse_header(self, header): """ Parses a header and stores information contained in it as necessary Returns (counterparty pub key sexp, symmetric key) throws Error """ assert type(header) == type('') try: hash = sha(header).digest() cached = self.__cached_headers.get(hash) if cached is not None: return cached u = Unpacker(header) # messages start with the hash of the recipient's public id recipient_id = u.unpack_fstring(SIZE_OF_UNIQS) if recipient_id != self.__my_public_key_id: raise Error, 'message not intended for me' # unpack PK encrypted public key encrypted_key = u.unpack_string() self.__key.set_value_string(encrypted_key) self.__key.decrypt() # PKop decrypted = self.__key.get_value() try: symmetric_key = cryptutil.oaep_decode(decrypted[1:]) # Leave off the initial 0 byte. ### XX check whether it really is 0 and raise bad-encoding error if not. --Zooko 2000-07-29 except cryptutil.OAEPError, le: raise Error, 'bad encryption -- pad badding: padded: %s, Error: %s' % (`decrypted`, `le.args`) iv = u.unpack_fstring(8) # first half of the MAC # XXX A.K.A. the key? --Zooko 2000-07-29 prefix = header[:u.get_position()] # all data except the symmetric key and recipient, encrypted encrypted = u.unpack_string() u.done() decrypted = tripledescbc.new(symmetric_key).decrypt(iv, encrypted) u = Unpacker(decrypted) # the full public key of the sender sender_key = u.unpack_string() full_key = MojoKey.makePublicRSAKeyForCommunicating(modval.new(sender_key, HARDCODED_RSA_PUBLIC_EXPONENT)) full_key_id = idlib.make_id(full_key, 'broker') # the session id for messages sent 'here' id_in = _mix_counterparties(full_key_id, self.__my_public_key_id, u.unpack_fstring(SIZE_OF_UNIQS)) # the session id for messages sent 'there' id_out = _mix_counterparties(full_key_id, self.__my_public_key_id, u.unpack_fstring(SIZE_OF_UNIQS)) # check that the pk encrypted symmetric key used to send this message is the same was generated properly strl = u.unpack_fstring(SIZE_OF_UNIQS) sr = HashRandom.SHARandom(_mix_counterparties(full_key_id, self.__my_public_key_id, strl)) spaml = sr.get(SIZE_OF_SYMMETRIC_KEYS) if symmetric_key != spaml: raise Error, 'improperly generated key' # the second half of what's in the MAC # XXX A.K.A. the message? --Zooko 2000-07-29 end = decrypted[:u.get_position()] # the signature of everything signature = u.unpack_fstring(len(sender_key)) u.done() # debugprint("------ ------ ------ ------ hmachish(key=%s, message=%s)\n" % (`symmetric_key`, `end`)) summary = cryptutil.hmacish(key=symmetric_key, message=end) x = modval.new(sender_key, HARDCODED_RSA_PUBLIC_EXPONENT, signature) x.undo_signature() # PKop signed_value = x.get_value() try: thingie = cryptutil.oaep_decode(signed_value[1:]) # Leave off the initial 0 byte. ### XX check whether it really is 0 and raise bad-encoding error if not. --Zooko 2000-07-29 except cryptutil.OAEPError, le: raise Error, 'bad encryption -- pad badding: padded: %s, Error: %s' % (`signed_value`, `le.args`)
### Counter Records End ### ### Counter Sample End ### ### Something else ### else: logging.warning("Unknown [Enterprise, Format] " + str(enterprise_format_num) + " not defined in sFlow standard - FAIL") ### Sample Parsing Finish ### # Verify XDR data has been completely unpacked try: unpacked_data.done() except Exception as unpack_done_error: logging.warning(str(unpack_done_error)) logging.warning("Failed to completely unpack sample data - FAIL") ### sFlow Samples End ### # Elasticsearch bulk upload if record_num >= bulk_insert_count: # Perform the bulk upload to the index try: helpers.bulk(es, sflow_data) # Call Elasticsearch bulk upload API logging.info( str(record_num) +