def decode(cls, data):
'''Deserialize the data and return an object.'''
with _convert_exceptions():
xdr = Unpacker(data)
ret = cls.decode_xdr(xdr)
xdr.done()
return ret
def gmetric_read(header_msg, data_msg):
header = Unpacker(header_msg)
data = Unpacker(data_msg)
values = dict()
header.unpack_int()
values['HOSTNAME'] = str(header.unpack_string().decode('ascii'))
values['NAME'] = str(header.unpack_string().decode('ascii'))
values['SPOOFENABLED'] = header.unpack_int()
values['TYPE'] = str(header.unpack_string().decode('ascii'))
values['NAME'] = str(header.unpack_string().decode('ascii'))
values['UNITS'] = str(header.unpack_string().decode('ascii'))
values['SLOPE'] = slope_int2str[header.unpack_int()]
values['TMAX'] = header.unpack_uint()
values['DMAX'] = header.unpack_uint()
if header.unpack_int() == 1:
header.unpack_string()
values['GROUP'] = str(header.unpack_string().decode('ascii'))
# Actual data in the second packet
data.unpack_int()
values['HOSTNAME'] = str(data.unpack_string().decode('ascii'))
values['NAME'] = str(data.unpack_string().decode('ascii'))
values['SPOOFENABLED'] = data.unpack_int()
data.unpack_string()
values['VAL'] = str(data.unpack_string().decode('ascii'))
header.done()
data.done()
return values
def gmetric_read(msg):
unpacker = Unpacker(msg)
values = dict()
unpacker.unpack_int()
values['TYPE'] = unpacker.unpack_string()
values['NAME'] = unpacker.unpack_string()
values['VAL'] = unpacker.unpack_string()
values['UNITS'] = unpacker.unpack_string()
values['SLOPE'] = slope_int2str[unpacker.unpack_int()]
values['TMAX'] = unpacker.unpack_uint()
values['DMAX'] = unpacker.unpack_uint()
unpacker.done()
return values
def gmetric_read(msg):
unpacker = Unpacker(msg)
values = dict()
unpacker.unpack_int()
values['TYPE'] = unpacker.unpack_string()
values['NAME'] = unpacker.unpack_string()
values['VAL'] = unpacker.unpack_string()
values['UNITS'] = unpacker.unpack_string()
values['SLOPE'] = slope_int2str[unpacker.unpack_int()]
values['TMAX'] = unpacker.unpack_uint()
values['DMAX'] = unpacker.unpack_uint()
unpacker.done()
return values
def gmetric_read(msg):
unpacker = Unpacker(msg)
values = dict()
unpacker.unpack_int()
values["TYPE"] = unpacker.unpack_string()
values["NAME"] = unpacker.unpack_string()
values["VAL"] = unpacker.unpack_string()
values["UNITS"] = unpacker.unpack_string()
values["SLOPE"] = slope_int2str[unpacker.unpack_int()]
values["TMAX"] = unpacker.unpack_uint()
values["DMAX"] = unpacker.unpack_uint()
unpacker.done()
return values
def _length_from_bytes(four_bytes):
"""
The RPC standard calls for the length of a message to be sent as the least
significant 31 bits of an XDR encoded unsigned integer. The most
significant bit encodes a True/False bit which indicates that this message
will be the last.
"""
from xdrlib import Unpacker
unpacker = Unpacker(four_bytes)
val = unpacker.unpack_uint()
unpacker.done()
if val < 2**31:
return (val, False)
return (val-2**31, True)
def handle(self):
data = self.request[0]
unpacker = Unpacker(data)
type = unpacker.unpack_int()
if type not in GANGLIA_DECODE: return
host = unpacker.unpack_string()
name = unpacker.unpack_string()
unpacker.unpack_int() # spoof boolean
unpacker.unpack_string() # format string
value = GANGLIA_DECODE[type](unpacker)
unpacker.done()
graphite.record_stat(name, value)
def handle(self):
data = self.request[0]
unpacker = Unpacker(data)
type = unpacker.unpack_int()
if type not in GANGLIA_DECODE: return
host = unpacker.unpack_string()
name = unpacker.unpack_string()
unpacker.unpack_int() # spoof boolean
unpacker.unpack_string() # format string
value = GANGLIA_DECODE[type](unpacker)
unpacker.done()
graphite.record_stat(name, value)
def read_flow_record(up, sample):
"""Reads a 'struct flow_record' (p. 29)"""
flow_format = up.unpack_uint()
flow_data = up.unpack_opaque()
up_flow_data = Unpacker(flow_data)
if flow_format == FLOW_DATA_RAW_HEADER:
res = FlowRecord(sample, read_sampled_header(up_flow_data))
elif flow_format == FLOW_DATA_ETHERNET_HEADER:
res = FlowRecord(sample, read_sampled_ethernet(up_flow_data))
elif flow_format == FLOW_DATA_IPV4_HEADER:
res = FlowRecord(sample, read_sampled_ipv4(up_flow_data))
else:
res = 'read_flow_record:Unknown data_format (%d)' % flow_format
up_flow_data.done()
return res
def read_flow_record(up, sample):
"""Reads a 'struct flow_record' (p. 29)"""
flow_format = up.unpack_uint()
flow_data = up.unpack_opaque()
up_flow_data = Unpacker(flow_data)
if flow_format == FLOW_DATA_RAW_HEADER:
res = FlowRecord(sample, read_sampled_header(up_flow_data))
elif flow_format == FLOW_DATA_ETHERNET_HEADER:
res = FlowRecord(sample, read_sampled_ethernet(up_flow_data))
elif flow_format == FLOW_DATA_IPV4_HEADER:
res = FlowRecord(sample, read_sampled_ipv4(up_flow_data))
else:
res = 'read_flow_record:Unknown data_format (%d)' % flow_format
up_flow_data.done()
return res
def decode_data(self, data):
''' Decodes 1 UDP Ganglia packet, returns a dict of the decoded data '''
ret = {}
xdr = Unpacker(data)
message_type = xdr.unpack_int()
if message_type == GANGLIA_METADATA_MESSAGE:
ret = {
'METRIC': 'METADATA',
'hostname': xdr.unpack_string(),
'name': xdr.unpack_string(),
'spoof': bool(xdr.unpack_int()),
'metric_type': xdr.unpack_string(),
'metric_name': xdr.unpack_string(),
'units': xdr.unpack_string(),
'slope': xdr.unpack_int(),
'time_max': xdr.unpack_int(),
'data_max': xdr.unpack_int()
}
assert xdr.unpack_int() == 1
assert xdr.unpack_string() == 'GROUP'
ret['group'] = xdr.unpack_string()
elif message_type == GANGLIA_METRIC_MESSAGE:
ret = {
'type': 'METRIC',
'hostname': xdr.unpack_string(),
'metric': xdr.unpack_string(),
'spoof': bool(xdr.unpack_int()),
'format': xdr.unpack_string(),
'value': xdr.unpack_string()
}
xdr.done()
return ret
def read_sample_record(up, sample_datagram):
# Unpack sample_record structure
# data_format sample_type;
# Specifies the type of sample data
# opaque sample_data<>;
# A structure corresponding to the sample_type
sample_type = up.unpack_uint()
sample_data = up.unpack_opaque()
up_sample_data = Unpacker(sample_data)
if sample_type == SAMPLE_DATA_FLOW_RECORD:
return read_flow_sample(up_sample_data, sample_datagram)
elif sample_type == SAMPLE_DATA_COUNTER_RECORD:
return read_counter_sample(up_sample_data, sample_datagram)
else:
raise Exception()
# Check if whole data block was unpacked
up_sample_data.done()
def read_sample_record(up, sample_datagram):
# Unpack sample_record structure
# data_format sample_type;
# Specifies the type of sample data
# opaque sample_data<>;
# A structure corresponding to the sample_type
sample_type = up.unpack_uint()
sample_data = up.unpack_opaque()
up_sample_data = Unpacker(sample_data)
if sample_type == SAMPLE_DATA_FLOW_RECORD:
return read_flow_sample(up_sample_data, sample_datagram)
elif sample_type == SAMPLE_DATA_COUNTER_RECORD:
return read_counter_sample(up_sample_data, sample_datagram)
else:
raise Exception()
# Check if whole data block was unpacked
up_sample_data.done()
def parse(self, wired_string):
"""
@returns (counterparty_pub_key_sexp, cleartext,)
@raises SessionInvalidated if the incoming message was an "invalidate session" message \000\000\000\002.
@raises UnknownSession error if the incoming message did not identify a known session key.
@precondition `wired_string' must be a string.: type(wired_string) == types.StringType: "wired_string: %s :: %s" % (hr(wired_string), hr(type(wired_string)))
@postcondition `counterparty_pub_key_sexp' is a public key.: MojoKey.publicRSAKeyForCommunicationSecurityIsWellFormed(counterparty_pub_key_sexp): "counterparty_pub_key_sexp: %s" % hr(counterparty_pub_key_sexp)
"""
assert type(wired_string) == types.StringType, "precondition: `wired_string' must be a string." + " -- " + "wired_string: %s :: %s" % (hr(wired_string), hr(type(wired_string)))
session = None
try:
u = Unpacker(wired_string)
mtype = u.unpack_fstring(4)
if mtype == '\000\000\000\000': # a message with a full PK header
header = u.unpack_string()
iv = u.unpack_fstring(8)
prefix = wired_string[:u.get_position()]
encrypted = u.unpack_string()
u.done()
counterparty_pub_key_sexp, symmetric_key = self._session_keeper.parse_header(header)
decrypted = tripledescbc.new(symmetric_key).decrypt(iv, encrypted)
u = Unpacker(decrypted)
message = u.unpack_string()
mac = u.unpack_fstring(SIZE_OF_UNIQS)
u.done()
# debugprint("------ ------ ------ ------ hmachish(key=%s, message=%s)\n" % (`symmetric_key`, `message`))
maccomp = cryptutil.hmacish(key=symmetric_key, message=message)
if mac != maccomp:
raise Error, 'incorrect MAC'
return (counterparty_pub_key_sexp, message)
elif mtype == '\000\000\000\001': # a message using an already established session id
session = u.unpack_fstring(SIZE_OF_UNIQS)
iv = u.unpack_fstring(8)
prefix = wired_string[:u.get_position()]
encrypted = u.unpack_string()
u.done()
counterparty_pub_key_sexp, symmetric_key, want_ack = self._session_keeper.get_session_info(session)
decrypted = tripledescbc.new(symmetric_key).decrypt(iv, encrypted)
u = Unpacker(decrypted)
message = u.unpack_string()
mac = u.unpack_fstring(SIZE_OF_UNIQS)
u.done()
counterparty_id = idlib.make_id(counterparty_pub_key_sexp, 'broker')
# debugprint("------ ------ ------ ------ hmachish(key=%s, message=%s)\n" % (`symmetric_key`, `message`))
maccomp = cryptutil.hmacish(key=symmetric_key, message=message)
if mac != maccomp:
raise Error, 'incorrect MAC'
if want_ack:
self._session_keeper.got_ack(counterparty_id)
return (counterparty_pub_key_sexp, message)
elif mtype == '\000\000\002\002': # a short "message" invalidating an outgoing session id
bad_session_id_out = u.unpack_fstring(SIZE_OF_UNIQS)
unverified_counterparty_id = u.unpack_fstring(SIZE_OF_UNIQS)
self._session_keeper.invalidate_session(bad_session_id_out, unverified_counterparty_id)
raise SessionInvalidated, 'session_id %s with %s invalidated' % (`bad_session_id_out`, idlib.to_ascii(unverified_counterparty_id))
else:
raise Error, 'unsupported message type'
except (modval.Error, tripledescbc.Error, xdrlib.Error, EOFError), le:
debugprint("got error in mesgen.parse(): %s", args=(le,), v=4, vs="debug")
if session is not None:
raise UnknownSession(session, self.get_id())
else:
raise Error, le
def __parse_header(self, header):
"""
Parses a header and stores information contained in it as necessary
Returns (counterparty pub key sexp, symmetric key) throws Error
"""
assert type(header) == type('')
try:
hash = sha(header).digest()
cached = self.__cached_headers.get(hash)
if cached is not None:
return cached
u = Unpacker(header)
# messages start with the hash of the recipient's public id
recipient_id = u.unpack_fstring(SIZE_OF_UNIQS)
if recipient_id != self.__my_public_key_id:
raise Error, 'message not intended for me'
# unpack PK encrypted public key
encrypted_key = u.unpack_string()
self.__key.set_value_string(encrypted_key)
self.__key.decrypt() # PKop
decrypted = self.__key.get_value()
try:
symmetric_key = cryptutil.oaep_decode(decrypted[1:]) # Leave off the initial 0 byte. ### XX check whether it really is 0 and raise bad-encoding error if not. --Zooko 2000-07-29
except cryptutil.OAEPError, le:
raise Error, 'bad encryption -- pad badding: padded: %s, Error: %s' % (`decrypted`, `le.args`)
iv = u.unpack_fstring(8)
# first half of the MAC # XXX A.K.A. the key? --Zooko 2000-07-29
prefix = header[:u.get_position()]
# all data except the symmetric key and recipient, encrypted
encrypted = u.unpack_string()
u.done()
decrypted = tripledescbc.new(symmetric_key).decrypt(iv, encrypted)
u = Unpacker(decrypted)
# the full public key of the sender
sender_key = u.unpack_string()
full_key = MojoKey.makePublicRSAKeyForCommunicating(modval.new(sender_key, HARDCODED_RSA_PUBLIC_EXPONENT))
full_key_id = idlib.make_id(full_key, 'broker')
# the session id for messages sent 'here'
id_in = _mix_counterparties(full_key_id, self.__my_public_key_id, u.unpack_fstring(SIZE_OF_UNIQS))
# the session id for messages sent 'there'
id_out = _mix_counterparties(full_key_id, self.__my_public_key_id, u.unpack_fstring(SIZE_OF_UNIQS))
# check that the pk encrypted symmetric key used to send this message is the same was generated properly
strl = u.unpack_fstring(SIZE_OF_UNIQS)
sr = HashRandom.SHARandom(_mix_counterparties(full_key_id, self.__my_public_key_id, strl))
spaml = sr.get(SIZE_OF_SYMMETRIC_KEYS)
if symmetric_key != spaml:
raise Error, 'improperly generated key'
# the second half of what's in the MAC # XXX A.K.A. the message? --Zooko 2000-07-29
end = decrypted[:u.get_position()]
# the signature of everything
signature = u.unpack_fstring(len(sender_key))
u.done()
# debugprint("------ ------ ------ ------ hmachish(key=%s, message=%s)\n" % (`symmetric_key`, `end`))
summary = cryptutil.hmacish(key=symmetric_key, message=end)
x = modval.new(sender_key, HARDCODED_RSA_PUBLIC_EXPONENT, signature)
x.undo_signature() # PKop
signed_value = x.get_value()
try:
thingie = cryptutil.oaep_decode(signed_value[1:]) # Leave off the initial 0 byte. ### XX check whether it really is 0 and raise bad-encoding error if not. --Zooko 2000-07-29
except cryptutil.OAEPError, le:
raise Error, 'bad encryption -- pad badding: padded: %s, Error: %s' % (`signed_value`, `le.args`)
### Counter Records End ###
### Counter Sample End ###
### Something else ###
else:
logging.warning("Unknown [Enterprise, Format] " +
str(enterprise_format_num) +
" not defined in sFlow standard - FAIL")
### Sample Parsing Finish ###
# Verify XDR data has been completely unpacked
try:
unpacked_data.done()
except Exception as unpack_done_error:
logging.warning(str(unpack_done_error))
logging.warning("Failed to completely unpack sample data - FAIL")
### sFlow Samples End ###
# Elasticsearch bulk upload
if record_num >= bulk_insert_count:
# Perform the bulk upload to the index
try:
helpers.bulk(es,
sflow_data) # Call Elasticsearch bulk upload API
logging.info(
str(record_num) +
