def datagramReceived(self, dgram, addr):
logger.trace("Packet from %s", addr)
if len(dgram) < 13:
logger.warn(
"Spoof packet too short (probably a stray one), only %s bytes",
len(dgram))
return
(magic, token, spoofed) = struct.unpack('!LQ?', dgram[:13])
if magic != 0x17ACEE43:
logger.warn(
"Wrong magic number in spoof packet (probably a stray one)")
return
tok = self.__spoof.get_token(token)
if not tok:
logger.warn("Token %s not known", token)
return
if spoofed:
tok.expect_spoofed = False
else:
tok.expect_ordinary = False
if not tok.expect_spoofed and not tok.expect_ordinary:
self.__spoof.drop_token(token)
reactor.callInThread(store_packet, tok, spoofed, (not spoofed)
or (addr[0] == self.__spoof.src_addr()), addr[0],
database.now())
activity.log_activity(tok.client(), 'spoof')
def message_from_client(self, message, client):
if message[0] == 'C':
if client in self.__delayed_config:
# The client asks for a second time in a short while. Don't send anything now, but do send it a bit later
logger.info('Delaying config for %s', client)
self.__delayed_config[client] = True
return
logger.debug('Sending config to %s', client)
self.__delayed_config[client] = False # We know about the client, but it hasn't asked twice yet.
if self.version(client) < 2:
self.send(self.__build_config(''), client)
else:
self.send(self.__build_config('-diff'), client)
for a in self._addresses:
self.send(self.__build_filter_version(a, self._addresses[a][0], self._addresses[a][1]), client)
elif message[0] == 'D':
logger.debug('Flows from %s', client)
activity.log_activity(client, 'flow')
if not self.__rate_limiter.check_rate(client, 1):
logger.warn("Storing flows for client %s blocked by rate limiter.", client)
return
# the limit for the number of records in a message is 2*max_flows because the client may buffer up to two times the number if he disconnects/reconnects
reactor.callInThread(store_flows, 2 * int(self._conf['max_flows']), client, message[1:], int(self._conf['version']), database.now())
elif message[0] == 'U':
self._provide_diff(message[1:], client)
def message_from_client(self, message, client):
kind = message[0]
if kind == 'C':
logger.debug('Config %s for client %s at %s',
self.__config_version, client, int(time.time()))
# It asks for config. Send some.
self.send('C' + self.__config(), client)
# And that makes it active.
self.__clients[client].activate()
elif kind == 'G':
# Generation data.
buckets.batch.submit(self.__process_generation, lambda x: None,
message, client)
activity.log_activity(client, "buckets")
elif kind == 'K':
# Got keys from the plugin
(req_id, ) = struct.unpack('!L', message[1:5])
logger.debug('Received keys from %s', client)
buckets.client.manager.response(req_id, message[5:])
self.__have_keys = True
elif kind == 'M':
(req_id, ) = struct.unpack('!L', message[1:5])
buckets.client.manager.missing(req_id)
else:
logger.error('Unknown data from plugin %s: %s', client,
repr(message))
def message_from_client(self, message, client):
if message[0] == 'L':
activity.log_activity(client, 'fake')
# max_size - sizeof(14)
if not self.__rate_limiter.check_rate(client, 1):
logger.warn(
"Storing fake server log events for client %s blocked by rate limiter.",
client)
return
# maximal number of records in one message is not specified for fake plugin (as far as I know)
# but the message shouldn't be much larger then max_size bytes (from documentation)
# so 3 times max_size should be fine (2 times when the client reconnects + reserve - "the message may be actually larger, usually by the last event")
if len(message[1:]) > 3 * int(self.__config['max_size']):
logger.warn(
"Unexpectedly long message for fake plugin from client %s - %s bytes, max expected size %s. Ignoring.",
client, len(message[1:]), 3 * self.__config['max_size'])
return
reactor.callInThread(store_logs, message[1:], client,
database.now(), self.version(client))
elif message[0] == 'C':
config = struct.pack(
'!IIIII',
*map(lambda name: int(self.__config[name]), [
'version', 'max_age', 'max_size', 'max_attempts',
'throttle_holdback'
]))
self.send('C' + config, client)
else:
logger.error("Unknown message from client %s: %s", client, message)
def message_from_client(self, message, client):
"""
This method parses message from client.
"""
# Parse message from client
# Message contains 64bit numbers - 3 numbers for every window
int_count = len(message) / 8
data = struct.unpack("!" + str(int_count) + "Q", message);
logger.debug("Bandwidth data from client %s: %s", client, data)
# Add client's record
if not client in self.__data:
self.__data[client] = ClientData(self.version(client));
# Extract timestamp from message and skip it
timestamp = data[0]
if timestamp < self.__last:
logger.info("Data of bandwidth snapshot on %s too old, ignoring (%s vs. %s)", client, timestamp, self.__last)
return
self.__data[client].timestamp_dbg = timestamp
if self.version(client) <= 1:
int_count -= 1
data = data[1:]
windows = int_count / PROTO_ITEMS_PER_WINDOW
for i in range(0, windows):
self.__data[client].add_window(
data[i*PROTO_ITEMS_PER_WINDOW],
data[i*PROTO_ITEMS_PER_WINDOW+1],
data[i*PROTO_ITEMS_PER_WINDOW+2]
)
elif self.version(client) >= 2:
win_cnt = data[1]
buckets_cnt_pos = 2 + PROTO_ITEMS_PER_WINDOW * win_cnt
data_windows = data[2:buckets_cnt_pos]
buckets_cnt = data[buckets_cnt_pos]
data_buckets = data[(buckets_cnt_pos+1):]
# Get data from message
for i in range(0, win_cnt):
self.__data[client].add_window(
data_windows[i*PROTO_ITEMS_PER_WINDOW],
data_windows[i*PROTO_ITEMS_PER_WINDOW+1],
data_windows[i*PROTO_ITEMS_PER_WINDOW+2]
)
for i in range(0, buckets_cnt):
self.__data[client].add_bucket(
data_buckets[i*PROTO_ITEMS_PER_BUCKET],
data_buckets[i*PROTO_ITEMS_PER_BUCKET+1],
data_buckets[i*PROTO_ITEMS_PER_BUCKET+2],
data_buckets[i*PROTO_ITEMS_PER_BUCKET+3],
data_buckets[i*PROTO_ITEMS_PER_BUCKET+4]
)
# Log client's activity
activity.log_activity(client, "bandwidth")
def message_from_client(self, message, client):
"""
This method parses message from client.
"""
# Parse message from client
# Message contains 64bit numbers - 3 numbers for every window
int_count = len(message) / 8
data = struct.unpack("!" + str(int_count) + "Q", message)
logger.debug("Bandwidth data from client %s: %s", client, data)
# Add client's record
if not client in self.__data:
self.__data[client] = ClientData(self.version(client))
# Extract timestamp from message and skip it
timestamp = data[0]
if timestamp < self.__last:
logger.info(
"Data of bandwidth snapshot on %s too old, ignoring (%s vs. %s)",
client, timestamp, self.__last)
return
self.__data[client].timestamp_dbg = timestamp
if self.version(client) <= 1:
int_count -= 1
data = data[1:]
windows = int_count / PROTO_ITEMS_PER_WINDOW
for i in range(0, windows):
self.__data[client].add_window(
data[i * PROTO_ITEMS_PER_WINDOW],
data[i * PROTO_ITEMS_PER_WINDOW + 1],
data[i * PROTO_ITEMS_PER_WINDOW + 2])
elif self.version(client) >= 2:
win_cnt = data[1]
buckets_cnt_pos = 2 + PROTO_ITEMS_PER_WINDOW * win_cnt
data_windows = data[2:buckets_cnt_pos]
buckets_cnt = data[buckets_cnt_pos]
data_buckets = data[(buckets_cnt_pos + 1):]
# Get data from message
for i in range(0, win_cnt):
self.__data[client].add_window(
data_windows[i * PROTO_ITEMS_PER_WINDOW],
data_windows[i * PROTO_ITEMS_PER_WINDOW + 1],
data_windows[i * PROTO_ITEMS_PER_WINDOW + 2])
for i in range(0, buckets_cnt):
self.__data[client].add_bucket(
data_buckets[i * PROTO_ITEMS_PER_BUCKET],
data_buckets[i * PROTO_ITEMS_PER_BUCKET + 1],
data_buckets[i * PROTO_ITEMS_PER_BUCKET + 2],
data_buckets[i * PROTO_ITEMS_PER_BUCKET + 3],
data_buckets[i * PROTO_ITEMS_PER_BUCKET + 4])
# Log client's activity
activity.log_activity(client, "bandwidth")
def message_from_client(self, message, client):
if message[0] == 'L':
activity.log_activity(client, 'fake')
reactor.callInThread(store_logs, message[1:], client, database.now(), self.version(client))
elif message[0] == 'C':
config = struct.pack('!IIIII', *map(lambda name: int(self.__config[name]), ['version', 'max_age', 'max_size', 'max_attempts', 'throttle_holdback']))
self.send('C' + config, client)
else:
logger.error("Unknown message from client %s: %s", client, message)
def message_from_client(self, message, client):
if message == 'C':
logger.debug("Sending config %s to client %s", self.__config['version'], client)
config = struct.pack('!IIIIQQ', *map(lambda name: int(self.__config[name]), ['version', 'finished_limit', 'send_limit', 'undecided_limit', 'timeout', 'max_age']))
self.send('C' + config, client)
elif message[0] == 'D':
activity.log_activity(client, 'refused')
reactor.callInThread(store_connections, message[1:], client, database.now())
else:
logger.error("Unknown message from client %s: %s", client, message)
def message_from_client(self, message, client):
if message == 'C':
logger.debug("Sending config %s to client %s", self.__config['version'], client)
config = struct.pack('!IIIIQQ', *map(lambda name: int(self.__config[name]), ['version', 'finished_limit', 'send_limit', 'undecided_limit', 'timeout', 'max_age']))
self.send('C' + config, client)
elif message[0] == 'D':
activity.log_activity(client, 'refused')
if not self.__rate_limiter.check_rate(client, 1):
logger.warn("Storing refused connections for client %s blocked by rate limiter.", client)
return
# the limit for the number of records in a message is 2*send_limit because the client may buffer up to two times the number if he disconnects/reconnects
reactor.callInThread(store_connections, 2 * int(self.__config['send_limit']), message[1:], client, database.now())
else:
logger.error("Unknown message from client %s: %s", client, message)
def message_from_client(self, message, client):
if message[0] == 'L':
activity.log_activity(client, 'fake')
reactor.callInThread(store_logs, message[1:], client,
database.now(), self.version(client))
elif message[0] == 'C':
config = struct.pack(
'!IIIII',
*map(lambda name: int(self.__config[name]), [
'version', 'max_age', 'max_size', 'max_attempts',
'throttle_holdback'
]))
self.send('C' + config, client)
else:
logger.error("Unknown message from client %s: %s", client, message)
def message_from_client(self, message, client):
if message[0] == 'C':
logger.debug('Sending config to %s', client)
if self.version(client) < 2:
self.send(self.__build_config(''), client)
else:
self.send(self.__build_config('-diff'), client)
for a in self._addresses:
self.send(self.__build_filter_version(a, self._addresses[a][0], self._addresses[a][1]), client)
elif message[0] == 'D':
logger.debug('Flows from %s', client)
activity.log_activity(client, 'flow')
reactor.callInThread(store_flows, client, message[1:], int(self._conf['version']), database.now())
elif message[0] == 'U':
self._provide_diff(message[1:], client)
def connectionLost(self, reason):
if not self.__connected:
return
self.__connected = False
if self.__logged_in:
logger.info("Connection lost from %s", self.cid())
self.__pinger.stop()
self.__plugins.unregister_client(self)
now = database.now()
def log_plugins(transaction):
logger.debug("Dropping plugin list of %s", self.cid())
transaction.execute("INSERT INTO plugin_history (client, name, timestamp, active) SELECT ap.client, ap.name, %s, false FROM active_plugins AS ap JOIN clients ON ap.client = clients.id WHERE clients.name = %s", (now, self.cid()))
transaction.execute('DELETE FROM active_plugins WHERE client IN (SELECT id FROM clients WHERE name = %s)', (self.cid(),))
return True
activity.push(log_plugins)
activity.log_activity(self.cid(), "logout")
self.transport.abortConnection()
def message_from_client(self, message, client):
if message[0] == 'C':
logger.debug('Sending config to %s', client)
if self.version(client) < 2:
self.send(self.__build_config(''), client)
else:
self.send(self.__build_config('-diff'), client)
for a in self._addresses:
self.send(
self.__build_filter_version(a, self._addresses[a][0],
self._addresses[a][1]),
client)
elif message[0] == 'D':
logger.debug('Flows from %s', client)
activity.log_activity(client, 'flow')
reactor.callInThread(store_flows, client, message[1:],
int(self._conf['version']), database.now())
elif message[0] == 'U':
self._provide_diff(message[1:], client)
def datagramReceived(self, dgram, addr):
logger.trace("Packet from %s", addr)
if len(dgram) < 13:
logger.warn("Spoof packet too short (probably a stray one), only %s bytes", len(dgram))
return
(magic, token, spoofed) = struct.unpack('!LQ?', dgram[:13])
if magic != 0x17ACEE43:
logger.warn("Wrong magic number in spoof packet (probably a stray one)")
return
tok = self.__spoof.get_token(token)
if not tok:
logger.warn("Token %s not known", token)
return
if spoofed:
tok.expect_spoofed = False
else:
tok.expect_ordinary = False
if not tok.expect_spoofed and not tok.expect_ordinary:
self.__spoof.drop_token(token)
reactor.callInThread(store_packet, tok, spoofed, (not spoofed) or (addr[0] == self.__spoof.src_addr()), addr[0], database.now())
activity.log_activity(tok.client(), 'spoof')
def message_from_client(self, message, client):
count = len(message) / 4 - 2 # 2 for the timestamp
dtype = 'L'
data = struct.unpack('!Q' + str(count) + 'L', message)
if data[0] < self.__last:
logger.info("Data snapshot on %s too old, ignoring (%s vs. %s)", client, data[0], self.__last)
return
if_count = data[1]
self.__stats[client] = data[2:2 + 3 * if_count]
d = data[2 + 3 * if_count:]
if len(d) > 32:
# TODO: Remove this hack. It is temporary for the time when we have both clients
# sending 32bit sizes and 64bit sizes. If it's too long, it is 64bit - reencode
# the data and decode as 64bit ints.
packed = struct.pack("!" + str(len(d)) + 'L', *d)
d = struct.unpack('!' + str(len(d) / 2) + 'Q', packed)
self.__data[client] = d
logger.debug("Data: %s", data)
if len(self.__data[client]) % 2:
logger.error("Odd count of data elements (%s) from %s", len(self.__data[client]), client)
activity.log_activity(client, "counts")
def connectionLost(self, reason):
if not self.__connected:
return
self.__connected = False
if self.__logged_in:
logger.info("Connection lost from %s", self.cid())
self.__pinger.stop()
self.__plugins.unregister_client(self)
now = database.now()
def log_plugins(transaction):
logger.debug("Dropping plugin list of %s", self.cid())
transaction.execute(
"INSERT INTO plugin_history (client, name, timestamp, active) SELECT ap.client, ap.name, %s, false FROM active_plugins AS ap JOIN clients ON ap.client = clients.id WHERE clients.name = %s",
(now, self.cid()))
transaction.execute(
'DELETE FROM active_plugins WHERE client IN (SELECT id FROM clients WHERE name = %s)',
(self.cid(), ))
return True
activity.push(log_plugins)
activity.log_activity(self.cid(), "logout")
self.transport.abortConnection()
def message_from_client(self, message, client):
count = len(message) / 4 - 2 # 2 for the timestamp
dtype = 'L'
data = struct.unpack('!Q' + str(count) + 'L', message)
if data[0] < self.__last:
logger.info("Data snapshot on %s too old, ignoring (%s vs. %s)",
client, data[0], self.__last)
return
if_count = data[1]
self.__stats[client] = data[2:2 + 3 * if_count]
d = data[2 + 3 * if_count:]
if len(d) > 32:
# TODO: Remove this hack. It is temporary for the time when we have both clients
# sending 32bit sizes and 64bit sizes. If it's too long, it is 64bit - reencode
# the data and decode as 64bit ints.
packed = struct.pack("!" + str(len(d)) + 'L', *d)
d = struct.unpack('!' + str(len(d) / 2) + 'Q', packed)
self.__data[client] = d
logger.debug("Data: %s", data)
if len(self.__data[client]) % 2:
logger.error("Odd count of data elements (%s) from %s",
len(self.__data[client]), client)
activity.log_activity(client, "counts")
def message_from_client(self, message, client):
if message == 'C':
logger.debug("Sending config %s to client %s",
self.__config['version'], client)
config = struct.pack(
'!IIIIQQ',
*map(lambda name: int(self.__config[name]), [
'version', 'finished_limit', 'send_limit',
'undecided_limit', 'timeout', 'max_age'
]))
self.send('C' + config, client)
elif message[0] == 'D':
activity.log_activity(client, 'refused')
if not self.__rate_limiter.check_rate(client, 1):
logger.warn(
"Storing refused connections for client %s blocked by rate limiter.",
client)
return
# the limit for the number of records in a message is 2*send_limit because the client may buffer up to two times the number if he disconnects/reconnects
reactor.callInThread(store_connections,
2 * int(self.__config['send_limit']),
message[1:], client, database.now())
else:
logger.error("Unknown message from client %s: %s", client, message)
def success(self, client, payload):
reactor.callInThread(submit_data, client, payload, self.__batch_time)
log_activity(client, 'nat')
def stringReceived(self, string):
if self.__wait_auth:
self.__auth_buffer.append(string)
return
(msg, params) = (string[0], string[1:])
logger.trace("Received from %s: %s", self.cid(), repr(string))
if not self.__logged_in:
def login_failure(msg):
logger.warn('Login failure from %s: %s', self.cid(), msg)
self.sendString('F')
self.__challenge = None # Prevent more attempts
# Keep the connection open, but idle. Prevents very fast
# reconnects.
if msg == 'L':
# Client wants to log in.
# Extract parameters.
(version, params) = (params[0], params[1:])
(cid, params) = extract_string(params)
(response, params) = extract_string(params)
self.__cid = cid
if version == 'O':
self.__cid = self.__cid.encode('hex')
logger.debug('Client %s sent login info', self.cid())
if params != '':
login_failure('Protocol violation')
return
log_info = None
if version != 'O':
login_failure('Login scheme not implemented')
return
# A callback once we receive decision if the client is allowed
def auth_finished(allowed):
self.__wait_auth = False
if allowed:
self.__authenticated = True
# Replay the bufferend messages
for message in self.__auth_buffer:
self.stringReceived(message)
else:
login_failure('Incorrect password')
self.__auth_buffer = None
# Ask the authenticator
if self.__challenge:
auth.auth(auth_finished, self.__cid,
self.__challenge.encode('hex'),
response.encode('hex'))
self.__wait_auth = True
elif msg == 'H':
if self.__authenticated:
if len(params) >= 1:
(self.__proto_version, ) = struct.unpack(
"!B", params[0])
if self.__proto_version >= 1:
self.__available_plugins = {}
if self.__plugins.register_client(self):
if self.__proto_version == 0:
# Activate all the clients in the old version.
# The new protocol handles activation on plugin-by-plugin basis
for p in self.__plugins.get_plugins():
self.__plugins.activate_client(p, self)
else:
# Please tell me when there're changes to the allowed plugins
plugin_versions.add_client(self)
self.__logged_in = True
self.__pinger = timers.timer(
self.__ping,
45 if self.cid() in self.__fastpings else 120,
False)
activity.log_activity(self.cid(), "login")
logger.info('Client %s logged in', self.cid())
else:
return
else:
login_failure('Asked for session before loging in')
return
elif msg == 'S':
if len(params) != 4:
logger.warn("Wrong session ID length on client %s: %s",
self.cid(), len(params))
return
(self.session_id, ) = struct.unpack("!I", params)
return
elif msg == 'P': # Ping. Answer pong.
self.sendString('p' + params)
elif msg == 'p': # Pong. Reset the watchdog count
self.__pings_outstanding = 0
self.last_pong = time.time()
elif msg == 'R': # Route data to a plugin
(plugin, data) = extract_string(params)
self.__plugins.route_to_plugin(plugin, data, self.cid())
# TODO: Handle the possibility the plugin doesn't exist somehow (#2705)
elif msg == 'V': # New list of versions of the client
if self.__proto_version == 0:
self.__available_plugins = {}
while len(params) > 0:
(name, params) = extract_string(params)
(version, ) = struct.unpack('!H', params[:2])
self.__available_plugins[name] = version
params = params[2:]
else:
self.__handle_versions(params)
else:
logger.warn("Unknown message from client %s: %s", self.cid(), msg)
def success(self, client, payload): reactor.callInThread(store_certs, client, payload, self.__hosts, self.__batch_time, database.now()) log_activity(client, 'certs')
def success(self, client, payload):
reactor.callInThread(submit_data, client, payload, self.__hosts,
self.__batch_time, database.now())
log_activity(client, 'pings')
def success(self, client, payload): reactor.callInThread(submit_data, client, payload, self.__batch_time) log_activity(client, 'nat')
def stringReceived(self, string):
if self.__wait_auth:
self.__auth_buffer.append(string)
return
(msg, params) = (string[0], string[1:])
logger.trace("Received from %s: %s", self.cid(), repr(string))
if not self.__logged_in:
def login_failure(msg):
logger.warn('Login failure from %s: %s', self.cid(), msg)
self.sendString('F')
self.__challenge = None # Prevent more attempts
# Keep the connection open, but idle. Prevents very fast
# reconnects.
if msg == 'L':
# Client wants to log in.
# Extract parameters.
(version, params) = (params[0], params[1:])
(cid, params) = extract_string(params)
(response, params) = extract_string(params)
self.__cid = cid
if version == 'O':
self.__cid = self.__cid.encode('hex')
logger.debug('Client %s sent login info', self.cid())
if params != '':
login_failure('Protocol violation')
return
log_info = None
if version != 'O':
login_failure('Login scheme not implemented')
return
# A callback once we receive decision if the client is allowed
def auth_finished(allowed):
self.__wait_auth = False
if allowed:
self.__authenticated = True
# Replay the bufferend messages
for message in self.__auth_buffer:
self.stringReceived(message)
else:
login_failure('Incorrect password')
self.__auth_buffer = None
# Ask the authenticator
if self.__challenge:
auth.auth(auth_finished, self.__cid, self.__challenge.encode('hex'), response.encode('hex'))
self.__wait_auth = True
elif msg == 'H':
if self.__authenticated:
if len(params) >= 1:
(self.__proto_version,) = struct.unpack("!B", params[0])
if self.__proto_version >= 1:
self.__available_plugins = {}
if self.__plugins.register_client(self):
if self.__proto_version == 0:
# Activate all the clients in the old version.
# The new protocol handles activation on plugin-by-plugin basis
for p in self.__plugins.get_plugins():
self.__plugins.activate_client(p, self)
else:
# Please tell me when there're changes to the allowed plugins
plugin_versions.add_client(self)
self.__logged_in = True
self.__pinger = timers.timer(self.__ping, 45 if self.cid() in self.__fastpings else 120, False)
activity.log_activity(self.cid(), "login")
logger.info('Client %s logged in', self.cid())
else:
return
else:
login_failure('Asked for session before loging in')
return
elif msg == 'S':
if len(params) != 4:
logger.warn("Wrong session ID length on client %s: %s", self.cid(), len(params))
return
(self.session_id,) = struct.unpack("!I", params)
return
elif msg == 'P': # Ping. Answer pong.
self.sendString('p' + params)
elif msg == 'p': # Pong. Reset the watchdog count
self.__pings_outstanding = 0
self.last_pong = time.time()
elif msg == 'R': # Route data to a plugin
(plugin, data) = extract_string(params)
self.__plugins.route_to_plugin(plugin, data, self.cid())
# TODO: Handle the possibility the plugin doesn't exist somehow (#2705)
elif msg == 'V': # New list of versions of the client
if self.__proto_version == 0:
self.__available_plugins = {}
while len(params) > 0:
(name, params) = extract_string(params)
(version,) = struct.unpack('!H', params[:2])
self.__available_plugins[name] = version
params = params[2:]
else:
self.__handle_versions(params)
else:
logger.warn("Unknown message from client %s: %s", self.cid(), msg)
def success(self, client, payload):
reactor.callInThread(store_certs, client, payload, self.__hosts,
self.__batch_time, database.now())
log_activity(client, 'certs')