def __handle_versions(self, params):
"""
Parse the client's message about the plugins it knows.
Activate and deactivate plugins accordingly.
"""
versions = {}
while len(params) > 0:
(name, params) = extract_string(params)
(version, md5_hash) = struct.unpack('!H16s', params[:18])
(lib, params) = extract_string(params[18:])
p_activity = params[0]
params = params[1:]
versions[name] = {
'name': name,
'version': version,
'hash': md5_hash,
'activity': (p_activity == 'A'),
'lib': lib
}
self.__check_versions(versions)
now = database.now()
def log_versions(transaction):
logger.debug("Replacing plugin list of %s", self.cid())
# The current state (override anything previous)
transaction.execute('DELETE FROM active_plugins WHERE client IN (SELECT id FROM clients WHERE name = %s)', (self.cid(),))
transaction.executemany("INSERT INTO active_plugins (client, name, updated, version, hash, libname, active) SELECT clients.id, %s, %s, %s, %s, %s, %s FROM clients WHERE clients.name = %s", map(lambda plug: (plug['name'], now, plug['version'], plug['hash'].encode('hex'), plug['lib'], plug['activity'], self.cid()), versions.values()))
# The history, just append (yes, there may be duplicates, but who cares)
transaction.executemany("INSERT INTO plugin_history (client, name, timestamp, version, hash, active) SELECT clients.id, %s, %s, %s, %s, %s FROM clients WHERE clients.name = %s", map(lambda plug: (plug['name'], now, plug['version'], plug['hash'].encode('hex'), plug['activity'], self.cid()), versions.values()))
return True
activity.push(log_versions)
def datagramReceived(self, dgram, addr):
logger.trace("Packet from %s", addr)
if len(dgram) < 13:
logger.warn(
"Spoof packet too short (probably a stray one), only %s bytes",
len(dgram))
return
(magic, token, spoofed) = struct.unpack('!LQ?', dgram[:13])
if magic != 0x17ACEE43:
logger.warn(
"Wrong magic number in spoof packet (probably a stray one)")
return
tok = self.__spoof.get_token(token)
if not tok:
logger.warn("Token %s not known", token)
return
if spoofed:
tok.expect_spoofed = False
else:
tok.expect_ordinary = False
if not tok.expect_spoofed and not tok.expect_ordinary:
self.__spoof.drop_token(token)
reactor.callInThread(store_packet, tok, spoofed, (not spoofed)
or (addr[0] == self.__spoof.src_addr()), addr[0],
database.now())
activity.log_activity(tok.client(), 'spoof')
def message_from_client(self, message, client):
if message[0] == 'L':
activity.log_activity(client, 'fake')
# max_size - sizeof(14)
if not self.__rate_limiter.check_rate(client, 1):
logger.warn(
"Storing fake server log events for client %s blocked by rate limiter.",
client)
return
# maximal number of records in one message is not specified for fake plugin (as far as I know)
# but the message shouldn't be much larger then max_size bytes (from documentation)
# so 3 times max_size should be fine (2 times when the client reconnects + reserve - "the message may be actually larger, usually by the last event")
if len(message[1:]) > 3 * int(self.__config['max_size']):
logger.warn(
"Unexpectedly long message for fake plugin from client %s - %s bytes, max expected size %s. Ignoring.",
client, len(message[1:]), 3 * self.__config['max_size'])
return
reactor.callInThread(store_logs, message[1:], client,
database.now(), self.version(client))
elif message[0] == 'C':
config = struct.pack(
'!IIIII',
*map(lambda name: int(self.__config[name]), [
'version', 'max_age', 'max_size', 'max_attempts',
'throttle_holdback'
]))
self.send('C' + config, client)
else:
logger.error("Unknown message from client %s: %s", client, message)
def message_from_client(self, message, client):
if message[0] == 'L':
activity.log_activity(client, 'fake')
reactor.callInThread(store_logs, message[1:], client, database.now(), self.version(client))
elif message[0] == 'C':
config = struct.pack('!IIIII', *map(lambda name: int(self.__config[name]), ['version', 'max_age', 'max_size', 'max_attempts', 'throttle_holdback']))
self.send('C' + config, client)
else:
logger.error("Unknown message from client %s: %s", client, message)
def __process(self):
if not self.__data:
return # No data to store.
# As manipulation with DM might be time consuming (as it may be blocking, etc),
# move it to a separate thread, so we don't block the communication. This is
# safe -- we pass all the needed data to it as parameters and get rid of our
# copy, passing the ownership to the task.
reactor.callInThread(store_bandwidth, self.__data, database.now())
self.__data = {}
def __process(self):
if not self.__data:
return # No data to store.
# As manipulation with DM might be time consuming (as it may be blocking, etc),
# move it to a separate thread, so we don't block the communication. This is
# safe -- we pass all the needed data to it as parameters and get rid of our
# copy, passing the ownership to the task.
reactor.callInThread(store_bandwidth, self.__data, database.now())
self.__data = {}
def message_from_client(self, message, client):
if message == 'C':
logger.debug("Sending config %s to client %s", self.__config['version'], client)
config = struct.pack('!IIIIQQ', *map(lambda name: int(self.__config[name]), ['version', 'finished_limit', 'send_limit', 'undecided_limit', 'timeout', 'max_age']))
self.send('C' + config, client)
elif message[0] == 'D':
activity.log_activity(client, 'refused')
reactor.callInThread(store_connections, message[1:], client, database.now())
else:
logger.error("Unknown message from client %s: %s", client, message)
def do_perf_v2_test(self, tasks, ip_to, ip_from, port_list):
'''
args:
tasks : multiprocessing.JoinableQueue, the task queue to consumers
ip_to : str, the ip address of test node
ip_from : str, the ip address of test node
port_list : list(int), the port list genreate by Producer
'''
idx = sum([self.nodes_ip.index(ip) for ip in [ip_to, ip_from]]) % len(self.nodes_ip)
port = port_list[idx]
tasks.put(Task(kind=TaskKind.PERFV2TEST, ip=[ip_to, ip_from], port=port))
self.db.update_top([ip_to, ip_from], TaskKind.PERFV2TEST, Result.WAIT, now())
def message_from_client(self, message, client):
if message[0] == 'L':
activity.log_activity(client, 'fake')
reactor.callInThread(store_logs, message[1:], client,
database.now(), self.version(client))
elif message[0] == 'C':
config = struct.pack(
'!IIIII',
*map(lambda name: int(self.__config[name]), [
'version', 'max_age', 'max_size', 'max_attempts',
'throttle_holdback'
]))
self.send('C' + config, client)
else:
logger.error("Unknown message from client %s: %s", client, message)
def connectionLost(self, reason):
if not self.__connected:
return
self.__connected = False
if self.__logged_in:
logger.info("Connection lost from %s", self.cid())
self.__pinger.stop()
self.__plugins.unregister_client(self)
now = database.now()
def log_plugins(transaction):
logger.debug("Dropping plugin list of %s", self.cid())
transaction.execute("INSERT INTO plugin_history (client, name, timestamp, active) SELECT ap.client, ap.name, %s, false FROM active_plugins AS ap JOIN clients ON ap.client = clients.id WHERE clients.name = %s", (now, self.cid()))
transaction.execute('DELETE FROM active_plugins WHERE client IN (SELECT id FROM clients WHERE name = %s)', (self.cid(),))
return True
activity.push(log_plugins)
activity.log_activity(self.cid(), "logout")
self.transport.abortConnection()
def __handle_versions(self, params):
"""
Parse the client's message about the plugins it knows.
Activate and deactivate plugins accordingly.
"""
versions = {}
while len(params) > 0:
(name, params) = extract_string(params)
(version, md5_hash) = struct.unpack('!H16s', params[:18])
(lib, params) = extract_string(params[18:])
p_activity = params[0]
params = params[1:]
versions[name] = {
'name': name,
'version': version,
'hash': md5_hash,
'activity': (p_activity == 'A'),
'lib': lib
}
self.__check_versions(versions)
now = database.now()
def log_versions(transaction):
logger.debug("Replacing plugin list of %s", self.cid())
# The current state (override anything previous)
transaction.execute(
'DELETE FROM active_plugins WHERE client IN (SELECT id FROM clients WHERE name = %s)',
(self.cid(), ))
transaction.executemany(
"INSERT INTO active_plugins (client, name, updated, version, hash, libname, active) SELECT clients.id, %s, %s, %s, %s, %s, %s FROM clients WHERE clients.name = %s",
map(
lambda plug:
(plug['name'], now, plug['version'], plug['hash'].encode(
'hex'), plug['lib'], plug['activity'], self.cid()),
versions.values()))
# The history, just append (yes, there may be duplicates, but who cares)
transaction.executemany(
"INSERT INTO plugin_history (client, name, timestamp, version, hash, active) SELECT clients.id, %s, %s, %s, %s, %s FROM clients WHERE clients.name = %s",
map(
lambda plug: (plug['name'], now, plug['version'], plug[
'hash'].encode('hex'), plug['activity'], self.cid()),
versions.values()))
return True
activity.push(log_versions)
def message_from_client(self, message, client):
if message[0] == 'C':
logger.debug('Sending config to %s', client)
if self.version(client) < 2:
self.send(self.__build_config(''), client)
else:
self.send(self.__build_config('-diff'), client)
for a in self._addresses:
self.send(
self.__build_filter_version(a, self._addresses[a][0],
self._addresses[a][1]),
client)
elif message[0] == 'D':
logger.debug('Flows from %s', client)
activity.log_activity(client, 'flow')
reactor.callInThread(store_flows, client, message[1:],
int(self._conf['version']), database.now())
elif message[0] == 'U':
self._provide_diff(message[1:], client)
def __process_keys(self):
"""
Extract the keys aggregated in groups and send them to storage.
"""
self.__aggregating_keys = False
if not self.__have_keys:
# We are done with NOP now.
self.__background_processing = False
return
def done(result):
self.__background_processing = False
if result is Failure:
logger.error("Failed to push anomalies to DB: %s",
result.value)
deferred = threads.deferToThread(store_keys, self.__groups,
database.now())
deferred.addCallback(done)
def connectionLost(self, reason):
if not self.__connected:
return
self.__connected = False
if self.__logged_in:
logger.info("Connection lost from %s", self.cid())
self.__pinger.stop()
self.__plugins.unregister_client(self)
now = database.now()
def log_plugins(transaction):
logger.debug("Dropping plugin list of %s", self.cid())
transaction.execute(
"INSERT INTO plugin_history (client, name, timestamp, active) SELECT ap.client, ap.name, %s, false FROM active_plugins AS ap JOIN clients ON ap.client = clients.id WHERE clients.name = %s",
(now, self.cid()))
transaction.execute(
'DELETE FROM active_plugins WHERE client IN (SELECT id FROM clients WHERE name = %s)',
(self.cid(), ))
return True
activity.push(log_plugins)
activity.log_activity(self.cid(), "logout")
self.transport.abortConnection()
def message_from_client(self, message, client):
if message == 'C':
logger.debug("Sending config %s to client %s",
self.__config['version'], client)
config = struct.pack(
'!IIIIQQ',
*map(lambda name: int(self.__config[name]), [
'version', 'finished_limit', 'send_limit',
'undecided_limit', 'timeout', 'max_age'
]))
self.send('C' + config, client)
elif message[0] == 'D':
activity.log_activity(client, 'refused')
if not self.__rate_limiter.check_rate(client, 1):
logger.warn(
"Storing refused connections for client %s blocked by rate limiter.",
client)
return
# the limit for the number of records in a message is 2*send_limit because the client may buffer up to two times the number if he disconnects/reconnects
reactor.callInThread(store_connections,
2 * int(self.__config['send_limit']),
message[1:], client, database.now())
else:
logger.error("Unknown message from client %s: %s", client, message)
def success(self, client, payload):
reactor.callInThread(store_certs, client, payload, self.__hosts,
self.__batch_time, database.now())
log_activity(client, 'certs')
def success(self, client, payload):
reactor.callInThread(submit_data, client, payload, self.__hosts,
self.__batch_time, database.now())
log_activity(client, 'pings')
def message_from_client(self, message, client):
if message[0] == 'C':
if client in self.__delayed_config:
# The client asks for a second time in a short while. Don't send anything now, but do send it a bit later
logger.info('Delaying config for %s', client)
self.__delayed_config[client] = True
return
logger.debug('Sending config to %s', client)
self.__delayed_config[client] = False # We know about the client, but it hasn't asked twice yet.
if self.version(client) < 2:
self.send(self.__build_config(''), client)
else:
self.send(self.__build_config('-diff'), client)
for a in self._addresses:
self.send(self.__build_filter_version(a, self._addresses[a][0], self._addresses[a][1]), client)
elif message[0] == 'D':
logger.debug('Flows from %s', client)
activity.log_activity(client, 'flow')
if not self.__rate_limiter.check_rate(client, 1):
logger.warn("Storing flows for client %s blocked by rate limiter.", client)
return
# the limit for the number of records in a message is 2*max_flows because the client may buffer up to two times the number if he disconnects/reconnects
reactor.callInThread(store_flows, 2 * int(self._conf['max_flows']), client, message[1:], int(self._conf['version']), database.now())
elif message[0] == 'U':
self._provide_diff(message[1:], client)
def message_from_client(self, message, client):
if message[0] == 'C':
logger.debug('Sending config to %s', client)
if self.version(client) < 2:
self.send(self.__build_config(''), client)
else:
self.send(self.__build_config('-diff'), client)
for a in self._addresses:
self.send(self.__build_filter_version(a, self._addresses[a][0], self._addresses[a][1]), client)
elif message[0] == 'D':
logger.debug('Flows from %s', client)
activity.log_activity(client, 'flow')
reactor.callInThread(store_flows, client, message[1:], int(self._conf['version']), database.now())
elif message[0] == 'U':
self._provide_diff(message[1:], client)
def message_from_client(self, message, client):
if message == 'C':
logger.debug("Sending config %s to client %s", self.__config['version'], client)
config = struct.pack('!IIIIQQ', *map(lambda name: int(self.__config[name]), ['version', 'finished_limit', 'send_limit', 'undecided_limit', 'timeout', 'max_age']))
self.send('C' + config, client)
elif message[0] == 'D':
activity.log_activity(client, 'refused')
if not self.__rate_limiter.check_rate(client, 1):
logger.warn("Storing refused connections for client %s blocked by rate limiter.", client)
return
# the limit for the number of records in a message is 2*send_limit because the client may buffer up to two times the number if he disconnects/reconnects
reactor.callInThread(store_connections, 2 * int(self.__config['send_limit']), message[1:], client, database.now())
else:
logger.error("Unknown message from client %s: %s", client, message)
def signal_handler(*args):
for c in consumers:
os.kill(c.pid, signal.SIGTERM)
db.update_info(-1, end=now())
sys.exit()
def run(self):
# Store pid, date into DataBase
self.db.update_info(self.pid, start=now())
# Init node info
nodes_info = {}
for ip in self.nodes_ip:
nodes_info[ip] = NodeInfo(ip)
# Establish communication queues
tasks = multiprocessing.JoinableQueue()
results = multiprocessing.Queue()
# Create result path
remove_dir(self.result_path)
make_dir(self.result_path)
for kind in TaskKind:
make_dir(os.path.join(self.result_path, kind.value))
# Start consumers
print('Creating {} consumers'.format(self.num_consumers))
consumers = [
Consumer(tasks, results, self.result_path, self.target_path)
for _ in range(self.num_consumers)
]
for w in consumers:
w.daemon = True
w.start()
db = self.db
def signal_handler(*args):
for c in consumers:
os.kill(c.pid, signal.SIGTERM)
db.update_info(-1, end=now())
sys.exit()
signal.signal(signal.SIGTERM, signal_handler)
# Start Time
print("Test Start at {}".format(now()))
# Number of Task have enqueue
ntasks = 0
# Enqueue task first - "no password check"
for ip in self.nodes_ip:
tasks.put(Task(kind=TaskKind.NOPWCHECK, ip=ip))
nodes_info[ip].status = TaskKind.NOPWCHECK
# +2 because there is ACCEPT and SUCC -> two result
ntasks += 2
self.db.update_top(ip, TaskKind.NOPWCHECK, Result.WAIT, now())
conn_check_waiting_list = []
# Generate port List for ucx test
# [2001, 2002, ..., 2000+len(nodes_ip)]
port_list = [2001+i for i in range(0, len(self.nodes_ip))]
while ntasks > 0:
result = results.get()
ntasks -= 1
print('Result: {}'.format(result))
if result.code == Result.SUCC:
# enqueue more task
if result.kind == TaskKind.NOPWCHECK:
tasks.put(Task(kind=TaskKind.ENVCHECK, ip=result.ip))
nodes_info[result.ip].status = TaskKind.ENVCHECK
ntasks += 2
self.db.update_top(result.ip, TaskKind.ENVCHECK, Result.WAIT, now())
elif result.kind == TaskKind.ENVCHECK:
tasks.put(Task(kind=TaskKind.SETUP, ip=result.ip))
nodes_info[result.ip].status = TaskKind.SETUP
ntasks += 2
self.db.update_top(result.ip, TaskKind.SETUP, Result.WAIT, now())
elif result.kind == TaskKind.SETUP:
self.db.delete_top(result.ip)
for ip in conn_check_waiting_list:
ip1, ip2 = (result.ip, ip) if IPAddress(result.ip) < IPAddress(ip) else (ip, result.ip)
tasks.put(Task(kind=TaskKind.CONNCHECK, ip=[ip1, ip2]))
ntasks += 2
self.db.update_top([ip1, ip2], TaskKind.CONNCHECK, Result.WAIT, now())
conn_check_waiting_list.append(result.ip)
nodes_info[result.ip].status = TaskKind.CONNCHECK
elif result.kind == TaskKind.CONNCHECK:
self.db.delete_top(result.ip)
ip1 = result.ip[0]
ip2 = result.ip[1]
if self.same_cluster(ip1, ip2):
continue
if nodes_info[ip1].occupied is False and \
nodes_info[ip2].occupied is False:
# both ip is available
self.do_ucx_test(tasks, ip1, ip2, port_list)
ntasks += 2
# Occupied
nodes_info[ip1].occupied = True
nodes_info[ip2].occupied = True
else:
# at lease have one ip is not available
nodes_info[ip1].dep_list.append(ip2)
nodes_info[ip2].dep_list.append(ip1)
elif result.kind == TaskKind.UCXTEST:
self.db.delete_top(result.ip)
ip1 = result.ip[0]
ip2 = result.ip[1]
self.handle_ucx_test_result(result)
# Just go ahead and do next two-IPs test
assert nodes_info[ip1].occupied == True
assert nodes_info[ip2].occupied == True
# Do perf_v2_test
self.do_perf_v2_test(tasks, ip1, ip2, port_list)
ntasks += 2
# Maybe have to enqueue more task
elif result.kind == TaskKind.PERFV2TEST:
self.db.delete_top(result.ip)
self.handle_perf_v2_test_result(result)
nodes_info[result.ip[0]].occupied = False
nodes_info[result.ip[1]].occupied = False
# Find dependence, and Enqueue new UCX task
for ipx in result.ip:
if nodes_info[ipx].occupied is True: continue
for ipy in nodes_info[ipx].dep_list :
if nodes_info[ipy].occupied is False:
ip1, ip2 = (ipx, ipy) if IPAddress(ipx) < IPAddress(ipy) else (ipy, ipx)
self.do_ucx_test(tasks, ip1, ip2, port_list)
ntasks += 2
# Occupied
nodes_info[ip1].occupied = True
nodes_info[ip2].occupied = True
# Remove from List
nodes_info[ip1].dep_list.remove(ip2)
nodes_info[ip2].dep_list.remove(ip1)
break
elif result.code == Result.FAILED:
if type(result.ip) == str:
nodes_info[result.ip].status = Result.FAILED
self.db.update_top(result.ip, result.kind, Result.FAILED, now())
else:
assert type(result.ip) == list
assert len(result.ip) == 2
nodes_info[result.ip[0]].status = Result.FAILED
nodes_info[result.ip[1]].status = Result.FAILED
self.db.update_top(result.ip, result.kind, Result.FAILED, now())
# release
nodes_info[result.ip[0]].ocupied = False
nodes_info[result.ip[1]].ocupied = False
elif result.code == Result.ACCEPT:
# Update database
self.db.update_top(result.ip, result.kind, Result.ACCEPT, now())
# CleanUp for every remote machine
for ip in self.nodes_ip:
tasks.put(Task(kind=TaskKind.CLEAN, ip=ip))
ntasks += 2
# Do not show CLEAN phase (it will overwrite previous failure information)
#self.db.update_top(ip, TaskKind.CLEAN, Result.WAIT, now())
while ntasks > 0:
result = results.get()
ntasks -= 1
print('Result: {}'.format(result))
if result.kind == TaskKind.CLEAN:
#self.db.delete_top(result.ip)
if result.code == Result.FAILED:
assert type(result.ip) == str, "CLEAN IP: {}".format(result.ip)
#self.db.update_top(result.ip, result.kind, Result.FAILED, now())
print("{} CLEAN FAILED".format(result.ip))
#elif result.code == Result.ACCEPT:
#self.db.update_top(result.ip, result.kind, Result.ACCEPT, now())
else:
raise Exception("Clean must be after all tasks finished")
# Wait for all of the tasks to finish
tasks.join()
# Add a poison pill for each consumer
for _ in range(self.num_consumers):
tasks.put(None)
self.db.update_info(-1, end=now())
def success(self, client, payload): reactor.callInThread(store_certs, client, payload, self.__hosts, self.__batch_time, database.now()) log_activity(client, 'certs')
def printutf8(s, ofile=sys.stdout.buffer):
outs = database.now() + " " + str(s) + "\n"
ofile.write(outs.encode('utf-8'))
ofile.flush()
def datagramReceived(self, dgram, addr):
logger.trace("Packet from %s", addr)
if len(dgram) < 13:
logger.warn("Spoof packet too short (probably a stray one), only %s bytes", len(dgram))
return
(magic, token, spoofed) = struct.unpack('!LQ?', dgram[:13])
if magic != 0x17ACEE43:
logger.warn("Wrong magic number in spoof packet (probably a stray one)")
return
tok = self.__spoof.get_token(token)
if not tok:
logger.warn("Token %s not known", token)
return
if spoofed:
tok.expect_spoofed = False
else:
tok.expect_ordinary = False
if not tok.expect_spoofed and not tok.expect_ordinary:
self.__spoof.drop_token(token)
reactor.callInThread(store_packet, tok, spoofed, (not spoofed) or (addr[0] == self.__spoof.src_addr()), addr[0], database.now())
activity.log_activity(tok.client(), 'spoof')