def __init__(self, hostname, port, home_dir, node_name='anonymous_node',
ks_path=None, ks_passwd=None, node_type=None,
bind_host='0.0.0.0', config={}):
self.hostname = hostname
self.bind_host = bind_host
self.port = port
self.home_dir = home_dir
self.node_name = node_name
self.node_type = node_type
self.config = config
self.__auth_key = hashlib.sha1(str(uuid.uuid4())).hexdigest()
config_file = os.path.join(home_dir, CONFIG_FILE_NAME)
Config.load(config_file)
self.config.update(Config.get_config_dict())
if ks_path:
self.keystore = init_keystore(ks_path, ks_passwd)
ca_files = self.keystore.autodetect_ca(home_dir)
if ca_files:
logger.info('CA certificates detected at: %s'%(' '.join(ca_files),))
else:
logger.warning('No one CA certificate autodetected')
else:
self.keystore = None
self.server = None
self.operator_process = None
self.osproc_stat = None
cur_thread = threading.current_thread()
cur_thread.setName('%s-main'%self.node_name)
self.operators_map = {'base': Operator}
self.operators_map.update( PluginsManager.get_operators() )
def create_net(self, nodes_count):
global PROCESSES
global ADDRESSES
for unuse in range(nodes_count):
if not ADDRESSES:
n_node = 'init-fabnet'
i = 1900
else:
n_node = random.choice(ADDRESSES)
i = int(ADDRESSES[-1].split(':')[-1])+1
self._wait_node(n_node)
address = '127.0.0.1:%s'%i
ADDRESSES.append(address)
home = '/tmp/node_%s'%i
if os.path.exists(home):
shutil.rmtree(home)
os.mkdir(home)
os.system('cp fabnet_core/tests/cert/test_certs.ca %s/'%home)
logger.warning('{SNP} STARTING NODE %s'%address)
if n_node == 'init-fabnet':
ntype = 'MGMT'
Config.load(os.path.join(home, 'fabnet.conf'))
Config.update_config({'db_engine': 'mongodb', \
'db_conn_str': "mongodb://127.0.0.1/%s"%MONITOR_DB,\
'COLLECT_NODES_STAT_TIMEOUT': 1, \
'mgmt_cli_port': 2323,
'AUTH_KEY_CHANGE_PERIOD': 2, \
'mgmt_rest_port': 9923})
print open(os.path.join(home, 'fabnet.conf')).read()
else:
ntype = 'Base'
args = ['/usr/bin/python', './fabnet_core/bin/fabnet-node', address, n_node, 'NODE%.02i'%i, home, ntype, \
KS_PATH, '--input-pwd', '--nodaemon']
if DEBUG:
args.append('--debug')
print ' '.join(args)
#p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,\
p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=sys.stdout,\
env={'FABNET_PLUGINS_CONF': 'tests/plugins.yaml', 'PYTHONPATH': os.path.abspath('.')})
p.stdin.write(KS_PASSWD+'\n')
logger.warning('{SNP} PROCESS STARTED')
time.sleep(1)
PROCESSES.append(p)
#if len(ADDRESSES) > 2:
# self._check_stat(address)
for address in ADDRESSES:
self._check_stat(address)
time.sleep(1.5)
print 'NETWORK STARTED'
def create_net(self, nodes_count):
global PROCESSES
global ADDRESSES
for unuse in range(nodes_count):
if not ADDRESSES:
n_node = "init-fabnet"
i = 1900
else:
n_node = random.choice(ADDRESSES)
i = int(ADDRESSES[-1].split(":")[-1]) + 1
self._wait_node(n_node)
address = "127.0.0.1:%s" % i
ADDRESSES.append(address)
home = "/tmp/node_%s" % i
if os.path.exists(home):
shutil.rmtree(home)
os.mkdir(home)
logger.warning("{SNP} STARTING NODE %s" % address)
if n_node == "init-fabnet":
ntype = "Monitor"
Config.load(os.path.join(home, "node_config"))
Config.update_config(
{"db_engine": "postgresql", "db_conn_str": "dbname=%s user=postgres" % MONITOR_DB}, "Monitor"
)
else:
ntype = "DHT"
args = [
"/usr/bin/python",
"./fabnet/bin/fabnet-node",
address,
n_node,
"%.02i" % i,
home,
ntype,
"--nodaemon",
]
if DEBUG:
args.append("--debug")
p = subprocess.Popen(args)
logger.warning("{SNP} PROCESS STARTED")
time.sleep(1)
PROCESSES.append(p)
# if len(ADDRESSES) > 2:
# self._check_stat(address)
for address in ADDRESSES:
self._check_stat(address)
time.sleep(1.5)
print "NETWORK STARTED"
def create_monitor(neigbour):
os.system('rm -rf %s'%monitoring_home)
os.system('mkdir %s'%monitoring_home)
Config.load(os.path.join(monitoring_home, 'node_config'))
Config.update_config({'db_engine': 'postgresql', \
'db_conn_str': "dbname=%s user=postgres"%MONITOR_DB}, 'Monitor')
address = '%s:%s'%('127.0.0.1', 1989)
logger.warning('{SNP} STARTING MONITORING NODE %s'%address)
mon_p = subprocess.Popen(['/usr/bin/python', './fabnet/bin/fabnet-node', address, neigbour, 'monitor', monitoring_home, 'Monitor', '--nodaemon'])
logger.warning('{SNP} PROCESS STARTED')
time.sleep(1)
return mon_p
def __init__(self, self_address, home_dir='/tmp/', key_storage=None, \
is_init_node=False, node_name='unknown-node', config={}):
config_file = os.path.join(home_dir, 'node_config')
Config.load(config_file)
self.update_config(config, self.OPTYPE)
self.msg_container = MessageContainer(MC_SIZE)
self.__lock = threading.RLock()
self.self_address = self_address
self.home_dir = home_dir
self.node_name = node_name
self.__neighbours = {NT_SUPERIOR: {}, NT_UPPER: {}}
self.__upper_keep_alives = {}
self.__superior_keep_alives = {}
if key_storage:
cert = key_storage.get_node_cert()
ckey = key_storage.get_node_cert_key()
else:
cert = ckey = None
self.fri_client = FriClient(bool(cert), cert, ckey)
self.__fri_agents_manager = WorkersManager(FriAgent, server_name=node_name, \
init_params=(self,))
self.__fri_agents_manager.start_carefully()
self.__async_requests = self.__fri_agents_manager.get_queue()
self.__check_neighbours_thread = CheckNeighboursThread(self)
self.__check_neighbours_thread.setName('%s-CheckNeighbours'%(node_name,))
self.__check_neighbours_thread.start()
self.start_datetime = datetime.now()
self.is_init_node = is_init_node
self.__session_manager = SessionsManager(home_dir)
self.__discovery = NeigboursDiscoveryRoutines(self)
self.__api_workers_mgr = None
self.__stat = Statistic()
init_oper_stat = {}
for opclass in self.OPERATIONS_LIST:
init_oper_stat[opclass.get_name()] = 0
self.__null_oper_stat = init_oper_stat
self.stopped = threading.Event()
def check_database(self):
db_conn_str = Config.get('db_conn_str', self.OPTYPE)
if self.__db_api and db_conn_str == self.__db_conn_str:
return
if self.__db_api:
try:
self.__db_api.close()
except Exception, err:
logger.error('DBAPI closing failed with error "%s"'%err)
def get_config(self):
return Config.get_config_dict()
def update_config(self, config, section=None):
Config.update_config(config, section=section)
import unittest
import threading
import time
import os
import logging
import json
import shutil
from datetime import datetime
from fabnet.utils.logger import logger
from fabnet.dht_mgmt.fs_mapped_ranges import FSHashRanges, TmpFile
from fabnet.core.config import Config
Config.update_config({'WAIT_FILE_MD_TIMEDELTA': 0.1}, 'DHT')
logger.setLevel(logging.DEBUG)
TEST_FS_RANGE_DIR = '/tmp/test_fs_ranges'
START_RANGE_HASH = '%040x'%0
END_RANGE_HASH = '%040x'%10000000000
def tmpdata(data, f_end=''):
fname = '/tmp/tmpdata' + f_end
tmp = TmpFile(fname, data)
return fname, tmp
class WriteThread(threading.Thread):
def __init__(self, fs_ranges, start, end):
threading.Thread.__init__(self)
def update_config(self, config):
Config.update_config(config)
def get_wait_time(cls):
return int(Config.get('AUTH_KEY_CHANGE_PERIOD', 43200))
def __init__(self, self_address, home_dir='/tmp/', key_storage=None, is_init_node=False, node_name='unknown', config={}):
cur_cfg = {}
cur_cfg.update(DEFAULT_MONITOR_CONFIG)
cur_cfg.update(config)
Operator.__init__(self, self_address, home_dir, key_storage, is_init_node, node_name, cur_cfg)
self.__db_conn_str = None
self.__db_api = None
self.check_database()
client = FriClient(key_storage)
db_conn_str = Config.get('db_conn_str')
self.__dbm = MgmtDatabaseManager(db_conn_str)
ca_host = Config.get('mgmt_ca_host', None)
ca_port = int(Config.get('mgmt_ca_port', 8080))
self.mgmt_api = mgmt_api = ManagementEngineAPI(self.__dbm, self_address, init_scheduler=True, \
ca_host=ca_host, ca_port=ca_port)
BaseMgmtCLIHandler.mgmtManagementAPI = mgmt_api
host = Config.get('mgmt_cli_host', '0.0.0.0')
cli_port = int(Config.get('mgmt_cli_port', '23'))
logger.info('Starting CLI at %s:%s'%(host, cli_port))
cli_server = TelnetServer((host, cli_port), BaseMgmtCLIHandler)
#if key_storage:
# cert, key = self.get_ssl_keycert('cli_api')
# cli_server.socket = ssl.wrap_socket(cli_server.socket, certfile=cert, keyfile=key, server_side=True)
self.__cli_api_thrd = ExternalAPIThread(cli_server, host, cli_port)
self.__cli_api_thrd.setName('%s-CLIAPIThread'%self.node_name)
self.__cli_api_thrd.start()
RESTHandler.setup_mgmt_api(mgmt_api)
host = Config.get('mgmt_rest_host', '0.0.0.0')
rest_port = int(Config.get('mgmt_rest_port', '8080'))
logger.info('Starting REST at %s:%s'%(host, rest_port))
rest_server = BaseHTTPServer.HTTPServer((host, rest_port), RESTHandler)
if key_storage:
cert, key = self.get_ssl_keycert('rest_api')
rest_server.socket = ssl.wrap_socket(rest_server.socket, certfile=cert, keyfile=key, server_side=True)
self.__rest_api_thrd = ExternalAPIThread(rest_server, host, rest_port)
self.__rest_api_thrd.setName('%s-RESTAPIThread'%self.node_name)
self.__rest_api_thrd.start()
if not self.__cli_api_thrd.started():
self.__rest_api_thrd.stop()
raise Exception('CLI API does not started!')
if not self.__rest_api_thrd.started():
self.__cli_api_thrd.stop()
raise Exception('CLI API does not started!')
self.__collect_up_nodes_stat_thread = CollectNodeStatisticsThread(self, client, STATUS_UP)
self.__collect_up_nodes_stat_thread.setName('%s-UP-CollectNodeStatisticsThread'%self.node_name)
self.__collect_up_nodes_stat_thread.start()
self.__collect_dn_nodes_stat_thread = CollectNodeStatisticsThread(self, client, STATUS_DOWN)
self.__collect_dn_nodes_stat_thread.setName('%s-DN-CollectNodeStatisticsThread'%self.node_name)
self.__collect_dn_nodes_stat_thread.start()
self.__discovery_topology_thrd = DiscoverTopologyThread(self)
self.__discovery_topology_thrd.setName('%s-DiscoverTopologyThread'%self.node_name)
self.__discovery_topology_thrd.start()
