def get_nodeid_str_from_nodeid(get_path):
"""[get nodeid string]
Arguments:
get_path {[file]} -- [description]
Raises:
MCError -- [description]
Returns:
[string] -- [nodeid]
"""
# openssl x509 -text -in ./node.crt | sed -n '15,20p' | sed 's/://g' |
# tr "\n" " " | sed 's/ //g' | sed 's/pub//g' | cut -c 3-130
LOGGER.info("get_nodeid start! get path is %s", get_path)
if not os.path.isfile(get_path):
LOGGER.error(' node cert doesn\'t existed! Need %s', get_path)
raise MCError(' node cert doesn\'t existed! Need %s' % get_path)
try:
(status,
result) = utils.getstatusoutput('cat {} | head -n 1'.format(get_path))
if status != 0:
LOGGER.error(
' create nodeid failed! status is %d, output is %s, dir is %s.',
status, result, get_path)
LOGGER.info(
' create nodeid success! status is %d, output is %s, dir is %s.',
status, result, get_path)
except Exception as node_id_exp:
LOGGER.error(
' create nodeid failed! status is %d, output is %s, dir is %s.',
status, result, get_path)
raise MCError(' create nodeid failed! excepion is %s.' % node_id_exp)
LOGGER.info("get_nodeid success! get path is %s", get_path)
return result
def build(self):
logger.info(' build temp dir, dir is %s', self.dir())
try:
os.makedirs(self.dir())
shutil.copytree(path.get_path() + '/tpl/web3sdk',
self.dir() + '/web3sdk')
if self.is_gm():
shutil.move(self.dir() + '/web3sdk/conf/applicationContext_GM.xml',
self.dir() + '/web3sdk/conf/applicationContext.xml')
shutil.copytree(path.get_path() +
'/tpl/GM_temp_node', self.dir() + '/node')
shutil.copy(self.dir() + '/node/data/sdk/ca.crt',
self.dir() + '/web3sdk/conf')
shutil.copy(self.dir() + '/node/data/sdk/client.keystore',
self.dir() + '/web3sdk/conf')
else:
shutil.move(self.dir() + '/web3sdk/conf/applicationContext_NB.xml',
self.dir() + '/web3sdk/conf/applicationContext.xml')
shutil.copytree(path.get_path() + '/tpl/temp_node',
self.dir() + '/node')
shutil.copy(self.dir() + '/node/sdk/ca.crt',
self.dir() + '/web3sdk/conf')
shutil.copy(self.dir() + '/node/sdk/client.keystore',
self.dir() + '/web3sdk/conf')
# copy fisco-bcos
shutil.copy(self.fisco.get_fisco_path(), self.dir() + '/node/')
# config.json for temp node
Config('12345', self.port.get_rpc_port(),
self.port.get_p2p_port(), self.port.get_channel_port(), self.is_gm()).writeFile(self.dir() + '/node/config.json')
# web3sdk config for temp node
utils.replace(self.dir() + '/web3sdk/conf/applicationContext.xml', 'WEB3SDK_NODES_LIST',
'<value>[email protected]:%s</value>' % str(self.port.get_channel_port()))
except Exception as e:
logger.error(
' temp node build opr failed , chain is %s, exception is %s .', self.chain, e)
raise MCError(
' build temp node failed, chain is %s, exception is %s ' % (self.chain, e))
def valid_chain_id(chain_id):
"""[Determine if the chain id is valid]
Arguments:
ip {[string]} -- [chain id]
Returns:
[bool] -- [true or false]
"""
try:
int(chain_id)
return True
except ValueError as utils_exp:
LOGGER.error('%s is not a valid chain_id', utils_exp)
except Exception as utils_exp:
LOGGER.error('%s is not a valid chain_id', utils_exp)
raise MCError('%s is not a valid chain_id' % utils_exp)
def valid_peer(peer):
"""[Determine if the peer is valid]
Arguments:
peer {[str]} -- [peers]
Returns:
[bool] -- [true or false]
"""
try:
peer = peer.split(':')
if not valid_ip(peer[0]):
return False
if valid_port(peer[1]):
return True
return False
except Exception as utils_exp:
LOGGER.error('%s is not a valid peer', utils_exp)
raise MCError('%s is not a valid peer' % utils_exp)
def add_group(_group, _node):
"""
Arguments:
_group {[type]} -- [description]
_node {[type]} -- [description]
"""
data_path = _group
node_send = []
utils.file_must_exists(data_path)
file_name = os.path.basename(data_path)
group_id = utils.valid_genesis(file_name)
if group_id == 0:
raise MCError(' paser %s file failed' % (data_path))
node_send = utils.get_all_nodes_dir(_node)
for node_file in node_send:
utils.file_must_not_exists('{}/conf/{}'.format(node_file, file_name))
shutil.copyfile(data_path, '{}/conf/{}'.format(node_file, file_name))
shutil.copyfile('{}/tpl/group.i.ini'.format(path.get_path()),
'{}/conf/group.{}.ini'.format(node_file, group_id))
def read_peers(data_path):
"""[read peers]
Arguments:
data_path {[file]} -- [peers file]
"""
# read and parser peer file
try:
for line in open(data_path):
peer = line.strip('\n')
if utils.valid_peer(peer):
MchainConf.peers.append(peer)
except Exception as ini_exp:
LOGGER.error(' open %s file failed, exception is %s', data_path,
ini_exp)
raise MCError(' open %s file failed, exception is %s' %
(data_path, ini_exp))
MchainConf.peers = list(set(MchainConf.peers))
LOGGER.info('peers is %s', MchainConf.peers)
def valid_expand_package(pkg):
"""[Determine if the string->package is valid]
Arguments:
pkg {[string]} -- [a string outside the function]
Returns:
[bool] -- [true or false]
"""
try:
pack = pkg.split()
if not valid_ip(pack[0]):
return False
if pack[1]:
return True
return False
except ValueError as utils_exp:
LOGGER.error(' invalied package , exception is %s', utils_exp)
except Exception as utils_exp:
LOGGER.error(' invalied package , exception is %s', utils_exp)
raise MCError('invalied package , exception is %s' % utils_exp)
def download_bin(_download_link, _package_name):
"""dowloand
"""
try:
if sys.version > '3':
urllib.request.urlretrieve(_download_link, _package_name,
_hook_func)
else:
if os.environ.get('https_proxy') or os.environ.get('http_proxy'):
url = _download_link
response = urllib2.urlopen(url)
data = chunk_read(response, report_hook=chunk_report)
# data = response.read()
with open(_package_name, 'wb') as code:
code.write(data)
else:
urllib.urlretrieve(_download_link, _package_name, _hook_func)
except Exception as download_err:
LOGGER.error('download %s failed! exception is %s', _package_name,
download_err)
raise MCError('download %s failed! exception is %s' %
(_package_name, download_err))
def get_console_file(_file):
"""[get console file]
Arguments:
_file {[type]} -- [description]
"""
data = _file
utils.file_must_exists(data)
# Note: the default ip the console connects to is 127.0.0.1
rpc_ip = mconf.MchainConf.rpc_ip
channel_listen_port = mconf.MchainConf.channel_listen_port
channel_addr = []
group_id = mconf.MchainConf.group_id
utils.replace(data,
'"group1',
'"group{}'.format(group_id))
utils.replace(data,
'name="groupId" value="1"',
'name="groupId" value="{}"'.format(group_id))
for ip_idx, rpc_get in enumerate(rpc_ip):
channel_addr.append('{}:{}'.format(
rpc_get, channel_listen_port[ip_idx]))
cmd = "cat {} | grep -n connectionsStr | awk '{{print $1}}'".format(data)
(status, result) = utils.getstatusoutput(cmd)
result = result.strip('\n').strip(':')
if bool(status):
LOGGER.error(
' append console channel_addr failed, result is %s.', result)
raise MCError(
' append console channel_addr failed, result is %s.' % result)
line_num = int(result) + 1
for channel in channel_addr:
(status, result) \
= utils.getstatusoutput('sed -i "{} a'
'<value>{}</value>" {}'
.format(line_num, channel, data))
line_num = line_num + 1
CONSOLER.info('get console file end')
def check_fisco(_file):
"""checkfisco
"""
bin_fisco = _file
CONSOLER.info(" Checking fisco-bcos binary...")
LOGGER.info(" Checking fisco-bcos binary...")
(status, bin_version)\
= getstatusoutput('{} -v'.format(bin_fisco))
if bool(status):
LOGGER.error(
'Checking fisco-bcos failed! status is %d,'
' output is %s, dir is %s.', status, bin_version, bin_fisco)
raise MCError('Checking fisco-bcos failed!'
' status is %d, output is %s, dir is %s.' %
(status, bin_version, bin_fisco))
if not 'FISCO-BCOS' in bin_version:
LOGGER.error("%s is wrong. Please correct it and try again.",
bin_fisco)
raise Exception("%s is wrong. Please correct it and try again." %
bin_fisco)
# if Status.gm_option:
# if not 'gm' in bin_version:
# LOGGER.error(
# 'Checking fisco-bcos failed! %s isn\'t '
# 'gm version. Please correct it and try again.', bin_fisco)
# raise MCError(
# 'Checking fisco-bcos failed! %s isn\'t '
# 'gm version. Please correct it and try again' % bin_version)
# else:
# if 'gm' in bin_version:
# LOGGER.error(
# 'Checking fisco-bcos failed! %s isn\'t '
# 'standard version. Please correct it and try again.', bin_fisco)
# raise MCError(
# 'Checking fisco-bcos failed! %s isn\'t '
# 'standard version. Please correct it and try again.' % bin_version)
CONSOLER.info(' Binary check passed.')
LOGGER.info(' Binary check passed.')
def expand(cfg, dir):
"""expand operation
Arguments:
cfg {string} -- config file path
dir {string} -- dir with fisco-bcos, genesis.json, bootstrapsnode.json
"""
logger.debug(' cfg is %s, dir is %s', cfg, dir)
try:
# check java env
java = JAVA()
try:
# parser and check config if valid
cc = ConfigConf(cfg)
except Exception as e:
raise MCError(
' parser config failed, invalid format, config is %s, exception is %s'
% (cfg, e))
if os.path.exists(cc.get_chain().data_dir()):
ns = Names()
expand_chain.expand_on_exist_chain(cc)
ns.write()
else:
expand_chain.expand_on_nonexist_chain(cc, dir)
consoler.info(
' expand install package for chain %s version %s success.',
cc.get_chain().get_id(),
cc.get_chain().get_version())
except MCError as me:
consoler.error(me)
consoler.error(' \033[1;31m \t %s \033[0m', me)
def generator_node_ca(agent, dir, node, gm=False):
"""[generate node cert ]
Arguments:
agent {[path]} -- [agency cert path]
node {[string]} -- [node name]
dir {[path]} -- [node cert path]
"""
_dir = os.path.abspath(dir)
agent = os.path.abspath(agent)
try:
if gm:
(status, result
) = utils.getstatusoutput('bash ' + path.get_path() +
'/scripts/ca/gm/cts.sh gen_node_cert ' +
agent + ' ' + _dir + '/ ' + node)
else:
(status, result
) = utils.getstatusoutput('bash ' + path.get_path() +
'/scripts/ca/cts.sh gen_node_cert ' +
agent + ' ' + _dir + '/ ' + node)
if not status:
logger.info(' Generate %s cert successful! dir is %s.', node,
_dir + '/' + node)
else:
consoler.error(
' \033[1;31m Generate node cert failed! Please check your network, and try to check your opennssl version.\033[0m'
)
logger.error(' Generate %s cert failed! Result is %s' %
(node, result))
raise MCError(' Generate %s cert failed! Result is %s' %
(node, result))
except MCError as me:
consoler.error(' \033[1;31m %s \033[0m', me)
except Exception as e:
consoler.error(
' \033[1;31m Generate root cert failed! excepion is %s.\033[0m', e)
logger.error(' Generate root cert failed! Result is %s' % result)
def generator_agent_ca(dir, ca, agent, gm=False):
"""[generate agency cert]
Arguments:
dir {[path]} -- [agency cert path]
ca {[path]} -- [root cert path]
agent {[string]} -- [agency name]
"""
try:
ca = os.path.abspath(ca)
dir = os.path.abspath(dir)
if gm:
(status, result) = utils.getstatusoutput(
'bash ' + path.get_path() +
'/scripts/ca/gm/cts.sh gen_agency_cert ' + ca + ' ' + dir +
' ' + agent)
else:
(status, result
) = utils.getstatusoutput('bash ' + path.get_path() +
'/scripts/ca/cts.sh gen_agency_cert ' +
ca + ' ' + dir + ' ' + agent)
if not status:
logger.info(' Generate %s cert successful! dir is %s.' %
(agent, dir + '/' + agent))
else:
consoler.error(
' \033[1;31m Generate %s cert failed! Please check your network, and try to check your opennssl version.\033[0m'
)
logger.error(' Generate %s cert failed! Result is %s' %
(agent, result))
raise MCError(' Generate %s cert failed! Result is %s' %
(agent, result))
except MCError as me:
consoler.error(' \033[1;31m %s \033[0m', me)
except Exception as e:
consoler.error(
' \033[1;31m Generate root cert failed! excepion is %s.\033[0m', e)
logger.error(' Generate root cert failed! Result is %s' % result)
def export(self):
try:
if self.fisco.is_gm():
shutil.move(get_gm_god_path() + '/godInfo.txt',
get_gm_god_path() + '/godInfo.txt.bak')
cmd = self.fisco.get_fisco_path() + ' --newaccount ' + get_gm_god_path() + '/godInfo.txt'
status, result = utils.getstatusoutput(cmd)
logger.debug(' start status, status is %d, output is %s', status, result)
else:
shutil.move(get_god_path() + '/godInfo.txt',
get_god_path() + '/godInfo.txt.bak')
cmd = self.fisco.get_fisco_path() + ' --newaccount ' + get_god_path() + '/godInfo.txt'
status, result = utils.getstatusoutput(cmd)
logger.debug(' start status, status is %d, output is %s', status, result)
if status != 0:
logger.warn(' export godInfo.txt failed! status is %d, output is %s.', status, result)
raise MCError('godInfo.txt failed! status is %d, output is %s.' % (status, result))
logger.info(' export godInfo.txt status is %d, output is %s.', status, result)
consoler.info(' export godInfo.txt success')
except MCError as me:
consoler.error(' \033[1;31m %s \033[0m', me)
except Exception as e:
consoler.error(' \033[1;31m export godInfo.txt failed! excepion is %s.\033[0m', e)
logger.error(' export godInfo.txt failed!')
def write_to_file(self):
#if len(self.nodes) == 0:
# logger.debug('nodes empty, write return')
# return
if not data.meta_dir_exist(self.chain_id):
data.create_meta_dir(self.chain_id)
meta_file = data.meta_dir(self.chain_id) + '/meta.json'
meta_bak_file = meta_file + '_bak_' + \
time.strftime("%Y-%m-%d_%H-%M%S", time.localtime())
if os.path.exists(meta_file):
shutil.copy(meta_file, meta_bak_file)
logger.info(
'meta.json is exist, backup it, name is ' + meta_bak_file)
try:
with open(data.meta_dir(self.chain_id) + '/meta.json', "w+") as f:
f.write(self.to_json())
logger.info(
'write info meta.json, content is ' + self.to_json())
except Exception as e:
logger.error(
' write meta failed, chaind id is %s, exception is %s', self.chain_id, e)
# raise or not ???
raise MCError(' write meta.json failed, chain id is %s, exception is %s' % (self.chain_id, e))
def build_config_ini(_data_dir):
"""[-- build create config_ini]
Keyword Arguments:
_meta_dir {[PATH]} -- [input dir] (default: {meta})
_data_dir {[PATH]} -- [output dir] (default: {data})
Raises:
MCError -- [description]
MCError -- [description]
MCError -- [description]
MCError -- [description]
"""
LOGGER.info("build_config_ini start ")
p2p_listen_port = mconf.MchainConf.p2p_listen_port
jsonrpc_listen_port = mconf.MchainConf.jsonrpc_listen_port
channel_listen_port = mconf.MchainConf.channel_listen_port
p2p_ip = mconf.MchainConf.p2p_ip
rpc_ip = mconf.MchainConf.rpc_ip
peers = mconf.MchainConf.peers
meta_dir = '{}/meta'.format(path.get_path())
conf_dir = meta_dir
package_dir = _data_dir
gm_opr = utils.Status.gm_option
group_id = mconf.MchainConf.group_id
utils.file_must_exists('{}/group.{}.genesis'.format(meta_dir, group_id))
if os.path.exists(package_dir):
LOGGER.error(' %s existed, maybe u had created it!', package_dir)
raise MCError(' %s existed, maybe u had created it!' % package_dir)
os.mkdir(package_dir)
default_cfg = configparser.ConfigParser()
if gm_opr:
shutil.copy('{}/tpl/config.ini.gm'.format(path.get_path()),
'{}/.config.ini'.format(conf_dir))
else:
shutil.copy('{}/tpl/config.ini'.format(path.get_path()),
'{}/.config.ini'.format(conf_dir))
try:
with codecs.open('{}/.config.ini'.format(conf_dir),
'r', encoding='utf-8') as config_file:
default_cfg.readfp(config_file)
except Exception as build_exp:
LOGGER.error(
' open config.ini file failed, exception is %s', build_exp)
raise MCError(
' open config.ini file failed, exception is %s' % build_exp)
fin_p2p_ip = []
if not peers:
LOGGER.warning('section peers not existed!')
CONSOLER.warn('section peers not existed!')
else:
for _, peer in enumerate(peers):
fin_p2p_ip.append(peer)
# default_cfg.set("p2p", "node.{}".format(node_id + len(p2p_listen_port)),
# peer)
# with open('{}/.config.ini'.format(conf_dir), 'w') as config_file:
# default_cfg.write(config_file)
# init config.ini & node package
for my_node_index, node_ip in enumerate(p2p_ip):
LOGGER.info("p2p_ip -> %s", node_ip)
try:
if utils.Status.gm_option:
utils.file_must_exists('{}/gmcert_{}_{}.crt'.format(conf_dir,
node_ip,
p2p_listen_port[my_node_index]))
else:
utils.file_must_exists('{}/cert_{}_{}.crt'.format(conf_dir,
node_ip,
p2p_listen_port[my_node_index]))
except Exception as build_exp:
LOGGER.error('%s', build_exp)
raise MCError('%s' % build_exp)
CONSOLER.info(' Generate %s/node_%s_%s ',
package_dir, node_ip, p2p_listen_port[my_node_index])
node_dir = '{}/node_{}_{}'.format(package_dir,
node_ip, p2p_listen_port[my_node_index])
os.mkdir(node_dir)
shutil.copy('{}/tpl/start.sh'.format(path.get_path()),
'{}/start.sh'.format(node_dir))
shutil.copy('{}/tpl/stop.sh'.format(path.get_path()),
'{}/stop.sh'.format(node_dir))
shutil.copy('{}/fisco-bcos'.format(meta_dir),
'{}/fisco-bcos'.format(node_dir))
os.mkdir('{}/conf'.format(node_dir))
try:
# get node cert
shutil.copy('{}/.config.ini'.format(conf_dir),
'{}/config.ini'.format(node_dir))
shutil.copy('{}/group.{}.genesis'.format(conf_dir, group_id),
'{}/conf/group.{}.genesis'.format(node_dir, group_id))
shutil.copy('{}/tpl/group.i.ini'.format(path.get_path()),
'{}/conf/group.{}.ini'.format(node_dir, group_id))
if gm_opr:
get_node_cert('{}/gmcert_{}_{}.crt'.format(meta_dir, node_ip,
p2p_listen_port[my_node_index]),
'{}/conf/gmnode.crt'.format(node_dir))
# get_nodeid('{}/conf/gmnode.crt'.format(node_dir),
# '{}/conf/gmnode.nodeid'.format(node_dir))
shutil.copyfile('{}/gmca.crt'.format(meta_dir),
'{}/conf/gmca.crt'.format(node_dir))
else:
get_node_cert('{}/cert_{}_{}.crt'.format(meta_dir, node_ip,
p2p_listen_port[my_node_index]),
'{}/conf/node.crt'.format(node_dir))
# get_nodeid('{}/conf/node.crt'.format(node_dir),
# '{}/conf/node.nodeid'.format(node_dir))
shutil.copyfile('{}/ca.crt'.format(meta_dir),
'{}/conf/ca.crt'.format(node_dir))
except Exception as build_exp:
LOGGER.error(' get node.crt failed ! exception is %s', build_exp)
utils.delete_data(package_dir)
raise MCError(' get node.crt failed! exception is %s' % build_exp)
node_cfg = configparser.ConfigParser()
try:
with codecs.open('{}/config.ini'.format(node_dir),
'r', encoding='utf-8') as config_file:
node_cfg.readfp(config_file)
except Exception as build_exp:
LOGGER.error(
' open config.ini file failed, exception is %s', build_exp)
utils.delete_data(package_dir)
raise MCError(
' open config.ini file failed, exception is %s' % build_exp)
node_cfg.set("rpc", "listen_ip", rpc_ip[my_node_index])
node_cfg.set("rpc", "channel_listen_port",
channel_listen_port[my_node_index])
node_cfg.set("rpc", "jsonrpc_listen_port",
jsonrpc_listen_port[my_node_index])
# node_cfg.set("p2p", "listen_ip", p2p_ip[my_node_index])
node_cfg.set("p2p", "listen_port", p2p_listen_port[my_node_index])
with open('{}/config.ini'.format(node_dir), 'w') as config_file:
node_cfg.write(config_file)
config_file.close()
# set p2p ip in config.ini
for my_node_index, ip_item in enumerate(p2p_ip):
node_cfg = configparser.ConfigParser()
if not utils.valid_ip(ip_item):
LOGGER.error(
' init config.ini file failed, found ip => %s', ip_item)
utils.delete_data(package_dir)
raise MCError(
' init config.ini file failed, found ip => %s' % ip_item)
node_dir = '{}/node_{}_{}'.format(package_dir,
ip_item, p2p_listen_port[my_node_index])
try:
with codecs.open('{}/config.ini'.format(node_dir),
'r', encoding='utf-8') as config_file:
node_cfg.readfp(config_file)
except Exception as build_exp:
LOGGER.error(
' open config.ini file failed, exception is %s', build_exp)
utils.delete_data(package_dir)
raise MCError(
' open config.ini file failed, exception is %s' % build_exp)
# write p2pip:port into config.ini
for ip_idx, set_item in enumerate(p2p_ip):
fin_p2p_ip.append("{}:{}".format(set_item, p2p_listen_port[ip_idx]))
fin_p2p_ip = list(set(fin_p2p_ip))
for index, p2p_section in enumerate(fin_p2p_ip):
node_cfg.set("p2p", "node.{}".format(index),
'{}'.format(p2p_section))
with open('{}/config.ini'.format(node_dir), 'w') as config_file:
node_cfg.write(config_file)
os.mkdir(package_dir + '/scripts/')
shutil.copy('{}/scripts/install.sh'.format(path.get_path()),
package_dir + '/scripts/')
shutil.copy('{}/scripts/pack.sh'.format(path.get_path()),
package_dir + '/scripts/')
shutil.copy('{}/tpl/start_all.sh'.format(path.get_path()), package_dir)
shutil.copy('{}/tpl/stop_all.sh'.format(path.get_path()), package_dir)
shutil.copytree('{}/scripts/monitor'.format((path.get_path())),
'{}/monitor'.format(package_dir))
LOGGER.info("build_config_ini end!")
def gen_build_cert(_dir):
"""[gen_build_cert]
Arguments:
_dir {[PATH]} -- [cert output]
Raises:
MCError -- [description]
"""
meta_path = '{}/meta'.format(path.get_path())
cert_path = _dir
data_path = meta_path
p2p_listen_port = mconf.MchainConf.p2p_listen_port
p2p_ip = mconf.MchainConf.p2p_ip
utils.file_must_not_exists('{}/peers.txt'.format(cert_path))
if not os.path.exists(cert_path):
os.mkdir(cert_path)
if utils.Status.gm_option:
if not os.path.exists('{}/gmca.crt'.format(meta_path)):
CONSOLER.error(" gmca.crt not existed")
utils.delete_data(cert_path)
raise MCError(' gmca.crt not founded!')
else:
if not os.path.exists('{}/ca.crt'.format(meta_path)):
CONSOLER.error(" ca.crt not existed")
utils.delete_data(cert_path)
raise MCError(' ca.crt not founded!')
if utils.Status.gm_option:
if not (os.path.exists('{}/gmagency.key'.format(meta_path))
and os.path.exists(('{}/gmagency.crt'.format(meta_path)))):
CONSOLER.error(" gmagency.crt or gmagency.key not existed")
utils.delete_data(cert_path)
raise MCError(' gmagency.crt or gmagency.key not found in %s!' %
meta_path)
else:
if not (os.path.exists('{}/agency.key'.format(meta_path))
and os.path.exists(('{}/agency.crt'.format(meta_path)))):
CONSOLER.error(" agency.crt or agency.key not existed")
utils.delete_data(cert_path)
raise MCError(' agency.crt or agency.key not found in %s!' %
meta_path)
for my_node_index, node_ip in enumerate(p2p_ip):
LOGGER.info("p2p_ip -> %s", node_ip)
CONSOLER.info(' Generate %s/node_%s_%s ', data_path, node_ip,
p2p_listen_port[my_node_index])
if utils.Status.gm_option:
utils.file_must_not_exists('{}/gmcert_{}_{}.crt'.format(
meta_path, node_ip, p2p_listen_port[my_node_index]))
else:
utils.file_must_not_exists('{}/cert_{}_{}.crt'.format(
meta_path, node_ip, p2p_listen_port[my_node_index]))
ca.generator_node_ca(
data_path, '{}/'.format(meta_path),
'node_{}_{}'.format(node_ip, p2p_listen_port[my_node_index]))
if utils.Status.gm_option:
utils.off_gm()
if os.path.isdir('./.origin_cert'):
shutil.rmtree('./.origin_cert')
ca.generator_node_ca('./', meta_path, '.origin_cert')
shutil.copytree(
'./.origin_cert', '{}/node_{}_{}/origin_cert'.format(
data_path, node_ip, p2p_listen_port[my_node_index]))
shutil.rmtree('./.origin_cert')
utils.set_gm()
shutil.copyfile(
'{}/node_{}_{}/gmnode.crt'.format(
data_path, node_ip, p2p_listen_port[my_node_index]),
'{}/gmcert_{}_{}.crt'.format(meta_path, node_ip,
p2p_listen_port[my_node_index]))
shutil.copyfile(
'{}/gmcert_{}_{}.crt'.format(meta_path, node_ip,
p2p_listen_port[my_node_index]),
'{}/gmcert_{}_{}.crt'.format(cert_path, node_ip,
p2p_listen_port[my_node_index]))
else:
shutil.copyfile(
'{}/node_{}_{}/node.crt'.format(
data_path, node_ip, p2p_listen_port[my_node_index]),
'{}/cert_{}_{}.crt'.format(meta_path, node_ip,
p2p_listen_port[my_node_index]))
shutil.copyfile(
'{}/cert_{}_{}.crt'.format(meta_path, node_ip,
p2p_listen_port[my_node_index]),
'{}/cert_{}_{}.crt'.format(cert_path, node_ip,
p2p_listen_port[my_node_index]))
(status, result) = \
utils.getstatusoutput('echo {}:{} >> {}/peers.txt'
.format(node_ip,
p2p_listen_port[my_node_index],
cert_path))
LOGGER.info(" status is %s, result is %s", status, result)
CONSOLER.info(" Generate cert by node_installation.ini successful!")
def concatenate_cfg(cfg_file, cfg_file_get):
"""[combine two config.ini]
Arguments:
cfg_file {[type]} -- [description]
cfg_file_get {[type]} -- [description]
Raises:
MCError -- [description]
"""
LOGGER.info("concatenate two config.ini now!")
meta = cfg_file
data = cfg_file_get
utils.file_must_exists(meta)
utils.file_must_exists(data)
p2p_get = []
p2p_get_ip = []
p2p_send = []
p2p_send_ip = []
p2p_cfg = configparser.ConfigParser(allow_no_value=True)
try:
with codecs.open(meta, 'r', encoding='utf-8') as config_file:
p2p_cfg.readfp(config_file)
except Exception as build_exp:
LOGGER.error(' open config.ini file failed, exception is %s',
build_exp)
raise MCError(' open config.ini file failed, exception is %s' %
build_exp)
p2p_get = p2p_cfg.items('p2p')
p2p_get.pop(0)
p2p_get.pop(0)
LOGGER.info("get node is %s!", p2p_get)
for node_tuple in p2p_get:
p2p_get_ip.append(node_tuple[1])
LOGGER.info("get node ip is %s!", p2p_get_ip)
try:
with codecs.open(data, 'r', encoding='utf-8') as config_file:
p2p_cfg.readfp(config_file)
except Exception as build_exp:
LOGGER.error(' open config.ini file failed, exception is %s',
build_exp)
raise MCError(' open config.ini file failed, exception is %s' %
build_exp)
p2p_send = p2p_cfg.items('p2p')
p2p_send.pop(0)
p2p_send.pop(0)
LOGGER.info("send node is %s!", p2p_send)
for node_tuple in p2p_send:
p2p_send_ip.append(node_tuple[1])
LOGGER.info("get node ip is %s!", p2p_send_ip)
p2p_send_ip = list(set(p2p_get_ip + p2p_send_ip))
LOGGER.info("final node ip is %s!", p2p_send_ip)
for ip_idx, p2p_ip in enumerate(p2p_send_ip):
p2p_cfg.set("p2p", "node.{}".format(ip_idx), p2p_ip)
p2p_cfg.set('certificate_whitelist',
'; cal.0 should be nodeid, nodeid\'s length is 128')
p2p_cfg.set('certificate_whitelist', ';cal.0=')
p2p_cfg.set('certificate_blacklist',
'; crl.0 should be nodeid, nodeid\'s length is 128')
p2p_cfg.set('certificate_blacklist', ';crl.0=')
with open(data, 'w') as config_file:
p2p_cfg.write(config_file)
LOGGER.info("concatenate two config.ini now! output => %s/conf/config.ini",
data)
def build(cc, fisco):
"""build all install package for one chain base on cc
Arguments:
cc {ConfigConf} -- ConfigConf object
fisco {Fisco} -- Fisco object
Raises:
Exception -- exception description
"""
logger.info('building, cc is %s, fisco is %s', cc, fisco)
port = cc.get_port()
chain = cc.get_chain()
chain_id = chain.get_id()
chain_version = chain.get_version()
# create dir base on version of the chain.
if os.path.isdir(chain.data_dir()):
raise MCError('chain_id:%s chain_version:%s aleady exist !!!.' %
(chain_id, chain_version))
try:
temp = None
acp = AllChainPort()
# port check
for node in cc.get_nodes():
for index in range(node.get_node_num()):
# create dir for every node on the server
acp.port_conflicts_outside_chain(chain.get_id(),
node.get_host_ip(),
port.to_port(index))
dir = chain.data_dir()
os.makedirs(dir)
# generate bootstrapsnode.json
cc.to_p2p_nodes().writeFile(dir + '/bootstrapnodes.json')
# create common dir
build_pkg.build_common_dir(chain, fisco)
# build and start temp node for node info register
temp = Temp(chain, fisco, port)
# build install dir for every server
for node in cc.get_nodes():
build_pkg.build_host_dir(chain, node, port, fisco, temp)
# export for genesis.json file
temp.export()
temp.clean()
# copy genesis.json bootstrapnodes.json
for node in cc.get_nodes():
for index in range(node.get_node_num()):
shutil.copy(
dir + '/genesis.json',
dir + '/' + node.get_host_ip() + '/node' + str(index))
# web3sdk conf
web3_conf_by_chain(chain, fisco.is_gm())
logger.info(' build end ok, chain is %s', chain)
except Exception as e:
if not temp is None:
temp.clean()
if os.path.exists(dir):
shutil.rmtree(dir)
raise MCError(
' build package for chain %s version %s failed, exception is %s' %
(chain_id, chain_version, e))
def get_by_chain_id(self, chain_id):
if chain_id in self.ports:
return self.ports[chain_id]
raise MCError(' not found, chain id is %s' % (chain_id))
def create_group_genesis(data_dir='{}/meta'.format(path.get_path())):
'''
create group in meta
'''
LOGGER.info('create_group_genesis start')
package_dir = data_dir
gm_opr = utils.Status.gm_option
group_id = mgroup.MgroupConf.group_id
p2p_ip = mgroup.MgroupConf.p2p_ip
p2p_listen_port = mgroup.MgroupConf.p2p_listen_port
utils.file_must_not_exists('{}/group.{}.genesis'.format(
data_dir, group_id))
if not os.path.exists(package_dir):
LOGGER.warning(' %s not existed!', package_dir)
os.mkdir(data_dir)
shutil.copy('{}/tpl/group.i.genesis'.format(path.get_path()),
'{}/group.{}.genesis'.format(package_dir, group_id))
shutil.copy('{}/tpl/group.i.ini'.format(path.get_path()),
'{}/group.{}.ini'.format(package_dir, group_id))
# uptes second to ms
(status, time_stamp) = utils.getstatusoutput('echo $(date +%s"000")')
if not bool(status):
CONSOLER.info('generate %s/group.%s.genesis, successful', package_dir,
group_id)
else:
LOGGER.error(
' Generate %s/group.%s.genesis failed! Please check your network.',
package_dir, group_id)
raise MCError(
' Generate %s/group.%s.genesis failed, timestamp is %s! Please check your network.'
% (package_dir, group_id, time_stamp))
# CONSOLER.info('generate %s/group.%s.ini', package_dir, group_id)
group_cfg = configparser.ConfigParser()
with open('{}/group.{}.genesis'.format(package_dir, group_id),
'r') as config_file:
group_cfg.readfp(config_file)
for node_idx, _in in enumerate(p2p_ip):
try:
if gm_opr:
node_id = config.get_nodeid_str(
'{}/meta/gmcert_{}_{}.crt'.format(
path.get_path(), p2p_ip[node_idx],
p2p_listen_port[node_idx]))
node_id.strip('\n').strip(
'WARNING: can\'t open config file: /home/asherli/TASSL/ssl/openssl.cnf'
)
LOGGER.info('resolve %s/meta/gmcert_%s_%s.crt',
path.get_path(), p2p_ip[node_idx],
p2p_listen_port[node_idx])
LOGGER.info("nodeid -> %s", node_id)
group_cfg.set("consensus", "node.{}".format(node_idx), node_id)
else:
node_id = config.get_nodeid_str(
'{}/meta/cert_{}_{}.crt'.format(path.get_path(),
p2p_ip[node_idx],
p2p_listen_port[node_idx]))
LOGGER.info('resolve %s/meta/cert_%s_%s.crt', path.get_path(),
p2p_ip[node_idx], p2p_listen_port[node_idx])
LOGGER.info("nodeid -> %s", node_id)
group_cfg.set("consensus", "node.{}".format(node_idx), node_id)
except Exception as group_exp:
LOGGER.error('create group genesis failed! exception is %s',
group_exp)
raise MCError('create group genesis failed! exception is %s' %
group_exp)
group_cfg.set("group", "id", group_id)
group_cfg.set("group", "timestamp", time_stamp)
with open('{}/group.{}.genesis'.format(package_dir, group_id),
'w') as config_file:
group_cfg.write(config_file)
shutil.copy('{}/group.{}.genesis'.format(package_dir, group_id),
'{}/meta/group.{}.genesis'.format(path.get_path(), group_id))
LOGGER.info('create_group_genesis end')
def get_meta_by_chain_id(self, chain_id):
if chain_id in self.metas:
return self.metas[chain_id]
# raise or not ???
raise MCError(' not found meta, chain_id is %s' % (chain_id))
def parser(mchain):
"""resolve node_installation.ini
Arguments:
mchain {string} -- path of node_installation.ini
Raises:
MCError -- exception description
"""
LOGGER.info('node_installation.ini is %s', mchain)
# resolve configuration
if not utils.valid_string(mchain):
LOGGER.error(
' node_installation.ini not invalid path, node_installation.ini is %s',
mchain)
raise MCError(
' node_installation.ini not invalid path, node_installation.ini is %s'
% mchain)
# read and parser config file
config_parser = configparser.ConfigParser(allow_no_value=True)
try:
with codecs.open(mchain, 'r', encoding='utf-8') as file_mchain:
config_parser.readfp(file_mchain)
except Exception as ini_exp:
LOGGER.error(
' open node_installation.ini file failed, exception is %s',
ini_exp)
raise MCError(
' open node_installation.ini file failed, exception is %s' %
ini_exp)
for idx in range(0, 128):
node_index = ('node{}'.format(idx))
if config_parser.has_section('group'):
MchainConf.group_id = config_parser.get('group', 'group_id')
else:
LOGGER.error(
' invalid node_installation.ini format, group id is %s',
MchainConf.group_id)
raise MCError(
' invalid node_installation.ini format, group id is %s' %
MchainConf.group_id)
if config_parser.has_section(node_index):
p2p_ip = config_parser.get(node_index, 'p2p_ip')
rpc_ip = config_parser.get(node_index, 'rpc_ip')
if not utils.valid_ip(p2p_ip):
LOGGER.error(
' invalid node_installation.ini format, p2p_ip is %s',
p2p_ip)
raise MCError(
' invalid node_installation.ini format, p2p_ip is %s' %
p2p_ip)
# if rpc_ip == "0.0.0.0" and utils.Status.allow_unsecure_cfg:
if rpc_ip == "0.0.0.0":
LOGGER.warning('Your rpc_ip is %s, this is an unsecurity way',
rpc_ip)
CONSOLER.warning(
' \033[1;31m Your rpc_ip is %s, this is an unsecurity way \033[0m',
rpc_ip)
elif not utils.valid_ip(rpc_ip):
LOGGER.error(
' invalid node_installation.ini format, rpc_ip is %s',
rpc_ip)
raise MCError(
' invalid node_installation.ini format, rpc_ip is %s' %
rpc_ip)
p2p_listen_port = config_parser.get(node_index, 'p2p_listen_port')
jsonrpc_listen_port = config_parser.get(node_index,
'jsonrpc_listen_port')
channel_listen_port = config_parser.get(node_index,
'channel_listen_port')
if not (utils.valid_string(p2p_listen_port)
and utils.valid_string(jsonrpc_listen_port)
and utils.valid_string(channel_listen_port)):
LOGGER.error(
'mchain bad format, p2p_listen_port is %s, '
'jsonrpc_port is %s, channel_port is %s', p2p_listen_port,
jsonrpc_listen_port, channel_listen_port)
raise MCError('mchain bad format, p2p_listen_port is %s, '
'jsonrpc_port is %s, channel_port is %s' %
(p2p_listen_port, jsonrpc_listen_port,
channel_listen_port))
MchainConf.p2p_ip.append(p2p_ip)
MchainConf.rpc_ip.append(rpc_ip)
MchainConf.p2p_listen_port.append(p2p_listen_port)
MchainConf.jsonrpc_listen_port.append(jsonrpc_listen_port)
MchainConf.channel_listen_port.append(channel_listen_port)
else:
LOGGER.warning(' node%s not existed, break!', idx)
break
# if config_parser.has_section('peers'):
# for peer in config_parser.items('peers'):
# MchainConf.peers.append(peer[1])
# else:
# LOGGER.warning(' section peers not existed!')
LOGGER.info('group_id is %s', MchainConf.group_id)
LOGGER.info('p2p_ip is %s', MchainConf.p2p_ip)
LOGGER.info('rpc_ip is %s', MchainConf.rpc_ip)
LOGGER.info('p2p_listen_port is %s', MchainConf.p2p_listen_port)
LOGGER.info('jsonrpc_listen_port is %s', MchainConf.jsonrpc_listen_port)
LOGGER.info('channel_listen_port is %s', MchainConf.channel_listen_port)
LOGGER.info('peers is %s', MchainConf.peers)
LOGGER.info('node_installation.ini end, result is %s', MchainConf())
def get_by_host(self, host):
if host in self.ports:
return self.ports[host]
raise MCError(
' not found, chain id is %s, chain version is %s, host is %s' %
(self.chain_id, self.chain_version, host))
def get_by_chain_version(self, version):
if version in self.ports:
return self.ports[version]
raise MCError(' not found, chain id is %s, chain version is %s' %
(self.chain_id, version))
def download_fisco(_dir):
"""[download fisco-bcos]
Arguments:
_dir {[type]} -- [description]
"""
dir_must_exists(_dir)
bin_path = _dir
package_name = "fisco-bcos.tar.gz"
# (status, version)\
# = getstatusoutput(r'curl -s https://api.github.com/repos/FISCO-BCOS/FISCO-BCOS/releases '
# r'| grep "tag_name" | grep "\"v2\.[0-9]\.[0-9]\""'
# r' | sort -u | tail -n 1 | cut -d \" -f 4 | sed "s/^[vV]//"')
# if bool(status):
# LOGGER.error(
# ' get fisco-bcos verion failed, result is %s.', version)
# raise MCError(' get fisco-bcos verion failed, result is %s.' % version)
version = "2.7.2"
download_link = 'https://github.com/FISCO-BCOS/FISCO-BCOS/releases/download/v{}/{}'.format(
version.strip('\n'), package_name.strip('\n'))
cnd_link = 'https://osp-1257653870.cos.ap-guangzhou.myqcloud.com/FISCO-BCOS/FISCO-BCOS/releases/v{}/{}'.format(
version.strip('\n'), package_name.strip('\n'))
if Status.use_cdn:
if valid_url(cnd_link):
LOGGER.info("Downloading fisco-bcos binary from %s", cnd_link)
CONSOLER.info("Downloading fisco-bcos binary from %s", cnd_link)
download_bin(cnd_link, package_name)
elif valid_url(download_link):
LOGGER.info("Downloading fisco-bcos binary from %s", download_link)
CONSOLER.info("Downloading fisco-bcos binary from %s",
download_link)
download_bin(download_link, package_name)
else:
LOGGER.error(
' Download fisco-bcos failed, Please check your network!')
raise MCError(
' Download fisco-bcos failed, Please check your network!')
else:
if valid_url(download_link):
LOGGER.info("Downloading fisco-bcos binary from %s", download_link)
CONSOLER.info("Downloading fisco-bcos binary from %s",
download_link)
download_bin(download_link, package_name)
else:
LOGGER.error(
' Download fisco-bcos failed, Please check your network!')
raise MCError(
' Download fisco-bcos failed, Please check your network!')
(status, result)\
= getstatusoutput('tar -zxf {} -C {} && rm {}'.format(package_name,
bin_path,
package_name))
if bool(status):
LOGGER.error(' Decompress fisco-bcos failed, result is %s.', result)
raise MCError(' Decompress fisco-bcos failed, result is %s.' % result)
(status, result) = getstatusoutput('chmod a+x {}'.format(bin_path))
if bool(status):
LOGGER.error(' exec fisco-bcos failed, result is %s.', result)
raise MCError(' exec fisco-bcos failed, result is %s.' % result)
LOGGER.info("Downloading fisco-bcos successful, fisco-bcos at %s",
bin_path)
CONSOLER.info("Downloading fisco-bcos successful, fisco-bcos at %s",
bin_path)
def get_by_index(self, index):
if index in self.ports:
return self.ports[index]
raise MCError(
' not found, chain id is %s, chain version is %s, host is %s, index is %d'
% (self.chain_id, self.chain_version, self.host, index))
def download_console(_dir):
"""[summary]
Arguments:
_dir {[type]} -- [description]
Raises:
MCError -- [description]
MCError -- [description]
"""
dir_must_exists(_dir)
bin_path = _dir
meta = '{}/meta'.format(path.get_path())
file_must_exists('{}/ca.crt'.format(meta))
file_must_exists('{}/agency.crt'.format(meta))
file_must_exists('{}/agency.key'.format(meta))
package_name = "console.tar.gz"
dir_must_not_exists('{}/console'.format(bin_path))
(status, version) = getstatusoutput('curl -s https://api.github.com/repos/FISCO-BCOS/'
'console/releases | grep "tag_name" '
'| sort -u | tail -n 1 | cut -d \\" -f 4 | sed "s/^[vV]//"')
if bool(status):
LOGGER.error(
' get fisco-bcos verion failed, result is %s.', version)
raise MCError(' get fisco-bcos verion failed, result is %s.' % version)
download_link = 'https://github.com/FISCO-BCOS/console/releases/download/v{}/{}'.format(
version.strip('\n'), package_name.strip('\n'))
cnd_link = 'https://www.fisco.com.cn/cdn/console/releases/download/v{}/{}'.format(
version.strip('\n'), package_name.strip('\n'))
if Status.use_cdn:
if valid_url(cnd_link):
LOGGER.info("Downloading console binary from %s", cnd_link)
CONSOLER.info("Downloading console binary from %s", cnd_link)
download_bin(cnd_link, package_name)
elif valid_url(download_link):
LOGGER.info("Downloading console binary from %s", download_link)
CONSOLER.info("Downloading console binary from %s", download_link)
download_bin(download_link, package_name)
else:
LOGGER.error(
' Download console failed, Please check your network!')
raise MCError(
' Download console failed, Please check your network!')
else:
if valid_url(download_link):
LOGGER.info("Downloading console binary from %s", download_link)
CONSOLER.info("Downloading console binary from %s", download_link)
download_bin(download_link, package_name)
else:
LOGGER.error(
' Download console failed, Please check your network!')
raise MCError(
' Download console failed, Please check your network!')
(status, result)\
= getstatusoutput('tar -zxf {} -C {} && '
'rm {}'.format(package_name,
bin_path,
package_name))
if bool(status):
LOGGER.error(
' Decompress console failed, result is %s.', result)
raise MCError(
' Decompress console failed, result is %s.' % result)
(status, result) = getstatusoutput(
'chmod a+x {}/console/start.sh'.format(bin_path))
if bool(status):
LOGGER.error(
'chmod console failed! status is %d,'
' output is %s.', status, result)
raise MCError('chmod console failed!'
' status is %d, output is %s.' % (
status, result))
def generator_node_ca(_dir, agent, node):
"""[generate node cert ]
Arguments:
agent {[path]} -- [agency cert path]
node {[string]} -- [node name]
dir {[path]} -- [node cert path]
"""
node_dir = os.path.abspath(_dir)
agent = os.path.abspath(agent)
try:
if utils.Status.gm_option:
os.chdir('{}/scripts/gm/'.format(path.get_path()))
(status, result) = utils.getstatusoutput('./cts.sh'
' gen_node_cert {} {}/{}'
.format(
agent, node_dir, node))
os.chdir('{}'.format(path.get_path()))
else:
os.chdir('{}/scripts/'.format(path.get_path()))
(status, result) = utils.getstatusoutput('./cts.sh'
' gen_node_cert {} {}/{}'
.format(
agent, node_dir, node))
os.chdir('{}'.format(path.get_path()))
if not bool(status):
LOGGER.info(' Generate %s cert successful! dir is %s/%s.',
node, node_dir, node)
os.chdir('{}'.format(path.get_path()))
if utils.Status.gm_option:
(status, result) = utils.getstatusoutput('cat {}/{}/gmagency.crt '
'>> {}/{}/gmnode.crt'.format(
_dir, node, _dir, node))
os.remove('{}/{}/gmagency.crt'.format(_dir, node))
os.remove('{}/{}/gmnode.serial'.format(_dir, node))
else:
(status, result) = utils.getstatusoutput('cat {}/{}/agency.crt '
'>> {}/{}/node.crt'.format(
_dir, node, _dir, node))
os.remove('{}/{}/agency.crt'.format(_dir, node))
os.remove('{}/{}/node.ca'.format(_dir, node))
os.remove('{}/{}/node.json'.format(_dir, node))
os.remove('{}/{}/node.private'.format(_dir, node))
os.remove('{}/{}/node.serial'.format(_dir, node))
os.remove('{}/{}/node.param'.format(_dir, node))
os.remove('{}/{}/node.pubkey'.format(_dir, node))
else:
# console_error(
# ' Generate node cert failed! Please check your network,'
# ' and try to check your opennssl version.')
LOGGER.error(' Generate %s cert failed! Result is %s',
node, result)
raise MCError(' Generate %s cert failed! Result is %s' %
(node, result))
except MCError as cert_exp:
console_error(' %s ' % cert_exp)
except Exception as gen_cert_exp:
console_error(
' Generate node cert failed! excepion is %s.' % gen_cert_exp)
LOGGER.error(' Generate node cert failed! Result is %s', result)
raise MCError(
'Generate node failed! Result is %s' % gen_cert_exp)
def get_cc(self, chain):
if self.exist(chain):
return self.ccs[chain.get_id() + '_' + chain.get_version()]
raise MCError(' cc not exist, chain %s' % chain)