def valid_url(_url):
"""check valid url
"""
baseURL = _url
# print fullURL
try:
if sys.version > '3':
try:
resp = urllib.request.urlopen(baseURL)
return True
except urllib.error.HTTPError as e:
# Return code error (e.g. 404, 501, ...)
# ...
LOGGER.warning('HTTPError: {}'.format(e.code))
return False
except urllib.error.URLError as e:
# Not an HTTP-specific error (e.g. connection refused)
# ...
LOGGER.warning('URLError: {}'.format(e.reason))
return False
LOGGER.warning('Maybe others err')
return False
else:
req = urllib2.Request(baseURL)
resp = urllib2.urlopen(req)
if resp.getcode() == 404:
# Do whatever you want if 404 is found
LOGGER.warning("404 Found!")
return False
# Do your normal stuff here if page is found.
LOGGER.info("URL: {0} Response: {1}".format(
baseURL, resp.getcode()))
return True
except Exception as download_err:
LOGGER.error("Could not connect to URL: %s ,err is %s",
baseURL, download_err)
return False
def create_group_genesis(data_dir='{}/meta'.format(path.get_path())):
'''
create group in meta
'''
LOGGER.info('create_group_genesis start')
package_dir = data_dir
gm_opr = utils.Status.gm_option
group_id = mgroup.MgroupConf.group_id
p2p_ip = mgroup.MgroupConf.p2p_ip
p2p_listen_port = mgroup.MgroupConf.p2p_listen_port
utils.file_must_not_exists('{}/group.{}.genesis'.format(
data_dir, group_id))
if not os.path.exists(package_dir):
LOGGER.warning(' %s not existed!', package_dir)
os.mkdir(data_dir)
shutil.copy('{}/tpl/group.i.genesis'.format(path.get_path()),
'{}/group.{}.genesis'.format(package_dir, group_id))
shutil.copy('{}/tpl/group.i.ini'.format(path.get_path()),
'{}/group.{}.ini'.format(package_dir, group_id))
# uptes second to ms
(status, time_stamp) = utils.getstatusoutput('echo $(date +%s"000")')
if not bool(status):
CONSOLER.info('generate %s/group.%s.genesis, successful', package_dir,
group_id)
else:
LOGGER.error(
' Generate %s/group.%s.genesis failed! Please check your network.',
package_dir, group_id)
raise MCError(
' Generate %s/group.%s.genesis failed, timestamp is %s! Please check your network.'
% (package_dir, group_id, time_stamp))
# CONSOLER.info('generate %s/group.%s.ini', package_dir, group_id)
group_cfg = configparser.ConfigParser()
with open('{}/group.{}.genesis'.format(package_dir, group_id),
'r') as config_file:
group_cfg.readfp(config_file)
for node_idx, _in in enumerate(p2p_ip):
try:
if gm_opr:
node_id = config.get_nodeid_str(
'{}/meta/gmcert_{}_{}.crt'.format(
path.get_path(), p2p_ip[node_idx],
p2p_listen_port[node_idx]))
node_id.strip('\n').strip(
'WARNING: can\'t open config file: /home/asherli/TASSL/ssl/openssl.cnf'
)
LOGGER.info('resolve %s/meta/gmcert_%s_%s.crt',
path.get_path(), p2p_ip[node_idx],
p2p_listen_port[node_idx])
LOGGER.info("nodeid -> %s", node_id)
group_cfg.set("consensus", "node.{}".format(node_idx), node_id)
else:
node_id = config.get_nodeid_str(
'{}/meta/cert_{}_{}.crt'.format(path.get_path(),
p2p_ip[node_idx],
p2p_listen_port[node_idx]))
LOGGER.info('resolve %s/meta/cert_%s_%s.crt', path.get_path(),
p2p_ip[node_idx], p2p_listen_port[node_idx])
LOGGER.info("nodeid -> %s", node_id)
group_cfg.set("consensus", "node.{}".format(node_idx), node_id)
except Exception as group_exp:
LOGGER.error('create group genesis failed! exception is %s',
group_exp)
raise MCError('create group genesis failed! exception is %s' %
group_exp)
group_cfg.set("group", "id", group_id)
group_cfg.set("group", "timestamp", time_stamp)
with open('{}/group.{}.genesis'.format(package_dir, group_id),
'w') as config_file:
group_cfg.write(config_file)
shutil.copy('{}/group.{}.genesis'.format(package_dir, group_id),
'{}/meta/group.{}.genesis'.format(path.get_path(), group_id))
LOGGER.info('create_group_genesis end')
def build_config_ini(_data_dir):
"""[-- build create config_ini]
Keyword Arguments:
_meta_dir {[PATH]} -- [input dir] (default: {meta})
_data_dir {[PATH]} -- [output dir] (default: {data})
Raises:
MCError -- [description]
MCError -- [description]
MCError -- [description]
MCError -- [description]
"""
LOGGER.info("build_config_ini start ")
p2p_listen_port = mconf.MchainConf.p2p_listen_port
jsonrpc_listen_port = mconf.MchainConf.jsonrpc_listen_port
channel_listen_port = mconf.MchainConf.channel_listen_port
p2p_ip = mconf.MchainConf.p2p_ip
rpc_ip = mconf.MchainConf.rpc_ip
peers = mconf.MchainConf.peers
meta_dir = '{}/meta'.format(path.get_path())
conf_dir = meta_dir
package_dir = _data_dir
gm_opr = utils.Status.gm_option
group_id = mconf.MchainConf.group_id
utils.file_must_exists('{}/group.{}.genesis'.format(meta_dir, group_id))
if os.path.exists(package_dir):
LOGGER.error(' %s existed, maybe u had created it!', package_dir)
raise MCError(' %s existed, maybe u had created it!' % package_dir)
os.mkdir(package_dir)
default_cfg = configparser.ConfigParser()
if gm_opr:
shutil.copy('{}/tpl/config.ini.gm'.format(path.get_path()),
'{}/.config.ini'.format(conf_dir))
else:
shutil.copy('{}/tpl/config.ini'.format(path.get_path()),
'{}/.config.ini'.format(conf_dir))
try:
with codecs.open('{}/.config.ini'.format(conf_dir),
'r', encoding='utf-8') as config_file:
default_cfg.readfp(config_file)
except Exception as build_exp:
LOGGER.error(
' open config.ini file failed, exception is %s', build_exp)
raise MCError(
' open config.ini file failed, exception is %s' % build_exp)
fin_p2p_ip = []
if not peers:
LOGGER.warning('section peers not existed!')
CONSOLER.warn('section peers not existed!')
else:
for _, peer in enumerate(peers):
fin_p2p_ip.append(peer)
# default_cfg.set("p2p", "node.{}".format(node_id + len(p2p_listen_port)),
# peer)
# with open('{}/.config.ini'.format(conf_dir), 'w') as config_file:
# default_cfg.write(config_file)
# init config.ini & node package
for my_node_index, node_ip in enumerate(p2p_ip):
LOGGER.info("p2p_ip -> %s", node_ip)
try:
if utils.Status.gm_option:
utils.file_must_exists('{}/gmcert_{}_{}.crt'.format(conf_dir,
node_ip,
p2p_listen_port[my_node_index]))
else:
utils.file_must_exists('{}/cert_{}_{}.crt'.format(conf_dir,
node_ip,
p2p_listen_port[my_node_index]))
except Exception as build_exp:
LOGGER.error('%s', build_exp)
raise MCError('%s' % build_exp)
CONSOLER.info(' Generate %s/node_%s_%s ',
package_dir, node_ip, p2p_listen_port[my_node_index])
node_dir = '{}/node_{}_{}'.format(package_dir,
node_ip, p2p_listen_port[my_node_index])
os.mkdir(node_dir)
shutil.copy('{}/tpl/start.sh'.format(path.get_path()),
'{}/start.sh'.format(node_dir))
shutil.copy('{}/tpl/stop.sh'.format(path.get_path()),
'{}/stop.sh'.format(node_dir))
shutil.copy('{}/fisco-bcos'.format(meta_dir),
'{}/fisco-bcos'.format(node_dir))
os.mkdir('{}/conf'.format(node_dir))
try:
# get node cert
shutil.copy('{}/.config.ini'.format(conf_dir),
'{}/config.ini'.format(node_dir))
shutil.copy('{}/group.{}.genesis'.format(conf_dir, group_id),
'{}/conf/group.{}.genesis'.format(node_dir, group_id))
shutil.copy('{}/tpl/group.i.ini'.format(path.get_path()),
'{}/conf/group.{}.ini'.format(node_dir, group_id))
if gm_opr:
get_node_cert('{}/gmcert_{}_{}.crt'.format(meta_dir, node_ip,
p2p_listen_port[my_node_index]),
'{}/conf/gmnode.crt'.format(node_dir))
# get_nodeid('{}/conf/gmnode.crt'.format(node_dir),
# '{}/conf/gmnode.nodeid'.format(node_dir))
shutil.copyfile('{}/gmca.crt'.format(meta_dir),
'{}/conf/gmca.crt'.format(node_dir))
else:
get_node_cert('{}/cert_{}_{}.crt'.format(meta_dir, node_ip,
p2p_listen_port[my_node_index]),
'{}/conf/node.crt'.format(node_dir))
# get_nodeid('{}/conf/node.crt'.format(node_dir),
# '{}/conf/node.nodeid'.format(node_dir))
shutil.copyfile('{}/ca.crt'.format(meta_dir),
'{}/conf/ca.crt'.format(node_dir))
except Exception as build_exp:
LOGGER.error(' get node.crt failed ! exception is %s', build_exp)
utils.delete_data(package_dir)
raise MCError(' get node.crt failed! exception is %s' % build_exp)
node_cfg = configparser.ConfigParser()
try:
with codecs.open('{}/config.ini'.format(node_dir),
'r', encoding='utf-8') as config_file:
node_cfg.readfp(config_file)
except Exception as build_exp:
LOGGER.error(
' open config.ini file failed, exception is %s', build_exp)
utils.delete_data(package_dir)
raise MCError(
' open config.ini file failed, exception is %s' % build_exp)
node_cfg.set("rpc", "listen_ip", rpc_ip[my_node_index])
node_cfg.set("rpc", "channel_listen_port",
channel_listen_port[my_node_index])
node_cfg.set("rpc", "jsonrpc_listen_port",
jsonrpc_listen_port[my_node_index])
# node_cfg.set("p2p", "listen_ip", p2p_ip[my_node_index])
node_cfg.set("p2p", "listen_port", p2p_listen_port[my_node_index])
with open('{}/config.ini'.format(node_dir), 'w') as config_file:
node_cfg.write(config_file)
config_file.close()
# set p2p ip in config.ini
for my_node_index, ip_item in enumerate(p2p_ip):
node_cfg = configparser.ConfigParser()
if not utils.valid_ip(ip_item):
LOGGER.error(
' init config.ini file failed, found ip => %s', ip_item)
utils.delete_data(package_dir)
raise MCError(
' init config.ini file failed, found ip => %s' % ip_item)
node_dir = '{}/node_{}_{}'.format(package_dir,
ip_item, p2p_listen_port[my_node_index])
try:
with codecs.open('{}/config.ini'.format(node_dir),
'r', encoding='utf-8') as config_file:
node_cfg.readfp(config_file)
except Exception as build_exp:
LOGGER.error(
' open config.ini file failed, exception is %s', build_exp)
utils.delete_data(package_dir)
raise MCError(
' open config.ini file failed, exception is %s' % build_exp)
# write p2pip:port into config.ini
for ip_idx, set_item in enumerate(p2p_ip):
fin_p2p_ip.append("{}:{}".format(set_item, p2p_listen_port[ip_idx]))
fin_p2p_ip = list(set(fin_p2p_ip))
for index, p2p_section in enumerate(fin_p2p_ip):
node_cfg.set("p2p", "node.{}".format(index),
'{}'.format(p2p_section))
with open('{}/config.ini'.format(node_dir), 'w') as config_file:
node_cfg.write(config_file)
os.mkdir(package_dir + '/scripts/')
shutil.copy('{}/scripts/install.sh'.format(path.get_path()),
package_dir + '/scripts/')
shutil.copy('{}/scripts/pack.sh'.format(path.get_path()),
package_dir + '/scripts/')
shutil.copy('{}/tpl/start_all.sh'.format(path.get_path()), package_dir)
shutil.copy('{}/tpl/stop_all.sh'.format(path.get_path()), package_dir)
shutil.copytree('{}/scripts/monitor'.format((path.get_path())),
'{}/monitor'.format(package_dir))
LOGGER.info("build_config_ini end!")
def parser(mchain):
"""resolve node_installation.ini
Arguments:
mchain {string} -- path of node_installation.ini
Raises:
MCError -- exception description
"""
LOGGER.info('node_installation.ini is %s', mchain)
# resolve configuration
if not utils.valid_string(mchain):
LOGGER.error(
' node_installation.ini not invalid path, node_installation.ini is %s',
mchain)
raise MCError(
' node_installation.ini not invalid path, node_installation.ini is %s'
% mchain)
# read and parser config file
config_parser = configparser.ConfigParser(allow_no_value=True)
try:
with codecs.open(mchain, 'r', encoding='utf-8') as file_mchain:
config_parser.readfp(file_mchain)
except Exception as ini_exp:
LOGGER.error(
' open node_installation.ini file failed, exception is %s',
ini_exp)
raise MCError(
' open node_installation.ini file failed, exception is %s' %
ini_exp)
for idx in range(0, 128):
node_index = ('node{}'.format(idx))
if config_parser.has_section('group'):
MchainConf.group_id = config_parser.get('group', 'group_id')
else:
LOGGER.error(
' invalid node_installation.ini format, group id is %s',
MchainConf.group_id)
raise MCError(
' invalid node_installation.ini format, group id is %s' %
MchainConf.group_id)
if config_parser.has_section(node_index):
p2p_ip = config_parser.get(node_index, 'p2p_ip')
rpc_ip = config_parser.get(node_index, 'rpc_ip')
if not utils.valid_ip(p2p_ip):
LOGGER.error(
' invalid node_installation.ini format, p2p_ip is %s',
p2p_ip)
raise MCError(
' invalid node_installation.ini format, p2p_ip is %s' %
p2p_ip)
# if rpc_ip == "0.0.0.0" and utils.Status.allow_unsecure_cfg:
if rpc_ip == "0.0.0.0":
LOGGER.warning('Your rpc_ip is %s, this is an unsecurity way',
rpc_ip)
CONSOLER.warning(
' \033[1;31m Your rpc_ip is %s, this is an unsecurity way \033[0m',
rpc_ip)
elif not utils.valid_ip(rpc_ip):
LOGGER.error(
' invalid node_installation.ini format, rpc_ip is %s',
rpc_ip)
raise MCError(
' invalid node_installation.ini format, rpc_ip is %s' %
rpc_ip)
p2p_listen_port = config_parser.get(node_index, 'p2p_listen_port')
jsonrpc_listen_port = config_parser.get(node_index,
'jsonrpc_listen_port')
channel_listen_port = config_parser.get(node_index,
'channel_listen_port')
if not (utils.valid_string(p2p_listen_port)
and utils.valid_string(jsonrpc_listen_port)
and utils.valid_string(channel_listen_port)):
LOGGER.error(
'mchain bad format, p2p_listen_port is %s, '
'jsonrpc_port is %s, channel_port is %s', p2p_listen_port,
jsonrpc_listen_port, channel_listen_port)
raise MCError('mchain bad format, p2p_listen_port is %s, '
'jsonrpc_port is %s, channel_port is %s' %
(p2p_listen_port, jsonrpc_listen_port,
channel_listen_port))
MchainConf.p2p_ip.append(p2p_ip)
MchainConf.rpc_ip.append(rpc_ip)
MchainConf.p2p_listen_port.append(p2p_listen_port)
MchainConf.jsonrpc_listen_port.append(jsonrpc_listen_port)
MchainConf.channel_listen_port.append(channel_listen_port)
else:
LOGGER.warning(' node%s not existed, break!', idx)
break
# if config_parser.has_section('peers'):
# for peer in config_parser.items('peers'):
# MchainConf.peers.append(peer[1])
# else:
# LOGGER.warning(' section peers not existed!')
LOGGER.info('group_id is %s', MchainConf.group_id)
LOGGER.info('p2p_ip is %s', MchainConf.p2p_ip)
LOGGER.info('rpc_ip is %s', MchainConf.rpc_ip)
LOGGER.info('p2p_listen_port is %s', MchainConf.p2p_listen_port)
LOGGER.info('jsonrpc_listen_port is %s', MchainConf.jsonrpc_listen_port)
LOGGER.info('channel_listen_port is %s', MchainConf.channel_listen_port)
LOGGER.info('peers is %s', MchainConf.peers)
LOGGER.info('node_installation.ini end, result is %s', MchainConf())
def build_package_only(_data_dir):
"""[-- build create config_ini]
Keyword Arguments:
_meta_dir {[PATH]} -- [input dir] (default: {meta})
_data_dir {[PATH]} -- [output dir] (default: {data})
Raises:
MCError -- [description]
MCError -- [description]
MCError -- [description]
MCError -- [description]
"""
LOGGER.info("build_package_only start ")
p2p_listen_port = mconf.MchainConf.p2p_listen_port
jsonrpc_listen_port = mconf.MchainConf.jsonrpc_listen_port
channel_listen_port = mconf.MchainConf.channel_listen_port
p2p_ip = mconf.MchainConf.p2p_ip
rpc_ip = mconf.MchainConf.rpc_ip
channel_ip = mconf.MchainConf.channel_ip
peers = mconf.MchainConf.peers
meta_dir = '{}/meta'.format(path.get_path())
conf_dir = meta_dir
package_dir = _data_dir
gm_opr = utils.Status.gm_option
if os.path.exists(package_dir):
LOGGER.error(' %s existed, maybe u had created it!', package_dir)
raise MCError(' %s existed, maybe u had created it!' % package_dir)
os.mkdir(package_dir)
if gm_opr:
shutil.copy('{}/tpl/config.ini.gm'.format(path.get_path()),
'{}/.config.ini'.format(conf_dir))
else:
shutil.copy('{}/tpl/config.ini'.format(path.get_path()),
'{}/.config.ini'.format(conf_dir))
fin_p2p_ip = []
if not peers:
LOGGER.warning('section peers not existed!')
CONSOLER.warn('section peers not existed!')
else:
for _, peer in enumerate(peers):
fin_p2p_ip.append(peer)
# init config.ini & node package
for my_node_index, node_ip in enumerate(p2p_ip):
LOGGER.info("p2p_ip -> %s", node_ip)
CONSOLER.info(' Generate %s/node_%s_%s ',
package_dir, node_ip, p2p_listen_port[my_node_index])
node_dir = '{}/node_{}_{}'.format(package_dir,
node_ip, p2p_listen_port[my_node_index])
os.mkdir(node_dir)
os.mkdir('{}/scripts'.format(node_dir))
shutil.copy('{}/tpl/start.sh'.format(path.get_path()),
'{}/start.sh'.format(node_dir))
shutil.copy('{}/tpl/stop.sh'.format(path.get_path()),
'{}/stop.sh'.format(node_dir))
shutil.copy('{}/tpl/load_new_groups.sh'.format(path.get_path()),
'{}/scripts/load_new_groups.sh'.format(node_dir))
shutil.copy('{}/tpl/reload_whitelist.sh'.format(path.get_path()),
'{}/scripts/reload_whitelist.sh'.format(node_dir))
shutil.copy('{}/fisco-bcos'.format(meta_dir),
'{}/fisco-bcos'.format(node_dir))
os.mkdir('{}/conf'.format(node_dir))
try:
# cp config.ini
shutil.copy('{}/.config.ini'.format(conf_dir),
'{}/config.ini'.format(node_dir))
except Exception as build_exp:
LOGGER.error(' exception is %s', build_exp)
utils.delete_data(package_dir)
raise MCError(' exception is %s' % build_exp)
node_cfg = configparser.ConfigParser(allow_no_value=True)
try:
with codecs.open('{}/config.ini'.format(node_dir),
'r', encoding='utf-8') as config_file:
node_cfg.readfp(config_file)
except Exception as build_exp:
LOGGER.error(
' open config.ini file failed, exception is %s', build_exp)
utils.delete_data(package_dir)
raise MCError(
' open config.ini file failed, exception is %s' % build_exp)
if len(rpc_ip) > my_node_index:
node_cfg.set("rpc", "jsonrpc_listen_ip", rpc_ip[my_node_index])
else:
node_cfg.set("rpc", "jsonrpc_listen_ip", "127.0.0.1")
if len(channel_ip) > my_node_index:
node_cfg.set("rpc", "channel_listen_ip", channel_ip[my_node_index])
else:
node_cfg.set("rpc", "channel_listen_ip", "0.0.0.0")
node_cfg.set("rpc", "channel_listen_port",
channel_listen_port[my_node_index])
node_cfg.set("rpc", "jsonrpc_listen_port",
jsonrpc_listen_port[my_node_index])
node_cfg.set("p2p", "listen_port", p2p_listen_port[my_node_index])
with open('{}/config.ini'.format(node_dir), 'w') as config_file:
node_cfg.write(config_file)
config_file.close()
# set p2p ip in config.ini
for my_node_index, ip_item in enumerate(p2p_ip):
node_cfg = configparser.ConfigParser(allow_no_value=True)
if not utils.valid_ip(ip_item):
LOGGER.error(
' init config.ini file failed, found ip => %s', ip_item)
utils.delete_data(package_dir)
raise MCError(
' init config.ini file failed, found ip => %s' % ip_item)
node_dir = '{}/node_{}_{}'.format(package_dir,
ip_item, p2p_listen_port[my_node_index])
try:
with codecs.open('{}/config.ini'.format(node_dir),
'r', encoding='utf-8') as config_file:
node_cfg.readfp(config_file)
except Exception as build_exp:
LOGGER.error(
' open config.ini file failed, exception is %s', build_exp)
utils.delete_data(package_dir)
raise MCError(
' open config.ini file failed, exception is %s' % build_exp)
# write p2pip:port into config.ini
for ip_idx, set_item in enumerate(p2p_ip):
fin_p2p_ip.append("{}:{}".format(
set_item, p2p_listen_port[ip_idx]))
fin_p2p_ip = list(set(fin_p2p_ip))
for index, p2p_section in enumerate(fin_p2p_ip):
node_cfg.set("p2p", "node.{}".format(index),
'{}'.format(p2p_section))
node_cfg.set('certificate_whitelist',
'; cal.0 should be nodeid, nodeid\'s length is 128')
node_cfg.set('certificate_whitelist', ';cal.0=')
node_cfg.set('certificate_blacklist',
'; crl.0 should be nodeid, nodeid\'s length is 128')
node_cfg.set('certificate_blacklist', ';crl.0=')
with open('{}/config.ini'.format(node_dir), 'w') as config_file:
node_cfg.write(config_file)
os.mkdir(package_dir + '/scripts/')
shutil.copy('{}/scripts/install.sh'.format(path.get_path()),
package_dir + '/scripts/')
shutil.copy('{}/scripts/pack.sh'.format(path.get_path()),
package_dir + '/scripts/')
shutil.copy('{}/tpl/start_all.sh'.format(path.get_path()), package_dir)
shutil.copy('{}/tpl/stop_all.sh'.format(path.get_path()), package_dir)
shutil.copytree('{}/scripts/monitor'.format((path.get_path())),
'{}/monitor'.format(package_dir))
LOGGER.info("build_package_only end!")