def addNode(self, name: str) -> TestNode:
if name in self.nodes:
error("{} already added".format(name))
assert name in self.nodeReg
ha, cliname, cliha = self.nodeReg[name]
config_helper = PNodeConfigHelper(name, self.config, chroot=self.tmpdir)
seed = randomSeed()
if self.keyshare:
learnKeysFromOthers(config_helper.keys_dir, name, self.nodes.values())
testNodeClass = self.testNodeClass
node = self.enter_context(
testNodeClass(name=name,
ha=ha,
cliname=cliname,
cliha=cliha,
nodeRegistry=copy(self.nodeReg),
ledger_dir=config_helper.ledger_dir,
keys_dir=config_helper.keys_dir,
genesis_dir=config_helper.genesis_dir,
plugins_dir=config_helper.plugins_dir,
primaryDecider=self.primaryDecider,
pluginPaths=self.pluginPaths,
seed=seed))
if self.keyshare:
tellKeysToOthers(node, self.nodes.values())
self.nodes[name] = node
self.__dict__[name] = node
return node
def createOrientDbInMemStore(config, name, dbType):
"""
Create and return an OrientDb in-memory store used for test cases.
"""
host = "localhost"
port = 2424
try:
client = pyorient.OrientDB(host=host, port=port)
client.connect(user=config.OrientDB['user'],
password=config.OrientDB['password'])
# except ValueError:
# client.connect(user=config.OrientDB['user'],
# password=config.OrientDB['password'])
except pyorient.exceptions.PyOrientConnectionException:
error("OrientDB connection failed. Check if DB is running "
"on port {}".format(port))
try:
if client.db_exists(name, pyorient.STORAGE_TYPE_MEMORY):
client.db_drop(name, type=pyorient.STORAGE_TYPE_MEMORY)
# This is to avoid a known bug in OrientDb.
except pyorient.exceptions.PyOrientDatabaseException:
client.db_drop(name, type=pyorient.STORAGE_TYPE_MEMORY)
return OrientDbStore(user=config.OrientDB["user"],
password=config.OrientDB["password"],
dbName=name,
dbType=dbType,
storageType=pyorient.STORAGE_TYPE_MEMORY)
def checkDblImp():
"""
Added this because I spent the better part of an evening troubleshooting an
issue cause by double import. We were importing test.helper in one
place, and test_helper in another, and python sees them as two different
modules, and imported the same file twice. This caused genHa to be loaded
twice, which caused overlapping ports to be assigned. Took a long time to
track this down. I'm sure there's a better way to do this, but this seems
to work for the basic testing I did.
"""
logger.info("-------------checking for double imports-------------")
ignore = {
'posixpath.py', 'helpers/pydev/pydevd.py', 'importlib/_bootstrap.py',
'importlib/_bootstrap_external.py', 'helpers/pycharm/pytestrunner.py',
'test/__init__.py', 'site-packages/pytest.py', 'python3.5/os.py',
'python3.5/re.py'
}
files = [
x.__file__ for x in list(sys.modules.values())
if hasattr(x, "__file__")
]
dups = set([x for x in files if files.count(x) > 1])
ignoreddups = {d for d in dups for i in ignore if i in d}
filtereddups = dups - ignoreddups
if filtereddups:
error("Doubly imported files detected {}".format(filtereddups))
def __init__(self, *args, **kwargs):
checkPortAvailable(kwargs['ha'])
basedirpath = kwargs.get('basedirpath')
keep = RoadKeep(basedirpath=basedirpath,
stackname=kwargs['name'],
auto=kwargs.get('auto'),
baseroledirpath=basedirpath) # type: RoadKeep
kwargs['keep'] = keep
localRoleData = keep.loadLocalRoleData()
sighex = kwargs.pop('sighex', None) or localRoleData['sighex']
if not sighex:
(sighex, _), (prihex, _) = getEd25519AndCurve25519Keys()
else:
prihex = ed25519SkToCurve25519(sighex, toHex=True)
kwargs['sigkey'] = sighex
kwargs['prikey'] = prihex
self.msgHandler = kwargs.pop('msgHandler', None) # type: Callable
super().__init__(*args, **kwargs)
if self.ha[1] != kwargs['ha'].port:
error("the stack port number has changed, likely due to "
"information in the keep. {} passed {}, actual {}".
format(kwargs['name'], kwargs['ha'].port, self.ha[1]))
self.created = time.perf_counter()
self.coro = None
config = getConfig()
try:
self.messageTimeout = config.RAETMessageTimeout
except AttributeError:
# if no timeout is set then message will never timeout
self.messageTimeout = 0
def checkDblImp():
"""
Added this because I spent the better part of an evening troubleshooting an
issue cause by double import. We were importing test.helper in one
place, and test_helper in another, and python sees them as two different
modules, and imported the same file twice. This caused genHa to be loaded
twice, which caused overlapping ports to be assigned. Took a long time to
track this down. I'm sure there's a better way to do this, but this seems
to work for the basic testing I did.
"""
logger.info("-------------checking for double imports-------------")
ignore = {'posixpath.py',
'helpers/pydev/pydevd.py',
'importlib/_bootstrap.py',
'importlib/_bootstrap_external.py',
'helpers/pycharm/pytestrunner.py',
'test/__init__.py',
'site-packages/pytest.py',
'python3.5/os.py',
'python3.5/re.py'}
files = [x.__file__ for x in list(sys.modules.values())
if hasattr(x, "__file__")]
dups = set([x for x in files if files.count(x) > 1])
ignoreddups = {d for d in dups for i in ignore if i in d}
filtereddups = dups - ignoreddups
if filtereddups:
error("Doubly imported files detected {}".format(filtereddups))
def addNode(self, name: str) -> TestNode:
if name in self.nodes:
error("{} already added".format(name))
assert name in self.nodeReg
ha, cliname, cliha = self.nodeReg[name]
seed = randomSeed()
if self.keyshare:
learnKeysFromOthers(self.tmpdir, name, self.nodes.values())
testNodeClass = self.testNodeClass
node = self.enter_context(
testNodeClass(name=name,
ha=ha,
cliname=cliname,
cliha=cliha,
nodeRegistry=copy(self.nodeReg),
basedirpath=self.tmpdir,
base_data_dir=self.tmpdir,
primaryDecider=self.primaryDecider,
pluginPaths=self.pluginPaths,
seed=seed))
if self.keyshare:
tellKeysToOthers(node, self.nodes.values())
self.nodes[name] = node
self.__dict__[name] = node
return node
def __init__(self, length: int, **kwargs):
if not isinstance(length, int):
error('length should be integer', TypeError)
if length < 1:
error('should be greater than 0', ValueError)
self.length = length
super().__init__(**kwargs)
def __init__(self, *args, **kwargs):
checkPortAvailable(kwargs['ha'])
basedirpath = kwargs.get('basedirpath')
keep = RoadKeep(basedirpath=basedirpath,
stackname=kwargs['name'],
auto=kwargs.get('auto'),
baseroledirpath=basedirpath) # type: RoadKeep
kwargs['keep'] = keep
localRoleData = keep.loadLocalRoleData()
sighex = kwargs.pop('sighex', None) or localRoleData['sighex']
if not sighex:
(sighex, _), (prihex, _) = getEd25519AndCurve25519Keys()
else:
prihex = ed25519SkToCurve25519(sighex, toHex=True)
kwargs['sigkey'] = sighex
kwargs['prikey'] = prihex
self.msgHandler = kwargs.pop('msgHandler', None) # type: Callable
super().__init__(*args, **kwargs)
if self.ha[1] != kwargs['ha'].port:
error("the stack port number has changed, likely due to "
"information in the keep. {} passed {}, actual {}".format(
kwargs['name'], kwargs['ha'].port, self.ha[1]))
self.created = time.perf_counter()
self.coro = None
config = getConfig()
try:
self.messageTimeout = config.RAETMessageTimeout
except AttributeError:
# if no timeout is set then message will never timeout
self.messageTimeout = 0
def genTestClient(nodes=None,
nodeReg=None,
tmpdir=None,
testClientClass=TestClient,
identifier: Identifier = None,
verkey: str = None,
bootstrapKeys=True,
ha=None,
usePoolLedger=False,
name=None,
sighex=None) -> (TestClient, Wallet):
if not usePoolLedger:
nReg = nodeReg
if nodeReg:
assert isinstance(nodeReg, dict)
elif hasattr(nodes, "nodeReg"):
nReg = nodes.nodeReg.extractCliNodeReg()
else:
error("need access to nodeReg")
for k, v in nReg.items():
assert type(k) == str
assert (type(v) == HA or type(v[0]) == HA)
else:
logger.debug("TestClient using pool ledger")
nReg = None
ha = genHa() if not ha else ha
name = name or "testClient{}".format(ha.port)
tc = testClientClass(name,
nodeReg=nReg,
ha=ha,
basedirpath=tmpdir,
sighex=sighex)
if not usePoolLedger and nodes:
for node in nodes:
stack = node.clientstack
# TODO: Remove this if condition once raet is removed
if isinstance(stack, ZStack):
initRemoteKeys(tc.name,
stack.name,
tmpdir,
stack.verhex,
override=True)
w = None # type: Wallet
if bootstrapKeys and nodes:
if not identifier or not verkey:
# no identifier or verkey were provided, so creating a wallet
w = Wallet("test")
w.addIdentifier()
identifier = w.defaultId
verkey = w.getVerkey()
bootstrapClientKeys(identifier, verkey, nodes)
return tc, w
def cleanSeed(seed=None):
if seed:
bts = seedFromHex(seed)
if not bts:
if isinstance(seed, str):
seed = seed.encode('utf-8')
bts = bytes(seed)
if len(seed) != 32:
error('seed length must be 32 bytes')
return bts
def cleanSeed(seed=None):
if seed:
bts = seedFromHex(seed)
if not bts:
if isinstance(seed, str):
seed = seed.encode('utf-8')
bts = bytes(seed)
if len(seed) != 32:
error('seed length must be 32 bytes')
return bts
def __init__(self,
names: Iterable[str] = None,
count: int = None,
nodeReg=None,
tmpdir=None,
keyshare=True,
primaryDecider=None,
pluginPaths: Iterable[str] = None,
testNodeClass=TestNode):
super().__init__()
self.tmpdir = tmpdir
self.keyshare = keyshare
self.primaryDecider = primaryDecider
self.pluginPaths = pluginPaths
self.testNodeClass = testNodeClass
self.nodes = OrderedDict() # type: Dict[str, TestNode]
# Can use just self.nodes rather than maintaining a separate dictionary
# but then have to pluck attributes from the `self.nodes` so keeping
# it simple a the cost of extra memory and its test code so not a big
# deal
if nodeReg:
self.nodeReg = nodeReg
else:
nodeNames = (names if names is not None and count is None else
genNodeNames(count) if count is not None else error(
"only one of either names or count is required"))
self.nodeReg = genNodeReg(
names=nodeNames) # type: Dict[str, NodeDetail]
for name in self.nodeReg.keys():
self.addNode(name)
# The following lets us access the nodes by name as attributes of the
# NodeSet. It's not a problem unless a node name shadows a member.
self.__dict__.update(self.nodes)
def serialize(obj, level=0, objname=None, topLevelKeysToIgnore=None):
"""
Create a string representation of the given object.
Examples:
::
>>> serialize("str")
'str'
>>> serialize([1,2,3,4,5])
'1,2,3,4,5'
>>> signing.serlize({1:'a', 2:'b'})
'1:a|2:b'
>>> signing.serlize({1:'a', 2:'b', 3:[1,{2:'k'}]})
'1:a|2:b|3:1,2:k'
:param obj: the object to serlize
:param level: a parameter used internally for recursion to serialize nested
data structures
:param topLevelKeysToIgnore: the list of top level keys to ignore for
serialization
:return: a string representation of `obj`
"""
if not isinstance(obj, acceptableTypes):
error("invalid type found {}: {}".format(objname, obj))
if isinstance(obj, str):
return obj
if isinstance(obj, dict):
if level > 0:
keys = list(obj.keys())
else:
topLevelKeysToIgnore = topLevelKeysToIgnore or []
keys = [k for k in obj.keys() if k not in topLevelKeysToIgnore]
keys.sort()
strs = []
for k in keys:
onm = ".".join([objname, k]) if objname else k
strs.append(str(k) + ":" + serialize(obj[k], level + 1, onm))
return "|".join(strs)
if isinstance(obj, Iterable):
strs = []
for o in obj:
strs.append(serialize(o, level + 1, objname))
return ",".join(strs)
if obj is None:
return ""
else:
return str(obj)
def __init__(self, user, password, dbName, host="localhost", port=2424,
dbType=pyorient.DB_TYPE_GRAPH,
storageType=pyorient.STORAGE_TYPE_MEMORY):
self.dbType = dbType
try:
self.client = pyorient.OrientDB(host=host, port=port)
self.session_id = self.client.connect(user, password)
except pyorient.exceptions.PyOrientConnectionException:
raise OrientDBNotRunning("OrientDB connection failed. Check if DB is running "
"on port {}".format(port))
if not self.client.db_exists(dbName, storageType):
self.createDb(dbName, dbType, storageType)
self.client.db_open(dbName, user, password)
if not (self.serverVersion and self.serverVersion[0] >= 2 and
self.serverVersion[1] >= 2):
error("OrientDB version should be atleast 2.2. Current version is {}"
.format(".".join(map(str, self.serverVersion))))
def __init__(self, user, password, dbName, host="localhost", port=2424,
dbType=pyorient.DB_TYPE_GRAPH,
storageType=pyorient.STORAGE_TYPE_MEMORY):
self.dbType = dbType
try:
self.client = pyorient.OrientDB(host=host, port=port)
self.session_id = self.client.connect(user, password)
except pyorient.exceptions.PyOrientConnectionException:
error("OrientDB connection failed. Check if DB is running "
"on port {}".format(port))
if not self.client.db_exists(dbName, storageType):
self.createDb(dbName, dbType, storageType)
self.client.db_open(dbName, user, password)
if not (self.serverVersion and self.serverVersion[0] >= 2 and
self.serverVersion[1] >= 2):
error("OrientDB version should be atleast 2.2. Current version is {}"
.format(".".join(map(str, self.serverVersion))))
def addNode(self, name: str) -> TestNode:
if name in self.nodes:
error("{} already added".format(name))
assert name in self.nodeReg
ha, cliname, cliha = self.nodeReg[name]
testNodeClass = self.testNodeClass
node = self.enter_context(
testNodeClass(name=name,
ha=ha,
cliname=cliname,
cliha=cliha,
nodeRegistry=copy(self.nodeReg),
basedirpath=self.tmpdir,
primaryDecider=self.primaryDecider,
pluginPaths=self.pluginPaths))
self.nodes[name] = node
self.__dict__[name] = node
return node
def getSymmetricallyEncryptedVal(val, secretKey: Union[str, bytes]=None) -> Tuple[str, str]:
"""
Encrypt the provided value with symmetric encryption
:param val: the value to encrypt
:param secretKey: Optional key, if provided should be either in hex or bytes
:return: Tuple of the encrypted value and secret key encoded in hex
"""
if isinstance(val, str):
val = val.encode("utf-8")
if secretKey:
if isHex(secretKey):
secretKey = bytes(bytearray.fromhex(secretKey))
elif not isinstance(secretKey, bytes):
error("Secret key must be either in hex or bytes")
box = libnacl.secret.SecretBox(secretKey)
else:
box = libnacl.secret.SecretBox()
return box.encrypt(val).hex(), box.sk.hex()
def getSymmetricallyEncryptedVal(val, secretKey: Union[str, bytes]=None) -> Tuple[str, str]:
"""
Encrypt the provided value with symmetric encryption
:param val: the value to encrypt
:param secretKey: Optional key, if provided should be either in hex or bytes
:return: Tuple of the encrypted value and secret key encoded in hex
"""
if isinstance(val, str):
val = val.encode("utf-8")
if secretKey:
if isHex(secretKey):
secretKey = bytes(bytearray.fromhex(secretKey))
elif not isinstance(secretKey, bytes):
error("Secret key must be either in hex or bytes")
box = libnacl.secret.SecretBox(secretKey)
else:
box = libnacl.secret.SecretBox()
return box.encrypt(val).hex(), box.sk.hex()
def serialize(obj, level=0, objname=None):
"""
Create a string representation of the given object.
Examples:
::
>>> serialize("str")
'str'
>>> serialize([1,2,3,4,5])
'1,2,3,4,5'
>>> signing.serlize({1:'a', 2:'b'})
'1:a|2:b'
>>> signing.serlize({1:'a', 2:'b', 3:[1,{2:'k'}]})
'1:a|2:b|3:1,2:k'
:param obj: the object to serlize
:param level: a parameter used internally for recursion to serialize nested data structures
:return: a string representation of `obj`
"""
if not isinstance(obj, acceptableTypes):
error("invalid type found {}: {}".format(objname, obj))
if isinstance(obj, str):
return obj
if isinstance(obj, dict):
keys = [k for k in obj.keys() if level > 0 or k != f.SIG.nm] # remove signature if top level
keys.sort()
strs = []
for k in keys:
onm = ".".join([objname, k]) if objname else k
strs.append(str(k) + ":" + serialize(obj[k], level + 1, onm))
return "|".join(strs)
if isinstance(obj, Iterable):
strs = []
for o in obj:
strs.append(serialize(o, level + 1, objname))
return ",".join(strs)
if obj is None:
return ""
else:
return str(obj)
def createOrientDbInMemStore(config, name, dbType):
"""
Create and return an OrientDb in-memory store used for test cases.
"""
host = "localhost"
port = 2424
try:
client = pyorient.OrientDB(host=host, port=port)
client.connect(user=config.OrientDB['user'],
password=config.OrientDB['password'])
except pyorient.exceptions.PyOrientConnectionException:
error("OrientDB connection failed. Check if DB is running "
"on port {}".format(port))
try:
if client.db_exists(name, pyorient.STORAGE_TYPE_MEMORY):
client.db_drop(name, type=pyorient.STORAGE_TYPE_MEMORY)
# This is to avoid a known bug in OrientDb.
except pyorient.exceptions.PyOrientDatabaseException:
client.db_drop(name, type=pyorient.STORAGE_TYPE_MEMORY)
return OrientDbStore(user=config.OrientDB["user"],
password=config.OrientDB["password"],
dbName=name,
dbType=dbType,
storageType=pyorient.STORAGE_TYPE_MEMORY)
def getNodeName(node: NodeRef) -> str:
return node if isinstance(node, str) \
else node.name if isinstance(node, Node) \
else error("Expected a node or node name")
def getNode(self, node: NodeRef) -> TestNode:
return node if isinstance(node, Node) \
else self.nodes.get(node) if isinstance(node, str) \
else error("Expected a node or node name")
def core_authenticator(self):
if not self._authenticators:
error('No authenticator registered yet', RuntimeError)
return self._authenticators[0]
def register_hook(self, hook_id, hook: Callable):
if hook_id not in self.hooks:
error('Unknown hook id', KeyError)
self.hooks[hook_id].append(hook)