def get_client(
src: typing.Union[Node, RunContext, NetworkIdentifier]
) -> pyclx.CasperLabsClient:
"""Factory method to return configured clabs client.
:param src: The source form which a network node will be derived.
:returns: A configured clabs client ready for use.
"""
# Pull node information from cache.
if isinstance(src, Node):
node = src
elif isinstance(src, NetworkIdentifier):
node = cache.get_node_by_network_id(src)
elif isinstance(src, RunContext):
node = cache.get_run_node(src)
if not node:
raise ValueError(
"Network nodeset is empty, therefore cannot dispatch a deploy.")
logger.log(
f"PYCLX :: connecting to node :: {node.network} :: {node.host}:{node.port}"
)
# TODO: get node id / client ssl cert.
return pyclx.CasperLabsClient(
host=node.host,
port=node.port,
)
def main(args):
"""Entry point.
:param args: Parsed CLI arguments.
"""
# Unpack.
host = args.address.split(':')[0]
index = int(args.node.split(':')[-1])
network = args.node.split(':')[0]
port = int(args.address.split(':')[-1])
typeof = NodeType[args.typeof.upper()]
# Instantiate.
node = factory.create_node(host=host,
index=index,
network_id=factory.create_network_id(network),
port=port,
typeof=typeof)
# Push.
cache.set_network_node(node)
# Notify.
logger.log(f"Node {args.node} was successfully registered")
def main(args):
"""Entry point.
:param args: Parsed CLI arguments.
"""
# Unpack.
network_id = factory.create_network_id(args.node.split(':')[0])
node_id = factory.create_node_id(network_id, int(args.node.split(':')[-1]))
# Pull.
node = cache.get_node(node_id)
if node is None:
raise ValueError("Unregistered node.")
# Set key pair.
pvk, pbk = crypto.get_key_pair_from_pvk_pem_file(args.pem_path,
crypto.KeyEncoding.HEX)
# Set bonding account.
node.account = factory.create_account(index=-node_id.index,
private_key=pvk,
public_key=pbk,
status=AccountStatus.ACTIVE,
typeof=AccountType.BOND)
# Push.
cache.set_network_node(node)
# Inform.
logger.log(f"Node {args.node} bonding key was successfully registered")
def main(args):
"""Entry point.
:param args: Parsed CLI arguments.
"""
# Pull data.
network_id = factory.create_network_id(args.network)
data = cache.orchestration.get_info_list(network_id, args.run_type)
data = [i for i in data if i.aspect == ExecutionAspect.RUN]
if not data:
logger.log("No run information found.")
return
# Set cols/rows.
cols = ["Network", "Type", "ID", "Start Time", "Duration (s)", "Status"]
rows = map(
lambda i: [
network_id.name, i.run_type,
i.index_label.strip(), i.ts_start, i.tp_elapsed_label, i.
status_label
], sorted(data, key=lambda i: f"{i.run_type}.{i.index_label}"))
# Set table.
t = get_table(cols, rows)
t.column_alignments['Start Time'] = BeautifulTable.ALIGN_LEFT
t.column_alignments['Duration (s)'] = BeautifulTable.ALIGN_RIGHT
# Render.
print(t)
print(f"total runs = {len(data)}")
def on_run_end(ctx: ExecutionContext):
"""Ends a workflow.
:param ctx: Execution context information.
"""
# Update ctx.
ctx.status = ExecutionStatus.COMPLETE
# Set info/state.
run_state = factory.create_state(ExecutionAspect.RUN, ctx)
# Update cache.
cache.orchestration.set_context(ctx)
cache.orchestration.set_state(run_state)
cache.orchestration.update_run_info(ctx)
# Locks can now be flushed.
cache.orchestration.flush_locks(ctx)
# Inform.
logger.log(f"WFLOW :: {ctx.run_type} :: {ctx.run_index_label} -> ends")
# Loop.
if ctx.loop_count != 0:
do_run_loop(ctx)
def main(args):
"""Entry point.
:param args: Parsed CLI arguments.
"""
# Pull data.
network_id = factory.create_network_id(args.network)
data = cache.state.get_deploys(network_id, args.run_type, args.run_index)
if not data:
logger.log("No run deploys found.")
return
# Set table cols/rows.
cols = [i for i, _ in COLS]
rows = map(lambda i: [
i.deploy_hash,
i.typeof.name,
i.status.name,
i.dispatch_node,
i.dispatch_ts,
i.label_finalization_time,
i.block_hash or "--"
], sorted(data, key=lambda i: i.dispatch_ts))
# Set table.
t = get_table(cols, rows, max_width=1080)
# Set table alignments.
for key, aligmnent in COLS:
t.column_alignments[key] = aligmnent
# Render.
print(t)
print(f"{network_id.name} - {args.run_type} - Run {args.run_index}")
def main(args):
"""Entry point.
:param args: Parsed CLI arguments.
"""
# Pull.
network = cache.get_network_by_name(args.network)
if network is None:
raise ValueError("Unregistered network.")
# Set key pair.
pvk, pbk = crypto.get_key_pair_from_pvk_pem_file(args.pem_path,
crypto.KeyEncoding.HEX)
# Set faucet.
network.faucet = factory.create_account(index=0,
private_key=pvk,
public_key=pbk,
typeof=AccountType.FAUCET,
status=AccountStatus.ACTIVE)
# Push.
cache.set_network(network)
# Inform.
logger.log(
f"Network {args.network} faucet key was successfully registered")
def do_step_end(ctx: ExecutionContext):
"""Ends a workflow step.
:param ctx: Execution context information.
"""
# Set step.
step = Workflow.get_phase_step(ctx, ctx.phase_index, ctx.step_index)
# Set info/state.
step_state = factory.create_state(ExecutionAspect.STEP, ctx,
ExecutionStatus.COMPLETE)
# Update cache.
cache.orchestration.set_state(step_state)
cache.orchestration.update_step_info(ctx, ExecutionStatus.COMPLETE)
# Inform.
logger.log(
f"WFLOW :: {ctx.run_type} :: {ctx.run_index_label} :: {ctx.phase_index_label} :: {ctx.step_index_label} :: {step.label} -> end"
)
# Enqueue either end of phase or next step.
if step.is_last:
on_phase_end.send(ctx)
else:
do_step.send(ctx)
def do_run(ctx: ExecutionContext):
"""Runs a workflow.
:param ctx: Execution context information.
"""
# Escape if unexecutable.
if not predicates.can_start_run(ctx):
return
# Update ctx.
ctx.status = ExecutionStatus.IN_PROGRESS
# Set info/state.
run_info = factory.create_info(ExecutionAspect.RUN, ctx)
run_state = factory.create_state(ExecutionAspect.RUN, ctx)
# Update cache.
cache.flush_by_run(ctx)
cache.orchestration.set_context(ctx)
cache.orchestration.set_info(run_info)
cache.orchestration.set_state(run_state)
# Inform.
logger.log(f"WFLOW :: {ctx.run_type} :: {ctx.run_index_label} -> starts")
# Run phase.
do_phase.send(ctx)
def do_step(ctx: ExecutionContext):
"""Runs a workflow step.
:param ctx: Execution context information.
"""
# Escape if unexecutable.
if not predicates.can_start_step(ctx):
return
# Set step.
step = Workflow.get_phase_step(ctx, ctx.phase_index, ctx.step_index + 1)
# Update ctx.
ctx.step_index += 1
ctx.step_label = step.label
# Set info/state.
step_info = factory.create_info(ExecutionAspect.STEP, ctx)
step_state = factory.create_state(ExecutionAspect.STEP, ctx,
ExecutionStatus.IN_PROGRESS)
# Update cache.
cache.orchestration.set_context(ctx)
cache.orchestration.set_info(step_info)
cache.orchestration.set_state(step_state)
# Inform.
logger.log(
f"WFLOW :: {ctx.run_type} :: {ctx.run_index_label} :: {ctx.phase_index_label} :: {ctx.step_index_label} :: {step.label} -> starts"
)
# Execute.
do_step_execute(ctx, step)
def on_phase_end(ctx: ExecutionContext):
"""Ends a workflow phase.
:param ctx: Execution context information.
"""
# Set phase.
phase = Workflow.get_phase_(ctx, ctx.phase_index)
# Set info/state.
phase_state = factory.create_state(ExecutionAspect.PHASE,
ctx,
status=ExecutionStatus.COMPLETE)
# Update cache.
cache.orchestration.set_state(phase_state)
cache.orchestration.update_phase_info(ctx, ExecutionStatus.COMPLETE)
# Inform.
logger.log(
f"WFLOW :: {ctx.run_type} :: {ctx.run_index_label} :: {ctx.phase_index_label} -> ends"
)
# Enqueue either end of workflow or next phase.
if phase.is_last:
on_run_end.send(ctx)
else:
do_phase.send(ctx)
def main(args: argparse.Namespace):
"""Entry point.
"""
# Import initialiser to setup upstream services / actors.
import stests.initialiser
# Unpack args.
network_id = factory.create_network_id(args.network_name)
node_id = factory.create_node_id(network_id, args.node_index)
# Set execution context.
ctx = factory.create_run_info(args=Arguments.create(args),
loop_count=args.loop_count,
loop_interval=args.loop_interval,
network_id=network_id,
node_id=node_id,
run_index=args.run_index,
run_type=constants.TYPE,
use_stored_contracts=True)
# Abort if a run lock cannot be acquired.
if is_run_locked(ctx):
logger.log_warning(
f"{constants.TYPE} :: run {args.run_index} aborted as it is currently executing."
)
# Start run.
else:
from stests.orchestration.actors import do_run
do_run.send(ctx)
logger.log(f"{constants.TYPE} :: run {args.run_index} started")
def main(args):
"""Entry point.
:param args: Parsed CLI arguments.
"""
# Pull data.
network_id = factory.create_network_id(args.network)
data = cache.orchestration.get_info_list(network_id, args.run_type, args.run_index)
if not data:
logger.log("No run information found.")
return
# Set cols/rows.
cols = ["Phase / Step", "Start Time", "Duration (s)", "Action", "Status"]
rows = map(lambda i: [
i.index_label,
i.ts_start,
i.tp_elapsed_label,
i.step_label if i.step_label else '--' ,
i.status.name,
], sorted(data, key=lambda i: i.index_label))
# Set table.
t = get_table(cols, rows)
t.column_alignments['Phase / Step'] = BeautifulTable.ALIGN_LEFT
t.column_alignments['Start Time'] = BeautifulTable.ALIGN_LEFT
t.column_alignments['Duration (s)'] = BeautifulTable.ALIGN_RIGHT
t.column_alignments['Action'] = BeautifulTable.ALIGN_RIGHT
t.column_alignments['Status'] = BeautifulTable.ALIGN_RIGHT
# Render.
print(t)
print(f"{network_id.name} - {args.run_type} - Run {args.run_index}")
def main(args: argparse.Namespace):
"""Entry point.
"""
# Set run context.
network_id = factory.create_network_id(args.network)
node_id = factory.create_node_id(network_id, args.node)
ctx = factory.create_run_context(args=Arguments.create(args),
network_id=network_id,
node_id=node_id,
run=args.run,
run_type=constants.TYPE)
# Flush previous cache data.
cache.flush_run(ctx)
# Initialise MQ broker.
mq.initialise()
# Import actors.
import stests.generators.correlator
import stests.generators.wg_100.phase_1
import stests.generators.wg_100.phase_2
import stests.generators.wg_100.step_incrementor
# Start workflow.
logger.log("... workload generator begins")
# Execute first actor in pipeline.
from stests.generators.wg_100.step_incrementor import PIPELINE
PIPELINE[0].send(ctx)
def do_phase(ctx: ExecutionContext):
"""Runs a workflow phase.
:param ctx: Execution context information.
"""
# Escape if unexecutable.
if not predicates.can_start_phase(ctx):
return
# Update ctx.
ctx.phase_index += 1
ctx.step_index = 0
# Set info/state.
phase_info = factory.create_info(ExecutionAspect.PHASE, ctx)
phase_state = factory.create_state(ExecutionAspect.PHASE, ctx,
ExecutionStatus.IN_PROGRESS)
# Update cache.
cache.orchestration.set_context(ctx)
cache.orchestration.set_info(phase_info)
cache.orchestration.set_state(phase_state)
# Inform.
logger.log(
f"WFLOW :: {ctx.run_type} :: {ctx.run_index_label} :: {ctx.phase_index_label} -> starts"
)
# Run step.
do_step.send(ctx)
def do_transfer(
ctx: RunContext,
cp1: Account,
cp2: Account,
amount: int,
is_refundable: bool = True
) -> typing.Tuple[Deploy, Transfer]:
"""Executes a transfer between 2 counter-parties & returns resulting deploy hash.
:param ctx: Generator run contextual information.
:param cp1: Account information of counter party 1.
:param cp2: Account information of counter party 2.
:param amount: Amount in motes to be transferred.
:param is_refundable: Flag indicating whether a refund is required.
:returns: Dispatched deploy.
"""
deploy_hash = get_client(ctx).transfer(
amount=amount,
from_addr=cp1.public_key,
private_key=cp1.private_key_as_pem_filepath,
target_account_hex=cp2.public_key,
# TODO: allow these to be passed in via standard arguments
payment_amount=defaults.CLX_TX_FEE,
gas_price=defaults.CLX_TX_GAS_PRICE
)
logger.log(f"PYCLX :: transfer :: {amount} CLX :: {cp1.public_key[:8]} -> {cp2.public_key[:8]} :: {deploy_hash}")
return (
factory.create_deploy_for_run(ctx, deploy_hash, DeployStatus.DISPATCHED, DeployType.TRANSFER),
factory.create_transfer(ctx, amount, "CLX", cp1, cp2, deploy_hash, is_refundable)
)
def before_process_message(self, broker, message):
"""Called before a message is processed.
:param broker: Message broker to which message was dispatched.
:param message: A message being processed.
"""
msg = f"ACTOR :: {_get_actor_name(message)} :: executing ..."
_logger.log(msg)
def do_transfer(
ctx: ExecutionContext,
cp1: Account,
cp2: Account,
amount: int,
contract: ClientContract = None,
is_refundable: bool = True,
deploy_type: DeployType = DeployType.TRANSFER
) -> typing.Tuple[Deploy, Transfer]:
"""Executes a transfer between 2 counter-parties & returns resulting deploy hash.
:param ctx: Execution context information.
:param cp1: Account information of counter party 1.
:param cp2: Account information of counter party 2.
:param amount: Amount in motes to be transferred.
:param contract: The transfer contract to call (if any).
:param is_refundable: Flag indicating whether a refund is required.
:param deploy_type: The type of deploy to dispatch.
:returns: Dispatched deploy & transfer.
"""
# Set client.
node, client = utils.get_client(ctx)
# Transfer using called contract - does not dispatch wasm.
if contract:
session_args = ABI.args([
ABI.account("address", cp2.public_key),
ABI.big_int("amount", amount)
])
dhash = client.deploy(
session_hash=bytes.fromhex(contract.chash),
session_args=session_args,
from_addr=cp1.public_key,
private_key=cp1.private_key_as_pem_filepath,
# TODO: allow these to be passed in via standard arguments
payment_amount=defaults.CLX_TX_FEE,
gas_price=defaults.CLX_TX_GAS_PRICE)
# Transfer using stored contract - dispatches wasm.
else:
dhash = client.transfer(
amount=amount,
from_addr=cp1.public_key,
private_key=cp1.private_key_as_pem_filepath,
target_account_hex=cp2.public_key,
# TODO: allow these to be passed in via standard arguments
payment_amount=defaults.CLX_TX_FEE,
gas_price=defaults.CLX_TX_GAS_PRICE)
logger.log(
f"PYCLX :: transfer :: {dhash} :: {amount} CLX :: {cp1.public_key[:8]} -> {cp2.public_key[:8]}"
)
return (node, dhash)
def _yield_events(network_id: NetworkIdentifier, on_block_added,
on_block_finalized):
"""Yields events from event source (i.e. a CLX chain).
"""
# TODO: handle client disconnects.
logger.log(f"PYCLX :: stream_events :: connecting ...")
for event in get_client(network_id).stream_events(
block_added=on_block_added is not None,
block_finalized=on_block_finalized is not None):
yield event
def main(args):
"""Entry point.
:param args: Parsed CLI arguments.
"""
# Instantiate.
network = factory.create_network(args.network)
# Push.
cache.infra.set_network(network)
# Inform.
logger.log(f"Network {args.network} was successfully registered")
def do_deploy_contract(ctx: RunContext, account: Account, wasm_filepath: str):
"""Deploys a smart contract to chain.
:param ctx: Generator run contextual information.
:param account: Account to be associated with contract.
:param wasm_filepath: Path to smart contract's wasm file.
:returns: Deploy hash (in hex format).
"""
pyclx = get_client(ctx)
logger.log(f"TODO :: deploy-contract :: {account.key_pair.public_key.as_hex} :: {wasm_filepath}")
return "TODO: dispatch contract deploy"
def get_broker() -> Broker:
"""Returns an MQ broker instance for integration with dramatiq framework.
:returns: A configured message broker.
"""
# factory = FACTORIES["REDIS"]
try:
factory = FACTORIES[EnvVars.TYPE]
except KeyError:
raise InvalidEnvironmentVariable("BROKER_TYPE", EnvVars.TYPE, FACTORIES)
broker = factory.get_broker()
logger.log(f"... established connection to {EnvVars.TYPE} MQ broker")
return broker
def main(args):
"""Entry point.
:param args: Parsed CLI arguments.
"""
network_id = factory.create_network_id(args.network)
network = cache.infra.get_network(network_id)
if network is None:
logger.log_warning(f"Network {args.network} is unregistered.")
return
logger.log(
f"""NETWORK: {network.name} -> faucet pvk {network.faucet.private_key}"""
)
logger.log(
f"""NETWORK: {network.name} -> faucet pbk {network.faucet.public_key}"""
)
def do_step_execute(ctx: ExecutionContext, step: WorkflowStep):
"""Performs step execution.
:param ctx: Execution context information.
:param step: Step related execution information.
"""
# Execute step.
step.execute()
# Process errors.
if step.error:
do_step_error.send(ctx, str(step.error))
# Async steps are processed by deploy listeners.
elif step.is_async:
if step.result is not None:
if inspect.isfunction(step.result):
message_factory = step.result()
group = dramatiq.group(message_factory)
group.run()
else:
raise TypeError(
"Expecting either none or a message factory from a step function."
)
logger.log(
f"WFLOW :: {ctx.run_type} :: {ctx.run_index_label} :: {ctx.phase_index_label} :: {ctx.step_index_label} :: {step.label} -> listening for chain events"
)
# Sync steps are processed inline.
else:
if step.result is None:
do_step_end.send(ctx)
elif inspect.isfunction(step.result):
message_factory = step.result()
group = dramatiq.group(message_factory)
group.add_completion_callback(do_step_end.message(ctx))
group.run()
else:
raise TypeError(
"Expecting either none or a message factory from a step function."
)
def wrapper(*args, **kwargs):
# Pre log.
messages = {
"get_block": lambda args: f"bhash={args[-1]}",
"get_deploys": lambda args: f"bhash={args[-1]}",
"get_balance": lambda args: f"pbk={args[-1].public_key}",
}
try:
message = messages[func.__name__]
except KeyError:
logger.log(f"PYCLX :: {func.__name__} :: executing ...")
else:
logger.log(f"PYCLX :: {func.__name__} :: {message(args)}")
try:
return func(*args, **kwargs)
except Exception as err:
logger.log_error(f"PYCLX :: {err}")
raise err
def on_finalized_deploy(network_id: NetworkIdentifier, bhash: str, dhash: str,
finalization_ts: float):
"""Event: raised whenever a deploy is finalized.
:param network_id: Identifier of network upon which a block has been finalized.
:param bhash: Hash of finalized block.
:param dhash: Hash of finalized deploy.
:param finalization_ts: Moment in time when finalization occurred.
"""
# Set network deploy.
deploy = factory.create_deploy(network_id, bhash, dhash,
DeployStatus.FINALIZED)
# Encache - skip duplicates.
_, encached = cache.monitoring.set_deploy(deploy)
if not encached:
return
logger.log(f"processing finalized deploy: {bhash} :: {dhash}")
# Pull run deploy - escape if none found.
deploy = cache.state.get_run_deploy(dhash)
if not deploy:
return
# Update deploy.
deploy.update_on_finalization(bhash, finalization_ts)
cache.state.set_run_deploy(deploy)
# Increment deploy counts.
ctx = cache.orchestration.get_context(deploy.network, deploy.run_index,
deploy.run_type)
cache.orchestration.increment_deploy_counts(ctx)
# Update transfers.
transfer = cache.state.get_run_transfer(dhash)
if transfer:
transfer.update_on_completion()
cache.state.set_run_transfer(transfer)
# Signal to orchestrator.
on_step_deploy_finalized.send(ctx, dhash)
def after_process_message(self,
broker,
message,
*,
result=None,
exception=None):
"""Called after a message has been processed.
:param broker: Message broker to which message was dispatched.
:param message: A message being processed.
"""
if exception is None:
return
msg = f"ACTOR :: {_get_actor_name(message)} :: complete"
_logger.log(msg)
else:
msg = f"ACTOR :: {_get_actor_name(message)} :: ERROR :: err={exception}"
_logger.log_error(msg)
def main(args):
"""Entry point.
:param args: Parsed CLI arguments.
"""
# Unpack.
network_id = factory.create_network_id(args.node.split(':')[0])
node_id = factory.create_node_id(network_id, int(args.node.split(':')[-1]))
# Pull.
node = cache.infra.get_node(node_id)
if node is None:
raise ValueError("Unregistered node.")
# Inform.
logger.log(
f"""NODE: {node.label} -> bonding pvk {node.account.private_key}""")
logger.log(
f"""NODE: {node.label} -> bonding pbk {node.account.public_key}""")
def main(args):
"""Entry point.
:param args: Parsed CLI arguments.
"""
# Pull.
network = cache.infra.get_network_by_name(args.network)
if network is None:
raise ValueError("Unregistered network.")
# Update.
network.status = NetworkStatus[args.status.upper()]
# Push.
cache.infra.set_network(network)
# Notify.
logger.log(
f"Network {args.network} status was updated --> {network.status}")
def stream_events(src: typing.Union[NodeIdentifier, NetworkIdentifier],
on_block_added: typing.Callable = None,
on_block_finalized: typing.Callable = None):
"""Hooks upto network streaming events.
:param src: The source from which a network node will be derived.
:param on_block_added: Callback to invoke whenever a block is added to chain.
:param on_block_finalized: Callback to invoke whenever a block is finalized.
"""
for node, event in _yield_events(src, on_block_added, on_block_finalized):
if on_block_added and event.HasField("block_added"):
bhash = event.block_added.block.summary.block_hash.hex()
logger.log(f"PYCLX :: stream_events :: block added :: {bhash}")
on_block_added(node, bhash)
elif on_block_finalized and event.HasField("new_finalized_block"):
bhash = event.new_finalized_block.block_hash.hex()
logger.log(f"PYCLX :: stream_events :: block finalized :: {bhash}")
on_block_finalized(node, bhash)
