def test_rpc_method_raise_exception(self): # pylint: disable=R0201
"""
Tests what happens if a rpc method raises an exception
"""
@Rpc.method
@asyncio.coroutine
def raises_async(): # pylint: disable=R0201,W0612
"""
Simple async rpc function, that raises an Exception.
"""
raise Exception('foo')
cmd = Command('raises_async')
status = Status.err({
'method': 'raises_async',
'result': str(Exception('foo'))
})
status.uuid = cmd.uuid
Server(
[
cmd.to_json(),
],
[
status.to_json(),
],
).run()
def test_math_add(self): # pylint: disable=R0201
"""
Testing simple math add function with async features.
"""
@Rpc.method
def math_add(integer1, integer2): # pylint: disable=R0201,W0612
"""
Simple add function with async.
Arguments
---------
integer1: first operand
integer2: second operand
"""
res = (integer1 + integer2)
return res
cmd = Command("math_add", integer1=1, integer2=2)
status = Status.ok({'method': 'math_add', 'result': 3})
status.uuid = cmd.uuid
Server(
[
cmd.to_json(),
],
[
status.to_json(),
],
).run()
def _createFile(self):
super(CommandLock, self)._createFile()
# For now, we default to permitting shared locks from non-root
# processes. The sysadmin may change this; we won't change the
# permissions once the file has been created.
#
# N.B.: The names may not be sanitized for use with a shell!
cmd = Command(["chmod", "644", self.path])
cmd.run()
def runVagrantDeployment(self):
# It is expected the ansible directory is completely set up and ready for use.
# If needed, ANSIBLE_EXTRA_VARS can be used to set extra vars to set or override existing ansible variables.
os.chdir("%s/vagrant" % self._config_data["general"]["ansible_dir"])
# construct ansible extra vars
if self._config_data["deployment"]["vagrant"]["ansible"] != None:
pairs = []
for key in self._config_data["deployment"]["vagrant"]["ansible"]:
pairs.append("%s=%s" % (key, self._config_data["deployment"]
["vagrant"]["ansible"][key]))
os.environ["ANSIBLE_EXTRA_VARS"] = " ".join(pairs)
print "ANSIBLE_EXTRA_VARS=\"%s\"" % os.environ["ANSIBLE_EXTRA_VARS"]
# set image, number of nodes
if self._config_data["deployment"]["vagrant"] != None:
for key in self._config_data["deployment"]["vagrant"]:
os.environ[key.upper()] = str(
self._config_data["deployment"]["vagrant"][key])
print "%s=\"%s\"" % (key.upper(), os.environ[key.upper()])
# It is expected Vagrant technology is completely set up and ready for use.
# If needed, OS_IMAGE and other envs can be set to alter provisioning.
self._command.run("vagrant up")
# collect kubeconfig from master node
ssh_command = Command(dry=self._dry_run).run("sudo vagrant ssh-config")
if ssh_command.rc() > 0:
logging.error("Unable to retrieve vagrant ssh-config: %s" %
ssh_command.err())
exit(1)
lines = ssh_command.out().split("\n")
master_line = True
vagrant_master_ip = ""
for line in lines:
if line.startswith("Host kube-master-1"):
master_line = True
continue
if master_line and line.startswith(" HostName "):
vagrant_master_ip = line.split(" HostName ")[1].strip()
print vagrant_master_ip
break
if line == "":
master_line = False
self._command.run(
"scp -i ~/.vagrant.d/insecure_private_key -o StrictHostKeyChecking=no vagrant@%s:/etc/kubernetes/kubectl.kubeconfig %s"
% (vagrant_master_ip,
self._config_data["general"]["kubeconfig_dest_dir"]))
def test_command_with_kwargs(self):
"""
Uses map arguments.
"""
cmd = Command("test_func", a=2, b="vier")
string = '{{"{}": "test_func", "{}": {{"a": 2, "b": "vier" }}, "{}": "{}"}}'.format(
Command.ID_METHOD, Command.ID_ARGUMENTS, Command.ID_UUID, cmd.uuid)
cmd_string = Command.from_json(string)
cmd_new = Command.from_json(cmd.to_json())
self.assertEqual(cmd, cmd_string)
self.assertEqual(cmd, cmd_new)
self.assertEqual(cmd_string, cmd_new)
def run_on_ios(self, cmd='', shell=False, process=False, timeout=5, retry=2):
Log.d('Running on iOS: {cmd}'.format(cmd=cmd))
try:
cmd = cmd.replace("\ ", " ") # fix for backslashes
full_cmd = '{ssh} \'{cmd}\''.format(ssh=settings.ssh_ios, cmd=cmd) if shell else [settings.ssh_ios, cmd]
Log.d('Full command: {cmd}'.format(cmd=full_cmd))
if process:
return Popen(full_cmd, stdout=PIPE, stderr=None, shell=shell)
result = Command(full_cmd).run(shell, timeout)
if result:
result = (result[0].split('\r\n', 2)[2], result[1]) if 'password:'******'\r\n', 1)[1], result[1])
if 'ssh_exchange_identification: Connection closed by remote host' in result[0] and retry > 0:
return self.run_on_ios(cmd, shell, process, timeout, retry-1)
return result
except Exception:
Log.d(traceback.format_exc())
if retry > 0:
return self.run_on_ios(cmd, shell, process, timeout, retry-1)
return ['', '']
def setScriptSource(scriptId, scriptSource):
params = {}
params['scriptId'] = scriptId
params['scriptSource'] = scriptSource
command = Command('Debugger.setScriptSource', params)
return command
def preflight(self, args):
"""Perform checks prior to actually executing the command.
Arguments:
args (dict): The command line arguments (used by subclasses)
Raises:
OperationError
"""
if self.requiresRoot and (os.getuid() != 0):
msg = _("You must be root to use the \"{0}\" command").format(
self.name)
raise OperationError(msg)
if self.checkBinaries:
for executable in [
'vdodumpconfig', 'vdoforcerebuild', 'vdoformat'
]:
if not MgmntUtils.which(executable):
msg = _("executable '{0}' not found in $PATH").format(
executable)
raise OperationError(msg)
if self.requiresRunMode and Command.noRunMode():
msg = _("{0} command not available with --noRun").format(self.name)
raise OperationError(msg)
def prog_log_disable(program):
"""
This function disables the log transfer for a `ProgramModel` by sending a
command to the slave. If the slave is offline an error will be returned.
Parameters
----------
program: ProgramModel
A valid `ProgramModel`.
Raises
------
SlaveOfflineError
TypeError:
If `program` is not an `ProgramModel`
"""
ensure_type("program", program, ProgramModel)
LOGGER.info(
"Disabling logging for program %s on slave %s",
program.name,
program.slave.name,
)
if not program.slave.is_online:
raise SlaveOfflineError('', '', 'log_disable', program.slave.name)
notify_slave(
Command(
method="disable_logging",
target_uuid=program.programstatus.command_uuid,
),
program.slave.id,
)
def getProperties(objectId, ownProperties=False):
params = {}
params['objectId'] = str(objectId)
params['ownProperties'] = ownProperties
command = Command('Runtime.getProperties', params)
return command
def run_asp_solver(commands: [Command], files: iter) -> [Command]:
commands = tuple(commands)
memory = list(next(c for c in commands if c.type == 'MEMORY').args)
atoms = (
''.join('{}({}).'.format(type.lower(), ','.join(
map(str.lower, map(str, args)))) for type, args in commands)
# + ''.join('memory({},{}).'.format(idx, mem) # access to memory
# for idx, mem in enumerate(memory))
)
log('ASP ATOMS:', atoms)
for model in clyngor.solve(files, inline=atoms).by_predicate:
log('ASP MODEL:', model)
for idx, increment in model.get('inc_memory', ()):
memory[idx] = (int(memory[idx]) + increment) % 256
for type, *args in model.get('do', ()):
yield Command(type, tuple(args))
yield Command('SET_MEMORY', memory)
def test_rpc_method_cancel_method(self): # pylint: disable=R0201
"""
Tests what happens if you cancel a running method
"""
@Rpc.method
@asyncio.coroutine
def sleep(sec): # pylint: disable=R0201,W0612
"""
Simple async rpc function, that raises an Exception.
"""
try:
yield from asyncio.sleep(sec)
except asyncio.CancelledError:
return -15
sleep_cmd = Command(method='sleep', sec=1)
cancel_cmd = Command(method='sleep')
cancel_cmd.uuid = sleep_cmd.uuid
status = Status.ok({'method': 'sleep', 'result': -15})
status.uuid = sleep_cmd.uuid
Server(
[
sleep_cmd.to_json(),
cancel_cmd.to_json(),
],
[
status.to_json(),
],
).run()
def run_ant_by_state(commands):
# TODO: include DSL of run_ant
# determine the state
state = frozenset(
state for state, ok in {
'explorer searching for food':
antype == 1 and memory[0] == 0 and not foods,
'explorer finding food':
antype == 1 and memory[0] == 0 and foods,
'explorer going home':
antype == 1 and memory[0] == 1 and memory[1] == 1,
'explorer marking path':
antype == 1 and memory[0] == 1 and memory[1] == 0,
'scavenger searching for food (lost)':
antype == 2 and memory[0] == 0 and not food and not pheromones,
'scavenger searching for food (following marks)':
antype == 2 and memory[0] == 0 and not food and pheromones,
'scavenger finding food':
antype == 2 and memory[0] == 0 and food,
'scavenger going home':
antype == 2 and memory[0] == 1 and memory[1] ==
1, # should be divided into multiple states to catch unconsistent cases
}.items() if ok)
assert len(state) == 1
if state == 'explorer searching for food':
yield Command('EXPLORE')
if state == 'explorer finding food':
yield Command('TURN', [180])
memory[0], memory[1] = 1, 1 # food found
if state == 'explorer going home':
yield Command('EXPLORE')
memory[1] = 0
if state == 'explorer marking path':
yield Command('PUT_PHEROMONE', [Pheromone.FOODPATH])
memory[1] = 1
if state == 'scavenger finding food':
close_foods = tuple(p for p in foods
if p.zone == 'NEAR' and p.amount > 1)
if not any(p.zone == 'NEAR' for p in foods): # nothing near
target = max(foods, key=lambda p: p.amount)
yield goto(target)
else: # something here
getfrom = random.choice(close_foods)
yield Command('COLLECT',
(getfrom.idx, min(getfrom.amount - 1, 100 - stock)))
if state == 'scavenger searching for food (lost)':
yield Command('EXPLORE')
if state == 'scavenger searching for food (following marks)':
paths = tuple(p for p in see['pheromone']
if p.amount == Pheromone.FOODPATH)
target = max(paths, key=lambda p: persistance)
yield goto(target)
if state == 'scavenger going home':
friend_nests = tuple(n for n in see['nest'] if n.friend)
if friend_nests: # go to the closest
target = min(friend_nests, key=lambda p: p.dist)
yield goto(target)
else:
yield Command('EXPLORE')
def setBreakpoint(location, condition=None):
params = {}
params['location'] = location()
if condition:
params['condition'] = condition
command = Command('Debugger.setBreakpoint', params)
return command
def ws_rpc_connect(message):
"""
Handles incoming connections on the websocket `/commands`. Connections only
get accepted if the IP of the sender matches the IP of a known client. If
the client is known then a unique group will be created. The only member of
this group is the sender. The naming scheme for this group is
`client_<id>`. Now the sender has to response to the `online` method.
Parameters
----------
message: channels.message.Message
The first message which is send by the sender.
"""
ip_address, port = message.get('client')
message.channel_session['ip_address'] = ip_address
try:
slave = SlaveModel.objects.get(ip_address=ip_address)
# Accept the connection
message.reply_channel.send({"accept": True})
LOGGER.info(
"client connected with ip %s on port %s",
ip_address,
port,
)
# Add to the command group
Group('client_{}'.format(slave.id)).add(message.reply_channel)
LOGGER.debug('Added client to command group client_%s', slave.id)
cmd = Command(method='online')
# send/save online request
slave.command_uuid = cmd.uuid
slave.save()
Group('client_{}'.format(slave.id)).send({'text': cmd.to_json()})
LOGGER.info("send online request to %s", slave.name)
except SlaveModel.DoesNotExist:
LOGGER.error("Rejecting unknown client with ip %s!", ip_address)
message.reply_channel.send({"accept": False})
def persist(self):
"""Writes out the Configuration if necessary.
If the Configuration is read-only or has not been modified, this
method will silently return. If Command.noRunMode is True, any
new Configuration will be printed to stdout instead of the file.
This method will generate an assertion failure if the configuration
file is not open.
"""
if self._readonly:
return
if not self._dirty:
self.log.debug("Configuration is clean, not persisting")
return
self.log.debug("Writing configuration to {0}".format(self.filepath))
if self._empty():
self._removeFile()
return
s = yaml.dump({"config": self}, default_flow_style=False)
if Command.noRunMode():
print(_("New configuration (not written):"))
print(s)
self._dirty = False
return
newFile = self.filepath + ".new"
if os.path.exists(newFile):
os.remove(newFile)
with open(newFile, 'w') as fh:
# Write the warning about not editing the file.
fh.write(
"####################################################################"
)
fh.write(os.linesep)
fh.write("# {0}".format(
_("THIS FILE IS MACHINE GENERATED. DO NOT EDIT THIS FILE BY HAND."
)))
fh.write(os.linesep)
fh.write(
"####################################################################"
)
fh.write(os.linesep)
# Write the configuration, flush and sync.
fh.write(s)
fh.flush()
os.fsync(fh)
os.rename(newFile, self.filepath)
self._fsyncDirectory()
self._dirty = False
def test_connect_success(self):
slave = SlaveFactory()
ws_client = WSClient()
ws_client.send_and_consume(
'websocket.connect',
path='/commands',
content={'client': [slave.ip_address, slave.mac_address]},
)
self.assertEqual(
Command(method="online"),
Command.from_json(json.dumps(ws_client.receive())),
)
Group('client_{}'.format(slave.id)).send(
{'text': 'ok'},
immediately=True,
)
self.assertEqual(ws_client.receive(json=False), 'ok')
def _fsyncDirectory(self):
"""Open and issue an fsync on the directory containing the config file.
"""
dirname = os.path.dirname(self.filepath)
if Command.noRunMode():
runCommand(['fsync', dirname])
return
fd = os.open(dirname, os.O_RDONLY)
try:
os.fsync(fd)
finally:
os.close(fd)
def evaluate(expression, objectGroup=None, returnByValue=None):
params = {}
params['expression'] = expression
if(objectGroup):
params['objectGroup'] = objectGroup
if(returnByValue):
params['returnByValue'] = returnByValue
command = Command('Runtime.evaluate', params)
return command
def fs_restore(fs):
"""
This functions restores a given `fs` by sending a command to the slave to
restore the original state. If the slave is offline an error will be
returned.
Parameters
----------
fs: FilesystemModel
A valid `FilesystemModel`.
Raises
------
SlaveOfflineError
FilesystemNotMovedError
TypeError:
If `fs` is not an `FilesystemModel`
"""
ensure_type("fs", fs, FilesystemModel)
slave = fs.slave
if slave.is_online:
if not fs.is_moved:
raise FilesystemNotMovedError(
str(fs.name),
str(fs.slave.name),
)
cmd = Command(
method="filesystem_restore",
source_path=fs.source_path,
source_type=fs.source_type,
destination_path=fs.destination_path,
destination_type=fs.destination_type,
backup_ending=FILE_BACKUP_ENDING,
hash_value=fs.hash_value,
)
# send command to the client
notify_slave(cmd, slave.id)
fs.command_uuid = cmd.uuid
fs.save()
else:
raise SlaveOfflineError(
str(fs.name),
"filesystem",
str(fs.slave.name),
"restore",
)
def test_error_close(self): # pylint: disable=R0201
"""
Tests if RPC-Receiver doesn't raise an exception if closed during execution.
"""
@Rpc.method
@asyncio.coroutine
def async_sleep(): # pylint: disable=R0201,W0612
"""
Simple function that sleeps.
"""
yield from asyncio.sleep(10)
cmd = Command("async_sleep", integer1=1, integer2=2)
status = Status.ok({'method': 'async_sleep', 'result': ''})
status.uuid = cmd.uuid
CloseFailServer(
[
cmd.to_json(),
],
[
status.to_json(),
],
).run()
def _removeFile(self):
"""Deletes the current configuration file.
In noRun mode, pretend that we're doing an rm of the file."""
if Command.noRunMode():
runCommand(['rm', self.filepath])
return
if os.path.exists(self.filepath):
os.remove(self.filepath)
self._fsyncDirectory()
try:
with FileLock(self.singletonLock, "r+") as f:
del Configuration.modifiableSingltons[self.filepath]
except KeyError:
pass
def main():
args = parse_args()
jsonlogs.setup_logging(args.log_format, args.log_level)
try:
registry = Registry(args.deployment, args.namespace)
if not registry.is_readonly:
registry.readOnly(True)
Command('/bin/registry garbage-collect --delete-untagged=true /etc/docker/registry/config.yml',
timeout=args.timeout, graceful_period=args.graceful_period).run()
if registry.is_readonly:
registry.readOnly(False)
except Exception as e:
LOG.error(e)
sys.exit(1)
def sendRequests(self, num_requests):
for i in range(num_requests):
# define the request
operation = "operation %s.%d" % (self.id[-1], i)
cmd = Command(self.id, "%s.%d" % (self.id[-1], i), operation)
# send out request to all replicas (implementation asks for it, but it works sending to only one also)
for r in self.env.conf.replicas:
self.sendMessage(r, RequestMessage(self.id, cmd))
# wait for response(s)
count = 0
while True:
msg = self.getNextMessage()
if isinstance(msg, ResponseMessage):
if msg.decision == operation:
# count += 1
break # one confirmation is enough
def run(self):
# Need a place to put all logs
if not os.path.exists(self._config_data["general"]["results_dir"]):
os.mkdir(self._config_data["general"]["results_dir"])
log_file_fd = None
if "log_file" in self._config_data["general"]:
log_file = os.path.join(
self._config_data["general"]["results_dir"],
self._config_data["general"]["log_file"])
print "Logs available in %s" % log_file
log_file_fd = open(log_file, "w")
self._command = Command(dry=self._dry_run,
interactive=True,
log_file=log_file_fd)
# provision Kubernetes cluster via Vagrantfile
# from https://github.com/kubernetes/contrib under
# ansible/vagrant directory
# Need a place to put all logs
if not os.path.exists(self._config_data["general"]["results_dir"]):
os.mkdir(self._config_data["general"]["results_dir"])
if self._config_data["deployment"]["vagrant"]["enabled"]:
self.runVagrantDeployment()
elif self._config_data["deployment"]["openstack"]["enabled"]:
self.runOpenstackDeployment()
# collect logs
os.chdir(self._config_data["general"]["results_dir"])
# upload logs to GS Bucket
if self._config_data["results_uploader"][
"enabled"] and not self._config_data["results_uploader"][
"skip_success"]:
self.uploadResults()
# comment GH PR
if self._config_data["results_notifier"][
"enabled"] and not self._config_data["results_notifier"][
"skip_success"]:
self.notifyResults(SUCCESS)
def prog_stop(prog):
"""
This function stops a `prog` by sending a command to the slave.
The program can only be stoped if the program is currently running. If the
slave is offline an error will be returned.
Parameters
----------
prog: ProgramModel
A valid `ProgramModel`.
Exception
-------
SlaveOfflineError
ProgramNotRunningError
TypeError:
If `prog` is not an `ProgramModel`
"""
ensure_type("prog", prog, ProgramModel)
if prog.slave.is_online:
if not prog.is_running:
raise ProgramNotRunningError(str(prog.name), str(prog.slave.name))
LOGGER.info(
"Stoping program %s on slave %s",
prog.name,
prog.slave.name,
)
notify_slave(
Command(
method="execute",
uuid=prog.programstatus.command_uuid,
),
prog.slave.id,
)
else:
raise SlaveOfflineError(
str(prog.name),
"program",
str(prog.slave.name),
"stop",
)
def slave_shutdown(slave):
"""
This functions shutsdown a `slave` by a command to the slave.
Parameters
----------
slave: SlaveModel
A valid `SlaveModel`.
Raises
------
TypeError:
If `slave` is not an `SlaveModel`
"""
if slave.is_online:
notify_slave(Command(method="shutdown"), slave.id)
notify({"message": "Send shutdown Command to {}".format(slave.name)})
else:
raise SlaveOfflineError('', '', 'shutdown', slave.name)
def body(self):
if self.verbose:
print(f"Here I am: {self.id}")
next_cmd = i = 0
while i < self.num_requests:
cmd = Command(str(self), i, f"operation {self.id}")
for replica in self.replicas:
self.sendMessage(replica,
RequestMessage(f"{self.id}", cmd))
msg = self.getNextMessage()
while int(msg) <= next_cmd:
msg = self.getNextMessage()
next_cmd = int(msg)
i += 1
if self.verbose:
print(f"{self.id} is leaving")
def setBreakpointByUrl(lineNumber,
url=None,
urlRegex=None,
columnNumber=None,
condition=None):
params = {}
params['lineNumber'] = lineNumber - 1
if url:
params['url'] = url
if urlRegex:
params['urlRegex'] = urlRegex
if columnNumber:
params['columnNumber'] = columnNumber
if condition:
params['condition'] = condition
command = Command('Debugger.setBreakpointByUrl', params)
return command
def prog_log_get(program):
"""
This function is asking for a log for a `program` by sending a command to
the slave. If the slave is offline an error will be returned.
Parameters
----------
program: ProgramModel
A valid `ProgramModel`.
Raises
------
SlaveOfflineError
LogNotExistError
TypeError:
If `program` is not an `ProgramModel`
"""
ensure_type("program", program, ProgramModel)
LOGGER.info(
"Requesting log for program %s on slave %s",
program.name,
program.slave.name,
)
if not program.slave.is_online:
raise SlaveOfflineError('', '', 'get_log', program.slave.name)
if not (program.is_executed or program.is_running):
raise LogNotExistError(program.id)
notify_slave(
Command(
method="get_log",
target_uuid=program.programstatus.command_uuid,
),
program.slave.id,
)