def main():
parser = argparse.ArgumentParser("""List all or some subset of pi machines
e.g.
python list.py
python list.py 10.*
python list.py --me
"""
)
parser.add_argument("--me", help="unlock all machines locked by me", action="store_true", default=False)
parser.add_argument("ip_addresses", nargs="*", help="The address of the machine(s) to list")
args = parser.parse_args()
cluster = picluster.PiBoardTable(endpoint.url, endpoint.apikey)
machines = cluster.get_matching_machines(args.ip_addresses)
machines.sort(key=lambda x: x.ip_address)
for e in machines:
if args.me and e.current_user_name != cluster.username:
continue
msg = e.ip_address
if e.hostname:
msg += " " + e.hostname
if e.command == "Lock":
msg += " locked"
if e.current_user_name:
msg += " by " + e.current_user_name
if e.current_task_name:
msg += " for '" + e.current_task_name + "'"
if not e.alive:
msg += " is dead?"
print(msg)
def main():
parser = argparse.ArgumentParser("""Lock a given raspberry pi machine
e.g.
python lock.py 157.54.158.128 "perf experiments"
""")
parser.add_argument("ip_addresses",
nargs="+",
help="The address of the machine(s) to lock")
parser.add_argument("--reason",
"-r",
help="reason to show in the current_task_name field")
args = parser.parse_args()
reason = "Manual"
if args.reason:
reason = args.reason
cluster = picluster.PiBoardTable(endpoint.url, endpoint.apikey)
machines = cluster.get_matching_machines(args.ip_addresses)
for r in machines:
ip = r.ip_address
if r.command == 'Lock':
print("machine is already locked by someone else!")
else:
try:
cluster.lock(ip, reason)
print("machine %s is now locked by you!" % (ip))
except:
errorType, value, traceback = sys.exc_info()
print("### Exception locking machine {}: {}: {}".format(
ip, errorType, value))
def resolve_address(self, ipaddress, cluster):
"""Resolves the ip address of the target device and locks it if it is
part of a cluster"""
if cluster:
self.cluster = picluster.PiBoardTable(cluster, self.apikey)
task = " ".join((current_script, self.model_name))
if ipaddress:
# A specific machine is requested, try to lock it
self.machine = self.cluster.lock(ipaddress, task)
self.logger.info("Locked requested machine at " +
self.machine.ip_address)
else:
# No specific machine requested, find a free machine
self.machine = self.cluster.wait_for_free_machine(
task, rePlatform=self.rePlatform)
self.logger.info("Locked machine at " +
self.machine.ip_address)
# if any of the above fails, this line should throw
self.ipaddress = self.machine.ip_address
else:
if not ipaddress:
raise Exception("Missing ipaddress or pi cluster address")
self.ipaddress = ipaddress
def main():
parser = argparse.ArgumentParser(
"""Delete a given raspberry pi machine from the list
e.g.
python delete.py 157.54.158.128
""")
parser.add_argument("ip_addresses",
nargs="*",
help="One or more addresses of the machines to delete")
args = parser.parse_args()
cluster = picluster.PiBoardTable(endpoint.url, endpoint.apikey)
machines = cluster.get_matching_machines(args.ip_addresses)
machines.sort(key=lambda x: x.ip_address)
if len(machines) == 0:
print("no machines maching %s" % (args.ip_addresses))
for r in machines:
if r.command == 'Lock':
print("machine is locked!")
else:
try:
print("Deleting machine {}...".format(r.ip_address), end='')
cluster.delete(r.ip_address)
print("machine is deleted!")
except:
errorType, value, traceback = sys.exc_info()
print("### Exception: " + str(errorType) + ": " + str(value))
def resolve_address(self, ipaddress):
if len(ipaddress) > 4 and ipaddress[:4].lower() == "http":
# then this is a pi cluster server, so find a free machine
self.cluster = picluster.PiBoardTable(ipaddress)
self.machine = self.cluster.wait_for_free_machine(self.model_name)
print("Using machine at " + self.machine.ip_address)
return self.machine.ip_address
else:
return ipaddress
def __init__(self,
cluster=None,
ipaddress=None,
username=None,
password=None,
source_dir=None,
target_dir=None,
copyback_files=None,
copyback_dir=None,
command=None,
logfile=None,
start_clean=True,
cleanup=True,
timeout=None,
all=None,
source_files=None,
apikey=None):
self.cluster = cluster
if isinstance(cluster, str):
self.cluster = picluster.PiBoardTable(cluster, apikey)
self.ipaddress = ipaddress
self.username = username
self.password = password
self.source_dir = source_dir
self.source_files = source_files
self.target_dir = target_dir
self.copyback_files = copyback_files
self.copyback_dir = copyback_dir
self.command = command
self.start_clean = start_clean
self.cleanup = cleanup
self.logfile = logfile
self.timeout = timeout
self.all = all
self.machine = None
self.ssh = None
self.buffer = None
# global logger is hooked up to parent modules by module name and this
# logger can see all the remote command output from all commands, which
# will be formatted differently with "ThreadId: " prefix so user can
# make sense of the combined output when remote commands are running in
# parallel.
if self.logfile:
self.logger = logger.setup(self.logfile)
else:
self.logger = logger.get()
if not cluster and not ipaddress:
raise Exception("Error: required ipaddress or cluster or both")
# Sanity-check parameters
if self.target_dir and os.path.pathsep in self.target_dir:
raise Exception(
"Error: multilevel target directories not supported")
def _resolve_address(self, ipaddress, cluster):
"Resolves the ip address of the target device and locks it if it is part of a cluster"
if cluster:
# lock the machine in the cluster
task = " ".join((_current_script, self.model_name))
self.cluster = picluster.PiBoardTable(cluster)
self.machine = self.cluster.lock(ipaddress, task)
self.logger.info("Locked machine at " + self.machine.ip_address)
self.ipaddress = self.machine.ip_address
else:
self.ipaddress = ipaddress
def main():
parser = argparse.ArgumentParser("""Unlock a given raspberry pi machine
e.g.
python unlock.py 157.54.158.128
python unlock.py 157.*
python unlock.py --me
""")
parser.add_argument("ip_addresses",
nargs="*",
help="The address of the machine(s) to unlock")
parser.add_argument("--me",
help="unlock all machines locked by me",
action="store_true",
default=False)
parser.add_argument("--lock_override",
help="unlock even if it was locked by someone else",
action="store_true",
default=False)
args = parser.parse_args()
cluster = picluster.PiBoardTable(endpoint.url, endpoint.apikey)
machines = cluster.get_matching_machines(args.ip_addresses)
count = 0
if args.me:
for e in machines:
if e.current_user_name == cluster.username and e.command == 'Lock':
print("unlocking " + e.ip_address)
cluster.unlock(e.ip_address)
else:
for e in machines:
if e.command == 'Lock':
count += 1
if e.current_user_name != cluster.username:
if not args.lock_override:
print(
"machine {} was locked by {}, please talk first!".
format(e.ip_address, e.current_user_name))
continue
else:
print("overriding lock on machine {}".format(
e.ip_address))
saved = cluster.username
cluster.username = e.current_user_name
try:
cluster.unlock(e.ip_address)
except:
errorType, value, traceback = sys.exc_info()
print("### Exception locking machine {}: {}: {}".
format(e.ip_address, errorType, value))
cluster.username = saved
else:
print("unlocking " + e.ip_address)
cluster.unlock(e.ip_address)
if count == 0:
print("no machines matching your input were locked")
def _init(self, args):
self.ipaddress = args.ipaddress
self.validation_map = args.validation_map
self.validation_path = args.validation_path
self.maxfiles = args.maxfiles
self.labels = args.labels
self.username = args.username
self.password = args.password
self.target_dir = args.target_dir
if args.cluster:
# try to lock the machine in the cluster
self.cluster = picluster.PiBoardTable(args.cluster)
self.machine = self.cluster.lock(self.ipaddress, _current_script)
self.logger.info("Locked machine at " + self.machine.ip_address)
def __init__(self,
cluster=None,
ipaddress=None,
username=None,
password=None,
source_dir=None,
target_dir=None,
copyback_files=None,
copyback_dir=None,
command=None,
logfile=None,
verbose=True,
start_clean=True,
cleanup=True,
timeout=None,
all=None,
source_files=None):
if cluster:
self.cluster = picluster.PiBoardTable(cluster)
self.ipaddress = ipaddress
self.username = username
self.password = password
self.source_dir = source_dir
self.source_files = source_files
self.target_dir = target_dir
self.copyback_files = copyback_files
self.copyback_dir = copyback_dir
self.command = command
self.verbose = verbose
self.start_clean = start_clean
self.cleanup = cleanup
self.logfile = logfile
self.timeout = timeout
self.cluster = None
if not cluster and not ipaddress:
raise Exception("Error: required ipaddress or cluster or both")
self.all = all
self.machine = None
self.ssh = None
self.buffer = None
# Sanity-check parameters
if os.path.pathsep in self.target_dir:
raise Exception(
"Error: multilevel target directories not supported")
def run_profilers_with_profiler_data(all_profiler_data, output_path, cluster_address, ipaddress, api_key, password, username="******", target="pi3", platform_regex=None, parallel_run=24, logging_args=None):
if not platform_regex:
if target in default_platform_regex:
platform_regex = default_platform_regex[target]
if not platform_regex:
global_logger.error("Platform regex not specified and no default platform regex for {} exists in {}.".format(target, default_platform_regex))
return
if len(all_profiler_data) < parallel_run:
parallel_run = len(all_profiler_data)
cluster = None
if cluster_address:
cluster = picluster.PiBoardTable(cluster_address, api_key)
else:
global_logger.info("Cluster not specified, using device with ip {} for testing and setting parallel run count to 1".format(ipaddress))
parallel_run = 1
global_logger.info("Running {} profiles".format(len(all_profiler_data)))
runners = [RemoteProfileRunner(profiler_data,
cluster,
ipaddress,
username,
password,
os.path.join(output_path, profiler_data.model_tag),
platform_regex,
logging_args=logging_args)
for profiler_data in all_profiler_data]
values = [delayed(runner.run)() for runner in runners]
results = compute(*values, scheduler="threads", num_workers=parallel_run)
global_logger.info("Done running profiling")
if None in results:
global_logger.error("Run failures detected, results = {}".format(results))
tag_to_profile_options_and_files_dict = {}
for result in results:
if result:
profiler_data = result[0]
profiler_files = result[1]
tag = profiler_data.model_tag
profile_options = profiler_data.profile_options
if tag not in tag_to_profile_options_and_files_dict:
tag_to_profile_options_and_files_dict[tag] = []
tag_to_profile_options_and_files_dict[tag].append((profile_options, profiler_files))
return tag_to_profile_options_and_files_dict
def __init__(self):
self.cluster_address = endpoint.url
self.cluster = picluster.PiBoardTable(self.cluster_address, endpoint.apikey)
self.lock_override = False
self.locked_ip = None
def test(server):
# test api key security
try:
t = picluster.PiBoardTable(server, "123", user)
a = picluster.PiBoardEntity(entity)
r = t.update(a)
test_assert(False, "api key security is working")
except Exception as e:
test_assert(
str(e) == "api key mismatch", "api key security is working")
# add or update
t = picluster.PiBoardTable(server, endpoint.apikey, user)
# insert our test entity so we can play with it.
a = picluster.PiBoardEntity(entity)
r = t.update(a)
test_assert(r.ip_address == ip, "add or update entity")
# get all
r = t.get_all()
test_assert(len(r) > 0 and ip in [x.ip_address for x in r], "get_all")
# get the entity we added
r = t.get(ip)
test_assert(r and r.ip_address == ip, "get the entity we added")
# locking
r = t.lock(ip, 'Test')
test_assert(r and r.ip_address == ip and r.current_user_name == t.username,
"lock our machine")
# now try and free the device using wrong user name
saved = t.username
t.username = '******'
failed = False
try:
r = t.unlock(ip)
failed = False
except:
failed = True
t.username = saved
test_assert(failed, "try and free the device using wrong user name")
# double check this is really the case
r = t.get(ip)
test_assert(r and r.ip_address == ip, "ensure entity is still there")
# now try and free the device using correct user name
r = t.unlock(ip)
test_assert(r and r.ip_address == ip, "unlock our machine")
# check it really is not locked
r = t.get(ip)
test_assert(r and r.current_user_name != t.username, "lock is gone")
# delete
r = t.delete(ip)
test_assert(r and r.current_user_name != t.username, "delete our machine")
# create 10 machines so we can lock them all in parallel
for i in range(10):
a.ip_address = "255.255.255.{}".format(i)
r = t.update(a)
# now a multithreaded lock to make sure it is safe!
dask.config.set(scheduler='threads')
jobs = dask.bag.from_sequence(list(range(10)))
results = dask.bag.map(lambda id: get_free_machine(t, id), jobs).compute()
test_assert(len(results) == 10, "We have locked 10 machines in parallel")
addresses = [r.ip_address for r in results]
unique_list = set(addresses)
test_assert(
len(addresses) == len(unique_list), "locked machines are unique")
for m in unique_list:
if len([x for x in addresses if x == m]) > 1:
print("Machine {} locked twice, which should not be possible!!".
format(m))
t.unlock(m)
t.delete(m)
return "{}'C".format(float(tempstring) / 1000)
except:
return ''
def get_system_load():
try:
return '{} : {} : {}'.format(*os.getloadavg())
except:
return ''
# setup loop, waiting for network to be available
while True:
try:
cluster = picluster.PiBoardTable(endpoint.url, endpoint.apikey)
ip = get_local_ip()
r = cluster.get(ip)
if r:
if os.path.isfile("nounlock"):
print("nounlock override, leaving machine locked")
elif r.command == 'Lock':
try:
print("machine was locked when we rebooted, so free the lock now!")
cluster.username = r.current_user_name
cluster.unlock(ip)
except:
errorType, value, traceback = sys.exc_info()
print("unlock failed: {}: {}".format(str(errorType), str(value)))
else:
