def add_multi_chip(layout, max_num_neighbor_candidate_attempts,
num_chips_to_add):
utils.info(2, "Adding chips in multiples of " + str(num_chips_to_add))
new_chips = 0
add_attempts = 0
tmp_layout = Layout(layout.get_chip(), layout.get_chip_positions(),
layout.get_medium(), layout.get_overlap(),
layout.get_inductor_properties())
while new_chips < num_chips_to_add and add_attempts < max_num_neighbor_candidate_attempts:
add_attempts += 1
random_chip = utils.pick_random_element(
range(0, tmp_layout.get_num_chips()))
if (tmp_layout.get_longest_shortest_path_from_chip(random_chip) >=
utils.argv.diameter):
# utils.info(2, "Ooops, chip " + str(random_chip) + " won't work for the diameter");
continue
result = tmp_layout.get_random_feasible_neighbor_position(random_chip)
if result != None:
new_chips += 1
try:
tmp_layout.add_new_chip(result)
except:
#print "error in add_multi_chip()"
continue
if new_chips < num_chips_to_add:
utils.abort("Could not find any more feasible neighbors after " +
str(add_attempts) + " attempts")
candidate_list = tmp_layout.get_chip_positions()[-num_chips_to_add:]
return candidate_list
def async_send_command(command, host, port, timeout):
pid = os.getpid()
if os.fork() != 0:
return
# Avoid the pesky KeyboardInterrupts in the child
signal.signal(signal.SIGINT, signal.SIG_IGN)
command_deadline = datetime.now() + timedelta(seconds=timeout)
utils.pend("sending command %s to %s:%d" % (command.args, host, port))
while True:
try:
os.kill(pid, 0) # check if parent is still here
except OSError:
break
now = datetime.now()
if now < command_deadline:
try:
send_command(command, host, port, TIMEOUT_CMD_SEND)
except CommandError as e:
utils.pend(None, msg="%s, retrying for %d more seconds"
% (e, (command_deadline - now).seconds))
else:
utils.ok("command %s successfully sent" % command.args)
exit(0)
else:
utils.abort("command timeout")
break
time.sleep(1)
exit(1)
def optimize_layout():
# Compute continuous solution
layout_scheme = utils.argv.layout_scheme.split(":")[0]
if (layout_scheme == "stacked"):
solution = optimize_layout_stacked()
elif (layout_scheme == "rectilinear_straight"):
solution = optimize_layout_rectilinear("straight")
elif (layout_scheme == "rectilinear_diagonal"):
solution = optimize_layout_rectilinear("diagonal")
elif (layout_scheme == "checkerboard"):
solution = optimize_layout_checkerboard()
elif (layout_scheme == "cradle"):
solution = optimize_layout_cradle()
elif (layout_scheme == "double_helix"):
solution = optimize_layout_double_helix()
elif (layout_scheme == "bridge"):
solution = optimize_layout_bridge()
elif (layout_scheme == "linear_random_greedy"):
solution = optimize_layout_linear_random_greedy()
elif (layout_scheme == "random_greedy"):
if (utils.argv.mpi):
solution = optimize_layout_random_greedy_mpi()
# print "!!!!MPI!!!!!!!!!"
else:
solution = optimize_layout_random_greedy()
# print "$$$$$$$$$$$regular$$$$$$$$$$"
else:
utils.abort("Layout scheme '" + utils.argv.layout_scheme +
"' is not supported")
return solution
def validate_args(args):
"""
Validates command line arguments
"""
schema = Schema({
'--user': And(str, len, Use(str.lower), error="missing or invalid user"),
'--repo': And(str, len, Use(str.lower), error="missing or invalid repo"),
'--type': And(str, len, Use(str.lower), lambda s: s in ('deb', 'rpm'), error="type must be one of deb or rpm"),
'--distro': And(str, len, Use(str.lower), error="missing or invalid distro"),
'--distro_version': And(str, len, Use(str.lower), error="missing or invalid distro_version"),
'--arch': And(str, len, Use(str.lower), error="missing or invalid arch"),
'--pkg_name': And(str, len, Use(str.lower), error="missing or invalid pkg_name"),
'--filename': And(str, len, Use(str.lower), error="missing or invalid filename"),
'--timeout': And(Use(float), lambda f: f > 0, lambda f, i=float(args['--poll_interval']): f > i,
error="timeout must be a number greater than 1, and greater than poll_interval (default 30)"),
'--poll_interval': And(Use(float), lambda f: f > 0, error="poll_interval must be a number greater than 0"),
'--page_interval': And(Use(float), lambda f: f > 0, error="page_interval must be a number greater than 0"),
'--help': bool,
'--version': bool,
'--log-level': And(str, len, Use(str.upper),
lambda s: s in ('DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL'),
error="invalid log level specified")
})
try:
return schema.validate(args)
except SchemaError as ex:
utils.abort("Invalid argument: {}. ".format(ex.message))
def require(*keys, **kwargs):
"""
Check for given keys in the shared environment dict and abort if not found.
Positional arguments should be strings signifying what env vars should be
checked for. If any of the given arguments do not exist, Fabric will abort
execution and print the names of the missing keys.
The optional keyword argument ``used_for`` may be a string, which will be
printed in the error output to inform users why this requirement is in
place. ``used_for`` is printed as part of a string similar to::
"Th(is|ese) variable(s) (are|is) used for %s"
so format it appropriately.
The optional keyword argument ``provided_by`` may be a list of functions or
function names which the user should be able to execute in order to set the
key or keys; it will be included in the error output if requirements are
not met.
Note: it is assumed that the keyword arguments apply to all given keys as a
group. If you feel the need to specify more than one ``used_for``, for
example, you should break your logic into multiple calls to ``require()``.
"""
# If all keys exist, we're good, so keep going.
missing_keys = filter(lambda x: x not in env, keys)
if not missing_keys:
return
# Pluralization
if len(missing_keys) > 1:
variable = "variables were"
used = "These variables are"
else:
variable = "variable was"
used = "This variable is"
# Regardless of kwargs, print what was missing. (Be graceful if used outside
# of a command.)
if "command" in env:
prefix = "The command '%s' failed because the " % env.command
else:
prefix = "The "
msg = "%sfollowing required environment %s not defined:\n%s" % (prefix, variable, indent(missing_keys))
# Print used_for if given
if "used_for" in kwargs:
msg += "\n\n%s used for %s" % (used, kwargs["used_for"])
# And print provided_by if given
if "provided_by" in kwargs:
funcs = kwargs["provided_by"]
# Pluralize this too
if len(funcs) > 1:
command = "one of the following commands"
else:
command = "the following command"
to_s = lambda obj: getattr(obj, "__name__", str(obj))
provided_by = [to_s(obj) for obj in funcs]
msg += "\n\nTry running %s prior to this one, to fix the problem:\n%s" % (command, indent(provided_by))
abort(msg)
def display_command(command):
# Sanity check
if command not in commands:
abort("Command '%s' not found, exiting." % command)
# Print out nicely presented docstring
print("Displaying detailed information for command '%s':" % command)
print('')
print(indent(commands[command].__doc__, strip=True))
print('')
sys.exit(0)
def create_mesh(self) :
"""Create the mesh using a mesh generator : Triangle for 2D and
Tetgen for 3D."""
if self.dimension == 2 :
self.generate_triangle_input()
os.system("triangle -pqnea"+str(self.area_cells)+\
" mesh_input.poly")
else :
utils.abort("The mesh has to be 2D.")
def check_templates():
template_files = [None]*3
template_files[0] = f"body-templates/{args.template_body}.template.xml"
template_files[1] = f"body-templates/{args.template_body}.yaml"
template_files[2] = f"body-templates/{args.template_body}.py"
for f in template_files:
if not os.path.exists(f):
abort(f"Template file not found: {f}")
return template_files
def main():
# Load args
try:
args = docopt(__doc__, version=__version__)
except DocoptExit as ex:
# Sadly, docopt doesn't return exactly which argument was invalid.
sys.stderr.write('[{}] ERROR: Invalid argument.\n\n{}'.format(datetime.now(), ex))
sys.exit(1)
# Set up logger
logger = logging.getLogger(__package__)
log_level = getattr(logging, args['--log-level'].upper(), None)
# if log_level is DEBUG, set it for everything so Requests will log what
# it's doing too. Else just configure logger the normal way.
if log_level == logging.DEBUG:
logging.basicConfig(
level=log_level,
format='[%(asctime)s] %(levelname)s: %(message)s',
datefmt='%m/%d/%Y %I:%M:%S %p'
)
else:
handler = logging.StreamHandler()
formatter = logging.Formatter(
fmt='[%(asctime)s] %(levelname)s: %(message)s',
datefmt='%m/%d/%Y %I:%M:%S %p'
)
handler.setFormatter(formatter)
logger.addHandler(handler)
try:
logger.setLevel(log_level)
except TypeError:
sys.stderr.write('[{}] ERROR: Invalid log level specified: {}'.format(datetime.now(), args['--log-level']))
sys.exit(1)
try:
# validate command line args and check environment variables
args = config.validate_args(args)
env = config.load_env()
# engage
logger.info("ENGAGE")
logger.debug("Using arguments: \n{}".format(args))
if pkgcloudapi.look_for_package(env, args):
logger.info("Success. Found filename {}.".format(args['--filename']))
else:
utils.abort("Filename {} was not found during polling period.".format(args['--filename']))
except KeyboardInterrupt:
logger.info("\nOkay, bye.\n")
sys.exit(130)
# Done
sys.exit(0)
def read_dimension(self) :
"""Read the dimension of the problem : 2D or 3D."""
# We read the dimension of the problem
self.search("DIMENSION")
self.pos = self.begin
value,number = self.read_next()
if number :
if value == 2 or value == 3 :
self.dimension = int(value)
else :
utils.abort("Wrong dimension")
def post(self):
logger.debug('request.header: \n%s', request.headers)
login = request.headers.get('login')
passHash = request.headers.get('passHash')
Authentication = request.headers.get('Authentication')
if not login or not passHash or not Authentication:
return omitError(ErrorMsg=\
'Not Found: login(%s)/passHash(%s)/Authentication(%s)' %
( login, passHash, Authentication)
), 400
elif ('SAGITTARIUS' not in Authentication):
return omitError(ErrorMsg='SAGITTARIUS not in Authentication header(%s)'\
% Authentication), 400
user = Users.get_by_loginpass(login, passHash)
if not user:
logger.debug("user %s not found with password %s", login, passHash)
#return omitError(ErrorMsg='not Authentication ), 401
abort(401)
#elif not Users.sign(user.id, login, passHash):
# logger.debug("Users.sign fail")
# return omitError(ErrorMsg='not Authentication ), 401
else:
if user.passHash == "": # no passHash, not use email to login
logger.debug(
"user %s not found with password %s, use non email login",
login, passHash)
abort(401)
self.args, _resource_fields_wrap = signin(user)
# g.user = user
# _resource_fields = {}
# _resource_fields_wrap = resource_fields_wrap.copy()
# for col in resource_fields_wrap:
# if col not in ['type', 'subtype']:
# _resource_fields_wrap.pop(col, None)
#
# _resource_fields_wrap[field_inputs_wrap_head] = fields.Nested(resource_fields)
#
# self.args = {}
# self.args[field_inputs_wrap_head] = dict((col, getattr(user, col)) for col in user.__table__.columns.keys())
# #self.args[field_inputs_wrap_head]['sid'] = session['sid']
# token = user.generate_auth_token()
# self.args[field_inputs_wrap_head]['access_token'] = token.decode('ascii')
# self.args[field_inputs_wrap_head]['clientIp'] = request.environ['REMOTE_ADDR']
#
# logger.debug("self.args[field_inputs_wrap_head]: %s", self.args[field_inputs_wrap_head])
# self.args['type'] = "admin"
# self.args['subtype'] = "login"
return marshal(self.args, _resource_fields_wrap), 200
def read_xs_id(self) :
"""Read the cross sections ids. They are put in self.xs_id"""
# We read the cross section
self.search("XS IDS")
self.pos = self.begin
self.xs_id = []
for i in xrange(0,self.n_polygons) :
value,number = self.read_next()
if number :
self.xs_id.append(value)
else :
utils.abort("Problem while reading the cross sections id")
def optimize_layout_linear_random_greedy():
# Create an initial layout
layout = Layout(utils.argv.chip, [[1, 0.0, 0.0]], utils.argv.medium,
utils.argv.overlap, [])
max_num_random_trials = 5 # TODO: Don't hardcode this
while (layout.get_num_chips() != utils.argv.num_chips):
utils.info(
1, "* Generating " + str(max_num_random_trials) +
" candidate positions for chip #" +
str(1 + layout.get_num_chips()) + " in the layout")
num_random_trials = 0
candidate_random_trials = []
while (len(candidate_random_trials) < max_num_random_trials):
[picked_level, picked_x,
picked_y] = layout.get_random_feasible_neighbor_position(-1)
candidate_random_trials.append([picked_level, picked_x, picked_y])
# Pick a candidate
max_power = -1
picked_candidate = None
for candidate in candidate_random_trials:
layout.add_new_chip(candidate)
# print layout.get_chip_positions()
utils.info(1, "- Evaluating candidate " + str(candidate))
result = find_maximum_power_budget(layout)
if (result != None):
[power_distribution, temperature] = result
if (temperature <= utils.argv.max_allowed_temperature) and (
sum(power_distribution) > max_power):
picked_candidate = candidate
try:
layout.remove_chip(layout.get_num_chips() - 1)
except Exception:
utils.abort(
"Fatal error: Graph shouldn't be disconnected here!!")
# Add the candidate
utils.info(1, "Picked candidate: " + str(candidate))
layout.add_new_chip(picked_candidate)
# Do the final evaluation (which was already be done, but whatever)
result = find_maximum_power_budget(layout)
if (result == None):
return None
[power_distribution, temperature] = result
return [layout, power_distribution, temperature]
def pkgcloud_api_call(url, method, **kwargs):
"""
Generic method to make HTTP requests to packagecloud
"""
resp = None
attempt = 0
req = Request(method.upper(), url, **kwargs)
while True:
try:
attempt += 1
resp = Session().send(
Session().prepare_request(req), verify=True)
resp.raise_for_status()
break
except (HTTPError, ConnectionError, Timeout) as ex:
if attempt >= 10:
utils.abort(ex.message)
else:
time.sleep(3)
continue
except RequestException as ex:
utils.abort(ex.message)
try:
if "error" in resp.json():
raise Exception('{}'.format(resp.json()['error']))
return resp
except ValueError as ex:
utils.abort("Unexpected response from packagecloud API. "
"Expected json, got something else: {}".format(ex.message))
except Exception as ex:
utils.abort("Error making packagecloud API call: {}".format(ex.message))
def create_folder():
dataset_path = f"dataset/{args.template_body}_{args.num_bodies}_{args.body_variation_range}-v{args.seed_bodies}"
if os.path.exists(dataset_path):
if args.dataset_exist_ok:
output(f"Overwrite dataset {args.template_body}.", 0)
shutil.rmtree(dataset_path)
os.mkdir(dataset_path)
else:
abort(f"Dataset exists: {dataset_path}")
else:
os.mkdir(dataset_path)
os.mkdir(f"{dataset_path}/bodies")
os.mkdir(f"{dataset_path}/params")
return dataset_path
def bootstrap(ubuntu_only):
print(utils.make_bright("<bootstrap>"))
if not utils.query_yes_no(
"This operation will download a bunch of libcs into"
f" {utils.make_bright(utils.get_libcs_dirpath())}. Proceed?"):
utils.abort("Aborted by user.")
_add_ubuntu_libcs()
if not ubuntu_only:
_add_debian_libcs()
_add_arch_linux_libcs()
print(utils.make_bright("</bootstrap>"))
def display_command(command):
# Sanity check
if command not in commands:
abort("Command '%s' not found, exiting." % command)
cmd = commands[command]
# Print out nicely presented docstring if found
if cmd.__doc__:
print("Displaying detailed information for command '%s':" % command)
print('')
print(indent(cmd.__doc__, strip=True))
print('')
# Or print notice if not
else:
print("No detailed information available for command '%s':" % command)
sys.exit(0)
def load_env():
"""
Reads environment looking for token and returns config.
"""
token = os.environ.get('PACKAGECLOUD_TOKEN')
if token is None:
utils.abort('PACKAGECLOUD_TOKEN environment variable is not set. '
'Set this token to use packagecloud-poll.')
http_scheme = 'https'
api_domain = 'packagecloud.io'
api_version = 'v1'
return {
'url_base': '{}://{}:@{}/api/{}'.format(
http_scheme, token, api_domain, api_version),
'token': token
}
def find_maximum_power_budget_discrete(layout):
# Simple exhaustive search
if (utils.argv.powerdistopt == "exhaustive_discrete"):
return find_maximum_power_budget_discrete_exhaustive(layout)
elif (utils.argv.powerdistopt == "random_discrete"):
return find_maximum_power_budget_discrete_random(layout)
elif (utils.argv.powerdistopt == "greedy_random_discrete"):
return find_maximum_power_budget_discrete_greedy_random(layout)
elif (utils.argv.powerdistopt == "greedy_not_so_random_discrete"):
return find_maximum_power_budget_discrete_greedy_not_so_random(layout)
elif (utils.argv.powerdistopt == "uniform_discrete"):
return find_maximum_power_budget_discrete_uniform(layout)
else:
utils.abort("Unknown discrete power budget maximization method " +
utils.argv.powerdistopt)
def minimize_temperature(layout, total_power_budget, optimization_method,
num_iterations):
if (optimization_method == "simulated_annealing_gradient"):
return minimize_temperature_simulated_annealing_gradient(
layout, total_power_budget, num_iterations)
elif (optimization_method == "neighbor"):
return minimize_temperature_neighbor(layout, total_power_budget,
num_iterations)
elif (optimization_method == "gradient"):
return minimize_temperature_gradient(layout, total_power_budget,
num_iterations)
elif (optimization_method == "random"):
return minimize_temperature_random_continuous(layout,
total_power_budget,
num_iterations)
elif (optimization_method == "uniform"):
return minimize_temperature_uniform(layout, total_power_budget,
num_iterations)
else:
utils.abort("Error: Unknown optimization method '" +
optimization_method)
def evaluate_candidate(args):
[layout, candidate] = args
utils.info(1, " - Evaluating candidate " + str(candidate))
#print 'layout recieved ',layout
#print '\ncandidates ', candidate
dummy_layout = Layout(layout.get_chip(), layout.get_chip_positions(),
layout.get_medium(), layout.get_overlap(),
layout.get_inductor_properties())
#print 'layout chips ', dummy_layout.get_chip_positions(),'\n'
#print 'dummy layout is ',dummy_layout
for chip in candidate:
#print "HERE"
try:
dummy_layout.add_new_chip(chip)
except:
#print 'chip is ',chip
#print 'layout chips ', dummy_layout.get_chip_positions()
utils.abort("Add error in evaluate candidates")
if (dummy_layout.get_diameter() > utils.argv.diameter):
utils.abort(
"Layout diameter is too big (this should never happen here!)")
return [dummy_layout, find_maximum_power_budget(dummy_layout)]
def optimize_layout_rectilinear(mode):
if (utils.argv.verbose == 0):
sys.stderr.write("o")
utils.info(1, "Constructing a " + mode + " rectilinear layout")
if (mode == "straight"):
layout = LayoutBuilder.compute_rectilinear_straight_layout(
utils.argv.num_chips)
elif (mode == "diagonal"):
layout = LayoutBuilder.compute_rectilinear_diagonal_layout(
utils.argv.num_chips)
else:
utils.abort("Unknown rectilinear layout mode '" + mode + "'")
# layout.draw_in_3D(None, True);
result = find_maximum_power_budget(layout)
if (result == None):
return None
[power_distribution, temperature] = result
return [layout, power_distribution, temperature]
def connect(user, host, port):
"""
Create and return a new SSHClient instance connected to given host.
"""
from state import env
#
# Initialization
#
# Init client
client = ssh.SSHClient()
# Load known host keys (e.g. ~/.ssh/known_hosts)
client.load_system_host_keys()
# Unless user specified not to, accept/add new, unknown host keys
if not env.reject_unknown_keys:
client.set_missing_host_key_policy(ssh.AutoAddPolicy())
#
# Connection attempt loop
#
# Initialize loop variables
connected = False
password = env.password
# Loop until successful connect (keep prompting for new password)
while not connected:
# Attempt connection
try:
client.connect(host, int(port), user, password, key_filename=env.key_filename, timeout=10)
connected = True
return client
# Prompt for new password to try on auth failure
except (ssh.AuthenticationException, ssh.SSHException):
# TODO: tie this into global prompting (i.e. right now both uses of
# prompt_for_password() do the same "if not env.password" stuff.
# may want to roll that into prompt_for_password() itself?
password = prompt_for_password(None, password)
# Update env.password if it was empty
if not env.password:
env.password = password
# Ctrl-D / Ctrl-C for exit
except (EOFError, TypeError):
# Print a newline (in case user was sitting at prompt)
print("")
sys.exit(0)
# Handle timeouts
except socket.timeout:
abort("Timed out trying to connect to %s" % host)
# Handle DNS error / name lookup failure
except socket.gaierror:
abort("Name lookup failed for %s" % host)
# Handle generic network-related errors
# NOTE: In 2.6, socket.error subclasses IOError
except socket.error, e:
abort("Low level socket error connecting to host %s: %s" % (host, e[1]))
def search(self,keyword) :
"""Search for the given keyword. The beginning and the end of the
interesting section are saved in self.begin and self.end."""
# We search for the keyword in the file
begin = "# BEGIN "
end = "# END "
# We look for the beginning of the section
self.begin = self.input.find(begin+keyword)
if self.begin == -1 :
utils.abort("Cannot find "+begin+keyword)
else :
self.begin += len(begin+keyword) + 1
# We look for the end of the section
self.end = self.input.find(end+keyword)
if self.end == -1 :
utils.abort("Cannot find "+end+keyword)
# We look for an error
error = self.input[self.begin:self.end-1].find(end)
if error != -1 :
utils.abort("Nested sections.")
def connect(user, host, port):
"""
Create and return a new SSHClient instance connected to given host.
"""
from state import env
#
# Initialization
#
# Init client
client = ssh.SSHClient()
# Load known host keys (e.g. ~/.ssh/known_hosts) unless user says not to.
if not env.disable_known_hosts:
client.load_system_host_keys()
# Unless user specified not to, accept/add new, unknown host keys
if not env.reject_unknown_hosts:
client.set_missing_host_key_policy(ssh.AutoAddPolicy())
#
# Connection attempt loop
#
# Initialize loop variables
connected = False
password = env.password
# Loop until successful connect (keep prompting for new password)
while not connected:
# Attempt connection
try:
client.connect(host, int(port), user, password,
key_filename=env.key_filename, pkey=env.key_value, timeout=10)
connected = True
return client
# BadHostKeyException corresponds to key mismatch, i.e. what on the
# command line results in the big banner error about man-in-the-middle
# attacks.
except ssh.BadHostKeyException:
abort("Host key for %s did not match pre-existing key! Server's key was changed recently, or possible man-in-the-middle attack." % env.host)
# Prompt for new password to try on auth failure
except (
ssh.AuthenticationException,
ssh.PasswordRequiredException,
ssh.SSHException
), e:
# TODO: tie this into global prompting (i.e. right now both uses of
# prompt_for_password() do the same "if not env.password" stuff.
# may want to roll that into prompt_for_password() itself?
# For whatever reason, empty password + no ssh key or agent results
# in an SSHException instead of an AuthenticationException. Since
# it's difficult to do otherwise, we must assume empty password +
# SSHException == auth exception. Conversely: if we get
# SSHException and there *was* a password -- it is probably
# something non auth related, and should be sent upwards.
if e.__class__ is ssh.SSHException and password:
abort(str(e))
# Otherwise, assume an auth exception, and prompt for new/better
# password.
# Paramiko doesn't handle prompting for locked private keys (i.e.
# keys with a passphrase and not loaded into an agent) so we have
# to detect this and tweak our prompt slightly. (Otherwise,
# however, the logic flow is the same, because Paramiko's connect()
# method overrides the password argument to be either the login
# password OR the private key passphrase. Meh.)
#
# NOTE: This will come up if you normally use a
# passphrase-protected private key with ssh-agent, and enter an
# incorrect remote username, because Paramiko:
# * Tries the agent first, which will fail as you gave the wrong
# username, so obviously any loaded keys aren't gonna work for a
# nonexistent remote account;
# * Then tries the on-disk key file, which is passphrased;
# * Realizes there's no password to try unlocking that key with,
# because you didn't enter a password, because you're using
# ssh-agent;
# * In this condition (trying a key file, password is None)
# Paramiko raises PasswordRequiredException.
text = None
if e.__class__ is ssh.PasswordRequiredException:
# NOTE: we can't easily say WHICH key's passphrase is needed,
# because Paramiko doesn't provide us with that info, and
# env.key_filename may be a list of keys, so we can't know
# which one raised the exception. Best not to try.
text = "Please enter passphrase for private key"
password = prompt_for_password(None, password, text)
# Update env.password if it was empty
if not env.password:
env.password = password
# Ctrl-D / Ctrl-C for exit
except (EOFError, TypeError):
# Print a newline (in case user was sitting at prompt)
print('')
sys.exit(0)
# because Paramiko doesn't provide us with that info, and
# env.key_filename may be a list of keys, so we can't know
# which one raised the exception. Best not to try.
text = "Please enter passphrase for private key"
password = prompt_for_password(None, password, text)
# Update env.password if it was empty
if not env.password:
env.password = password
# Ctrl-D / Ctrl-C for exit
except (EOFError, TypeError):
# Print a newline (in case user was sitting at prompt)
print('')
sys.exit(0)
# Handle timeouts
except socket.timeout:
abort('Timed out trying to connect to %s' % host)
# Handle DNS error / name lookup failure
except socket.gaierror:
abort('Name lookup failed for %s' % host)
# Handle generic network-related errors
# NOTE: In 2.6, socket.error subclasses IOError
except socket.error, e:
abort('Low level socket error connecting to host %s: %s' % (
host, e[1])
)
def prompt_for_password(output=None, previous_password=None, text=None):
"""
Prompts for and returns a new password if required; otherwise, returns None.
def main():
"""
Main command-line execution loop.
"""
try:
# Parse command line options
parser, options, arguments = parse_options()
# Update env with any overridden option values
# NOTE: This needs to remain the first thing that occurs
# post-parsing, since so many things hinge on the values in env.
for option in env_options:
state.env[option.dest] = getattr(options, option.dest)
# Handle --hosts, --roles (comma separated string => list)
for key in ['hosts', 'roles']:
if key in state.env and isinstance(state.env[key], str):
state.env[key] = state.env[key].split(',')
# Handle output control level show/hide
update_output_levels(show=options.show, hide=options.hide)
# Handle version number option
if options.show_version:
print("Fabric %s" % state.env.version)
sys.exit(0)
# Load settings from user settings file, into shared env dict.
state.env.update(load_settings(state.env.rcfile))
# Find local fabfile path or abort
fabfile = find_fabfile()
if not fabfile:
abort("Couldn't find any fabfiles!")
# Store absolute path to fabfile in case anyone needs it
state.env.real_fabfile = fabfile
# Load fabfile (which calls its module-level code, including
# tweaks to env values) and put its commands in the shared commands
# dict
commands.update(load_fabfile(fabfile))
# Abort if no commands found
if not commands:
abort("Fabfile didn't contain any commands!")
# Now that we're settled on a fabfile, inform user.
if state.output.debug:
print("Using fabfile '%s'" % fabfile)
# Handle list-commands option (now that commands are loaded)
if options.list_commands:
list_commands()
# Handle show (command-specific help) option
if options.display:
display_command(options.display)
# If user didn't specify any commands to run, show help
if not arguments:
parser.print_help()
sys.exit(0) # Or should it exit with error (1)?
# Parse arguments into commands to run (plus args/kwargs/hosts)
commands_to_run = parse_arguments(arguments)
# Figure out if any specified names are invalid
unknown_commands = []
for tup in commands_to_run:
if tup[0] not in commands:
unknown_commands.append(tup[0])
# Abort if any unknown commands were specified
if unknown_commands:
abort("Command(s) not found:\n%s" \
% indent(unknown_commands))
# At this point all commands must exist, so execute them in order.
for name, args, kwargs, cli_hosts, cli_roles in commands_to_run:
# Get callable by itself
command = commands[name]
# Set current command name (used for some error messages)
state.env.command = name
# Set host list (also copy to env)
state.env.all_hosts = hosts = get_hosts(
command, cli_hosts, cli_roles)
# If hosts found, execute the function on each host in turn
for host in hosts:
username, hostname, port = normalize(host)
state.env.host_string = host
state.env.host = hostname
# Preserve user
prev_user = state.env.user
state.env.user = username
state.env.port = port
# Actually run command
commands[name](*args, **kwargs)
# Put old user back
state.env.user = prev_user
# If no hosts found, assume local-only and run once
if not hosts:
commands[name](*args, **kwargs)
# If we got here, no errors occurred, so print a final note.
if state.output.status:
print("\nDone.")
except SystemExit:
# a number of internal functions might raise this one.
raise
except KeyboardInterrupt:
if state.output.status:
print >>sys.stderr, "\nStopped."
sys.exit(1)
except:
sys.excepthook(*sys.exc_info())
# we might leave stale threads if we don't explicitly exit()
sys.exit(1)
finally:
# Explicitly disconnect from all servers
for key in connections.keys():
if state.output.status:
print "Disconnecting from %s..." % denormalize(key),
connections[key].close()
if state.output.status:
print "done."
sys.exit(0)
def connect(user, host, port):
"""
Create and return a new SSHClient instance connected to given host.
"""
from state import env
#
# Initialization
#
# Init client
client = ssh.SSHClient()
# Load known host keys (e.g. ~/.ssh/known_hosts) unless user says not to.
if not env.disable_known_hosts:
client.load_system_host_keys()
# Unless user specified not to, accept/add new, unknown host keys
if not env.reject_unknown_hosts:
client.set_missing_host_key_policy(ssh.AutoAddPolicy())
#
# Connection attempt loop
#
# Initialize loop variables
connected = False
password = env.password
# Loop until successful connect (keep prompting for new password)
while not connected:
# Attempt connection
try:
client.connect(host, int(port), user, password,
key_filename=env.key_filename, timeout=10)
connected = True
return client
# Prompt for new password to try on auth failure
except (
ssh.AuthenticationException,
ssh.PasswordRequiredException,
ssh.SSHException
), e:
# TODO: tie this into global prompting (i.e. right now both uses of
# prompt_for_password() do the same "if not env.password" stuff.
# may want to roll that into prompt_for_password() itself?
# For whatever reason, empty password + no ssh key or agent results
# in an SSHException instead of an AuthenticationException. Since
# it's difficult to do otherwise, we must assume empty password +
# SSHException == auth exception. Conversely: if we get
# SSHException and there *was* a password -- it is probably
# something non auth related, and should be sent upwards.
if e.__class__ is ssh.SSHException and password:
abort(str(e))
# Otherwise, assume an auth exception, and prompt for new/better
# password.
# Paramiko doesn't handle prompting for locked private keys (i.e.
# keys with a passphrase and not loaded into an agent) so we have
# to detect this and tweak our prompt slightly. (Otherwise,
# however, the logic flow is the same, because Paramiko's connect()
# method overrides the password argument to be either the login
# password OR the private key passphrase. Meh.)
#
# TODO: this block below results in passphrase prompt for some
# errors such as when the wrong user was given. See if this can be
# fixed.
text = None
if e.__class__ is ssh.PasswordRequiredException:
# NOTE: we can't easily say WHICH key's passphrase is needed,
# because Paramiko doesn't provide us with that info, and
# env.key_filename may be a list of keys, so we can't know
# which one raised the exception. Best not to try.
text = "Please enter passphrase for private key"
password = prompt_for_password(None, password, text)
# Update env.password if it was empty
if not env.password:
env.password = password
# Ctrl-D / Ctrl-C for exit
except (EOFError, TypeError):
# Print a newline (in case user was sitting at prompt)
print('')
sys.exit(0)
def __find_available_power_levels(chip_name, benchmark_name):
power_levels = {}
power_level_ptrace_files = {}
if (chip_name == "base2" or chip_name == "base3"):
benchmarks = [""]
benchmark_name = ""
else:
benchmarks = ["bc", "cg", "dc", "ep", "is", "lu", "mg", "sp", "ua", "stress"]
power_levels_frequency_file = {}
# Get all the benchmark, frequency, power, file info
for benchmark in benchmarks:
power_levels_frequency_file[benchmark] = []
filenames = glob("./PTRACE/" + chip_name + "-" + benchmark + "*.ptrace")
for filename in filenames:
f = open(filename, "r")
lines = f.readlines()
f.close()
sum_power = sum([float(x) for x in lines[1].rstrip().split(" ")])
tokens = filename.split('.')
tokens = tokens[1].split('-')
last_part = tokens[-1]
from string import ascii_letters
last_part = last_part.replace(' ', '')
for i in ascii_letters:
last_part = last_part.replace(i, '')
frequency = float(last_part)
power_levels_frequency_file[benchmark].append((frequency, sum_power, filename))
power_levels_frequency_file[benchmark] = sorted(power_levels_frequency_file[benchmark])
# Select the relevant data
if (benchmark_name in benchmarks):
return power_levels_frequency_file[benchmark]
elif (benchmark_name == "overall_max"): # Do the "max" stuff
lengths = [len(power_levels_frequency_file[x]) for x in power_levels_frequency_file]
if (max(lengths) != min(lengths)):
utils.abort(
"Cannot use the \"overall_max\" benchmark mode for power levels because some benchmarks have more power measurements than others")
maxima_power_levels_frequency_file = []
for i in xrange(0, min(lengths)):
max_benchmark = None
max_power = None
for benchmark in benchmarks:
(frequency, sum_power, filename) = power_levels_frequency_file[benchmark][i]
if (max_power == None) or (max_power < sum_power):
max_power = sum_power
max_benchmark = benchmark
maxima_power_levels_frequency_file.append(power_levels_frequency_file[max_benchmark][i])
return maxima_power_levels_frequency_file
else:
utils.abort("Unknown benchmark " + benchmark_name + " for computing power levels")
def optimize_layout_random_greedy_mpi():
##########################################
###TODO: add check for --mpi flag
###TODO: maybe run exp overlap vs ex time
##########################################
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
if size < 2:
comm.bcast([0, 0, 0, 0, 1], root=0)
utils.abort(
"Need to invoke with 'mpirun-np #' where # is int processes greater than 1"
)
if rank == 0:
layout = LayoutBuilder.compute_cradle_layout(3)
num_neighbor_candidates = 20 # Default value
max_num_neighbor_candidate_attempts = 1000 # default value
num_chips_to_add = 1 # Default value
add_scheme = ''
if (len(utils.argv.layout_scheme.split(":")) == 2):
num_neighbor_candidates = int(
utils.argv.layout_scheme.split(":")[1])
if (len(utils.argv.layout_scheme.split(":")) == 3):
num_neighbor_candidates = int(
utils.argv.layout_scheme.split(":")[1])
max_num_neighbor_candidate_attempts = int(
utils.argv.layout_scheme.split(":")[2])
if (len(utils.argv.layout_scheme.split(":")) == 4):
num_neighbor_candidates = int(
utils.argv.layout_scheme.split(":")[1])
max_num_neighbor_candidate_attempts = int(
utils.argv.layout_scheme.split(":")[2])
try:
num_chips_to_add = int(utils.argv.layout_scheme.split(":")[3])
add_scheme = "multi"
except:
add_scheme = utils.argv.layout_scheme.split(":")[3]
num_chips_to_add = 3
#utils.abort("add scheme is "+str(add_scheme))
results = []
picked_index = 0
while (layout.get_num_chips() != utils.argv.num_chips):
###############################################
### Create Candidates
##########################################
#candidate_random_trials = []
#candidate_random_trials = generate_candidates(layout, candidate_random_trials, num_neighbor_candidates,
utils.info(
2, "* Generating " + str(num_neighbor_candidates) +
" candidate positions for chip #" +
str(1 + layout.get_num_chips()) + " in the layout")
if num_chips_to_add > utils.argv.num_chips - layout.get_num_chips(
): #preprocessing
num_chips_to_add = utils.argv.num_chips - layout.get_num_chips(
)
utils.info(
2,
"Warning num_chips_to_add too great\nCandidates will be generated in "
+ str(num_chips_to_add) + "'s")
candidate_random_trials = generate_multi_candidates(
layout, [], num_neighbor_candidates,
max_num_neighbor_candidate_attempts, num_chips_to_add,
add_scheme)
###############################################
### Evaluate all Candidates
###############################################
###############################################
### TODO
### - implement add cradles
### - find feasible cradle function, in layout
### - look into merging this function with sequential version
###############################################
worker_list = [False] * (size - 1)
results = [None] * len(candidate_random_trials)
end = 0
i = 0
while None in results:
if False in worker_list and end < 1:
if i < len(candidate_random_trials):
worker = worker_list.index(False)
worker_list[worker] = True
data_to_worker = [
layout, candidate_random_trials[i], i, worker, end
]
# data_to_worker[layout, candidate, index in results list, index in worker list, worker stop variable]
# print 'SENT layout is ', layout
comm.send(data_to_worker, dest=worker + 1)
i += 1
else:
end = 1 # when no more candidates and workers arent working, and alive
else:
data_from_worker = comm.recv(source=MPI.ANY_SOURCE)
# data_from_worker[[[power_distribution, temperature]], index in results list, index in worker list]
results[data_from_worker[1]] = data_from_worker[0]
worker_list[data_from_worker[2]] = False
# print "RESULTS = ", results
# picked_candidate = pick_candidates(layout, results,candidate_random_trials)
picked_candidate, picked_index = pick_candidates(
results, candidate_random_trials)
if picked_candidate == None:
send_stop_signals(worker_list, comm)
utils.abort(
"Could not find a candidate that met the temperature constraint"
)
utils.info(1, "Picked candidate: " + str(picked_candidate))
for chip in picked_candidate: ###TODO: should we just make a deep copy form results[picked_index][0]???
#print "ADDING HERE "
try:
layout.add_new_chip(chip)
except:
send_stop_signals(worker_list, comm)
utils.abort("Final add doesnt work")
# Do the final evaluation (which was already be done, but whatever)
# result = find_maximum_power_budget(layout)
# saved_result = results[picked_index]
# print '\n\result is ',result,'\nsaved_result is ',saved_result
if not results:
return None
result = results[picked_index]
if (result == None):
return None
[power_distribution, temperature] = result[-1]
# print 'Random greedy layout optimization returning ',[layout, power_distribution, temperature]
# send stop signal to all worker ranks
send_stop_signals(worker_list, comm)
# for k in range(0, len(worker_list)):
# stop_worker = [0, 0, 0, 0, 1]
# comm.send(stop_worker,dest = k+1)
#comm.Disconnect()
return [layout, power_distribution, temperature]
else:
while True:
data_from_master = comm.recv(source=0)
# data_from_master[layout, candidate,index of restult, index of worker,stop worker variable]
if data_from_master[4] > 0:
#print '\n\n!!!!!!worker rank ', rank,' exiting layout is ', data_from_master[0]
#comm.Disconnect()
sys.exit(0)
# print '>>>>>>>>EXIT val is',data_from_master[4], ' for rank ', rank
# layout = data_from_master[0]
# candidate = data_from_master[1]
# result_index = data_from_master[2]
# worker_index = data_from_master[3]
powerdisNtemp = evaluate_candidate(data_from_master[:2])
# data_to_master = [powerdisNtemp,result_index,worker_index]
data_to_master = [
powerdisNtemp, data_from_master[2], data_from_master[3]
]
# data_to_master[[power_distribution, temperature], candidate,index of restult, index of worker,stop worker variable]
comm.send(data_to_master, dest=0)
def optimize_layout_random_greedy():
layout = LayoutBuilder.compute_cradle_layout(3)
num_neighbor_candidates = 10 # Default value
max_num_neighbor_candidate_attempts = 1000 # default value
num_chips_to_add = 1 # Default value
add_scheme = None
#print utils.argv.layout_scheme
if (len(utils.argv.layout_scheme.split(":")) == 2):
num_neighbor_candidates = int(utils.argv.layout_scheme.split(":")[1])
if (len(utils.argv.layout_scheme.split(":")) == 3):
num_neighbor_candidates = int(utils.argv.layout_scheme.split(":")[1])
max_num_neighbor_candidate_attempts = int(
utils.argv.layout_scheme.split(":")[2])
"""
if (len(utils.argv.layout_scheme.split(":")) == 4):
num_neighbor_candidates = int(utils.argv.layout_scheme.split(":")[1])
max_num_neighbor_candidate_attempts = int(utils.argv.layout_scheme.split(":")[2])
num_chips_to_add = int(utils.argv.layout_scheme.split(":")[3])
"""
if (len(utils.argv.layout_scheme.split(":")) == 4):
num_neighbor_candidates = int(utils.argv.layout_scheme.split(":")[1])
max_num_neighbor_candidate_attempts = int(
utils.argv.layout_scheme.split(":")[2])
try:
num_chips_to_add = int(utils.argv.layout_scheme.split(":")[3])
add_scheme = "multi"
except:
add_scheme = utils.argv.layout_scheme.split(":")[3]
num_chips_to_add = 3
results = []
picked_index = 0
if layout.get_num_chips() == utils.argv.num_chips:
result = find_maximum_power_budget(layout)
if result == None:
return None
[power_distribution, temperature] = result
return [layout, power_distribution, temperature]
while (layout.get_num_chips() != utils.argv.num_chips):
###############################################
### Create Candidates
###############################################
utils.info(
2, "* Generating " + str(num_neighbor_candidates) +
" candidate positions for chip #" +
str(1 + layout.get_num_chips()) + " in the layout")
if num_chips_to_add > utils.argv.num_chips - layout.get_num_chips(
): #preprocessing
num_chips_to_add = utils.argv.num_chips - layout.get_num_chips()
utils.info(
2,
"Warning num_chips_to_add too great\nCandidates will be generated in "
+ str(num_chips_to_add) + "'s")
candidate_random_trials = generate_multi_candidates(
layout, [], num_neighbor_candidates,
max_num_neighbor_candidate_attempts, num_chips_to_add, add_scheme)
utils.info(3, "\n\nCandidates picked\n\n")
list_of_args = []
for index in xrange(0, len(candidate_random_trials)):
#print 'layout to evaluate ',layout
list_of_args.append([layout, candidate_random_trials[index]])
results = map(evaluate_candidate, list_of_args)
#print "RESULTS = ", results
###############################################
### Pick the best candidate
################################################
picked_candidate, picked_index = pick_candidates(
results, candidate_random_trials)
#utils.abort("candidate random trial is \n"+str(candidate_random_trials))
# Add the candidate
if picked_candidate == None:
utils.abort(
"Could not find a candidate that met the temperature constraint"
)
utils.info(1, "Picked candidate: " + str(picked_candidate))
for chip in picked_candidate: ###TODO: should we just make a deep copy form results[picked_index][0]???
try:
layout.add_new_chip(chip)
except:
print 'ADD error'
#print "---> ", layout.get_num_chips(), utils.argv.num_chips
# Do the final evaluation (which was already be done, but whatever)
# result = find_maximum_power_budget(layout)
#print 'picked index is ',picked_index,' results len is ',results
result = results[picked_index]
#print 'resuts are ', result
# print "RESULTS: ", result
if (result == None):
return None
[power_distribution, temperature] = result[-1]
return [layout, power_distribution, temperature]
def patch(binary_filepath, supplied_libc_filepath):
print(utils.make_bright("<patch>"))
binary_dirpath = os.path.dirname(binary_filepath)
# identify the supplied libc
matches = identify(supplied_libc_filepath)
if not matches:
utils.abort("The supplied libc is not in the local library.")
# TODO pick the first for now
libc = matches[0]
libc_filepath = os.path.join(utils.get_libcs_dirpath(), libc["relpath"])
libc_architecture = libc["architecture"]
libc_patch = libc["patch"]
libc_version = libc["version"]
ld_filepath = os.path.join(
os.path.dirname(libc_filepath),
os.path.basename(libc_filepath).replace("libc-", "ld-"),
)
# if the dynamic loader does not exist, abort (don't care about race conditions)
if not os.path.isfile(ld_filepath):
utils.abort(
"The dynamic loader corresponding to the libc to use cannot be found."
f" It should reside at {utils.make_bright(ld_filepath)}"
)
# copy the dynamic loader and the libc to the directory where the binary is located
libs_dirpath = os.path.join(
binary_dirpath, "libs", libc_architecture, libc_version, libc_patch
)
ld_proper_filename = f"ld-{libc_version}.so"
ld_proper_filepath = os.path.join(libs_dirpath, ld_proper_filename)
libc_proper_filename = f"libc-{libc_version}.so"
libc_proper_filepath = os.path.join(libs_dirpath, libc_proper_filename)
if not utils.query_yes_no(
"Copy:\n"
f"- {utils.make_bright(ld_filepath)}\n"
f"- {utils.make_bright(libc_filepath)}\n"
"to:\n"
f"- {utils.make_bright(ld_proper_filepath)}\n"
f"- {utils.make_bright(libc_proper_filepath)}\n"
"?"
):
utils.abort("Aborted by user.")
os.makedirs(libs_dirpath, exist_ok=True)
shutil.copy2(ld_filepath, ld_proper_filepath)
shutil.copy2(libc_filepath, libc_proper_filepath)
print()
# if debug symbols exist, copy them also
libc_dbg_filepath = f"{libc_filepath}.debug"
if os.path.isfile(libc_dbg_filepath):
libs_debug_dirpath = os.path.join(libs_dirpath, ".debug")
libc_dbg_proper_filename = utils.get_libc_dbg_proper_filename(libc_filepath)
libc_dbg_proper_filepath = os.path.join(
libs_debug_dirpath, libc_dbg_proper_filename
)
if utils.query_yes_no(
"Copy:\n"
f"- {utils.make_bright(libc_dbg_filepath)}\n"
"to:\n"
f"- {utils.make_bright(libc_dbg_proper_filepath)}\n"
"?"
):
os.makedirs(libs_debug_dirpath, exist_ok=True)
shutil.copy2(libc_dbg_filepath, libc_dbg_proper_filepath)
print()
# patch the binary to use the new dynamic loader and libc
patched_binary_filepath = (
f"{binary_filepath}-{libc_architecture}-{libc_version}-{libc_patch}"
)
if not utils.query_yes_no(
"Copy:\n"
f"- {utils.make_bright(binary_filepath)}\n"
"to:\n"
f"- {utils.make_bright(patched_binary_filepath)}\n"
"and patch the latter?"
):
utils.abort("Aborted by user.")
shutil.copy2(binary_filepath, patched_binary_filepath)
ld_basename = os.path.basename(ld_proper_filename)
libc_basename = os.path.basename(libc_proper_filename)
subprocess.run(
(
f"patchelf --set-interpreter {shlex.quote(os.path.relpath(ld_proper_filepath, binary_dirpath))} {shlex.quote(patched_binary_filepath)}"
f" && patchelf --add-needed {shlex.quote(os.path.relpath(libc_proper_filepath, binary_dirpath))} {shlex.quote(patched_binary_filepath)}"
),
check=True,
shell=True,
)
print(utils.make_bright("</patch>"))
def read_geometry(self) :
"""Read the geometry of the problem and put all the coordinates of the
vertices of the polygons in self.polygons. The size of the cells is
computed here."""
# We read the geometry of the problem
self.search("GEOMETRY")
self.pos = self.begin
self.polygons = []
self.n_vertices = []
while self.pos < self.end :
n_vertices,number = self.read_next()
n_vertices = int(n_vertices)
self.n_vertices.append(n_vertices)
counter = 0.0
polygon = []
for i in xrange(0,n_vertices) :
coord = []
for j in xrange(0,self.dimension) :
value,number = self.read_next()
coord.append(value)
if number :
counter += 1.0
polygon.append(coord)
else :
utils.abort("Problem while reading the geometry.")
self.polygons.append(polygon)
self.pos += 1
# We read the number of cells that the user wants
self.search("CELLS")
self.pos = self.begin
value,number = self.read_next()
if number :
self.n_cells = value
else :
utils.abort("Problem while reading the numbers of cells")
# We compute the maximum area of the cells.
"""The area of the domain is approximated by the area of the smaller
rectangle which contains the domain."""
left = 10^100
right = -10^100
bottom = 10e100
top = -10^100
self.n_polygons = len(self.polygons)
for i in xrange(0,self.n_polygons) :
for j in xrange(0,self.n_vertices[i]) :
if self.polygons[i][j][0] < left :
left = self.polygons[i][j][0]
if self.polygons[i][j][0] > right :
right = self.polygons[i][j][0]
if self.polygons[i][j][1] < bottom :
bottom = self.polygons[i][j][1]
if self.polygons[i][j][1] > top :
top = self.polygons[i][j][1]
# We convert the list in a float
top = float(top)
bottom = float(bottom)
right = float(right)
left = float(left)
# We compute the area
self.area = (top-bottom)*(right-left)
if self.dimension == 3 :
down = 10^100
up = -10^100
for i in xrange(0,self.n_polygons) :
for j in xrange(0,self.n_vertices[i]) :
if self.polygons[i][j][2] < down :
down = self.polygons[0][i][2]
if self.polygons[i][j][2] > up :
up = self.polygons[0][i][2]
self.volume = self.area*(up-down)
self.area_cells = self.area/self.n_cells
required=False,
dest='show_in_3D',
help='opens up an interactive matplotlib visualization')
return parser.parse_args()
if __name__ == '__main__':
# Parse command-line arguments
argv = parse_arguments()
utils.argv = argv
if not (argv.chip_name in ["e5-2667v4", "phi7250", "base2", "base3"]):
utils.abort("Chip '" + argv.chip_name + "' not supported")
else:
argv.chip = Chip(argv.chip_name, argv.power_benchmark)
if (argv.num_chips < 1):
utils.abort("The number of chips (--numchips, -n) should be >0")
if (argv.layout_scheme == "stacked") or (
argv.layout_scheme == "rectilinear_straight") or (
argv.layout_scheme == "rectilinear_diagonal") or (
argv.layout_scheme == "linear_random_greedy") or (
argv.layout_scheme == "checkerboard") or (
argv.layout_scheme
== "cradle") or (argv.layout_scheme == "bridge"):
argv.diameter = argv.num_chips
from Node import Node
from utils import abort
parser = argparse.ArgumentParser(description='Parse all log files')
parser.add_argument('directory', help='The experiment directory (EXPID)')
args = parser.parse_args()
node_pkl = args.directory + "/node-info.pkl"
db_file = args.directory + "/cplo.db"
try:
with open(node_pkl, 'rb') as fp:
data = pickle.load(fp)
except IOError as e:
abort(e, os.EX_IOERR, "Could not load pickle: " + node_pkl)
try:
conn = sqlite3.connect(db_file)
except db_Error as e:
abort(e, os.EX_IOERR, "Could not read DB: " + db_file)
cursor = conn.cursor()
cursor.execute("SELECT subplan_id FROM runhist;")
while True:
d = cursor.fetchone()
if d == None:
break
print(str(d[0]))
cursor.close()
def start_experiment_multi(script):
percentage_train = 0.8
num_exp = args.replicates
exp_path = f"outputs/{g_exp_name}"
os.makedirs(exp_path, exist_ok=True)
data = {
"num-exp": num_exp,
"num-bodies": g_total_bodies,
"args": args,
}
write_yaml(f"{exp_path}/config.yml", data)
repeated_train_bodies = {}
for i in range(
num_exp
): # no matter how many bodies are there in the dataset, we only do random experiment 30 times.
train_bodies, test_bodies = [], []
for j in range(
100
): # retry 100 times if we have chose the same training bodies before. probably will happen when num-bodies is small.
all_bodies = np.arange(0, g_total_bodies)
np.random.shuffle(all_bodies)
train_bodies = all_bodies[:int(percentage_train * g_total_bodies)]
test_bodies = all_bodies[int(percentage_train * g_total_bodies):]
train_bodies = np.sort(train_bodies)
test_bodies = np.sort(test_bodies)
_t = tuple(train_bodies)
if _t not in repeated_train_bodies:
repeated_train_bodies[tuple(train_bodies)] = 1
break
if j >= 100:
abort(
"Can't generate enough unique replicates. Maybe there are not enough bodies in the dataset."
)
output(f"train_bodies {train_bodies}", 2)
output(f"test_bodies {test_bodies}", 2)
train_seed = args.num_bodies * 100000 + args.body_variation_range * 10000 + args.seed_bodies * 100 + i # Unique seeds suggested by Sam
data = {
"train_bodies": train_bodies.tolist(),
"not_train_bodies": test_bodies.tolist(),
# "test_bodies": all_bodies.tolist(),
"test_bodies": test_bodies.tolist(
), # only evaluate on test set to save training time, so things can be done on partition short.
"train_seed": train_seed,
}
write_yaml(f"{exp_path}/exp_multi_{i}_bodies.yml", data)
str_train_bodies = np.array2string(train_bodies, separator=',')[1:-1]
str_train_bodies = str_train_bodies.replace(' ', '').replace('\n', '')
output(f"str_train_bodies: {str_train_bodies}", 2)
str_test_bodies = np.array2string(
test_bodies, separator=','
)[1:
-1] # only evaluate on test set to save training time, so things can be done on partition short.
str_test_bodies = str_test_bodies.replace(' ', '').replace('\n', '')
output(f"str_test_bodies: {str_test_bodies}", 2)
# calculate a propriate seed smaller than 2**32-1
output(f"Starting {script} with exp-idx {i} seed {train_seed}", 1)
if args.vacc:
bash = "sbatch"
else:
bash = "bash"
cmd_w = [
bash, script, g_exp_name,
str(i), str_train_bodies, str_test_bodies,
str(train_seed), "--with-bodyinfo", f"{args.n_timesteps}"
]
cmd_wo = [
bash, script, g_exp_name,
str(i), str_train_bodies, str_test_bodies,
str(train_seed), "", f"{args.n_timesteps}"
]
output(" ".join(cmd_w), 2)
output(" ".join(cmd_wo), 2)
if args.in_parallel:
Popen(cmd_w)
Popen(cmd_wo)
else:
call(cmd_w)
call(cmd_wo)
def main():
"""
Main command-line execution loop.
"""
try:
try:
# Parse command line options
parser, options, arguments = parse_options()
# Update env with any overridden option values
# NOTE: This needs to remain the first thing that occurs
# post-parsing, since so many things hinge on the values in env.
for option in env_options:
state.env[option.dest] = getattr(options, option.dest)
# Handle version number option
if options.show_version:
print "Fabric " + state.env.version
sys.exit(0)
# Load settings from user settings file, into shared env dict.
state.env.update(load_settings(rc_path()))
# Find local fabfile path or abort
fabfile = find_fabfile()
if not fabfile:
abort("Couldn't find any fabfiles!")
# Store absolute path to fabfile in case anyone needs it
state.env.real_fabfile = fabfile
# Load fabfile (which calls its module-level code, including
# tweaks to env values) and put its commands in the shared commands
# dict
commands.update(load_fabfile(fabfile))
# Abort if no commands found
# TODO: continue searching for fabfiles if one we selected doesn't
# contain any callables? Bit of an edge case, but still...
if not commands:
abort("Fabfile didn't contain any commands!")
# Handle list-commands option (now that commands are loaded)
if options.list_commands:
list_commands()
# Handle show (command-specific help) option
if options.display:
show_command(options.display)
# If user didn't specify any commands to run, show help
if not arguments:
parser.print_help()
sys.exit(0) # Or should it exit with error (1)?
# Parse arguments into commands to run (plus args/kwargs/hosts)
commands_to_run = parse_arguments(arguments)
# Figure out if any specified names are invalid
unknown_commands = []
for tup in commands_to_run:
if tup[0] not in commands:
unknown_commands.append(tup[0])
# Abort if any unknown commands were specified
if unknown_commands:
abort("Command(s) not found:\n%s" \
% indent(unknown_commands))
# At this point all commands must exist, so execute them in order.
for name, args, kwargs, cli_hosts in commands_to_run:
# Get callable by itself
command = commands[name]
# Set current command name (used for some error messages)
state.env.command = name
# Set host list
hosts = get_hosts(cli_hosts, command)
# If hosts found, execute the function on each host in turn
for host in hosts:
username, hostname, port = normalize(host)
state.env.host_string = host
state.env.host = hostname
# Preserve user
prev_user = state.env.user
state.env.user = username
state.env.port = port
# Actually run command
commands[name](*args, **kwargs)
# Put old user back
state.env.user = prev_user
# If no hosts found, assume local-only and run once
if not hosts:
commands[name](*args, **kwargs)
# If we got here, no errors occurred, so print a final note.
print("\nDone.")
finally:
# TODO: explicit disconnect?
pass
except SystemExit:
# a number of internal functions might raise this one.
raise
except KeyboardInterrupt:
print >>sys.stderr, "\nStopped."
sys.exit(1)
except:
sys.excepthook(*sys.exc_info())
# we might leave stale threads if we don't explicitly exit()
sys.exit(1)
sys.exit(0)
