def run_shark(particle, *args):
opts, space, subvols, statTest = args
pid = multiprocessing.current_process().pid
shark_output_base = os.path.join(opts.outdir, 'output_%d' % pid)
_, simu, model, _ = common.read_configuration(opts.config)
modeldir = common.get_shark_output_dir(shark_output_base, simu, model)
cmdline = [
opts.shark_binary, opts.config, '-o',
'execution.output_directory=%s' % shark_output_base, '-o',
'execution.simulation_batches=%s' % ' '.join(map(str, subvols))
]
for option in _to_shark_options(particle, space):
cmdline += ['-o', option]
_exec_shark('Executing shark instance', cmdline)
total = 0
for constraint in opts.constraints:
y_obs, y_mod, err = constraint.get_data(modeldir, subvols)
total += statTest(y_obs, y_mod, err)
logger.info('Particle %r evaluated to %f', particle, total)
if not opts.keep:
shutil.rmtree(shark_output_base)
return total
def main():
prefs = read_configuration(os.path.join(os.path.dirname(__file__),'fritz-speed.ini'))
fc = fritzconnection.FritzConnection()
status = fc.call_action('WANCommonInterfaceConfig', 'GetTotalBytesSent')
bytes_up = status['NewTotalBytesSent']
status = fc.call_action('WANCommonInterfaceConfig', 'GetTotalBytesReceived')
bytes_down = status['NewTotalBytesReceived']
update_rra(prefs['rra_filename'], str(bytes_up), str(bytes_down))
def main():
prefs = read_configuration(
os.path.join(os.path.dirname(__file__), "fritz-speed.ini"))
fc = fritzconnection.FritzConnection(address=prefs["fritz_ip"])
status = fc.call_action("WANCommonIFC1", "GetAddonInfos")
bytes_down = status["NewX_AVM_DE_TotalBytesReceived64"]
bytes_up = status["NewX_AVM_DE_TotalBytesSent64"]
update_rra(prefs["rra_filename"], str(bytes_up), str(bytes_down))
def main():
prefs = read_configuration(
os.path.join(os.path.dirname(__file__), "fritz-speed.ini"))
for g in prefs["graphs"]:
# graph the data for information about parameters see https://oss.oetiker.ch/rrdtool/doc/rrdgraph.en.html
rrdtool.graph(
g["filename"],
"-t",
g["title"],
"-w",
g["graph_width"],
"-h",
g["graph_height"],
"-s",
"end-" + g["interval"] + "s",
"DEF:bytes-up=" + g["rra_filename"] + ":bytes-up:AVERAGE",
"DEF:bytes-down=" + g["rra_filename"] + ":bytes-down:AVERAGE",
prefs["graph_type_down"] + ":bytes-down#" +
prefs["graph_color_down"] + ":Downloadrate",
prefs["graph_type_up"] + ":bytes-up#" + prefs["graph_color_up"] +
":Uploadrate",
"-v",
"Transfer rate [B/s]",
)
four datasources which are counters
three RRA for one day, one week and one month
"""
import os
import rrdtool
from common import read_configuration
def main():
up_speed_max = int(prefs["uplink_mbit"]) * 1024 * 1024 * 1.1
down_speed_max = int(prefs["downlink_mbit"]) * 1024 * 1024 * 1.1
rrdtool.create(
prefs["rra_filename"],
"--step",
"60",
"DS:bytes-up:COUNTER:500:0:" + str(up_speed_max),
"DS:bytes-down:COUNTER:500:0:" + str(down_speed_max),
"RRA:AVERAGE:0.8:1:1440",
"RRA:AVERAGE:0.8:10:1008",
"RRA:AVERAGE:0.8:60:5040",
)
if __name__ == "__main__":
prefs = read_configuration(
os.path.join(os.path.dirname(__file__), "fritz-speed.ini"))
main()
def run_shark_hpc(particles, *args):
"""
- Handler function for running PSO on Shark on a SLURM based cluster.
- Swarm size and number of iterations need to be set within the script for now
- Function needs the relative path to a Shark config file under the -c option
- For now the subprocess call within must be altered if you are changing shark submit options
- To find appropriate memory allocations peruse the initial output lines of each particle
"""
global count
opts, space, subvols, statTest = args
# Prepare the file that will be used by the shark submission scripts
# to determine which values shark will be run for. We put a final \n so the
# final line gets properly counted by wc (used by shark-submit)
shark_options = [
' '.join([
'-o "%s"' % option
for option in _to_shark_options(particle, space)
]) for particle in particles
]
positions_fname = tempfile.mktemp('particle_positions.txt')
logger.info('Creating particle positions file at %s', positions_fname)
with open(positions_fname, 'wt') as f:
f.write('\n'.join(shark_options) + '\n')
# Submit the execution of multiple shark instances, one for each particle
job_name = 'PSOSMF_%d' % count
shark_output_base = os.path.join(opts.outdir, job_name)
cmdline = [
'./shark-submit', '-S', opts.shark_binary, '-w', opts.walltime, '-n',
job_name, '-O', shark_output_base, '-E', positions_fname, '-V',
' '.join(map(str, subvols))
]
if opts.account:
cmdline += ['-a', opts.account]
if opts.queue:
cmdline += ['-Q', opts.queue]
if opts.nodes:
cmdline += ['-N', str(opts.nodes)]
else:
cmdline += ['-m', opts.memory, '-c', str(opts.cpus)]
cmdline.append(opts.config)
_exec_shark('Queueing PSO particles', cmdline)
# Actually wait for the jobs to finish...
while count_jobs(job_name) > 0:
time.sleep(10)
ss = len(particles)
fx = np.zeros([ss, 3])
for i in range(ss):
_, simu, model, _ = common.read_configuration(opts.config)
particle_outdir = os.path.join(shark_output_base, str(i))
modeldir = common.get_shark_output_dir(particle_outdir, simu, model)
for j, constraint in enumerate(opts.constraints):
y_obs, y_mod, err = constraint.get_data(modeldir, subvols)
fx[i, j] = statTest(y_obs, y_mod, err)
if not opts.keep:
shutil.rmtree(particle_outdir)
fx = np.sum(fx, 1)
logger.info('Particles %r evaluated to %r', particles, fx)
# this global count just tracks the number of iterations so they can be saved to different files
count += 1
return fx
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'-c',
'--config',
help='Configuration file used as the basis for running shark',
type=_abspath)
parser.add_argument(
'-v',
'--subvolumes',
help='Comma- and dash-separated list of subvolumes to process',
default='0')
parser.add_argument(
'-b',
'--shark-binary',
help=
'The shark binary to use, defaults to either "shark" or "../build/shark"',
default=None,
type=_abspath)
parser.add_argument('-o',
'--outdir',
help='Auxiliary output directory, defaults to .',
default=_abspath('.'),
type=_abspath)
parser.add_argument('-k',
'--keep',
help='Keep temporary output files',
action='store_true')
pso_opts = parser.add_argument_group('PSO options')
pso_opts.add_argument(
'-s',
'--swarm-size',
help=
'Size of the particle swarm. Defaults to 10 + sqrt(D) * 2 (D=number of dimensions)',
type=int,
default=None)
pso_opts.add_argument(
'-m',
'--max-iterations',
help=
'Maximum number of iterations to reach before giving up, defaults to 20',
default=10,
type=int)
pso_opts.add_argument(
'-S',
'--space-file',
help='File with the search space specification, defaults to space.txt',
default='space.txt',
type=_abspath)
pso_opts.add_argument(
'-t',
'--stat-test',
help=
'Stat function used to calculate the value of a particle, defaults to student-t',
default='student-t',
choices=list(analysis.stat_tests.keys()))
pso_opts.add_argument(
'-x',
'--constraints',
default='HIMF,SMF_z0,SMF_z1',
help=
("Comma-separated list of constraints, any of HIMF, SMF_z0 or SMF_z1, defaults to 'HIMF,SMF_z0,SMF_z1'. "
"Can specify a domain range after the name (e.g., 'SMF_z0(8-11)')"))
hpc_opts = parser.add_argument_group('HPC options')
hpc_opts.add_argument('-H',
'--hpc-mode',
help='Enable HPC mode',
action='store_true')
hpc_opts.add_argument('-C',
'--cpus',
help='Number of CPUs per shark instance',
default=1,
type=int)
hpc_opts.add_argument('-M',
'--memory',
help='Memory needed by each shark instance',
default='1500m')
hpc_opts.add_argument('-N',
'--nodes',
help='Number of nodes to use',
default=None,
type=int)
hpc_opts.add_argument('-a',
'--account',
help='Submit jobs using this account',
default=None)
hpc_opts.add_argument('-q',
'--queue',
help='Submit jobs to this queue',
default=None)
hpc_opts.add_argument(
'-w',
'--walltime',
help='Walltime for each submission, defaults to 1:00:00',
default='1:00:00')
opts = parser.parse_args()
if not opts.config:
parser.error('-c option is mandatory but missing')
if opts.shark_binary and not common.has_program(opts.shark_binary):
parser.error(
"shark binary '%s' not found, specify a correct one via -b" %
opts.shark_binary)
elif not opts.shark_binary:
for candidate in ['shark', '../build/shark']:
if not common.has_program(candidate):
continue
opts.shark_binary = _abspath(candidate)
break
if not opts.shark_binary:
parser.error("No shark binary found, specify one via -b")
_, _, _, redshift_file = common.read_configuration(opts.config)
redshift_table = common._redshift_table(redshift_file)
subvols = common.parse_subvolumes(opts.subvolumes)
setup_logging(opts.outdir)
opts.constraints = constraints.parse(opts.constraints)
for c in opts.constraints:
c.redshift_table = redshift_table
# Read search space specification, which is a comma-separated multiline file,
# each line containing the following elements:
#
# param_name, plot_label, is_log, lower_bound, upper_bound
space = analysis.load_space(opts.space_file)
ss = opts.swarm_size
if ss is None:
ss = 10 + int(2 * math.sqrt(len(space)))
args = (opts, space, subvols, analysis.stat_tests[opts.stat_test])
if opts.hpc_mode:
procs = 0
f = run_shark_hpc
else:
n_cpus = multiprocessing.cpu_count()
procs = min(n_cpus, ss)
f = run_shark
logger.info('-----------------------------------------------------')
logger.info('Runtime information')
logger.info(' shark binary: %s', opts.shark_binary)
logger.info(' Base configuration file: %s', opts.config)
logger.info(' Subvolumes to use: %r', subvols)
logger.info(' Output directory: %s', opts.outdir)
logger.info(' Keep temporary output files: %d', opts.keep)
logger.info("PSO information:")
logger.info(' Search space parameters: %s', ' '.join(space['name']))
logger.info(' Swarm size: %d', ss)
logger.info(' Maximum iterations: %d', opts.max_iterations)
logger.info(' Lower bounds: %r', space['lb'])
logger.info(' Upper bounds: %r', space['ub'])
logger.info(' Test function: %s', opts.stat_test)
logger.info('Constraints:')
for c in opts.constraints:
logger.info('%10s [%.1f - %.1f]' %
(c.__class__.__name__, c.domain[0], c.domain[1]))
logger.info('HPC mode: %d', opts.hpc_mode)
if opts.hpc_mode:
logger.info(' Account used to submit: %s',
opts.account if opts.account else '')
logger.info(' Queue to submit: %s',
opts.queue if opts.queue else '')
logger.info(' Walltime per submission: %s', opts.walltime)
logger.info(' CPUs per instance: %d', opts.cpus)
logger.info(' Memory per instance: %s', opts.memory)
logger.info(' Nodes to use: %s', opts.nodes)
while True:
answer = raw_input('\nAre these parameters correct? (Yes/no): ')
if answer:
if answer.lower() in ('n', 'no'):
logger.info(
'Not starting PSO, check your configuration and try again')
return
print("Please answer 'yes' or 'no'")
continue
break
# Directory where we store the intermediate results
tracksdir = os.path.join(opts.outdir, 'tracks')
try:
os.makedirs(tracksdir)
except OSError:
pass
# Go, go, go!
logger.info('Starting PSO now')
tStart = time.time()
if opts.hpc_mode:
os.chdir('../hpc')
xopt, fopt = pso.pso(f,
space['lb'],
space['ub'],
args=args,
swarmsize=ss,
maxiter=opts.max_iterations,
processes=procs,
dumpfile_prefix=os.path.join(tracksdir, 'track_%03d'))
tEnd = time.time()
global count
logger.info('Number of iterations = %d', count)
logger.info('xopt = %r', xopt)
logger.info('fopt = %r', fopt)
logger.info('PSO finished in %.3f [s]', tEnd - tStart)
four datasources which are counters
three RRA for one day, one week and one mont
"""
import fritzconnection
import rrdtool
import os
from common import read_configuration
def get_link_speed():
fc = fritzconnection.FritzConnection()
status = fc.call_action('WANCommonInterfaceConfig', 'GetCommonLinkProperties')
downstream = status['NewLayer1DownstreamMaxBitRate'] / 8.
upstream = status['NewLayer1UpstreamMaxBitRate'] / 8.
return (upstream, downstream)
def main():
link_speeds = get_link_speed()
max_speeds = tuple([speed*1.1 for speed in link_speeds])
rrdtool.create(prefs['rra_filename'], '--step', '60',
'DS:bytes-up:COUNTER:500:0:'+str(max_speeds[0]),
'DS:bytes-down:COUNTER:500:0:'+str(max_speeds[1]),
'RRA:AVERAGE:0.8:1:1440',
'RRA:AVERAGE:0.8:10:1008',
'RRA:AVERAGE:0.8:60:5040' )
if __name__ == '__main__':
prefs = read_configuration('fritz-speed.ini')
main()
