def main():
import logging
import os
import sys
# Add parent directory to path.
# Note that these imports cannot be at the top level because the worker
# does not have these modules.
sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..'))
from qjam.master import Master, RemoteWorker
from qjam.dataset import BaseDataSet
from qjam import DataSet
class WorkersDataSet(BaseDataSet):
'''Workers DataSet.'''
def __init__(self, workers):
workers = int(workers)
BaseDataSet.__init__(self, workers)
self._workers = workers
def __len__(self):
return self._workers
def chunks(self):
return self._workers
def slice(self, index):
return DataSet([index])
_fmt = '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
logging.basicConfig(level=logging.INFO, format=_fmt)
if len(sys.argv) < 3 or len(sys.argv[2].split(':')) > 2:
print "runs adam framework slaves"
print "usage: %s <path to fwk> <master> [<worker> ...]" % sys.argv[0]
exit(1)
params = {}
params['path'] = sys.argv[1]
master_address = sys.argv[2]
if ':' in master_address:
host, port = master_address.split(':')
params['host'] = host
params['port'] = int(port)
else:
params['host'] = master_address
params['port'] = 12345
cluster = sys.argv[3:]
if not cluster:
cluster = ['localhost']
mod = sys.modules[__name__]
master = Master([RemoteWorker(host) for host in cluster])
dataset = WorkersDataSet(len(cluster))
print 'Running fwk in %(path)s at %(host)s:%(port)d' % params
print master.run(mod, params=params, dataset=dataset)
def main():
import logging
import os
import sys
# Add parent directory to path.
# Note that these imports cannot be at the top level because the worker
# does not have these modules.
sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..'))
from qjam import DataSet
from qjam.master import Master, RemoteWorker
_fmt = '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
logging.basicConfig(level=logging.INFO, format=_fmt)
if len(sys.argv) == 1:
print "approximates sqrt(n)"
print "usage: %s <n> [<hostname> ...]" % sys.argv[0]
exit(1)
n = float(sys.argv[1])
cluster = sys.argv[2:]
if not cluster:
cluster = ['localhost']
master = Master([RemoteWorker(host) for host in cluster])
x0 = DataSet([float(10)])
mod = sys.modules[__name__]
eps = 0.0001
xs = [float(10.0)]
i = 1
while True:
x = master.run(mod, params=n, dataset=DataSet([xs[-1]]))
dx = abs(x - xs[-1])
xs.append(x)
print "%d. %f (delta=%f)" % (i, x, dx)
if dx < eps:
break
i += 1
def main():
_fmt = '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
logging.basicConfig(level=logging.DEBUG, format=_fmt)
# parse args
options, args = parse_args()
module_name = args[0]
dataset_name = args[1]
params_name = args[2] if len(args) == 3 else None
# set up cluster
workers = [RemoteWorker(host) for host in options.workers.split()]
print_workers(workers)
master = Master(workers)
# set up job
mod = resolve_module(module_name)
dataset = resolve_module_attr(dataset_name)
params = resolve_module_attr(params_name) if params_name else None
# run
dataset = DataSet(list(dataset))
result = master.run(mod, params=params, dataset=dataset)
print result