Exemple #1
0
def main():

  import logging
  import os
  import sys

  # Add parent directory to path.
  # Note that these imports cannot be at the top level because the worker
  # does not have these modules.
  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..'))
  from qjam.master import Master, RemoteWorker
  from qjam.dataset import BaseDataSet
  from qjam import DataSet

  class WorkersDataSet(BaseDataSet):
    '''Workers DataSet.'''
    def __init__(self, workers):
      workers = int(workers)
      BaseDataSet.__init__(self, workers)
      self._workers = workers
    def __len__(self):
      return self._workers

    def chunks(self):
      return self._workers

    def slice(self, index):
      return DataSet([index])


  _fmt = '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
  logging.basicConfig(level=logging.INFO, format=_fmt)

  if len(sys.argv) < 3 or len(sys.argv[2].split(':')) > 2:
    print "runs adam framework slaves"
    print "usage: %s <path to fwk> <master> [<worker> ...]" % sys.argv[0]
    exit(1)

  params = {}
  params['path'] = sys.argv[1]
  master_address = sys.argv[2]
  if ':' in master_address:
    host, port = master_address.split(':')
    params['host'] = host
    params['port'] = int(port)
  else:
    params['host'] = master_address
    params['port'] = 12345

  cluster = sys.argv[3:]
  if not cluster:
    cluster = ['localhost']

  mod = sys.modules[__name__]
  master = Master([RemoteWorker(host) for host in cluster])
  dataset = WorkersDataSet(len(cluster))

  print 'Running fwk in %(path)s at %(host)s:%(port)d' % params
  print master.run(mod, params=params, dataset=dataset)
Exemple #2
0
def main():
    import logging
    import os
    import sys

    # Add parent directory to path.
    # Note that these imports cannot be at the top level because the worker
    # does not have these modules.
    sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..'))
    from qjam import DataSet
    from qjam.master import Master, RemoteWorker

    _fmt = '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
    logging.basicConfig(level=logging.INFO, format=_fmt)

    if len(sys.argv) == 1:
        print "approximates sqrt(n)"
        print "usage: %s <n> [<hostname> ...]" % sys.argv[0]
        exit(1)
    n = float(sys.argv[1])
    cluster = sys.argv[2:]
    if not cluster:
        cluster = ['localhost']

    master = Master([RemoteWorker(host) for host in cluster])
    x0 = DataSet([float(10)])
    mod = sys.modules[__name__]

    eps = 0.0001
    xs = [float(10.0)]
    i = 1
    while True:
        x = master.run(mod, params=n, dataset=DataSet([xs[-1]]))
        dx = abs(x - xs[-1])
        xs.append(x)
        print "%d. %f (delta=%f)" % (i, x, dx)
        if dx < eps:
            break
        i += 1
Exemple #3
0
def main():
    _fmt = '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
    logging.basicConfig(level=logging.DEBUG, format=_fmt)

    # parse args
    options, args = parse_args()
    module_name = args[0]
    dataset_name = args[1]
    params_name = args[2] if len(args) == 3 else None

    # set up cluster
    workers = [RemoteWorker(host) for host in options.workers.split()]
    print_workers(workers)
    master = Master(workers)

    # set up job
    mod = resolve_module(module_name)
    dataset = resolve_module_attr(dataset_name)
    params = resolve_module_attr(params_name) if params_name else None

    # run
    dataset = DataSet(list(dataset))
    result = master.run(mod, params=params, dataset=dataset)
    print result