def _str(self): return ','.join(['%s:%d' % (host, count) for host, count in self.val]) class AssignMode(object): BY_CORE = 1 BY_NODE = 2 class AssignModeFlag(config.Flag): def parse(self, option_str): self.val = getattr(AssignMode, option_str) def _str(self): if self.val == AssignMode.BY_CORE: return 'BY_CORE' return 'BY_NODE' FLAGS.add(HostListFlag('hosts', default=[('localhost', 8)])) FLAGS.add(BoolFlag('xterm', default=False, help='Run workers in xterm')) FLAGS.add(BoolFlag('oprofile', default=False, help='Run workers inside of operf')) FLAGS.add(AssignModeFlag('assign_mode', default=AssignMode.BY_NODE)) FLAGS.add(BoolFlag('use_single_core', default=True)) FLAGS.add(BoolFlag( 'use_threads', help='When running locally, use threads instead of forking. (slow, for debugging)', default=True)) FLAGS.add(IntFlag('heartbeat_interval', default=3, help='Heartbeat Interval in each worker')) FLAGS.add(IntFlag('worker_failed_heartbeat_threshold', default=10, help='the max number of heartbeat that a worker can delay')) def start_remote_worker(worker, st, ed): ''' Start processes on a worker machine.
from contextlib import contextmanager import logging from math import ceil import os from os.path import basename import socket import sys import threading import time import traceback import numpy as np from spartan.config import FLAGS, BoolFlag FLAGS.add(BoolFlag('dump_timers', default=False)) HOSTNAME = socket.gethostname() PID = os.getpid() LOGGING_CONFIGURED = False def _setup_logger(): global LOGGING_CONFIGURED if logging.root is None: raise Exception, 'Log attempt before logging was configured.' logging.RootLogger.findCaller = findCaller LOGGING_CONFIGURED = True def log_debug(*args, **kw):
import atexit import os from os.path import basename, splitext import signal from spartan import util, config from spartan.cluster import start_cluster from spartan.config import FLAGS, StrFlag, BoolFlag import cProfile import imp import spartan import sys import time import types import unittest FLAGS.add(StrFlag('worker_list', default='4,8,16,32,64,80')) FLAGS.add(BoolFlag('test_optimizations', default=False)) def millis(t1, t2): dt = t2 - t1 ms = (dt.days * 24 * 60 * 60 + dt.seconds) * 1000 + dt.microseconds / 1000.0 return ms def sig_handler(sig, frame): import threading import sys import traceback for thread_id, stack in sys._current_frames().items(): print '-' * 100 traceback.print_stack(stack)
class AssignMode(object): BY_CORE = 1 BY_NODE = 2 class AssignModeFlag(config.Flag): def parse(self, option_str): self.val = getattr(AssignMode, option_str) def _str(self): if self.val == AssignMode.BY_CORE: return "BY_CORE" return "BY_NODE" FLAGS.add(HostListFlag("hosts", default=[("localhost", 8)])) FLAGS.add(BoolFlag("xterm", default=False, help="Run workers in xterm")) FLAGS.add(BoolFlag("oprofile", default=False, help="Run workers inside of operf")) FLAGS.add(AssignModeFlag("assign_mode", default=AssignMode.BY_NODE)) FLAGS.add(BoolFlag("use_single_core", default=True)) FLAGS.add( BoolFlag( "use_threads", help="When running locally, use threads instead of forking. (slow, for debugging)", default=True ) ) FLAGS.add(IntFlag("heartbeat_interval", default=3, help="Heartbeat Interval in each worker")) FLAGS.add( IntFlag("worker_failed_heartbeat_threshold", default=10, help="the max number of heartbeat that a worker can delay") )
import collections from contextlib import contextmanager import logging from math import ceil import os from os.path import basename import socket import sys import threading import time import traceback import numpy as np from spartan.config import FLAGS, BoolFlag FLAGS.add(BoolFlag('dump_timers', default=False)) HOSTNAME = socket.gethostname() PID = os.getpid() LOGGING_CONFIGURED = False def _setup_logger(): global LOGGING_CONFIGURED if logging.root is None: raise Exception, 'Log attempt before logging was configured.' logging.RootLogger.findCaller = findCaller LOGGING_CONFIGURED = True
return module._jit_fn class ParakeetExpr(LocalExpr): deps = PythonValue source = PythonValue def fn_name(self): return 'parakeet' def pretty_str(self): return 'parakeet_op' def evaluate(self, ctx): names = self.input_names() fn = compile_parakeet_source(self.source) kw_args = {} for var in names: value = ctx.inputs[var] kw_args[var] = value if FLAGS.use_cuda: return fn(_backend='cuda', **kw_args) else: return fn(**kw_args) from spartan.config import FLAGS, BoolFlag FLAGS.add(BoolFlag('use_cuda', default=False))
class AssignMode(object): BY_CORE = 1 BY_NODE = 2 class AssignModeFlag(config.Flag): def parse(self, option_str): self.val = getattr(AssignMode, option_str) def _str(self): if self.val == AssignMode.BY_CORE: return 'BY_CORE' return 'BY_NODE' FLAGS.add(HostListFlag('hosts', default=[('localhost', 8)])) FLAGS.add(BoolFlag('xterm', default=False, help='Run workers in xterm')) FLAGS.add( BoolFlag('oprofile', default=False, help='Run workers inside of operf')) FLAGS.add(AssignModeFlag('assign_mode', default=AssignMode.BY_NODE)) FLAGS.add(BoolFlag('use_single_core', default=True)) FLAGS.add( BoolFlag( 'use_threads', help= 'When running locally, use threads instead of forking. (slow, for debugging)', default=True)) FLAGS.add( IntFlag('heartbeat_interval', default=3,
import atexit import os from os.path import basename, splitext import signal from spartan import util, config from spartan.cluster import start_cluster from spartan.config import FLAGS, StrFlag, BoolFlag import cProfile import imp import spartan import sys import time import types import unittest FLAGS.add(StrFlag('worker_list', default='4,8,16,32,64,80')) FLAGS.add(BoolFlag('test_optimizations', default=False)) def millis(t1, t2): dt = t2 - t1 ms = (dt.days * 24 * 60 * 60 + dt.seconds) * 1000 + dt.microseconds / 1000.0 return ms def sig_handler(sig, frame): import threading import sys import traceback
class ParakeetExpr(LocalExpr): deps = PythonValue source = PythonValue def fn_name(self): return "parakeet" def pretty_str(self): return "parakeet_op" def evaluate(self, ctx): names = self.input_names() fn = compile_parakeet_source(self.source) kw_args = {} for var in names: value = ctx.inputs[var] kw_args[var] = value if FLAGS.use_cuda: return fn(_backend="cuda", **kw_args) else: return fn(**kw_args) from spartan.config import FLAGS, BoolFlag FLAGS.add(BoolFlag("use_cuda", default=False))