class ManticoreMultiprocessing(ManticoreBase):
_worker_type = WorkerProcess
def __init__(self, *args, **kwargs):
# This is the global manager that will handle all shared memory access
# See. https://docs.python.org/3/library/multiprocessing.html#multiprocessing.managers.SyncManager
self._manager = SyncManager()
self._manager.start(raise_signal)
# The main manticore lock. Acquire this for accessing shared objects
# THINKME: we use the same lock to access states lists and shared contexts
self._lock = self._manager.Condition()
self._killed = self._manager.Value(bool, False)
self._running = self._manager.Value(bool, False)
# List of state ids of States on storage
self._ready_states = self._manager.list()
self._terminated_states = self._manager.list()
self._busy_states = self._manager.list()
self._killed_states = self._manager.list()
self._shared_context = self._manager.dict()
self._context_value_types = {
list: self._manager.list,
dict: self._manager.dict
}
super().__init__(*args, **kwargs)
def _finalize_manager(process, address, authkey, state, _Client):
try:
SyncManager._finalize_manager(process, address, authkey, state,
_Client)
except WindowsError:
# http://stackoverflow.com/questions/17076679/windowserror-access-is-denied-on-calling-process-terminate
pass
def dconnect(self):
"""
Attempts to connect to the DFS on the host/port for which the Node was initialized for.
If no connection can be made, Node will keep attempting to connect until a connection
can be established. Once a connection can be established the remove methods requested
will be registered.
"""
# remove connection from cache:
# BaseProxy class has thread local storage which caches the connection
# which is reused for future connections causing "borken pipe" errors on
# creating new manager.
if self.dfs in BaseProxy._address_to_local:
if hasattr(BaseProxy._address_to_local[self.dfs][0], 'connection'):
del BaseProxy._address_to_local[self.dfs][0].connection
# register handlers
SyncManager.register("get_nodes")
print "connecting to dfs", self.dfs
while self.alive:
try:
self.impd= SyncManager(address= self.dfs, authkey= self.dauthkey)
self.impd.connect()
print "connected to dfs", self.dfs
break
except (EOFError, IOError, SocketError) as e:
print "could not connect ...trying again", str(e)
sleep(1)
def main_proc():
pid = os.getpid()
# initialize manager
mgr = SyncManager()
mgr.start(mgr_init)
try:
# Create share object between processes
shared_queue = mgr.Queue()
# Create subprocesses
put_proc = Process(target=put_data_proc, args=(shared_queue,))
put_proc_1 = Process(target=put_data_proc_1, args=(shared_queue,))
get_proc = Process(target=get_data_proc, args=(shared_queue,))
# Start the processes
put_proc.start()
put_proc_1.start()
get_proc.start()
# Join the processes until they finished
put_proc.join()
put_proc_1.join()
get_proc.join()
except KeyboardInterrupt:
print "Main process (pid=%s) was interruptted" % pid
finally:
mgr.shutdown()
def server_manager(address, authkey):
mgr = SyncManager(address, authkey)
setproctitle('process_mgr')
debug('manager server started.')
server = mgr.get_server()
server.serve_forever()
debug('manager server stopped.')
def _manticore_multiprocessing(self):
def raise_signal():
signal.signal(signal.SIGINT, signal.SIG_IGN)
self._worker_type = WorkerProcess
# This is the global manager that will handle all shared memory access
# See. https://docs.python.org/3/library/multiprocessing.html#multiprocessing.managers.SyncManager
self._manager = SyncManager()
self._manager.start(raise_signal)
# The main manticore lock. Acquire this for accessing shared objects
# THINKME: we use the same lock to access states lists and shared contexts
self._lock = self._manager.Condition()
self._killed = self._manager.Value(bool, False)
self._running = self._manager.Value(bool, False)
# List of state ids of States on storage
self._ready_states = self._manager.list()
self._terminated_states = self._manager.list()
self._busy_states = self._manager.list()
self._killed_states = self._manager.list()
# The multiprocessing queue is much slower than the deque when it gets full, so we
# triple the size in order to prevent that from happening.
self._log_queue = self._manager.Queue(15000)
self._shared_context = self._manager.dict()
self._context_value_types = {
list: self._manager.list,
dict: self._manager.dict
}
class CoordinatorFactory:
def __init__(self, num_processes: int, host: str, port: int,
max_simultaneous_connections: int, as_remote_client: bool):
"""
:param num_processes: number of parallel executions to be conducted
:param host: host name of main node that holds the test queue, result queue, etc
:param port: port of main node to connect to
:param max_simultaneous_connections: maximum allowed connections at one time to main node; throttled to
prevent overloading *multiprocsesing* module and deadlock
"""
self._host = host
self._port = port
self._num_processes = num_processes
self._max_simultaneous_connections = max_simultaneous_connections
self._is_local = not as_remote_client
def launch(self) -> "Coordinator":
if not self._is_local:
self._sm = SyncManager(authkey=AUTHKEY)
self._sm.start()
coordinator = self._sm.CoordinatorProxy(
self._num_processes, self._host, self._port,
self._max_simultaneous_connections, self._is_local)
else:
coordinator = Coordinator(self._num_processes, self._host,
self._port,
self._max_simultaneous_connections,
self._is_local)
client = Orchestrator.Manager(addr=(self._host, self._port))
client.register_client(coordinator, self._num_processes)
return coordinator
def main():
config = load_config()
cred = config["drive4data.influx"]
dry_run = bool(config.get("dry_run", False))
os.makedirs("out", exist_ok=True)
with ExitStack() as stack:
executor = concurrent.futures.ProcessPoolExecutor(max_workers=8)
stack.enter_context(executor)
manager = SyncManager()
manager.start(mgr_init)
stack.enter_context(manager)
client = InfluxDBClient(batched=False, async_executor=True, time_epoch=TIME_EPOCH, **cred)
stack.enter_context(closing(client))
try:
if not dry_run:
client.drop_measurement("trips")
preprocess_trips(client, executor, manager, dry_run=dry_run)
if not dry_run:
client.drop_measurement("charge_cycles")
preprocess_cycles(client, executor, manager, dry_run=dry_run)
except:
executor.shutdown(wait=False)
raise
def start_manager_server() -> SyncManager:
"""
Start a SyncManager
:return: SyncManager
"""
for port_nr in reversed(range(8080, 8100)):
try:
# If a port is already occupied the SyncManager process will spit out EOFError and OSError messages. The
# former can be catched, but the latter will still show up. So we first check if a port is available
# manually
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('', port_nr))
s.close()
# Create manager
sm = SyncManager(address=('', port_nr), authkey=b'mpire_dashboard')
sm.register('get_dashboard_tqdm_dict', get_dashboard_tqdm_dict)
sm.register('get_dashboard_tqdm_details_dict', get_dashboard_tqdm_details_dict)
sm.register('get_dashboard_tqdm_lock', get_dashboard_tqdm_lock)
sm.start()
# Set host and port number so other processes know where to connect to
DASHBOARD_MANAGER_HOST.value = b''
DASHBOARD_MANAGER_PORT.value = port_nr
return sm
except OSError:
# Port is occupied, ignore it and try another
pass
raise OSError("All ports (8080-8099) are in use")
def __init__(
self, kube_config, task_queue, result_queue, kube_client, worker_uuid, kube_dbnd
):
super(DbndKubernetesScheduler, self).__init__(
kube_config, task_queue, result_queue, kube_client, worker_uuid
)
self.kube_dbnd = kube_dbnd
# PATCH watcher communication manager
# we want to wait for stop, instead of "exit" inplace, so we can get all "not" received messages
from multiprocessing.managers import SyncManager
# TODO: why can't we use original SyncManager?
# Scheduler <-> (via _manager) KubeWatcher
# if _manager dies inplace, we will not get any "info" from KubeWatcher until shutdown
self._manager = SyncManager()
self._manager.start(mgr_init)
self.watcher_queue = self._manager.Queue()
self.current_resource_version = 0
self.kube_watcher = self._make_kube_watcher_dbnd()
# pod to airflow key (dag_id, task_id, execution_date)
self.submitted_pods = {} # type: Dict[str,SubmittedPodState]
# sending data to databand tracker
self.metrics_logger = KubernetesMetricsLogger()
# disappeared pods mechanism
self.last_disappeared_pods = {}
self.current_iteration = 1
# add `k8s-scheduler:` prefix to all log messages
self._log = PrefixLoggerAdapter("k8s-scheduler", self.log)
def __init__(
self, kube_config, task_queue, result_queue, kube_client, worker_uuid, kube_dbnd
):
super(DbndKubernetesScheduler, self).__init__(
kube_config, task_queue, result_queue, kube_client, worker_uuid
)
self.kube_dbnd = kube_dbnd
# PATCH watcher communication manager
# we want to wait for stop, instead of "exit" inplace, so we can get all "not" received messages
from multiprocessing.managers import SyncManager
# Scheduler <-> (via _manager) KubeWatcher
# if _manager dies inplace, we will not get any "info" from KubeWatcher until shutdown
self._manager = SyncManager()
self._manager.start(mgr_init)
self.watcher_queue = self._manager.Queue()
self.current_resource_version = 0
self.kube_watcher = self._make_kube_watcher_dbnd()
# will be used to low level pod interactions
self.failed_pods_to_ignore = []
self.running_pods = {}
self.pod_to_task = {}
self.metrics_logger = KubernetesMetricsLogger()
def joinable_returnable_process_manager(n, ):
'''
This decorator runs n processes to complete a list of tasks given to the decorated function. It waits until all compuations have completed, and returns a list of results.
:param n: Number of processes to run in parallel
:return: List of results
'''
manager = SyncManager()
manager.start()
return_object = manager.list()
def faux_decorator(f):
def wrapper(list_of_parameters):
q = manager.Queue()
lock = manager.Lock()
[q.put(item) for item in list_of_parameters]
processes = [[
q.put(None),
queued_start_process(f, q, lock, return_object)
] for i in range(n)]
[p[1].join() for p in processes]
return return_object
return wrapper
return faux_decorator
def run_local():
from multiprocessing.managers import SyncManager
manager = SyncManager()
manager.start()
flags.namespace = manager.Namespace()
flags.namespace.run = True
signal.signal(signal.SIGINT, signal_handler)
print(sys.argv)
ins = GetYp()
# ins.getCookie()
print(ins)
print("Instance: ", ins)
update_nl = True
if "no_namelist" in sys.argv:
update_nl = False
if "drain" in sys.argv:
update_nl = False
if not update_nl:
print("Not fetching new names from site!")
# ins.go(ctrlNamespace=flags.namespace, update_namelist=True)
ins.go(ctrlNamespace=flags.namespace,
update_namelist=update_nl,
local=True)
def main():
config = load_config(extra_args_func=gw_args)
init_logger(config)
log.info(f'Backend.AI Gateway {__version__}')
log.info(f'runtime: {env_info()}')
log_config = logging.getLogger('ai.backend.gateway.config')
log_config.debug('debug mode enabled.')
if config.debug:
aiohttp.log.server_logger.setLevel('DEBUG')
aiohttp.log.access_logger.setLevel('DEBUG')
else:
aiohttp.log.server_logger.setLevel('WARNING')
aiohttp.log.access_logger.setLevel('WARNING')
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
num_workers = os.cpu_count()
manager = SyncManager()
manager.start(lambda: signal.signal(signal.SIGINT, signal.SIG_IGN))
shared_states = manager.Namespace()
shared_states.lock = manager.Lock()
shared_states.barrier = manager.Barrier(num_workers)
shared_states.agent_last_seen = manager.dict()
try:
aiotools.start_server(server_main, num_workers=num_workers,
extra_procs=[event_router],
args=(config, shared_states))
finally:
manager.shutdown()
log.info('terminated.')
class DbndKubernetesExecutor(KubernetesExecutor):
def __init__(self, kube_dbnd=None):
# type: (DbndKubernetesExecutor, DbndKubernetesClient) -> None
super(DbndKubernetesExecutor, self).__init__()
from multiprocessing.managers import SyncManager
self._manager = SyncManager()
self.kube_dbnd = kube_dbnd
_update_airflow_kube_config(airflow_kube_config=self.kube_config,
engine_config=kube_dbnd.engine_config)
def start(self):
logger.info("Starting Kubernetes executor..")
self._manager.start(mgr_init)
dbnd_run = try_get_databand_run()
if dbnd_run:
self.worker_uuid = str(dbnd_run.run_uid)
else:
self.worker_uuid = (
KubeWorkerIdentifier.get_or_create_current_kube_worker_uuid())
self.log.debug("Start with worker_uuid: %s", self.worker_uuid)
# always need to reset resource version since we don't know
# when we last started, note for behavior below
# https://github.com/kubernetes-client/python/blob/master/kubernetes/docs
# /CoreV1Api.md#list_namespaced_pod
# KubeResourceVersion.reset_resource_version()
self.task_queue = self._manager.Queue()
self.result_queue = self._manager.Queue()
self.kube_client = self.kube_dbnd.kube_client
self.kube_scheduler = DbndKubernetesScheduler(
self.kube_config,
self.task_queue,
self.result_queue,
self.kube_client,
self.worker_uuid,
kube_dbnd=self.kube_dbnd,
)
if self.kube_dbnd.engine_config.debug:
self.log.setLevel(logging.DEBUG)
self.kube_scheduler.log.setLevel(logging.DEBUG)
self._inject_secrets()
self.clear_not_launched_queued_tasks()
self._flush_result_queue()
# override - by default UpdateQuery not working failing with
# sqlalchemy.exc.CompileError: Unconsumed column names: state
# due to model override
# + we don't want to change tasks statuses - maybe they are managed by other executors
@provide_session
def clear_not_launched_queued_tasks(self, *args, **kwargs):
# we don't clear kubernetes tasks from previous run
pass
def setUp(self):
if not hasattr(self, 'manager'):
self.manager = SyncManager()
self.manager.start(
lambda: signal.signal(signal.SIGINT, signal.SIG_IGN))
l = linux.Linux('/bin/ls')
self.state = State(ConstraintSet(), l)
self.lock = self.manager.Condition()
def _run_server(cls, *args):
# make sure that the server ignores SIGINT (KeyboardInterrupt)
signal.signal(signal.SIGINT, signal.SIG_IGN)
# prevent connection errors to trigger exceptions
try:
SyncManager._run_server(*args)
except socket.error:
pass
def _run_server(cls, *args):
# make sure that the server ignores SIGINT (KeyboardInterrupt)
signal.signal(signal.SIGINT, signal.SIG_IGN)
# prevent connection errors to trigger exceptions
try:
SyncManager._run_server(*args)
except socket.error:
pass
def __init__(self, args):
super(MultiprocessingContext, self).__init__(args)
# Configuration directory (i.e., INI files folder)
self.config_dir = args.i
# Command line action
if hasattr(args, 'action'):
self.action = args.action
# Input
self.directory = args.directory
if hasattr(args, 'dataset_list'):
self.dataset_list = args.dataset_list
if hasattr(args, 'dataset_id'):
self.dataset_id = args.dataset_id
if hasattr(args, 'incoming'):
self.incoming = args.incoming
# Multiprocessing configuration
self.processes = args.max_processes if args.max_processes <= cpu_count(
) else cpu_count()
self.use_pool = (self.processes != 1)
# Scan counters
self.scan_errors = 0
self.scan_data = 0
self.nbsources = 0
# Process manager
if self.use_pool:
self.manager = SyncManager()
self.manager.start()
self.progress = self.manager.Value('i', 0)
Print.BUFFER = self.manager.Value(c_char_p, '')
else:
self.progress = Value('i', 0)
# Stdout lock
self.lock = Lock()
# Directory filter (esgmapfile + esgcheckvocab)
if hasattr(args, 'ignore_dir'):
self.dir_filter = args.ignore_dir
# File filters (esgmapfile + esgcheckvocab)
self.file_filter = []
if hasattr(args, 'include_files'):
if args.include_file:
self.file_filter.extend([(f, True) for f in args.include_file])
else:
# Default includes netCDF only
self.file_filter.append(('^.*\.nc$', True))
if hasattr(args, 'exclude_file'):
if args.exclude_file:
self.file_filter.extend([(f, False)
for f in args.exclude_file])
else:
# Default exclude hidden files
self.file_filter.append(('^\..*$', False))
# Facet declaration (esgcheckvocab + esgdrs)
self.set_values = {}
if hasattr(args, 'set_value') and args.set_value:
self.set_values = dict(args.set_value)
self.set_keys = {}
if hasattr(args, 'set_key') and args.set_key:
self.set_keys = dict(args.set_key)
class StateTest(unittest.TestCase):
_multiprocess_can_split_ = True
def setUp(self):
if not hasattr(self, 'manager'):
self.manager = SyncManager()
self.manager.start(
lambda: signal.signal(signal.SIGINT, signal.SIG_IGN))
dirname = os.path.dirname(__file__)
l = linux.Linux(os.path.join(dirname, 'binaries', 'basic_linux_amd64'))
self.state = State(ConstraintSet(), l)
self.lock = self.manager.Condition()
def test_workspace_save_load(self):
self.state.constraints.add(True)
workspace = Workspace(self.lock, 'mem:')
id_ = workspace.save_state(self.state)
state = workspace.load_state(id_)
# Make sure our memory maps come back through serialization
for left, right in zip(sorted(self.state.mem._maps),
sorted(state.mem._maps)):
self.assertEqual(left.start, right.start)
self.assertEqual(left.end, right.end)
self.assertEqual(left.name, right.name)
# Check constraints
self.assertEqual(str(state.constraints), str(self.state.constraints))
def test_workspace_id_start_with_zero(self):
workspace = Workspace(self.lock, 'mem:')
id_ = workspace.save_state(self.state)
self.assertEquals(id_, 0)
def test_output(self):
out = ManticoreOutput('mem:')
name = 'mytest'
message = 'custom message'
out.save_testcase(self.state, name, message)
workspace = out._store._data
# Make sure names are constructed correctly
for entry, data in workspace.items():
self.assertTrue(entry.startswith(name))
if 'messages' in entry:
self.assertTrue(message in data)
keys = [x.split('.')[1] for x in workspace.keys()]
for key in self.state.platform.generate_workspace_files():
self.assertIn(key, keys)
# Make sure we log everything we should be logging
self.assertIn('smt', keys)
self.assertIn('trace', keys)
self.assertIn('messages', keys)
self.assertIn('input', keys)
self.assertIn('pkl', keys)
def __init__(self, args):
# Init print management
Print.init(log=args.log, debug=args.debug, all=args.all, cmd=args.prog)
# Print command-line
Print.command()
self._process_color_arg(args)
# Get project and related configuration
self.project = args.project
self.config_dir = args.i
self.processes = args.max_processes if args.max_processes <= cpu_count() else cpu_count()
self.use_pool = (self.processes != 1)
self.lock = Lock()
self.nbfiles = 0
self.nbskip = 0
self.nberrors = 0
self.file_filter = []
if args.include_file:
self.file_filter.extend([(f, True) for f in args.include_file])
else:
# Default includes netCDF only
self.file_filter.append(('^.*\.nc$', True))
if args.exclude_file:
# Default exclude hidden files
self.file_filter.extend([(f, False) for f in args.exclude_file])
else:
self.file_filter.append(('^\..*$', False))
self.dir_filter = args.ignore_dir
# Init process manager
if self.use_pool:
manager = SyncManager()
manager.start()
Print.BUFFER = manager.Value(c_char_p, '')
self.progress = manager.Value('i', 0)
else:
self.progress = Value('i', 0)
self.tunits_default = None
if self.project in DEFAULT_TIME_UNITS.keys():
self.tunits_default = DEFAULT_TIME_UNITS[self.project]
# Change frequency increment
if args.set_inc:
for table, frequency, increment, units in args.set_inc:
if table not in set(zip(*FREQ_INC.keys())[0]):
raise InvalidTable(table)
if frequency not in set(zip(*FREQ_INC.keys())[1]):
raise InvalidFrequency(frequency)
keys = [(table, frequency)]
if table == 'all':
keys = [k for k in FREQ_INC.keys() if k[1] == frequency]
if frequency == 'all':
keys = [k for k in FREQ_INC.keys() if k[0] == table]
for key in keys:
FREQ_INC[key] = [float(increment), str(units)]
# Get reference time properties if submitted
# Default is to deduce them from first file scanned
self.ref_calendar = args.calendar
self.ref_units = args.units
# Init collector
self.sources = None
def setUp(self):
if not hasattr(self, 'manager'):
self.manager = SyncManager()
self.manager.start(
lambda: signal.signal(signal.SIGINT, signal.SIG_IGN))
dirname = os.path.dirname(__file__)
l = linux.Linux(os.path.join(dirname, 'binaries', 'basic_linux_amd64'))
self.state = State(ConstraintSet(), l)
self.lock = self.manager.Condition()
def get_instance(shared_dictionary: dict = None) -> Configurator:
# print('PID:' + str(os.getpid()) + ' Config generated. __config Exist?' + str(ConfigurationFactory.__config is not None))
if ConfigurationFactory.__config is None:
SyncManager.register('ManagedObjectFactory', ManagedObjectFactory,
ManagedObjectFactoryProxy)
if ConfigurationFactory.__config is None or shared_dictionary is not None:
ConfigurationFactory.__config = _BaseConfigurator(
shared_dictionary=shared_dictionary)
return ConfigurationFactory.__config
def __init__(self, *args, **kwargs):
# init ingestor process, create tweet queue
manager = SyncManager()
manager.start(mgr_init)
self.tweet_queue = manager.Queue()
self.ingestion_process = multiprocessing.Process(target=do_ingestion, args=(self.tweet_queue,))
self.ingestion_process.start()
# call superclass init
tweepy.StreamListener.__init__(self, *args, **kwargs)
def __init__(self, queue_size=None):
#print "instantiating process manager"
SyncManager.__init__(self)
self.start()
self.ff = None
self.input_queue = self.Queue(queue_size)
self.output_queue = self.Queue(queue_size)
self.worker = None
def __init__(self, queue_size=None):
#print "instantiating process manager"
SyncManager.__init__(self)
self.start()
self.ff = None
self.input_queue = self.Queue(queue_size)
self.output_queue = self.Queue(queue_size)
self.worker = None
def connect(self):
"""
Attempts to connect to the Queue on the host/port for which the DFS was initialized for.
If no connection can be made, DFS will keep attempting to connect until a connection
can be established. One connection is established the remove methods requested will be
registered.
"""
# remove connection from cache:
# BaseProxy class has thread local storage which caches the connection
# which is reused for future connections causing "borken pipe" errors on
# creating new manager.
if self.queue in BaseProxy._address_to_local:
del BaseProxy._address_to_local[self.queue][0].connection
# register handlers
SyncManager.register("get_streams")
SyncManager.register("get_queue")
SyncManager.register("get_store")
SyncManager.register("get_properties")
print "connecting to queue", self.queue
while self.alive:
try:
self.impq= SyncManager(address= self.queue, authkey= self.qauthkey)
self.impq.connect()
break
except (EOFError, IOError, SocketError) as e:
print "could not connect ...trying again", str(e)
sleep(1)
def prepare_experiment(env, args):
# Manager to share PER between a learner and explorers
SyncManager.register('PrioritizedReplayBuffer', PrioritizedReplayBuffer)
manager = SyncManager()
manager.start()
kwargs = get_default_rb_dict(args.replay_buffer_size, env)
kwargs["check_for_update"] = True
global_rb = manager.PrioritizedReplayBuffer(**kwargs)
# queues to share network parameters between a learner and explorers
n_queue = 1 if args.n_env > 1 else args.n_explorer
n_queue += 1 # for evaluation
queues = [manager.Queue() for _ in range(n_queue)]
# Event object to share training status. if event is set True, all exolorers stop sampling transitions
is_training_done = Event()
# Lock
lock = manager.Lock()
# Shared memory objects to count number of samples and applied gradients
trained_steps = Value('i', 0)
return global_rb, queues, is_training_done, lock, trained_steps
def Manager():
'''
Returns a manager associated with a running server process
The managers methods such as `Lock()`, `Condition()` and `Queue()`
can be used to create shared objects.
'''
from multiprocessing.managers import SyncManager
m = SyncManager()
m.start()
return m
def __init__(self, kube_dbnd=None):
# type: (DbndKubernetesExecutor, DbndKubernetesClient) -> None
super(DbndKubernetesExecutor, self).__init__()
from multiprocessing.managers import SyncManager
self._manager = SyncManager()
self.kube_dbnd = kube_dbnd
_update_airflow_kube_config(airflow_kube_config=self.kube_config,
engine_config=kube_dbnd.engine_config)
def _finalize_manager(process, *args, **kwargs):
"""Shutdown the manager process."""
def _join(functor, *args, **kwargs):
timeout = kwargs.get('timeout')
if not timeout is None and timeout < 1:
kwargs['timeout'] = 1
functor(*args, **kwargs)
process.join = functools.partial(_join, process.join)
SyncManager._finalize_manager(process, *args, **kwargs)
def start_vision_process(manager: SyncManager) -> \
Tuple[Namespace, Event, Event]:
ns = manager.Namespace()
# init ns
setattr(ns, 'active_config', ConfigMode.GEARS)
evt = manager.Event()
sh_evt = manager.Event()
proc = mp.Process(target=vision_starter, args=(ns, evt, sh_evt))
proc.start()
return ns, evt, sh_evt
def Manager():
"""
Returns a manager associated with a running server process
The managers methods such as `Lock()`, `Condition()` and `Queue()`
can be used to create shared objects.
"""
from multiprocessing.managers import SyncManager
m = SyncManager()
m.start()
return m
def manager():
""" Get multiprocessing manager
Transparently creates a :obj:`multiprocessing.manager.SyncManager` the first time it's invoked. Used for sharing values across multipe concurrently executing Manticore instances.
"""
global _manager
if _manager is None:
_manager = SyncManager()
_manager.start(lambda: signal.signal(signal.SIGINT, signal.SIG_IGN))
return _manager
def func_server(self):
p = current_process()
print('{} id: {}'.format(p.name, id(p)))
host = '127.0.0.1'
port = 12345
authkey = 'authkey'
# SyncManager.register('get_list', list, proxytype=ListProxy)
mgr = SyncManager(address=(host, port),
authkey=authkey.encode('utf-8'))
server = mgr.get_server()
server.serve_forever()
def __init__(self, servo_id):
self.servo_id = servo_id
self.angle = Value('f', 0.0)
self.stop_signal = Value('b', False)
# http://jtushman.github.io/blog/2014/01/14/python-|-multiprocessing-and-interrupts/
manager = SyncManager() # instead of regular Manager because we want to ignore kb interrupt
manager.start(Servo.init_mgr) # start the manager explicitly
self.command_queue = manager.list([])
self.current_command = manager.dict()
self.finished = Value('b', False)
def __init__(self,
initial=None,
store=None,
policy='random',
context=None,
**kwargs):
super().__init__(**kwargs)
# Signals / Callbacks handlers will be invoked potentially at different
# worker processes. State provides a local context to save data.
self.subscribe('did_load_state', self._register_state_callbacks)
# This is the global manager that will handle all shared memory access among workers
self.manager = SyncManager()
self.manager.start(
lambda: signal.signal(signal.SIGINT, signal.SIG_IGN))
# The main executor lock. Acquire this for accessing shared objects
self._lock = self.manager.Condition()
# Shutdown Event
self._shutdown = self.manager.Event()
# States on storage. Shared dict state name -> state stats
self._states = self.manager.list()
# Number of currently running workers. Initially no running workers
self._running = self.manager.Value('i', 0)
self._workspace = Workspace(self._lock, store)
# Executor wide shared context
if context is None:
context = {}
self._shared_context = self.manager.dict(context)
# scheduling priority policy (wip)
# Set policy
policies = {
'random': Random,
'uncovered': Uncovered,
'branchlimited': BranchLimited,
}
self._policy = policies[policy](self)
assert isinstance(self._policy, Policy)
if self.load_workspace():
if initial is not None:
logger.error("Ignoring initial state")
else:
if initial is not None:
self.add(initial)
class DataSender:
def __init__(self,phantfile):
try:
self.phant = json.load(open(phantfile, 'r'))
except IOError:
raise ValueError("Invalid phantfile location")
self.running = True
self._manager = SyncManager()
def start(self):
self._manager.start(self._mgr_init)
self._que = self._manager.Queue()
self._process = Process(target=self.up, args=(self._que,))
self._process.start()
def _mgr_init(self):
signal.signal(signal.SIGINT, signal.SIG_IGN)
print("initialized manager")
def up(self,que):
def stop(val,val2):
print "process SIGINT stopping"
self.running = False
signal.signal(signal.SIGINT, stop)
print('datauploader started')
while self.running or not que.empty():
item = json.loads(que.get(True))
print("handling item={0}".format(item))
self.httpsend(item)
que.task_done()
time.sleep(2)
print("datauploader process terminating...")
def send(self, data):
self._que.put(data)
def httpsend(self, data):
postdata = urllib.urlencode(data)
headers = {'Phant-Private-Key': self.phant['privateKey'] }
req = urllib2.Request(self.phant['inputUrl'], postdata, headers)
res = urllib2.urlopen(req)
content = res.read()
print("response: {0}".format(content))
def stop(self):
print("shutting down sender")
self.running = False
self._que.join()
self._process.terminate()
def _finalize_manager(process, *args, **kwargs):
"""Shutdown the manager process."""
def _join(functor, *args, **kwargs):
timeout = kwargs.get('timeout')
if not timeout is None and timeout < 1:
kwargs['timeout'] = 1
functor(*args, **kwargs)
process.join = functools.partial(_join, process.join)
SyncManager._finalize_manager(process, *args, **kwargs)
def test_run_keyboard_interrupt(sync_manager: SyncManager) -> None:
"""Function: run should stop by keyboard interupt."""
queue_process_id = sync_manager.Queue()
replier = Replier(sync_manager.dict(), queue_process_id)
loop = asyncio.new_event_loop()
with ProcessPoolExecutor() as executor:
future = cast(
"Future[Any]",
loop.run_in_executor(executor, run, replier, None,
keyboard_interrupt))
queue_process_id.get()
assert not future.get_loop().is_running()
assert not future.done()
def __init__(self, mp_manager: SyncManager, stop_event: threading.Event,
closing_event: threading.Event, connect_q: queue.Queue,
result_q: queue.Queue):
self.connect_q = connect_q
self.result_q = result_q
self._stop_event = stop_event
self._closing_event = closing_event
self.all_connections_created = mp_manager.Event()
self.all_connections_closed = mp_manager.Event()
self.last_all_closed = time.time()
self.connections: Dict[int, Connection] = {}
self.sock_filenos: List[int] = [] # cache this, select() takes a list
self.connected_targets: Dict[Tuple[int, str, int], Connection] = {}
def __init__(self):
self._pid = os.getpid()
#abort event
self._mgr = SyncManager()
self._mgr.start(ignore_interrupt)
self._abort_event = self._mgr.Event()
#stdout/err
self._err = StreamEncoder(sys.stderr)
self._out = StreamEncoder(sys.stdout)
#connection information
self._port = None
self._con = None
self()
def get_binary_matrix_from_service(q):
global matrix_service
if matrix_service == None:
matrices = dict()
matrix_service = SyncManager(address=("localhost", 50000), authkey="")
SyncManager.register("get_matrix", lambda q: get_matrix(q, matrices))
Process(target=lambda: matrix_service.get_server().serve_forever()).start()
SyncManager.register("get_matrix")
matrix_service.connect()
return matrix_service.get_matrix(q)
def connect(self):
# register with Queue
SyncManager.register('getPipeline')
SyncManager.register('getStore')
self.qInstance= self.opts.qInstance
(self.qHost, self.qPort, self.qKey)= self.opts.queue.split(':')
queue= SyncManager(address= (self.qHost, int(self.qPort)), authkey= self.qKey)
queue.connect()
self.pipeline= queue.getPipeline()
self.store= queue.getStore()
def init_good_sync_manager():
from multiprocessing.managers import SyncManager
#handle SIGINT from SyncManager object
def mgr_sig_handler(signal, frame):
print 'not closing the mgr'
#initilizer for SyncManager
def mgr_init():
import signal
signal.signal(signal.SIGINT, mgr_sig_handler)
print 'initialized mananger'
#using syncmanager directly instead of letting Manager() do it for me
manager = SyncManager()
manager.start(mgr_init)
def get_server_queue():
#FIXME: some OSX users were getting "Can't assign requested address" errors
# if we use socket.gethostname() for the address. Changing it to
# 'localhost' seems to fix the issue, but I don't know why. We had to
# use socket.gethostname() in order to get our benchmark tests to run
# using qsub on a linux cluster, so with this 'fix', testflo benchmark tests
# will likely not work on a cluster of OSX machines.
if sys.platform == 'darwin':
addr = 'localhost'
else:
addr = socket.gethostname()
manager = SyncManager(address=(addr, 0), authkey=_testflo_authkey)
manager.start()
return manager, manager.Queue()
class Downloader(object):
def __init__(self, timeout=30, retries=100, wait=1):
self.timeout = timeout
self.retries = retries
self.wait = wait
self.manager = SyncManager()
self.manager.start()
def retry_fetch_data(self, url):
market_data = self.fetch_data(url)
retries = 1
while not market_data and retries < self.retries:
print "Retry #%s..." % str(retries)
market_data = self.fetch_data(url)
if market_data:
print "Fetched: " + str(len(market_data))
else:
print "Fetched nothing!"
retries += 1
return market_data
def fetch_data(self, url):
limit = 60
msg = "Downloading " + url[0: min(limit, len(url))]
if len(url) > limit:
msg += "(+" + str(len(url) - limit) + ")"
print msg
return_dict = self.manager.dict()
self.job = Process(target=get_page_data, args=(url, return_dict))
self.job.start()
self.job.join(self.timeout)
if self.job.is_alive():
self.job.terminate()
self.job = None
market_data = None
if 'page' in return_dict:
market_data = return_dict['page']
if self.wait > 0:
time.sleep(self.wait)
return market_data
def connect(self):
(qHost, qPort, qKey)= self.opts.queue.split(':')
self.queue= SyncManager(address= (qHost, int(qPort)), authkey= qKey)
self.queue.connect()
self.pipeline= self.queue.getPipeline()
self.store= self.queue.getStore()
# register with DFS
self.dfs = None
self.instances= dict()
if self.opts.dfs != None:
SyncManager.register('getInstances')
(dHost, dPort, dKey)= self.opts.dfs.split(':')
self.dfs= SyncManager(address= (dHost, int(dPort)), authkey= dKey)
self.dfs.connect()
self.instances= self.dfs.getInstances()
def __init__(self,phantfile):
try:
self.phant = json.load(open(phantfile, 'r'))
except IOError:
raise ValueError("Invalid phantfile location")
self.running = True
self._manager = SyncManager()
def __int__(self, name='default', address=None, authkey=None):
"""
This is the default constructor for the class.
:param name: The manager name
:param address: The address of the server
:param authkey: The auth key
:return:
"""
# Set internals
self.__name = name
# Override the manager
SyncManager.__init__(address, authkey)
return
def __init__(self, address, authkey, taskdir= "tasks", id= None, **properties):
"""Creates a stream and retrieves the streams priority queue and data-store."""
self.id= id if id else str(uuid1())
self.ipaddress= getipaddress()
self.address= address
self.taskdir= path.join(taskdir, self.id)
self.properties= properties
self.impq= SyncManager(address= self.address, authkey= authkey)
self.impq.register("get_streams")
self.impq.register("create_stream")
self.impq.register("delete_stream")
self.impq.register("get_store")
self.impq.register("get_queue")
self.impq.connect()
self.jobs= []
self.impq.create_stream(id= self.id, ipaddress= self.ipaddress, **properties)
self.store= self.impq.get_store(id= self.id)
self.queue= self.impq.get_queue(id= self.id)
self.alive= True
self._current_thread= None
self._lock= Lock()
self.threads= []
self.errors= {}
self.ready= {}
self._progress= {}
try:
makedirs(self.taskdir)
except:
pass
def __init__(self, timeout=30, retries=100, wait=1):
self.timeout = timeout
self.retries = retries
self.wait = wait
self.manager = SyncManager()
self.manager.start()
def __init__(self, cookie_file, url_queue_size, pg_queue_size, nr_downloadprocess, nr_parserprocess):
super(SpiderEngine, self).__init__()
self.logger = logging.getLogger(self.__class__.__name__)
self.multiprocess_manager = SyncManager()#SyncManager(('',58585))
self.multiprocess_manager.start()
self.lck4urlq=self.multiprocess_manager.Lock()
self.lck4pageq=self.multiprocess_manager.Lock()
# event for suprocess to initiative exit.
self.shutdown=self.multiprocess_manager.Event()
self.url_queue=Queue(url_queue_size)
self.page_queue=Queue(pg_queue_size)
self.url_hist=self.multiprocess_manager.dict()
self.urls= UrlScheduler(self.url_queue, self.url_hist, self.lck4urlq)
# init multiprocess log
self.mlog=get_logger()
mhandler=logging.StreamHandler()
mhandler.setFormatter(logging.Formatter('%(processName)s %(funcName)s() | %(message)s', '%H:%M:%S'))
self.mlog.addHandler(mhandler)
self.mlog.setLevel(logging.INFO)
self.pages= PageScheduler(self.urls, self.page_queue, self.lck4pageq)
self.downloader= PageDownloader(cookie_file, self.urls, self.pages, self.shutdown, self.multiprocess_manager, nr_downloadprocess, self.mlog)
self.parser=PageParser(self.urls, self.pages, self.shutdown, self.multiprocess_manager, nr_parserprocess, self.mlog)
def __int__(self, address=MANAGER_ADDRESS, authkey=MANAGER_PASSWORD):
"""
This is the default constructor for the
class that registers the valid methods that
can be called in the manager context.
:param address: The address as a tuple (address, port)
:param auth: The authentication for the server
:return:
"""
self.register("get_MqttRxBuffer")
self.register("get_MqttTxBuffer")
self.register("get_RawRxBuffer")
self.register("get_RawTxBuffer")
# Override the base class
SyncManager.__init__(address, authkey)
return
def __init__(self, address=MANAGER_ADDRESS, pwd=MANAGER_PASSWORD):
"""
This is the constructor for the class object that takes
both the managers address and password.
:param address: The address of the manager server
:param pwd: The managers password access
:return:
"""
# Override the base class
SyncManager.__init__(address, pwd)
Singleton.__init__(self)
# Register the classes needed
self.register("get_MqttRxBuffer", callable=lambda: self.__MqttRxBuffer)
self.register("get_MqttTxBuffer", callable=lambda: self.__MqttTxBuffer)
self.register("get_RawRxBuffer", callable=lambda: self.__RawTxBuffer)
self.register("get_RawTxBuffer", callable=lambda: self.__RawRxBuffer)
return
def __init__(self, name="TaskList", mix_ins=None,
prefix='ask_', *args, **kwds):
SyncManager.__init__(self)
self._name = name
self._mix_ins = mix_ins
self._prefix = prefix
self._args = args
self._kwds = kwds
x = []
if mix_ins:
for m in mix_ins:
for attr in dir(m):
if attr.startswith(prefix):
x.append(attr)
self.register(
'TaskList',
callable=TaskList,
exposed=(x))
def create(self):
"""Methods creates queue server
Args:
none
Returns:
void
Raises:
error: ValueError
"""
if self.__type != queue.QUEUE_TYPE_SERVER:
raise ValueError(
'This operation cannot be done on this queue type')
q = Queue()
SyncManager.register('get_queue', callable=lambda: q)
self.__manager = SyncManager(self.__address, self.__authkey)
self.__manager.start()
def connect(self):
"""Methods connects to queue
Args:
none
Returns:
void
Raises:
error: ValueError
"""
if self.__type != queue.QUEUE_TYPE_CLIENT:
raise ValueError(
'This operation cannot be done on this queue type')
q = Queue()
SyncManager.register('get_queue', callable=lambda: q)
self.__manager = SyncManager(self.__address, self.__authkey)
self.__manager.connect()
def __init__(self, address, authkey, logdir= curdir, piddir= curdir):
self.streams= {}
self.address= address
self.manager= SyncManager(address= self.address, authkey= authkey)
self.manager.register("create_stream", callable= self.create_stream)
self.manager.register("delete_stream", callable= self.delete_stream)
self.manager.register("get_streams", callable= lambda: self.streams, proxytype= DictProxy)
self.manager.register("get_store", callable= lambda id: self.streams[id].store, proxytype= DictProxy)
self.manager.register("get_queue", callable= lambda id: self.streams[id].queue, proxytype= PriorityQueue)
self.manager.register("get_properties", callable= lambda id: self.streams[id].properties, proxytype= DictProxy)
super(Queue, self).__init__(
pidfile= path.join(piddir, self.__class__.__name__ + ".pid"),
stdout= path.join(logdir, self.__class__.__name__ + ".out"),
stderr= path.join(logdir, self.__class__.__name__ + ".err"),
stdin= path.join(logdir, self.__class__.__name__ + ".in")
)
def __init__(self, address, authkey, queue, qauthkey, mnon, mpps, ec2= None, bootstrap= None, deploykey= None, logdir= curdir, piddir= curdir):
"""
Initializes the available remote methods
and I/O streams to be used for the method.
Establishes connection to the Queue.
"""
super(DFS, self).__init__(
pidfile= path.join(piddir, self.__class__.__name__ + ".pid"),
stdout= path.join(logdir, self.__class__.__name__ + ".out"),
stderr= path.join(logdir, self.__class__.__name__ + ".err"),
stdin= path.join(logdir, self.__class__.__name__ + ".in")
)
self.id= getipaddress()
self.address= address
self.authkey= authkey
self.queue= queue
self.qauthkey= qauthkey
self.mnon= mnon
self.mpps= mpps
self.bootstrap= bootstrap
self.deploykey= deploykey
self.nodes= {}
self.ec2= ec2
if self.ec2 != None:
(self.access_key, self.security_key, self.ami_id, self.security_group, self.key_name, self.instance_type)= self.ec2.split(',')
self.ec2= EC2Connection(self.access_key, self.security_key)
print "Connected to EC2", self.ec2
self.alive= True
self.manager= SyncManager(address= self.address, authkey= self.authkey)
self.manager.register("get_nodes", callable= lambda: self.nodes, proxytype= DictProxy)
self.connect()
