def _info_ask_clean(self):
SocketMultiprocess._info_ask_clean(self)
if self.subscribed_tasks:
print('subscribed_tasks is not clean: ')
print(' ' + str(sorted(self.subscribed_tasks)))
else:
print('subscribed_tasks is clean')
def __init__(self,
zmq_transport=None,
tornado_sockets=None,
display=None,
context=None,
io_threads=None,
db_num=0,
group_id='',
connection_info={}):
SocketMultiprocess.__init__(self,
zmq_transport,
tornado_sockets=False,
display=display,
context=context,
io_threads=io_threads,
db_num=db_num,
group_id=group_id,
connection_info=connection_info)
self._connect_db(db_num)
self.time_server_tasks = TIME_SERVER_TASKS
self.finish = False
#dictionary with server codes as keys and dictionary as values
#with the code of the subtasks as keys and GarbageData as values.
self.tasks = {}
# set of server tasks that are finished and can be deleted
self.server_delete = set()
def _info_ask_clean(self):
SocketMultiprocess._info_ask_clean(self)
if self.subscribed_tasks:
print('subscribed_tasks is not clean: ')
print(' ' + str(sorted(self.subscribed_tasks)))
else:
print('subscribed_tasks is clean')
def __init__(self, zmq_transport=None, tornado_sockets=None,
display=None, context=None, io_threads=None, db_num=0,
group_id='', connection_info={}):
SocketMultiprocess.__init__(self, zmq_transport, tornado_sockets,
display, context, io_threads, db_num,
group_id, connection_info)
self._connect_db(db_num)
self.tasks = {}
def __init__(self, zmq_transport=None, tornado_sockets=None,
display=None, context=None, io_threads=None, db_num=0,
group_id='', connection_info={}):
SocketMultiprocess.__init__(self, zmq_transport,
tornado_sockets=False, display=display,
context=context, io_threads=io_threads,
db_num=db_num, group_id=group_id,
connection_info=connection_info)
from tasks.models import MultiprocessTask
MultiprocessTask.set_environment(self)
def __init__(self, zmq_transport=None, tornado_sockets=None,
display=None, context=None, io_threads=None, db_num=0,
group_id='', connection_info={}, mongodb_name=''):
SocketMultiprocess.__init__(self, zmq_transport, tornado_sockets,
display, context, io_threads, db_num,
group_id, connection_info)
from pyople.tasks.models import MultiprocessTask
self.mongodb_name = mongodb_name
self.connect_mongodb(mongodb_name)
MultiprocessTask.set_environment(self)
self.result_class = Result
def __init__(self,
zmq_transport=None,
tornado_sockets=None,
display=None,
context=None,
io_threads=None,
db_num=0,
group_id='',
connection_info={}):
SocketMultiprocess.__init__(self, zmq_transport, tornado_sockets,
display, context, io_threads, db_num,
group_id, connection_info)
self._connect_db(db_num)
self.subscribed_tasks = []
def __init__(self, zmq_transport=None, tornado_sockets=None,
display=None, context=None, io_threads=None, db_num=0,
group_id='', connection_info={}):
SocketMultiprocess.__init__(self, zmq_transport,
tornado_sockets=False, display=display,
context=context, io_threads=io_threads,
db_num=db_num, group_id=group_id,
connection_info=connection_info)
self._connect_db(db_num)
self.time_server_tasks = TIME_SERVER_TASKS
self.finish = False
#dictionary with server codes as keys and dictionary as values
#with the code of the subtasks as keys and GarbageData as values.
self.tasks = {}
# set of server tasks that are finished and can be deleted
self.server_delete = set()
def _info_ask_clean(self):
SocketMultiprocess._info_ask_clean(self)
if self.tasks:
print('tasks_data is not clean of keys. Keys left: ')
for code, list_data in self.tasks.items():
print(code)
if not isinstance(list_data, list):
list_data = [list_data]
for data in list_data:
if hasattr(data, 'code'):
print(' ' + data.code + ' ; ')
if hasattr(data, 'address'):
print(' ' + data.address + ' ; ')
if hasattr(data, 'left'):
print(' ' + 'left : ' + str(data.left))
else:
print('tasks_data is clean')
def _info_ask_clean(self):
SocketMultiprocess._info_ask_clean(self)
if self.tasks:
print('tasks_data is not clean of keys. Keys left: ')
for code, list_data in self.tasks.items():
print(code)
if not isinstance(list_data, list):
list_data = [list_data]
for data in list_data:
if hasattr(data, 'code'):
print(' ' + data.code + ' ; ')
if hasattr(data, 'address'):
print(' ' + data.address + ' ; ')
if hasattr(data, 'left'):
print(' ' + 'left : ' + str(data.left))
else:
print('tasks_data is clean')
def __init__(self,
zmq_transport=None,
tornado_sockets=None,
display=None,
context=None,
io_threads=None,
db_num=0,
group_id='',
connection_info={},
mongodb_name=''):
SocketMultiprocess.__init__(self, zmq_transport, tornado_sockets,
display, context, io_threads, db_num,
group_id, connection_info)
from pyople.tasks.models import MultiprocessTask
self.mongodb_name = mongodb_name
self.connect_mongodb(mongodb_name)
MultiprocessTask.set_environment(self)
self.result_class = Result
def _sockets_path(self):
new_dict = SocketMultiprocess._sockets_path(self)
#tasks_pub: used for sending the status of the tasks.
#tasks_sub: used for receiving the status of the tasks.
#redis_manager: receive the dependencies from other instances.
new_dict.update({
'tasks_pub': ['to_tasks_publish', 'PUB', ''],
'redis_manager': ['redis_manager', 'PULL', '_recv_garbage'],
'tasks_sub': ['from_tasks_publish', 'SUB', '_update_zero_depend']
})
return new_dict
def __init__(self,
zmq_transport=None,
tornado_sockets=None,
display=None,
context=None,
io_threads=None,
db_num=0,
group_id='',
connection_info={}):
SocketMultiprocess.__init__(self,
zmq_transport,
tornado_sockets=False,
display=display,
context=context,
io_threads=io_threads,
db_num=db_num,
group_id=group_id,
connection_info=connection_info)
from tasks.models import MultiprocessTask
MultiprocessTask.set_environment(self)
def _sockets_path(self):
new_dict = SocketMultiprocess._sockets_path(self)
#tasks_pub: used for sending the status of the tasks.
#tasks_sub: used for receiving the status of the tasks.
#redis_manager: receive the dependencies from other instances.
new_dict.update({'tasks_pub': ['to_tasks_publish', 'PUB', ''],
'redis_manager': ['redis_manager',
'PULL', '_recv_garbage'],
'tasks_sub': ['from_tasks_publish', 'SUB',
'_update_zero_depend']})
return new_dict
def _sockets_path(self):
new_dict = SocketMultiprocess._sockets_path(self)
#from_socket: receive tasks to be executed in the instance.
#to_socket: send created tasks.
#data_socket: send data of created tasks.
#direct_task: send tasks that can be executed without needed data.
#redis_manager: send dependencies of the created tasks.
new_dict.update({'from_socket': ['from_new_task', 'PULL', '_work'],
'to_socket': ['to_done_task', 'PUSH', ''],
'data_socket': ['to_data', 'PUSH', ''],
'direct_task': ['to_new_task', 'PUSH', ''],
'redis_manager': ['redis_manager', 'PUSH', '']})
return new_dict
def _sockets_path(self):
new_dict = SocketMultiprocess._sockets_path(self)
#tasks_pub: send the message of done tasks.
#tasks_sub: receive the message to resend a task.
#tasks_del: receive the message to finish a task.
#data_socket: receive the data of a task.
new_dict.update({
'tasks_pub': ['to_tasks_publish', 'PUB', ''],
'tasks_sub': ['from_tasks_publish', 'SUB', '_actualize_task'],
'tasks_del': ['from_tasks_publish', 'SUB', '_del_task'],
'data_socket': ['from_data', 'PULL', '_create_data']
})
return new_dict
def _sockets_path(self):
new_dict = SocketMultiprocess._sockets_path(self)
#tasks_pub: send the message of done tasks.
#tasks_sub: receive the message to resend a task.
#tasks_del: receive the message to finish a task.
#data_socket: receive the data of a task.
new_dict.update({'tasks_pub': ['to_tasks_publish', 'PUB', ''],
'tasks_sub': ['from_tasks_publish',
'SUB', '_actualize_task'],
'tasks_del': ['from_tasks_publish',
'SUB', '_del_task'],
'data_socket': ['from_data',
'PULL', '_create_data']})
return new_dict
def _sockets_path(self):
new_dict = SocketMultiprocess._sockets_path(self)
#from_socket: receive tasks to be executed in the instance.
#to_socket: send created tasks.
#data_socket: send data of created tasks.
#direct_task: send tasks that can be executed without needed data.
#redis_manager: send dependencies of the created tasks.
new_dict.update({
'from_socket': ['from_new_task', 'PULL', '_work'],
'to_socket': ['to_done_task', 'PUSH', ''],
'data_socket': ['to_data', 'PUSH', ''],
'direct_task': ['to_new_task', 'PUSH', ''],
'redis_manager': ['redis_manager', 'PUSH', '']
})
return new_dict
def _sockets_path(self):
new_dict = SocketMultiprocess._sockets_path(self)
#tasks_pub: used for sending the status of the tasks.
#tasks_sub: used for receiving the status of the tasks.
#from_socket: receive new tasks created during execution.
#to_socket: send ready tasks to be executed.
#server: receive new tasks from the server.
#redis_manager: send dependencies of the server tasks.
#data_socket: send direct tasks data.
new_dict.update({
'tasks_pub': ['to_tasks_publish', 'PUB', ''],
'tasks_sub': ['from_tasks_publish', 'SUB', '_update_task'],
'from_socket': ['from_done_task', 'PULL', '_create_task'],
'to_socket': ['to_new_task', 'PUSH', ''],
'server': ['to_server', 'DEALER', '_receive_task_server'],
'redis_manager': ['redis_manager', 'PUSH', ''],
'data_socket': ['to_data', 'PUSH', '']
})
return new_dict
def _sockets_path(self):
new_dict = SocketMultiprocess._sockets_path(self)
#tasks_pub: used for sending the status of the tasks.
#tasks_sub: used for receiving the status of the tasks.
#from_socket: receive new tasks created during execution.
#to_socket: send ready tasks to be executed.
#server: receive new tasks from the server.
#redis_manager: send dependencies of the server tasks.
#data_socket: send direct tasks data.
new_dict.update({'tasks_pub': ['to_tasks_publish', 'PUB', ''],
'tasks_sub': ['from_tasks_publish',
'SUB', '_update_task'],
'from_socket': ['from_done_task', 'PULL',
'_create_task'],
'to_socket': ['to_new_task', 'PUSH', ''],
'server': ['to_server', 'DEALER',
'_receive_task_server'],
'redis_manager': ['redis_manager', 'PUSH', ''],
'data_socket': ['to_data', 'PUSH', '']})
return new_dict
def _connect_socket(self, sock, name, conn):
if name == 'redis_manager':
conn = 'bind'
SocketMultiprocess._connect_socket(self, sock, name, conn)
def _is_clean(self):
self_clean = not bool(self.tasks) and not bool(self.server_delete)
return self_clean and SocketMultiprocess._is_clean(self)
def _connect_socket(self, sock, name, conn):
if name == 'redis_manager':
conn = 'bind'
SocketMultiprocess._connect_socket(self, sock, name, conn)
def _list_subscribe(self):
subscribe = SocketMultiprocess._list_subscribe(self)
subscribe.extend(['Node', 'RedisManager',
'GroupManager', self.group_id])
return subscribe
def _list_subscribe(self):
subscribe = SocketMultiprocess._list_subscribe(self)
subscribe.extend(['Node', 'Worker', self.group_id])
return subscribe
def _is_clean(self):
self_clean = not bool(self.subscribed_tasks)
return self_clean and SocketMultiprocess._is_clean(self)
def _list_subscribe(self):
subscribe = SocketMultiprocess._list_subscribe(self)
subscribe.extend(
['Node', 'RedisManager', 'GroupManager', self.group_id])
return subscribe
def _list_subscribe(self):
subscribe = SocketMultiprocess._list_subscribe(self)
subscribe.extend(
['Node', 'Sink', 'TaskManager' + self.group_id, self.group_id])
return subscribe
def _list_subscribe(self):
subscribe = SocketMultiprocess._list_subscribe(self)
subscribe.extend(['Node', 'Ventilator',
'TaskManager' + self.group_id, self.group_id])
return subscribe
def _list_subscribe(self):
subscribe = SocketMultiprocess._list_subscribe(self)
subscribe.extend(['Node', 'Worker', self.group_id])
return subscribe
def _is_clean(self):
self_clean = not bool(self.tasks)
return self_clean and SocketMultiprocess._is_clean(self)
def _is_clean(self):
self_clean = not bool(self.tasks) and not bool(self.server_delete)
return self_clean and SocketMultiprocess._is_clean(self)