コード例 #1
0
 def _info_ask_clean(self):
     SocketMultiprocess._info_ask_clean(self)
     if self.subscribed_tasks:
         print('subscribed_tasks is not clean: ')
         print('    ' + str(sorted(self.subscribed_tasks)))
     else:
         print('subscribed_tasks is clean')
コード例 #2
0
 def __init__(self,
              zmq_transport=None,
              tornado_sockets=None,
              display=None,
              context=None,
              io_threads=None,
              db_num=0,
              group_id='',
              connection_info={}):
     SocketMultiprocess.__init__(self,
                                 zmq_transport,
                                 tornado_sockets=False,
                                 display=display,
                                 context=context,
                                 io_threads=io_threads,
                                 db_num=db_num,
                                 group_id=group_id,
                                 connection_info=connection_info)
     self._connect_db(db_num)
     self.time_server_tasks = TIME_SERVER_TASKS
     self.finish = False
     #dictionary with server codes as keys and  dictionary as values
     #with the code of the subtasks as keys and GarbageData as values.
     self.tasks = {}
     # set of server tasks that are finished and can be deleted
     self.server_delete = set()
コード例 #3
0
ファイル: sink.py プロジェクト: pyople/pyople
 def _info_ask_clean(self):
     SocketMultiprocess._info_ask_clean(self)
     if self.subscribed_tasks:
         print('subscribed_tasks is not clean: ')
         print('    ' + str(sorted(self.subscribed_tasks)))
     else:
         print('subscribed_tasks is clean')
コード例 #4
0
ファイル: ventilator.py プロジェクト: pyople/pyople
 def __init__(self, zmq_transport=None, tornado_sockets=None,
              display=None, context=None, io_threads=None, db_num=0,
              group_id='', connection_info={}):
     SocketMultiprocess.__init__(self, zmq_transport, tornado_sockets,
                                 display, context, io_threads, db_num,
                                 group_id, connection_info)
     self._connect_db(db_num)
     self.tasks = {}
コード例 #5
0
ファイル: server.py プロジェクト: pyople/pyople
 def __init__(self, zmq_transport=None, tornado_sockets=None,
              display=None, context=None, io_threads=None, db_num=0,
              group_id='', connection_info={}):
     SocketMultiprocess.__init__(self, zmq_transport,
                                 tornado_sockets=False, display=display,
                                 context=context, io_threads=io_threads,
                                 db_num=db_num, group_id=group_id,
                                 connection_info=connection_info)
     from tasks.models import MultiprocessTask
     MultiprocessTask.set_environment(self)
コード例 #6
0
ファイル: worker.py プロジェクト: pyople/pyople
 def __init__(self, zmq_transport=None, tornado_sockets=None,
              display=None, context=None, io_threads=None, db_num=0,
              group_id='', connection_info={}, mongodb_name=''):
     SocketMultiprocess.__init__(self, zmq_transport, tornado_sockets,
                                 display, context, io_threads, db_num,
                                 group_id, connection_info)
     from pyople.tasks.models import MultiprocessTask
     self.mongodb_name = mongodb_name
     self.connect_mongodb(mongodb_name)
     MultiprocessTask.set_environment(self)
     self.result_class = Result
コード例 #7
0
 def __init__(self,
              zmq_transport=None,
              tornado_sockets=None,
              display=None,
              context=None,
              io_threads=None,
              db_num=0,
              group_id='',
              connection_info={}):
     SocketMultiprocess.__init__(self, zmq_transport, tornado_sockets,
                                 display, context, io_threads, db_num,
                                 group_id, connection_info)
     self._connect_db(db_num)
     self.subscribed_tasks = []
コード例 #8
0
ファイル: redis_manager.py プロジェクト: pyople/pyople
 def __init__(self, zmq_transport=None, tornado_sockets=None,
              display=None, context=None, io_threads=None, db_num=0,
              group_id='', connection_info={}):
     SocketMultiprocess.__init__(self, zmq_transport,
                                 tornado_sockets=False, display=display,
                                 context=context, io_threads=io_threads,
                                 db_num=db_num, group_id=group_id,
                                 connection_info=connection_info)
     self._connect_db(db_num)
     self.time_server_tasks = TIME_SERVER_TASKS
     self.finish = False
     #dictionary with server codes as keys and  dictionary as values
     #with the code of the subtasks as keys and GarbageData as values.
     self.tasks = {}
     # set of server tasks that are finished and can be deleted
     self.server_delete = set()
コード例 #9
0
ファイル: ventilator.py プロジェクト: pyople/pyople
 def _info_ask_clean(self):
     SocketMultiprocess._info_ask_clean(self)
     if self.tasks:
         print('tasks_data is not clean of keys. Keys left: ')
         for code, list_data in self.tasks.items():
             print(code)
             if not isinstance(list_data, list):
                 list_data = [list_data]
             for data in list_data:
                 if hasattr(data, 'code'):
                     print('    ' + data.code + ' ; ')
                 if hasattr(data, 'address'):
                     print('        ' + data.address + ' ; ')
                 if hasattr(data, 'left'):
                     print('        ' + 'left   : ' + str(data.left))
     else:
         print('tasks_data is clean')
コード例 #10
0
ファイル: ventilator.py プロジェクト: pyople/pyople
 def _info_ask_clean(self):
     SocketMultiprocess._info_ask_clean(self)
     if self.tasks:
         print('tasks_data is not clean of keys. Keys left: ')
         for code, list_data in self.tasks.items():
             print(code)
             if not isinstance(list_data, list):
                 list_data = [list_data]
             for data in list_data:
                 if hasattr(data, 'code'):
                     print('    ' + data.code + ' ; ')
                 if hasattr(data, 'address'):
                     print('        ' + data.address + ' ; ')
                 if hasattr(data, 'left'):
                     print('        ' + 'left   : ' + str(data.left))
     else:
         print('tasks_data is clean')
コード例 #11
0
ファイル: worker.py プロジェクト: pyople/pyople
 def __init__(self,
              zmq_transport=None,
              tornado_sockets=None,
              display=None,
              context=None,
              io_threads=None,
              db_num=0,
              group_id='',
              connection_info={},
              mongodb_name=''):
     SocketMultiprocess.__init__(self, zmq_transport, tornado_sockets,
                                 display, context, io_threads, db_num,
                                 group_id, connection_info)
     from pyople.tasks.models import MultiprocessTask
     self.mongodb_name = mongodb_name
     self.connect_mongodb(mongodb_name)
     MultiprocessTask.set_environment(self)
     self.result_class = Result
コード例 #12
0
 def _sockets_path(self):
     new_dict = SocketMultiprocess._sockets_path(self)
     #tasks_pub: used for sending the status of the tasks.
     #tasks_sub: used for receiving the status of the tasks.
     #redis_manager: receive the dependencies from other instances.
     new_dict.update({
         'tasks_pub': ['to_tasks_publish', 'PUB', ''],
         'redis_manager': ['redis_manager', 'PULL', '_recv_garbage'],
         'tasks_sub': ['from_tasks_publish', 'SUB', '_update_zero_depend']
     })
     return new_dict
コード例 #13
0
 def __init__(self,
              zmq_transport=None,
              tornado_sockets=None,
              display=None,
              context=None,
              io_threads=None,
              db_num=0,
              group_id='',
              connection_info={}):
     SocketMultiprocess.__init__(self,
                                 zmq_transport,
                                 tornado_sockets=False,
                                 display=display,
                                 context=context,
                                 io_threads=io_threads,
                                 db_num=db_num,
                                 group_id=group_id,
                                 connection_info=connection_info)
     from tasks.models import MultiprocessTask
     MultiprocessTask.set_environment(self)
コード例 #14
0
ファイル: redis_manager.py プロジェクト: pyople/pyople
 def _sockets_path(self):
     new_dict = SocketMultiprocess._sockets_path(self)
     #tasks_pub: used for sending the status of the tasks.
     #tasks_sub: used for receiving the status of the tasks.
     #redis_manager: receive the dependencies from other instances.
     new_dict.update({'tasks_pub': ['to_tasks_publish', 'PUB', ''],
                      'redis_manager': ['redis_manager',
                                        'PULL', '_recv_garbage'],
                      'tasks_sub': ['from_tasks_publish', 'SUB',
                                    '_update_zero_depend']})
     return new_dict
コード例 #15
0
ファイル: worker.py プロジェクト: pyople/pyople
 def _sockets_path(self):
     new_dict = SocketMultiprocess._sockets_path(self)
     #from_socket: receive tasks to be executed in the instance.
     #to_socket: send created tasks.
     #data_socket: send data of created tasks.
     #direct_task: send tasks that can be executed without needed data.
     #redis_manager: send dependencies of the created tasks.
     new_dict.update({'from_socket': ['from_new_task', 'PULL', '_work'],
                      'to_socket': ['to_done_task', 'PUSH', ''],
                      'data_socket': ['to_data', 'PUSH', ''],
                      'direct_task': ['to_new_task', 'PUSH', ''],
                      'redis_manager': ['redis_manager', 'PUSH', '']})
     return new_dict
コード例 #16
0
 def _sockets_path(self):
     new_dict = SocketMultiprocess._sockets_path(self)
     #tasks_pub: send the message of done tasks.
     #tasks_sub: receive the message to resend a task.
     #tasks_del: receive the message to finish a task.
     #data_socket: receive the data of a task.
     new_dict.update({
         'tasks_pub': ['to_tasks_publish', 'PUB', ''],
         'tasks_sub': ['from_tasks_publish', 'SUB', '_actualize_task'],
         'tasks_del': ['from_tasks_publish', 'SUB', '_del_task'],
         'data_socket': ['from_data', 'PULL', '_create_data']
     })
     return new_dict
コード例 #17
0
ファイル: sink.py プロジェクト: pyople/pyople
 def _sockets_path(self):
     new_dict = SocketMultiprocess._sockets_path(self)
     #tasks_pub: send the message of done tasks.
     #tasks_sub: receive the message to resend a task.
     #tasks_del: receive the message to finish a task.
     #data_socket: receive the data of a task.
     new_dict.update({'tasks_pub': ['to_tasks_publish', 'PUB', ''],
                      'tasks_sub': ['from_tasks_publish',
                                    'SUB', '_actualize_task'],
                      'tasks_del': ['from_tasks_publish',
                                       'SUB', '_del_task'],
                      'data_socket': ['from_data',
                                      'PULL', '_create_data']})
     return new_dict
コード例 #18
0
ファイル: worker.py プロジェクト: pyople/pyople
 def _sockets_path(self):
     new_dict = SocketMultiprocess._sockets_path(self)
     #from_socket: receive tasks to be executed in the instance.
     #to_socket: send created tasks.
     #data_socket: send data of created tasks.
     #direct_task: send tasks that can be executed without needed data.
     #redis_manager: send dependencies of the created tasks.
     new_dict.update({
         'from_socket': ['from_new_task', 'PULL', '_work'],
         'to_socket': ['to_done_task', 'PUSH', ''],
         'data_socket': ['to_data', 'PUSH', ''],
         'direct_task': ['to_new_task', 'PUSH', ''],
         'redis_manager': ['redis_manager', 'PUSH', '']
     })
     return new_dict
コード例 #19
0
ファイル: ventilator.py プロジェクト: pyople/pyople
 def _sockets_path(self):
     new_dict = SocketMultiprocess._sockets_path(self)
     #tasks_pub: used for sending the status of the tasks.
     #tasks_sub: used for receiving the status of the tasks.
     #from_socket: receive new tasks created during execution.
     #to_socket: send ready tasks to be executed.
     #server: receive new tasks from the server.
     #redis_manager: send dependencies of the server tasks.
     #data_socket: send direct tasks data.
     new_dict.update({
         'tasks_pub': ['to_tasks_publish', 'PUB', ''],
         'tasks_sub': ['from_tasks_publish', 'SUB', '_update_task'],
         'from_socket': ['from_done_task', 'PULL', '_create_task'],
         'to_socket': ['to_new_task', 'PUSH', ''],
         'server': ['to_server', 'DEALER', '_receive_task_server'],
         'redis_manager': ['redis_manager', 'PUSH', ''],
         'data_socket': ['to_data', 'PUSH', '']
     })
     return new_dict
コード例 #20
0
ファイル: ventilator.py プロジェクト: pyople/pyople
 def _sockets_path(self):
     new_dict = SocketMultiprocess._sockets_path(self)
     #tasks_pub: used for sending the status of the tasks.
     #tasks_sub: used for receiving the status of the tasks.
     #from_socket: receive new tasks created during execution.
     #to_socket: send ready tasks to be executed.
     #server: receive new tasks from the server.
     #redis_manager: send dependencies of the server tasks.
     #data_socket: send direct tasks data.
     new_dict.update({'tasks_pub': ['to_tasks_publish', 'PUB', ''],
                      'tasks_sub': ['from_tasks_publish',
                                    'SUB', '_update_task'],
                      'from_socket': ['from_done_task', 'PULL',
                                       '_create_task'],
                      'to_socket': ['to_new_task', 'PUSH', ''],
                      'server': ['to_server', 'DEALER',
                                  '_receive_task_server'],
                      'redis_manager': ['redis_manager', 'PUSH', ''],
                      'data_socket': ['to_data', 'PUSH', '']})
     return new_dict
コード例 #21
0
 def _connect_socket(self, sock, name, conn):
     if name == 'redis_manager':
         conn = 'bind'
     SocketMultiprocess._connect_socket(self, sock, name, conn)
コード例 #22
0
ファイル: redis_manager.py プロジェクト: pyople/pyople
 def _is_clean(self):
     self_clean = not bool(self.tasks) and not bool(self.server_delete)
     return self_clean and SocketMultiprocess._is_clean(self)
コード例 #23
0
ファイル: redis_manager.py プロジェクト: pyople/pyople
 def _connect_socket(self, sock, name, conn):
     if name == 'redis_manager':
         conn = 'bind'
     SocketMultiprocess._connect_socket(self, sock, name, conn)
コード例 #24
0
ファイル: redis_manager.py プロジェクト: pyople/pyople
 def _list_subscribe(self):
     subscribe = SocketMultiprocess._list_subscribe(self)
     subscribe.extend(['Node', 'RedisManager',
                       'GroupManager', self.group_id])
     return subscribe
コード例 #25
0
ファイル: worker.py プロジェクト: pyople/pyople
 def _list_subscribe(self):
     subscribe = SocketMultiprocess._list_subscribe(self)
     subscribe.extend(['Node', 'Worker', self.group_id])
     return subscribe
コード例 #26
0
 def _is_clean(self):
     self_clean = not bool(self.subscribed_tasks)
     return self_clean and SocketMultiprocess._is_clean(self)
コード例 #27
0
 def _list_subscribe(self):
     subscribe = SocketMultiprocess._list_subscribe(self)
     subscribe.extend(
         ['Node', 'RedisManager', 'GroupManager', self.group_id])
     return subscribe
コード例 #28
0
 def _list_subscribe(self):
     subscribe = SocketMultiprocess._list_subscribe(self)
     subscribe.extend(
         ['Node', 'Sink', 'TaskManager' + self.group_id, self.group_id])
     return subscribe
コード例 #29
0
ファイル: ventilator.py プロジェクト: pyople/pyople
 def _list_subscribe(self):
     subscribe = SocketMultiprocess._list_subscribe(self)
     subscribe.extend(['Node', 'Ventilator',
                       'TaskManager' + self.group_id, self.group_id])
     return subscribe
コード例 #30
0
ファイル: worker.py プロジェクト: pyople/pyople
 def _list_subscribe(self):
     subscribe = SocketMultiprocess._list_subscribe(self)
     subscribe.extend(['Node', 'Worker', self.group_id])
     return subscribe
コード例 #31
0
ファイル: ventilator.py プロジェクト: pyople/pyople
 def _is_clean(self):
     self_clean = not bool(self.tasks)
     return self_clean and SocketMultiprocess._is_clean(self)
コード例 #32
0
 def _is_clean(self):
     self_clean = not bool(self.tasks) and not bool(self.server_delete)
     return self_clean and SocketMultiprocess._is_clean(self)