Beispiel #1
0
 def __init__(self, master_count=1, partitions=1, replicas=0, upstream=None,
                    adapter=os.getenv('NEO_TESTS_ADAPTER', 'SQLite'),
                    storage_count=None, db_list=None, clear_databases=True,
                    db_user=DB_USER, db_password='', compress=True,
                    importer=None, autostart=None):
     self.name = 'neo_%s' % self._allocate('name',
         lambda: random.randint(0, 100))
     self.compress = compress
     master_list = [MasterApplication.newAddress()
                    for _ in xrange(master_count)]
     self.master_nodes = ' '.join('%s:%s' % x for x in master_list)
     weak_self = weakref.proxy(self)
     kw = dict(cluster=weak_self, getReplicas=replicas, getAdapter=adapter,
               getPartitions=partitions, getReset=clear_databases,
               getSSL=self.SSL)
     if upstream is not None:
         self.upstream = weakref.proxy(upstream)
         kw.update(getUpstreamCluster=upstream.name,
             getUpstreamMasters=parseMasterList(upstream.master_nodes))
     self.master_list = [MasterApplication(getAutostart=autostart,
                                           address=x, **kw)
                         for x in master_list]
     if db_list is None:
         if storage_count is None:
             storage_count = replicas + 1
         index = count().next
         db_list = ['%s%u' % (DB_PREFIX, self._allocate('db', index))
                    for _ in xrange(storage_count)]
     if adapter == 'MySQL':
         setupMySQLdb(db_list, db_user, db_password, clear_databases)
         db = '%s:%s@%%s%s' % (db_user, db_password, DB_SOCKET)
     elif adapter == 'SQLite':
         db = os.path.join(getTempDirectory(), '%s.sqlite')
     else:
         assert False, adapter
     if importer:
         cfg = SafeConfigParser()
         cfg.add_section("neo")
         cfg.set("neo", "adapter", adapter)
         cfg.set("neo", "database", db % tuple(db_list))
         for name, zodb in importer:
             cfg.add_section(name)
             for x in zodb.iteritems():
                 cfg.set(name, *x)
         db = os.path.join(getTempDirectory(), '%s.conf')
         with open(db % tuple(db_list), "w") as f:
             cfg.write(f)
         kw["getAdapter"] = "Importer"
     self.storage_list = [StorageApplication(getDatabase=db % x, **kw)
                          for x in db_list]
     self.admin_list = [AdminApplication(**kw)]
     self.neoctl = NeoCTL(self.admin.getVirtualAddress(), ssl=self.SSL)
Beispiel #2
0
 def __init__(self, master_count=1, partitions=1, replicas=0, upstream=None,
                    adapter=os.getenv('NEO_TESTS_ADAPTER', 'SQLite'),
                    storage_count=None, db_list=None, clear_databases=True,
                    db_user=DB_USER, db_password='', compress=True,
                    importer=None, autostart=None):
     self.name = 'neo_%s' % self._allocate('name',
         lambda: random.randint(0, 100))
     self.compress = compress
     self.num_partitions = partitions
     master_list = [MasterApplication.newAddress()
                    for _ in xrange(master_count)]
     self.master_nodes = ' '.join('%s:%s' % x for x in master_list)
     kw = Node.convertInitArgs(replicas=replicas, adapter=adapter,
         partitions=partitions, reset=clear_databases)
     kw['cluster'] = weak_self = weakref.proxy(self)
     kw['getSSL'] = self.SSL
     if upstream is not None:
         self.upstream = weakref.proxy(upstream)
         kw.update(getUpstreamCluster=upstream.name,
             getUpstreamMasters=parseMasterList(upstream.master_nodes))
     self.master_list = [MasterApplication(getAutostart=autostart,
                                           address=x, **kw)
                         for x in master_list]
     if db_list is None:
         if storage_count is None:
             storage_count = replicas + 1
         index = count().next
         db_list = ['%s%u' % (DB_PREFIX, self._allocate('db', index))
                    for _ in xrange(storage_count)]
     if adapter == 'MySQL':
         setupMySQLdb(db_list, db_user, db_password, clear_databases)
         db = '%s:%s@%%s%s' % (db_user, db_password, DB_SOCKET)
     elif adapter == 'SQLite':
         db = os.path.join(getTempDirectory(), '%s.sqlite')
     else:
         assert False, adapter
     if importer:
         cfg = SafeConfigParser()
         cfg.add_section("neo")
         cfg.set("neo", "adapter", adapter)
         cfg.set("neo", "database", db % tuple(db_list))
         for name, zodb in importer:
             cfg.add_section(name)
             for x in zodb.iteritems():
                 cfg.set(name, *x)
         db = os.path.join(getTempDirectory(), '%s.conf')
         with open(db % tuple(db_list), "w") as f:
             cfg.write(f)
         kw["getAdapter"] = "Importer"
     self.storage_list = [StorageApplication(getDatabase=db % x, **kw)
                          for x in db_list]
     self.admin_list = [AdminApplication(**kw)]
Beispiel #3
0
    def __init__(self,
                 master_nodes,
                 name,
                 compress=True,
                 cache_size=None,
                 **kw):
        super(Application, self).__init__(parseMasterList(master_nodes), name,
                                          **kw)
        # Internal Attributes common to all thread
        self._db = None
        self.primary_master_node = None
        self.trying_master_node = None

        # no self-assigned NID, primary master will supply us one
        self._cache = ClientCache() if cache_size is None else \
                      ClientCache(max_size=cache_size)
        self._loading_oid = None
        self.new_oids = ()
        self.last_oid = '\0' * 8
        self.storage_event_handler = storage.StorageEventHandler(self)
        self.storage_bootstrap_handler = storage.StorageBootstrapHandler(self)
        self.storage_handler = storage.StorageAnswersHandler(self)
        self.primary_handler = master.PrimaryAnswersHandler(self)
        self.primary_bootstrap_handler = master.PrimaryBootstrapHandler(self)
        self.notifications_handler = master.PrimaryNotificationsHandler(self)
        self._txn_container = TransactionContainer()
        # Lock definition :
        # _load_lock is used to make loading and storing atomic
        lock = Lock()
        self._load_lock_acquire = lock.acquire
        self._load_lock_release = lock.release
        # _oid_lock is used in order to not call multiple oid
        # generation at the same time
        lock = Lock()
        self._oid_lock_acquire = lock.acquire
        self._oid_lock_release = lock.release
        lock = Lock()
        # _cache_lock is used for the client cache
        self._cache_lock_acquire = lock.acquire
        self._cache_lock_release = lock.release
        # _connecting_to_master_node is used to prevent simultaneous master
        # node connection attempts
        self._connecting_to_master_node = Lock()
        # same for storage nodes
        self._connecting_to_storage_node = Lock()
        self._node_failure_dict = {}
        self.compress = getCompress(compress)
Beispiel #4
0
 def __init__(self, master_count=1, partitions=1, replicas=0, upstream=None,
                    adapter=os.getenv('NEO_TESTS_ADAPTER', 'SQLite'),
                    storage_count=None, db_list=None, clear_databases=True,
                    compress=True,
                    importer=None, autostart=None, dedup=False, name=None):
     self.name = name or 'neo_%s' % self._allocate('name',
         lambda: random.randint(0, 100))
     self.compress = compress
     self.num_partitions = partitions
     master_list = [MasterApplication.newAddress()
                    for _ in xrange(master_count)]
     self.master_nodes = ' '.join('%s:%s' % x for x in master_list)
     kw = dict(replicas=replicas, adapter=adapter,
         partitions=partitions, reset=clear_databases, dedup=dedup)
     kw['cluster'] = weak_self = weakref.proxy(self)
     kw['ssl'] = self.SSL
     if upstream is not None:
         self.upstream = weakref.proxy(upstream)
         kw.update(upstream_cluster=upstream.name,
             upstream_masters=parseMasterList(upstream.master_nodes))
     self.master_list = [MasterApplication(autostart=autostart,
                                           address=x, **kw)
                         for x in master_list]
     if db_list is None:
         if storage_count is None:
             storage_count = replicas + 1
         index = count().next
         db_list = ['%s%u' % (DB_PREFIX, self._allocate('db', index))
                    for _ in xrange(storage_count)]
     if adapter == 'MySQL':
         db = setupMySQLdb(db_list, clear_databases)
     elif adapter == 'SQLite':
         db = os.path.join(getTempDirectory(), '%s.sqlite').__mod__
     else:
         assert False, adapter
     if importer:
         cfg = ImporterConfigParser(adapter, **importer)
         cfg.set("neo", "database", db(*db_list))
         db = os.path.join(getTempDirectory(), '%s.conf').__mod__
         with open(db(*db_list), "w") as f:
             cfg.write(f)
         kw["adapter"] = "Importer"
     kw['wait'] = 0
     self.storage_list = [StorageApplication(database=db(x), **kw)
                          for x in db_list]
     self.admin_list = [AdminApplication(**kw)]
Beispiel #5
0
 def __init__(self, cluster=None, address=None, **kw):
     if not address:
         address = self.newAddress()
     if cluster is None:
         master_nodes = kw.get("master_nodes", ())
         name = kw.get("name", "test")
     else:
         master_nodes = kw.get("master_nodes", cluster.master_nodes)
         name = kw.get("name", cluster.name)
     port = address[1]
     self._node_list[port] = weakref.proxy(self)
     self._init_args = init_args = kw.copy()
     init_args["cluster"] = cluster
     init_args["address"] = address
     threading.Thread.__init__(self)
     self.daemon = True
     self.node_name = "%s_%u" % (self.node_type, port)
     kw.update(getCluster=name, getBind=address, getMasters=master_nodes and parseMasterList(master_nodes, address))
     super(ServerNode, self).__init__(Mock(kw))
Beispiel #6
0
 def __init__(self, cluster=None, address=None, **kw):
     if not address:
         address = self.newAddress()
     if cluster is None:
         master_nodes = ()
         name = kw.get('name', 'test')
     else:
         master_nodes = cluster.master_nodes
         name = kw.get('name', cluster.name)
     port = address[1]
     if address is not BIND:
         self._node_list[port] = weakref.proxy(self)
     self._init_args = init_args = kw.copy()
     init_args['cluster'] = cluster
     init_args['address'] = address
     threading.Thread.__init__(self)
     self.daemon = True
     self.node_name = '%s_%u' % (self.node_type, port)
     kw.update(getCluster=name, getBind=address,
         getMasters=master_nodes and parseMasterList(master_nodes))
     super(ServerNode, self).__init__(Mock(kw))
Beispiel #7
0
    def __init__(self, master_nodes, name, compress=True, **kw):
        super(Application, self).__init__(parseMasterList(master_nodes),
                                          name, **kw)
        # Internal Attributes common to all thread
        self._db = None
        self.cp = ConnectionPool(self)
        self.primary_master_node = None
        self.trying_master_node = None

        # no self-assigned UUID, primary master will supply us one
        self._cache = ClientCache()
        self._loading_oid = None
        self.new_oid_list = ()
        self.last_oid = '\0' * 8
        self.last_tid = None
        self.storage_event_handler = storage.StorageEventHandler(self)
        self.storage_bootstrap_handler = storage.StorageBootstrapHandler(self)
        self.storage_handler = storage.StorageAnswersHandler(self)
        self.primary_handler = master.PrimaryAnswersHandler(self)
        self.primary_bootstrap_handler = master.PrimaryBootstrapHandler(self)
        self.notifications_handler = master.PrimaryNotificationsHandler( self)
        self._txn_container = TransactionContainer()
        # Lock definition :
        # _load_lock is used to make loading and storing atomic
        lock = Lock()
        self._load_lock_acquire = lock.acquire
        self._load_lock_release = lock.release
        # _oid_lock is used in order to not call multiple oid
        # generation at the same time
        lock = Lock()
        self._oid_lock_acquire = lock.acquire
        self._oid_lock_release = lock.release
        lock = Lock()
        # _cache_lock is used for the client cache
        self._cache_lock_acquire = lock.acquire
        self._cache_lock_release = lock.release
        # _connecting_to_master_node is used to prevent simultaneous master
        # node connection attemps
        self._connecting_to_master_node = Lock()
        self.compress = compress