def __init__(self, master_count=1, partitions=1, replicas=0, upstream=None, adapter=os.getenv('NEO_TESTS_ADAPTER', 'SQLite'), storage_count=None, db_list=None, clear_databases=True, db_user=DB_USER, db_password='', compress=True, importer=None, autostart=None): self.name = 'neo_%s' % self._allocate('name', lambda: random.randint(0, 100)) self.compress = compress master_list = [MasterApplication.newAddress() for _ in xrange(master_count)] self.master_nodes = ' '.join('%s:%s' % x for x in master_list) weak_self = weakref.proxy(self) kw = dict(cluster=weak_self, getReplicas=replicas, getAdapter=adapter, getPartitions=partitions, getReset=clear_databases, getSSL=self.SSL) if upstream is not None: self.upstream = weakref.proxy(upstream) kw.update(getUpstreamCluster=upstream.name, getUpstreamMasters=parseMasterList(upstream.master_nodes)) self.master_list = [MasterApplication(getAutostart=autostart, address=x, **kw) for x in master_list] if db_list is None: if storage_count is None: storage_count = replicas + 1 index = count().next db_list = ['%s%u' % (DB_PREFIX, self._allocate('db', index)) for _ in xrange(storage_count)] if adapter == 'MySQL': setupMySQLdb(db_list, db_user, db_password, clear_databases) db = '%s:%s@%%s%s' % (db_user, db_password, DB_SOCKET) elif adapter == 'SQLite': db = os.path.join(getTempDirectory(), '%s.sqlite') else: assert False, adapter if importer: cfg = SafeConfigParser() cfg.add_section("neo") cfg.set("neo", "adapter", adapter) cfg.set("neo", "database", db % tuple(db_list)) for name, zodb in importer: cfg.add_section(name) for x in zodb.iteritems(): cfg.set(name, *x) db = os.path.join(getTempDirectory(), '%s.conf') with open(db % tuple(db_list), "w") as f: cfg.write(f) kw["getAdapter"] = "Importer" self.storage_list = [StorageApplication(getDatabase=db % x, **kw) for x in db_list] self.admin_list = [AdminApplication(**kw)] self.neoctl = NeoCTL(self.admin.getVirtualAddress(), ssl=self.SSL)
def __init__(self, master_count=1, partitions=1, replicas=0, upstream=None, adapter=os.getenv('NEO_TESTS_ADAPTER', 'SQLite'), storage_count=None, db_list=None, clear_databases=True, db_user=DB_USER, db_password='', compress=True, importer=None, autostart=None): self.name = 'neo_%s' % self._allocate('name', lambda: random.randint(0, 100)) self.compress = compress self.num_partitions = partitions master_list = [MasterApplication.newAddress() for _ in xrange(master_count)] self.master_nodes = ' '.join('%s:%s' % x for x in master_list) kw = Node.convertInitArgs(replicas=replicas, adapter=adapter, partitions=partitions, reset=clear_databases) kw['cluster'] = weak_self = weakref.proxy(self) kw['getSSL'] = self.SSL if upstream is not None: self.upstream = weakref.proxy(upstream) kw.update(getUpstreamCluster=upstream.name, getUpstreamMasters=parseMasterList(upstream.master_nodes)) self.master_list = [MasterApplication(getAutostart=autostart, address=x, **kw) for x in master_list] if db_list is None: if storage_count is None: storage_count = replicas + 1 index = count().next db_list = ['%s%u' % (DB_PREFIX, self._allocate('db', index)) for _ in xrange(storage_count)] if adapter == 'MySQL': setupMySQLdb(db_list, db_user, db_password, clear_databases) db = '%s:%s@%%s%s' % (db_user, db_password, DB_SOCKET) elif adapter == 'SQLite': db = os.path.join(getTempDirectory(), '%s.sqlite') else: assert False, adapter if importer: cfg = SafeConfigParser() cfg.add_section("neo") cfg.set("neo", "adapter", adapter) cfg.set("neo", "database", db % tuple(db_list)) for name, zodb in importer: cfg.add_section(name) for x in zodb.iteritems(): cfg.set(name, *x) db = os.path.join(getTempDirectory(), '%s.conf') with open(db % tuple(db_list), "w") as f: cfg.write(f) kw["getAdapter"] = "Importer" self.storage_list = [StorageApplication(getDatabase=db % x, **kw) for x in db_list] self.admin_list = [AdminApplication(**kw)]
def __init__(self, master_nodes, name, compress=True, cache_size=None, **kw): super(Application, self).__init__(parseMasterList(master_nodes), name, **kw) # Internal Attributes common to all thread self._db = None self.primary_master_node = None self.trying_master_node = None # no self-assigned NID, primary master will supply us one self._cache = ClientCache() if cache_size is None else \ ClientCache(max_size=cache_size) self._loading_oid = None self.new_oids = () self.last_oid = '\0' * 8 self.storage_event_handler = storage.StorageEventHandler(self) self.storage_bootstrap_handler = storage.StorageBootstrapHandler(self) self.storage_handler = storage.StorageAnswersHandler(self) self.primary_handler = master.PrimaryAnswersHandler(self) self.primary_bootstrap_handler = master.PrimaryBootstrapHandler(self) self.notifications_handler = master.PrimaryNotificationsHandler(self) self._txn_container = TransactionContainer() # Lock definition : # _load_lock is used to make loading and storing atomic lock = Lock() self._load_lock_acquire = lock.acquire self._load_lock_release = lock.release # _oid_lock is used in order to not call multiple oid # generation at the same time lock = Lock() self._oid_lock_acquire = lock.acquire self._oid_lock_release = lock.release lock = Lock() # _cache_lock is used for the client cache self._cache_lock_acquire = lock.acquire self._cache_lock_release = lock.release # _connecting_to_master_node is used to prevent simultaneous master # node connection attempts self._connecting_to_master_node = Lock() # same for storage nodes self._connecting_to_storage_node = Lock() self._node_failure_dict = {} self.compress = getCompress(compress)
def __init__(self, master_count=1, partitions=1, replicas=0, upstream=None, adapter=os.getenv('NEO_TESTS_ADAPTER', 'SQLite'), storage_count=None, db_list=None, clear_databases=True, compress=True, importer=None, autostart=None, dedup=False, name=None): self.name = name or 'neo_%s' % self._allocate('name', lambda: random.randint(0, 100)) self.compress = compress self.num_partitions = partitions master_list = [MasterApplication.newAddress() for _ in xrange(master_count)] self.master_nodes = ' '.join('%s:%s' % x for x in master_list) kw = dict(replicas=replicas, adapter=adapter, partitions=partitions, reset=clear_databases, dedup=dedup) kw['cluster'] = weak_self = weakref.proxy(self) kw['ssl'] = self.SSL if upstream is not None: self.upstream = weakref.proxy(upstream) kw.update(upstream_cluster=upstream.name, upstream_masters=parseMasterList(upstream.master_nodes)) self.master_list = [MasterApplication(autostart=autostart, address=x, **kw) for x in master_list] if db_list is None: if storage_count is None: storage_count = replicas + 1 index = count().next db_list = ['%s%u' % (DB_PREFIX, self._allocate('db', index)) for _ in xrange(storage_count)] if adapter == 'MySQL': db = setupMySQLdb(db_list, clear_databases) elif adapter == 'SQLite': db = os.path.join(getTempDirectory(), '%s.sqlite').__mod__ else: assert False, adapter if importer: cfg = ImporterConfigParser(adapter, **importer) cfg.set("neo", "database", db(*db_list)) db = os.path.join(getTempDirectory(), '%s.conf').__mod__ with open(db(*db_list), "w") as f: cfg.write(f) kw["adapter"] = "Importer" kw['wait'] = 0 self.storage_list = [StorageApplication(database=db(x), **kw) for x in db_list] self.admin_list = [AdminApplication(**kw)]
def __init__(self, cluster=None, address=None, **kw): if not address: address = self.newAddress() if cluster is None: master_nodes = kw.get("master_nodes", ()) name = kw.get("name", "test") else: master_nodes = kw.get("master_nodes", cluster.master_nodes) name = kw.get("name", cluster.name) port = address[1] self._node_list[port] = weakref.proxy(self) self._init_args = init_args = kw.copy() init_args["cluster"] = cluster init_args["address"] = address threading.Thread.__init__(self) self.daemon = True self.node_name = "%s_%u" % (self.node_type, port) kw.update(getCluster=name, getBind=address, getMasters=master_nodes and parseMasterList(master_nodes, address)) super(ServerNode, self).__init__(Mock(kw))
def __init__(self, cluster=None, address=None, **kw): if not address: address = self.newAddress() if cluster is None: master_nodes = () name = kw.get('name', 'test') else: master_nodes = cluster.master_nodes name = kw.get('name', cluster.name) port = address[1] if address is not BIND: self._node_list[port] = weakref.proxy(self) self._init_args = init_args = kw.copy() init_args['cluster'] = cluster init_args['address'] = address threading.Thread.__init__(self) self.daemon = True self.node_name = '%s_%u' % (self.node_type, port) kw.update(getCluster=name, getBind=address, getMasters=master_nodes and parseMasterList(master_nodes)) super(ServerNode, self).__init__(Mock(kw))
def __init__(self, master_nodes, name, compress=True, **kw): super(Application, self).__init__(parseMasterList(master_nodes), name, **kw) # Internal Attributes common to all thread self._db = None self.cp = ConnectionPool(self) self.primary_master_node = None self.trying_master_node = None # no self-assigned UUID, primary master will supply us one self._cache = ClientCache() self._loading_oid = None self.new_oid_list = () self.last_oid = '\0' * 8 self.last_tid = None self.storage_event_handler = storage.StorageEventHandler(self) self.storage_bootstrap_handler = storage.StorageBootstrapHandler(self) self.storage_handler = storage.StorageAnswersHandler(self) self.primary_handler = master.PrimaryAnswersHandler(self) self.primary_bootstrap_handler = master.PrimaryBootstrapHandler(self) self.notifications_handler = master.PrimaryNotificationsHandler( self) self._txn_container = TransactionContainer() # Lock definition : # _load_lock is used to make loading and storing atomic lock = Lock() self._load_lock_acquire = lock.acquire self._load_lock_release = lock.release # _oid_lock is used in order to not call multiple oid # generation at the same time lock = Lock() self._oid_lock_acquire = lock.acquire self._oid_lock_release = lock.release lock = Lock() # _cache_lock is used for the client cache self._cache_lock_acquire = lock.acquire self._cache_lock_release = lock.release # _connecting_to_master_node is used to prevent simultaneous master # node connection attemps self._connecting_to_master_node = Lock() self.compress = compress