def setUpClass(cls): # noqa
"""Creates a kafka and mongo cluster instance
"""
cls.zk = ZookeeperFixture.instance()
cls.server = KafkaFixture.instance(0, cls.zk.host, cls.zk.port)
cls.client = KafkaClient(cls.server.host, cls.server.port)
cls.max_insertions = 100
cls.flag = start_cluster()
#Clear our config file
config = os.environ.get('CONFIG', "config.txt")
open(config, 'w').close
if cls.flag:
try:
cls.mongo_conn = Connection("%s:27217" %
(socket.gethostname()))
cls.mongo_db = cls.mongo_conn['test']['test']
except ConnectionFailure:
print("Cannot connect locally!")
cls.flag = False
if cls.flag:
cls.mongo_db = cls.mongo_conn['test']['test']
cls.conn = Connector(
("%s:27217" % (socket.gethostname())), config,
"%s:%s" % (cls.server.host, cls.server.port), ['test.test'],
'_id', None,
"/home/tyler/doc_managers/kafka_doc_manager/kafka_doc_manager.py",
None)
cls.conn.start()
def test_write_oplog_progress(self):
"""Test write_oplog_progress under several circumstances
"""
os.system('touch %s' % (TEMP_CONFIG))
config_file_path = TEMP_CONFIG
conn = Connector(MAIN_ADDRESS, config_file_path, None, ['test.test'],
'_id', None, None)
#test that None is returned if there is no config file specified.
self.assertEqual(conn.write_oplog_progress(), None)
conn.oplog_progress.get_dict()[1] = Timestamp(12, 34)
#pretend to insert a thread/timestamp pair
conn.write_oplog_progress()
data = json.load(open(config_file_path, 'r'))
self.assertEqual(1, int(data[0]))
self.assertEqual(long_to_bson_ts(int(data[1])), Timestamp(12, 34))
#ensure the temp file was deleted
self.assertFalse(os.path.exists(config_file_path + '~'))
#ensure that updates work properly
conn.oplog_progress.get_dict()[1] = Timestamp(44, 22)
conn.write_oplog_progress()
config_file = open(config_file_path, 'r')
data = json.load(config_file)
self.assertEqual(1, int(data[0]))
self.assertEqual(long_to_bson_ts(int(data[1])), Timestamp(44, 22))
os.system('rm ' + config_file_path)
config_file.close()
def setUp(self):
if not self.flag:
self.fail("Shards cannot be added to mongos")
self.connector = Connector(
"%s:%s" % (HOSTNAME, PORTS_ONE["MONGOS"]), CONFIG,
'%s:30000' % (HOSTNAME), ['test.test'], '_id', None,
'mongo_connector/doc_managers/mongo_doc_manager.py')
self.connector.start()
while len(self.connector.shard_set) == 0:
pass
self.conn['test']['test'].remove(safe=True)
while (len(self.mongo_doc._search()) != 0):
time.sleep(1)
def setUp(self):
""" Starts a new connector for every test
"""
if not self.flag:
self.fail("Shards cannot be added to mongos")
self.connector = Connector(
'%s:%s' % (HOSTNAME, PORTS_ONE['MONGOS']), CONFIG,
'http://localhost:9200', ['test.test'], '_id', None,
'mongo_connector/doc_managers/elastic_doc_manager.py')
self.connector.start()
while len(self.connector.shard_set) == 0:
pass
self.conn['test']['test'].remove(safe=True)
while (len(self.elastic_doc._search()) != 0):
time.sleep(1)
def test_connector(self):
"""Test whether the connector initiates properly
"""
if not self.flag:
self.fail("Shards cannot be added to mongos")
conn = Connector(MAIN_ADDRESS, CONFIG, None, ['test.test'], '_id',
None, None)
conn.start()
while len(conn.shard_set) != 1:
time.sleep(2)
conn.join()
self.assertFalse(conn.can_run)
time.sleep(5)
for thread in conn.shard_set.values():
self.assertFalse(thread.running)
def setUpClass(cls):
""" Initializes the cluster
"""
os.system('rm %s; touch %s' % (CONFIG, CONFIG))
use_mongos = True
if PORTS_ONE['MONGOS'] != "27217":
use_mongos = False
cls.flag = start_cluster(use_mongos=use_mongos)
if cls.flag:
cls.conn = Connection('%s:%s' % (HOSTNAME, PORTS_ONE['MONGOS']),
replicaSet="demo-repl")
timer = Timer(60, abort_test)
cls.connector = Connector(
"%s:%s" % (HOSTNAME, PORTS_ONE["MONGOS"]), CONFIG, None,
['test.test'], '_id', None, None)
cls.synchronizer = cls.connector.doc_manager
timer.start()
cls.connector.start()
while len(cls.connector.shard_set) == 0:
pass
timer.cancel()
def test_read_oplog_progress(self):
"""Test read_oplog_progress
"""
conn = Connector(MAIN_ADDRESS, None, None, ['test.test'], '_id', None,
None)
#testing with no file
self.assertEqual(conn.read_oplog_progress(), None)
os.system('touch %s' % (TEMP_CONFIG))
conn.oplog_checkpoint = TEMP_CONFIG
#testing with empty file
self.assertEqual(conn.read_oplog_progress(), None)
oplog_dict = conn.oplog_progress.get_dict()
#add a value to the file, delete the dict, and then read in the value
oplog_dict['oplog1'] = Timestamp(12, 34)
conn.write_oplog_progress()
del oplog_dict['oplog1']
self.assertEqual(len(oplog_dict), 0)
conn.read_oplog_progress()
self.assertTrue('oplog1' in oplog_dict.keys())
self.assertTrue(oplog_dict['oplog1'], Timestamp(12, 34))
oplog_dict['oplog1'] = Timestamp(55, 11)
#see if oplog progress dict is properly updated
conn.read_oplog_progress()
self.assertTrue(oplog_dict['oplog1'], Timestamp(55, 11))
os.system('rm ' + TEMP_CONFIG)
def setUp(self):
if not self.flag:
self.fail(self.err_msg)
self.connector = Connector(
('%s:%s' % (HOSTNAME, PORTS_ONE['MAIN'])), CONFIG,
'http://localhost:8983/solr', ['test.test'], '_id', None,
'mongo_connector/doc_managers/solr_doc_manager.py')
self.connector.start()
while len(self.connector.shard_set) == 0:
time.sleep(1)
count = 0
while (True):
try:
self.conn['test']['test'].remove(safe=True)
break
except (AutoReconnect, OperationFailure):
time.sleep(1)
count += 1
if count > 60:
unittest.SkipTest('Call to remove failed too '
'many times in setup')
while (len(self.solr_conn.search('*:*')) != 0):
time.sleep(1)
