def test_oplog5():
logging.basicConfig(level=logging.INFO,
stream=sys.stdout,
format='%(asctime)s %(levelname)-8s %(message)s')
save_loglevel()
# dataset test should not fail, oplog recover in action
oplog4 = {
'single-oplog': [
('test_data/oplog4/oplog1.js', None), # attempt 0
('test_data/oplog4/oplog2.js', None), # attempt 1
('test_data/oplog4/oplog3.js', None), # attempt 2
('test_data/oplog4/oplog4.js', None), # attempt 3
('test_data/oplog4/oplog5.js', None), # attempt 4
('test_data/oplog4/oplog6.js', None), # attempt 5
('test_data/oplog4/oplog7.js', None), # attempt 6
('test_data/oplog4/oplog8.js', None) # attempt 7
]
}
assert (check_dataset(
True,
'oplog4',
oplog4,
{'posts': None}, # don't raise error while reading posts
10 # - max attempts count to re-read oplog
) == True)
def test_oplog8():
logging.basicConfig(level=logging.INFO,
stream=sys.stdout,
format='%(asctime)s %(levelname)-8s %(message)s')
save_loglevel()
# dataset test
oplog31 = {'single-oplog': [('test_data/oplog3/oplog.js', None)]}
assert(check_dataset(False, 'oplog3', oplog31,
{'posts': pymongo.errors.AutoReconnect, 'posts2': None,
'rated_posts': None, 'guests': None}) == True)
# dataset test should fail
try:
oplog32 = {'single-oplog': [('test_data/oplog3/oplog.js',
pymongo.errors.InvalidURI)]}
check_dataset(False, 'oplog3', oplog32,
{'posts': None, 'posts2': None,
'rated_posts': None, 'guests': None})
except:
pass
else:
assert(0)
# dataset test should fail
try:
check_dataset(True, 'oplog1', oplog31,
{'posts': pymongo.errors.InvalidURI,
'guests': None})
except:
pass
else:
assert(0)
def test_oplog6():
logging.basicConfig(level=logging.INFO,
stream=sys.stdout,
format='%(asctime)s %(levelname)-8s %(message)s')
save_loglevel()
# dataset test
oplog21 = {'single-oplog': [('test_data/oplog2/oplog.js', None),
('test_data/oplog2/\
oplog_simulate_added_after_initload.js',
pymongo.errors.OperationFailure)]}
assert(check_dataset(False, 'oplog2', oplog21,
{'posts': None, 'guests': None}) == True)
# dataset test
oplog22 = {'single-oplog': [('test_data/oplog2/oplog.js', None),
('test_data/oplog2/\
oplog_simulate_added_after_initload.js',
pymongo.errors.NetworkTimeout)]}
assert(check_dataset(False, 'oplog2', oplog22,
{'posts': None, 'guests': None}) == True)
# dataset test
oplog32 = {'single-oplog': [('test_data/oplog3/oplog.js', None)]}
assert(check_dataset(False, 'oplog3', oplog32,
{'posts': pymongo.errors.OperationFailure, 'posts2': None,
'rated_posts': None, 'guests': None}) == True)
def test_oplog7():
logging.basicConfig(level=logging.INFO,
stream=sys.stdout,
format='%(asctime)s %(levelname)-8s %(message)s')
save_loglevel()
# dataset test
oplog7 = {'single-oplog': [('test_data/oplog7/oplog.js', None)]}
assert(check_dataset(False, 'oplog7', oplog7, {'posts': None}) == True)
def create_logger(logspath, name):
today = datetime.datetime.now()
logfilename='{date}-{name}.log'.format(name=name,
date=today.strftime('%Y-%m-%d'))
logfilename = os.path.join(logspath, logfilename)
logging.basicConfig(filename=logfilename,
level=logging.INFO,
format='%(asctime)s %(levelname)-8s %(message)s')
save_loglevel()
logger = getLogger(__name__)
logger.info('Created')
def create_logger(logspath, name):
today = datetime.datetime.now()
logfilename = '{date}-{name}.log'.format(name=name,
date=today.strftime('%Y-%m-%d'))
logfilename = os.path.join(logspath, logfilename)
logging.basicConfig(filename=logfilename,
level=logging.INFO,
format='%(asctime)s %(levelname)-8s %(message)s')
save_loglevel()
logger = getLogger(__name__)
logger.info('Created')
def test_oplog3():
logging.basicConfig(level=logging.INFO,
stream=sys.stdout,
format='%(asctime)s %(levelname)-8s %(message)s')
save_loglevel()
# dataset test
oplog3 = {'single-oplog': [('test_data/oplog3/oplog.js', None)]}
assert (check_dataset(False, 'oplog3', oplog3, {
'posts': None,
'posts2': None,
'rated_posts': None,
'guests': None
}) == True)
def test_oplog2():
logging.basicConfig(level=logging.INFO,
stream=sys.stdout,
format='%(asctime)s %(levelname)-8s %(message)s')
save_loglevel()
# dataset test
oplog2 = {'single-oplog': [('test_data/oplog2/oplog.js', None),
('test_data/oplog2/\
oplog_simulate_added_after_initload.js',
None)],
}
assert(check_dataset(False, 'oplog2', oplog2,
{'posts': None, 'guests': None}) == True)
def test_oplog1():
logging.basicConfig(level=logging.DEBUG,
stream=sys.stdout,
format='%(asctime)s %(levelname)-8s %(message)s')
save_loglevel()
# dataset test
oplog1 = {'shard1': [('test_data/oplog1/oplog1.js', None), # ntry 0
('test_data/oplog1/oplog2.js', None) # ntry 1
],
'shard2': [('test_data/oplog1/shard2-oplog1.js', None)
]
}
assert(check_dataset(False, 'oplog1', oplog1,
{'posts': None, 'guests': None}) == True)
def test_oplog9():
logging.basicConfig(level=logging.INFO,
stream=sys.stdout,
format='%(asctime)s %(levelname)-8s %(message)s')
save_loglevel()
# dataset test
oplog21 = {
'single-oplog': [('test_data/oplog2/oplog.js', None),
('test_data/oplog2/\
oplog_simulate_added_after_initload.js', pymongo.errors.OperationFailure)]
}
assert (check_dataset(False, 'oplog2', oplog21, {
'posts': None,
'guests': None
}) == True)
def test_oplog1():
logging.basicConfig(level=logging.DEBUG,
stream=sys.stdout,
format='%(asctime)s %(levelname)-8s %(message)s')
save_loglevel()
# dataset test
oplog1 = {
'shard1': [
('test_data/oplog1/oplog1.js', None), # attempt 0
('test_data/oplog1/oplog2.js', None) # attempt 1
],
'shard2': [('test_data/oplog1/shard2-oplog1.js', None)]
}
assert (check_dataset(False, 'oplog1', oplog1, {
'posts': None,
'guests': None
}) == True)
def test_oplog4():
logging.basicConfig(level=logging.DEBUG,
stream=sys.stdout,
format='%(asctime)s %(levelname)-8s %(message)s')
save_loglevel()
oplog4 = {'single-oplog': [('test_data/oplog4/oplog1.js', None), # ntry 0
('test_data/oplog4/oplog2.js', None), # ntry 1
('test_data/oplog4/oplog3.js', None), # ntry 2
('test_data/oplog4/oplog4.js', None), # ntry 3
('test_data/oplog4/oplog5.js', None), # ntry 4
('test_data/oplog4/oplog6.js', None), # ntry 5
('test_data/oplog4/oplog7.js', None), # ntry 6
('test_data/oplog4/oplog8.js', None), # ntry 7
('test_data/oplog4/oplog9.js', None) # ntry 8
]}
assert(check_dataset(False, 'oplog4', oplog4,
{'posts': None} # don't raise error while reading posts
) == True)
def test_oplog5():
logging.basicConfig(level=logging.INFO,
stream=sys.stdout,
format='%(asctime)s %(levelname)-8s %(message)s')
save_loglevel()
# dataset test should not fail, oplog recover in action
oplog4 = {'single-oplog': [('test_data/oplog4/oplog1.js', None), # ntry 0
('test_data/oplog4/oplog2.js', None), # ntry 1
('test_data/oplog4/oplog3.js', None), # ntry 2
('test_data/oplog4/oplog4.js', None), # ntry 3
('test_data/oplog4/oplog5.js', None), # ntry 4
('test_data/oplog4/oplog6.js', None), # ntry 5
('test_data/oplog4/oplog7.js', None), # ntry 6
('test_data/oplog4/oplog8.js', None) # ntry 7
]}
assert(check_dataset(True, 'oplog4', oplog4,
{'posts': None}, # don't raise error while reading posts
10 # - max ntrys count to re-read oplog
) == True)
def test_oplog8():
logging.basicConfig(level=logging.INFO,
stream=sys.stdout,
format='%(asctime)s %(levelname)-8s %(message)s')
save_loglevel()
# dataset test
oplog31 = {'single-oplog': [('test_data/oplog3/oplog.js', None)]}
assert (check_dataset(
False, 'oplog3', oplog31, {
'posts': pymongo.errors.AutoReconnect,
'posts2': None,
'rated_posts': None,
'guests': None
}) == True)
# dataset test should fail
try:
oplog32 = {
'single-oplog':
[('test_data/oplog3/oplog.js', pymongo.errors.InvalidURI)]
}
check_dataset(True, 'oplog3', oplog32, {
'posts': None,
'posts2': None,
'rated_posts': None,
'guests': None
})
except:
pass
else:
assert (0)
# dataset test should fail
try:
check_dataset(True, 'oplog1', oplog31, {
'posts': pymongo.errors.InvalidURI,
'guests': None
})
except:
pass
else:
assert (0)
def test_oplog4():
logging.basicConfig(level=logging.DEBUG,
stream=sys.stdout,
format='%(asctime)s %(levelname)-8s %(message)s')
save_loglevel()
oplog4 = {
'single-oplog': [
('test_data/oplog4/oplog1.js', None), # attempt 0
('test_data/oplog4/oplog2.js', None), # attempt 1
('test_data/oplog4/oplog3.js', None), # attempt 2
('test_data/oplog4/oplog4.js', None), # attempt 3
('test_data/oplog4/oplog5.js', None), # attempt 4
('test_data/oplog4/oplog6.js', None), # attempt 5
('test_data/oplog4/oplog7.js', None), # attempt 6
('test_data/oplog4/oplog8.js', None), # attempt 7
('test_data/oplog4/oplog9.js', None) # attempt 8
]
}
assert (check_dataset(
False,
'oplog4',
oplog4,
{'posts': None} # don't raise error while reading posts
) == True)
oplog_simulate_added_after_initload.js',
pymongo.errors.OperationFailure)]
}
assert(check_dataset(False, 'oplog2', oplog21,
{'posts': None, 'guests': None}) == True)
if __name__ == '__main__':
""" Test external data by providing path to schemas folder,
data folder as args """
logging.basicConfig(level=logging.DEBUG,
stream=sys.stdout,
format='%(asctime)s %(levelname)-8s %(message)s')
save_loglevel()
schemas_path = sys.argv[1]
data_path = sys.argv[2]
mongo_oplog = os.path.join(data_path, 'mongo_oplog.json')
empty_data_before = {}
data_after = {}
schema_engines = get_schema_engines_as_dict(schemas_path)
for schema_name in schema_engines:
path_with_data = os.path.join(data_path, 'mongo_%s.json' % schema_name)
data_after[schema_name] = (path_with_data, None)
oplog_test1 \
= OplogTest(empty_data_before,
{'oplog': [(mongo_oplog, None)]},
data_after,