def test_write_oplog_progress(self):
"""Test write_oplog_progress under several circumstances
"""
os.system('touch %s' % (TEMP_CONFIG))
config_file_path = TEMP_CONFIG
conn = Connector(MAIN_ADDRESS, config_file_path, None, ['test.test'],
'_id', None, None)
#test that None is returned if there is no config file specified.
self.assertEqual(conn.write_oplog_progress(), None)
conn.oplog_progress.get_dict()[1] = Timestamp(12, 34)
#pretend to insert a thread/timestamp pair
conn.write_oplog_progress()
data = json.load(open(config_file_path, 'r'))
self.assertEqual(1, int(data[0]))
self.assertEqual(long_to_bson_ts(int(data[1])), Timestamp(12, 34))
#ensure the temp file was deleted
self.assertFalse(os.path.exists(config_file_path + '~'))
#ensure that updates work properly
conn.oplog_progress.get_dict()[1] = Timestamp(44, 22)
conn.write_oplog_progress()
config_file = open(config_file_path, 'r')
data = json.load(config_file)
self.assertEqual(1, int(data[0]))
self.assertEqual(long_to_bson_ts(int(data[1])), Timestamp(44, 22))
os.system('rm ' + config_file_path)
config_file.close()
def test_write_oplog_progress(self):
"""Test write_oplog_progress under several circumstances
"""
os.system('touch temp_config.txt')
config_file_path = os.getcwd() + '/temp_config.txt'
c = Connector(main_address, config_file_path, None, ['test.test'],
'_id', None, None)
#test that None is returned if there is no config file specified.
self.assertEqual(c.write_oplog_progress(), None)
c.oplog_progress.get_dict()[1] = Timestamp(12, 34)
#pretend to insert a thread/timestamp pair
c.write_oplog_progress()
data = json.load(open(config_file_path, 'r'))
self.assertEqual(1, int(data[0]))
self.assertEqual(long_to_bson_ts(int(data[1])), Timestamp(12, 34))
#ensure the temp file was deleted
self.assertFalse(os.path.exists(config_file_path + '~'))
#ensure that updates work properly
c.oplog_progress.get_dict()[1] = Timestamp(44, 22)
c.write_oplog_progress()
config_file = open(config_file_path, 'r')
data = json.load(config_file)
self.assertEqual(1, int(data[0]))
self.assertEqual(long_to_bson_ts(int(data[1])), Timestamp(44, 22))
os.system('rm ' + config_file_path)
config_file.close()
print("PASSED TEST WRITE OPLOG PROGRESS")
def test_equality(self):
t = Timestamp(1, 1)
self.assertNotEqual(t, Timestamp(0, 1))
self.assertNotEqual(t, Timestamp(1, 0))
self.assertEqual(t, Timestamp(1, 1))
# Explicitly test inequality
self.assertFalse(t != Timestamp(1, 1))
def test_sort_11(monty_sort, mongo_sort):
docs = [{"a": Timestamp(0, 1)}, {"a": Timestamp(1, 1)}]
sort = [("a", -1)]
monty_c = monty_sort(docs, sort)
mongo_c = mongo_sort(docs, sort)
for i in range(len(docs)):
assert next(mongo_c)["_id"] == next(monty_c)["_id"]
def test_update_min_6(monty_update, mongo_update):
docs = [{"a": Timestamp(10, 5)}]
spec = {"$min": {"a": Timestamp(10, 0)}}
monty_c = monty_update(docs, spec)
mongo_c = mongo_update(docs, spec)
assert next(mongo_c) == next(monty_c)
monty_c.rewind()
assert next(monty_c) == {"a": Timestamp(10, 0)}
def object_hook(dct, compile_re=True):
if "$oid" in dct:
return ObjectId(str(dct["$oid"]))
if "$numberLong" in dct:
return int(dct["$numberLong"])
if "$decimal" in dct:
v = str(dct["$decimal"])
if "$precision" in dct:
precision = dct["$precision"][0]
scale = dct["$precision"][1]
d = Decimal(v, precision, scale)
else:
d = Decimal(v)
return d
if "$ref" in dct:
return DBRef(dct["$ref"], dct["$id"], dct.get("$db", None))
if "$date" in dct:
try:
secs = float(dct["$date"]) / 1000.0
return EPOCH_AWARE + datetime.timedelta(seconds=secs)
except ValueError:
return datetime.datetime.strptime(dct["$date"], "%Y-%m-%d")
if "$timestamp" in dct:
try:
ms = long_type(dct["$timestamp"])
return Timestamp(ms / 1000, ms % 1000 * 1000)
except ValueError:
dt = datetime.datetime.strptime(dct["$timestamp"],
"%Y-%m-%d-%H.%M.%S.%f")
secs = long_type(time.mktime(dt.timetuple()))
return Timestamp(secs, dt.microsecond)
if "$regex" in dct:
flags = 0
# PyMongo always adds $options but some other tools may not.
for opt in dct.get("$options", ""):
flags |= _RE_OPT_TABLE.get(opt, 0)
if compile_re:
return re.compile(dct["$regex"], flags)
else:
return Regex(dct["$regex"], flags)
if "$minKey" in dct:
return MinKey()
if "$maxKey" in dct:
return MaxKey()
if "$binary" in dct:
if isinstance(dct["$type"], int):
dct["$type"] = "%d" % dct["$type"]
subtype = int(dct["$type"])
return Binary(base64.b64decode(dct["$binary"].encode()), subtype)
if "$code" in dct:
return Code(dct["$code"], dct.get("$scope"))
if bson.has_uuid() and "$uuid" in dct:
return bson.uuid.UUID(dct["$uuid"])
return dct
def to_mongo(self, value):
if isinstance(value, Timestamp):
return value
elif isinstance(value, datetime.datetime):
return Timestamp(value, 0)
elif isinstance(value, string_types):
try:
return Timestamp(parse_datetime(value), 0)
except (ValueError, TypeError):
pass
raise ValidationError('%r cannot be converted to a Timestamp.' % value)
def test_qop_lt_11(monty_find, mongo_find):
ts_0 = Timestamp(0, 1)
ts_1 = Timestamp(1, 1)
docs = [{"a": ts_0}, {"a": ts_1}]
spec = {"a": {"$lt": ts_1}}
monty_c = monty_find(docs, spec)
mongo_c = mongo_find(docs, spec)
assert mongo_c.count() == 1
assert monty_c.count() == mongo_c.count()
assert next(mongo_c) == next(monty_c)
def test_qop_gt_11(monty_find, mongo_find):
ts_0 = Timestamp(0, 1)
ts_1 = Timestamp(1, 1)
docs = [{"a": ts_0}, {"a": ts_1}]
spec = {"a": {"$gt": ts_0}}
monty_c = monty_find(docs, spec)
mongo_c = mongo_find(docs, spec)
assert FieldWalker(docs[1]).go("a").get().value == [ts_1]
assert mongo_c.count() == 1
assert monty_c.count() == mongo_c.count()
assert next(mongo_c) == next(monty_c)
def format(self, record):
"""Formats LogRecord into python dictionary."""
# Standard document
document = {
'timestamp': Timestamp(int(record.created), int(record.msecs)),
'level': record.levelname,
'thread': record.thread,
'message': record.getMessage(),
'loggerName': record.name,
'fileName': record.pathname,
'method': record.funcName,
'lineNumber': record.lineno
}
# Standard document decorated with exception info
if record.exc_info is not None:
document.update({
'exception': {
'message': str(record.exc_info[1]),
'code': 0,
'stackTrace': self.formatException(record.exc_info)
}
})
# Standard document decorated with extra contextual information
if len(self.DEFAULT_PROPERTIES) != len(record.__dict__):
contextual_extra = set(record.__dict__).difference(set(self.DEFAULT_PROPERTIES))
if contextual_extra:
for key in contextual_extra:
document[key] = record.__dict__[key]
return document
def object_hook(dct, json_options=DEFAULT_JSON_OPTIONS):
if "$oid" in dct:
return ObjectId(str(dct["$oid"]))
if "$ref" in dct:
return DBRef(dct["$ref"], dct["$id"], dct.get("$db", None))
if "$date" in dct:
return _get_date(dct, json_options)
if "$regex" in dct:
flags = 0
# PyMongo always adds $options but some other tools may not.
for opt in dct.get("$options", ""):
flags |= _RE_OPT_TABLE.get(opt, 0)
return Regex(dct["$regex"], flags)
if "$minKey" in dct:
return MinKey()
if "$maxKey" in dct:
return MaxKey()
if "$binary" in dct:
return _get_binary(dct, json_options)
if "$code" in dct:
return Code(dct["$code"], dct.get("$scope"))
if "$uuid" in dct:
return uuid.UUID(dct["$uuid"])
if "$undefined" in dct:
return None
if "$numberLong" in dct:
return Int64(dct["$numberLong"])
if "$timestamp" in dct:
tsp = dct["$timestamp"]
return Timestamp(tsp["t"], tsp["i"])
if "$numberDecimal" in dct:
return Decimal128(dct["$numberDecimal"])
return dct
def test_qop_gte_11(monty_find, mongo_find):
ts_0 = Timestamp(0, 1)
ts_1 = Timestamp(1, 1)
docs = [
{"a": ts_0},
{"a": ts_1}
]
spec = {"a": {"$gte": ts_0}}
monty_c = monty_find(docs, spec)
mongo_c = mongo_find(docs, spec)
assert mongo_c.count() == 2
assert monty_c.count() == mongo_c.count()
for i in range(2):
assert next(mongo_c) == next(monty_c)
def __init__(self, client, db_map, queue, ts=None):
super(OpLogExecutor, self).__init__()
assert isinstance(client, MongoClient), type(client)
self.client = client
if isinstance(db_map, dict):
self.db_map = db_map
elif isinstance(db_map, Iterable):
self.db_map = {name: name for name in db_map}
else:
raise TypeError(
"Type of db_map should be dict or Iterable not %s" %
type(db_map))
self.methods = {
"i": self.insert,
"u": self.update,
"d": self.delete,
"c": self.create
}
self.looping = False
self.queue = queue
from datetime import datetime
self.ts = ts if isinstance(ts, Timestamp) else Timestamp(
int(datetime.now().timestamp()), 1)
self.daemon = True
def _get_timestamp(
data, position, as_class, tz_aware, uuid_subtype, compile_re):
inc, position = _get_int(data, position, unsigned=True)
inc = ctypes.c_int32(inc).value
timestamp, position = _get_int(data, position, unsigned=True)
timestamp = ctypes.c_int32(timestamp).value
return Timestamp(timestamp, inc), position
def long_to_bson_ts(val):
"""Convert integer into BSON timestamp.
"""
seconds = val >> 32
increment = val & 0xffffffff
return Timestamp(seconds, increment)
def test_timestamp_highorder_bits(self):
doc = {'a': Timestamp(0xFFFFFFFF, 0xFFFFFFFF)}
doc_bson = (b'\x10\x00\x00\x00'
b'\x11a\x00\xff\xff\xff\xff\xff\xff\xff\xff'
b'\x00')
self.assertEqual(doc_bson, BSON.encode(doc))
self.assertEqual(doc, BSON(doc_bson).decode())
def parse_currentDate(field, value, array_filters):
date_type = {
"date": datetime.utcnow(),
"timestamp": Timestamp(datetime.utcnow(), 1),
}
if not isinstance(value, bool):
if not is_duckument_type(value):
msg = ("{} is not valid type for $currentDate. Please use a "
"boolean ('true') or a $type expression ({{$type: "
"'timestamp/date'}}).".format(type(value).__name__))
raise WriteError(msg, code=2)
for k, v in value.items():
if k != "$type":
msg = "Unrecognized $currentDate option: {}".format(k)
raise WriteError(msg, code=2)
if v not in date_type:
msg = ("The '$type' string field is required to be 'date' "
"or 'timestamp': {$currentDate: {field : {$type: "
"'date'}}}")
raise WriteError(msg, code=2)
value = date_type[v]
else:
value = date_type["date"]
def _currentDate(fieldwalker):
parse_set(field, value, array_filters)(fieldwalker)
return _currentDate
def test_timestamp(self):
dct = {"ts": Timestamp(4, 13)}
res = json_util.dumps(dct, default=json_util.default)
self.assertEqual('{"ts": {"$timestamp": {"t": 4, "i": 13}}}', res)
rtdct = json_util.loads(res)
self.assertEqual(dct, rtdct)
def test_read_oplog_progress(self):
"""Test read_oplog_progress
"""
conn = Connector(address='%s:%d' % (mongo_host, self.primary_p),
oplog_checkpoint=None,
target_url=None,
ns_set=['test.test'],
u_key='_id',
auth_key=None)
#testing with no file
self.assertEqual(conn.read_oplog_progress(), None)
try:
os.unlink("temp_config.txt")
except OSError:
pass
open("temp_config.txt", "w").close()
conn.oplog_checkpoint = "temp_config.txt"
#testing with empty file
self.assertEqual(conn.read_oplog_progress(), None)
oplog_dict = conn.oplog_progress.get_dict()
#add a value to the file, delete the dict, and then read in the value
oplog_dict['oplog1'] = Timestamp(12, 34)
conn.write_oplog_progress()
del oplog_dict['oplog1']
self.assertEqual(len(oplog_dict), 0)
conn.read_oplog_progress()
oplog_dict = conn.oplog_progress.get_dict()
self.assertTrue('oplog1' in oplog_dict.keys())
self.assertTrue(oplog_dict['oplog1'], Timestamp(12, 34))
oplog_dict['oplog1'] = Timestamp(55, 11)
#see if oplog progress dict is properly updated
conn.read_oplog_progress()
self.assertTrue(oplog_dict['oplog1'], Timestamp(55, 11))
os.unlink("temp_config.txt")
def _resume_using_oplog_lastentry(self):
lastentry = Timestamp(
self.db["sync"].find_one(
{"collection": self._colname},
{"oplog_last_entry": 1})["oplog_last_entry"], 0)
logger.info("reading from last entry of op log: %s " % lastentry)
self._read_from_changestream(start_at_operation_time=lastentry)
def initial_sync(
self
): # Method that starts the initial collection dump and then spawns the writer
print self.dbnames
time_t = time.time()
time_log = Timestamp(int(time_t) - 1, 0)
times = Timestamp(int(time_t), 0)
curr_time = times.as_datetime()
self.target_mongos['sync_log']['init_sync'].insert({'ts': time_log})
self.master_mongos['sync_log']['init_sync'].insert({'ts': time_log})
for name in self.replnames:
print name
self.target_mongos['sync_log'][name].insert({'ts': time_log})
self.master_mongos['sync_log'][name].insert({'ts': time_log})
self.last_sync = time_log # set last time sync time to be current time, push to database
threads = []
for shard in self.master_shards:
for dbname in self.dbnames: # loop through all databases that you want to replicate
if dbname in shard.database_names(
): # if the database is on the shard
identity = shard.address
print("Replicating database: %s , on Shard: %s: %s" %
(dbname, identity[0], identity[1]))
db = shard[dbname]
colls = db.collection_names(
include_system_collections=False)
for coll in colls: # spawn collection dumper threads for all collections within the database
coll_dumper = Thread(target=self.dump_collection,
args=(
db,
dbname,
coll,
))
threads.append(coll_dumper)
coll_dumper.start()
for thread in threads: # wait on all dumper threads before moving on to write oplog operations
thread.join()
print("Finished inital sync, took")
print(time.time() - time_t)
self.start_listening() # start tailing on all shards
def test_timestamp(self):
dct = {"ts": Timestamp(4, 13)}
res = bsonjs_dumps(dct)
self.assertEqual('{ "ts" : { "$timestamp" : { "t" : 4, "i" : 13 } } }',
res)
rtdct = bsonjs_loads(res)
self.assertEqual(dct, rtdct)
def test_timestamp_pickling(self):
ts = Timestamp(0, 1)
pickled_with_3 = (b'\x80\x04\x95Q\x00\x00\x00\x00\x00\x00\x00\x8c'
b'\x0ebson.timestamp\x94\x8c\tTimestamp\x94\x93\x94)'
b'\x81\x94}\x94('
b'\x8c\x10_Timestamp__time\x94K\x00\x8c'
b'\x0f_Timestamp__inc\x94K\x01ub.')
self.round_trip_pickle(ts, pickled_with_3)
def test_read_oplog_progress(self):
"""Test read_oplog_progress
"""
conn = Connector(
mongo_address=self.repl_set.uri,
oplog_checkpoint=None,
**connector_opts
)
# testing with no file
self.assertEqual(conn.read_oplog_progress(), None)
try:
os.unlink("temp_oplog.timestamp")
except OSError:
pass
open("temp_oplog.timestamp", "w").close()
conn.oplog_checkpoint = "temp_oplog.timestamp"
# testing with empty file
self.assertEqual(conn.read_oplog_progress(), None)
oplog_dict = conn.oplog_progress.get_dict()
# add a value to the file, delete the dict, and then read in the value
oplog_dict['oplog1'] = Timestamp(12, 34)
conn.write_oplog_progress()
del oplog_dict['oplog1']
self.assertEqual(len(oplog_dict), 0)
conn.read_oplog_progress()
oplog_dict = conn.oplog_progress.get_dict()
self.assertTrue('oplog1' in oplog_dict.keys())
self.assertTrue(oplog_dict['oplog1'], Timestamp(12, 34))
oplog_dict['oplog1'] = Timestamp(55, 11)
# see if oplog progress dict is properly updated
conn.read_oplog_progress()
self.assertTrue(oplog_dict['oplog1'], Timestamp(55, 11))
os.unlink("temp_oplog.timestamp")
def timestamp(self):
"""
Arguments:
- `self`:
"""
return Timestamp(datetime.datetime.now(), 0)
def test_datetime_copy_pickle(self):
d = datetime.datetime(2010, 5, 5, tzinfo=utc)
t = Timestamp(d, 0)
dc = copy.deepcopy(d)
self.assertEqual(dc, t.as_datetime())
dp = pickle.loads(pickle.dumps(d))
self.assertEqual(dp, t.as_datetime())
def get_ts(time):
if isinstance(time, str):
return get_ts(datetime.strptime(time.replace("-", ""), "%Y%m%d"))
elif isinstance(time, datetime):
return get_ts(int(time.timestamp()))
elif isinstance(time, int):
return Timestamp(time, 1)
else:
return None
def get_consistent_end_ts(self):
ts = None
for host in self.tailed_oplogs:
for port in self.tailed_oplogs[host]:
instance = self.tailed_oplogs[host][port]
if 'last_ts' in instance and instance['last_ts'] is not None:
if ts is None or instance['last_ts'].time < ts.time:
ts = Timestamp(instance['last_ts'].time, 0)
return ts
def test_qop_type_17(monty_find, mongo_find):
docs = [{"a": Timestamp(0, 1)}]
spec = {"a": {"$type": 17}} # timestamp
monty_c = monty_find(docs, spec)
mongo_c = mongo_find(docs, spec)
assert mongo_c.count() == 1
assert monty_c.count() == mongo_c.count()
def test_exceptions(self):
self.assertRaises(TypeError, Timestamp)
self.assertRaises(TypeError, Timestamp, None, 123)
self.assertRaises(TypeError, Timestamp, 1.2, 123)
self.assertRaises(TypeError, Timestamp, 123, None)
self.assertRaises(TypeError, Timestamp, 123, 1.2)
self.assertRaises(ValueError, Timestamp, 0, -1)
self.assertRaises(ValueError, Timestamp, -1, 0)
self.assertTrue(Timestamp(0, 0))
