def test_aware_datetime(self):
aware = datetime.datetime(1993,
4,
4,
2,
tzinfo=FixedOffset(555, "SomeZone"))
as_utc = (aware - aware.utcoffset()).replace(tzinfo=utc)
self.assertEqual(datetime.datetime(1993, 4, 3, 16, 45, tzinfo=utc),
as_utc)
after = decode(encode({"date": aware}),
CodecOptions(tz_aware=True))["date"]
self.assertEqual(utc, after.tzinfo)
self.assertEqual(as_utc, after)
def children(self):
# Do not decode a BSONObj with an invalid size.
if not bson or self.size < 5 or self.size > 17 * 1024 * 1024:
return
inferior = gdb.selected_inferior()
buf = bytes(inferior.read_memory(self.ptr, self.size))
options = CodecOptions(document_class=collections.OrderedDict)
bsondoc = bson.BSON.decode(buf, codec_options=options)
for k, v in bsondoc.items():
yield 'key', k
yield 'value', bson.json_util.dumps(v)
def _initMongo(self, conf):
client = pymongo.MongoClient(
host=conf['host'],
port=conf['port']
)
db = client[conf['dbn']]
# 登陆
if conf.get('username'):
db.authenticate(conf['username'], conf['password'])
return db[conf['collection']].with_options(
codec_options=CodecOptions(tz_aware=True, tzinfo=LOCAL_TIMEZONE))
def getMongoData(self, reqRunId):
db = self.getMongoDbConnection()
opts = CodecOptions(document_class=SON)
collection_son = db.CrawlResponse.with_options(codec_options=opts)
finalRecordList = []
records = collection_son.find({'RequestRunId': int(reqRunId)},
{"_id": 0})
for val in records:
finalRecordList.append(val)
return finalRecordList
def test_write_response_raw_bson(self):
coll = self.client.get_database(
'pymongo_test',
codec_options=CodecOptions(
document_class=RawBSONDocument)).test_raw
# No Exceptions raised while handling write response.
coll.insert_one(self.document)
coll.delete_one(self.document)
coll.insert_many([self.document])
coll.delete_many(self.document)
coll.update_one(self.document, {'$set': {'a': 'b'}}, upsert=True)
coll.update_many(self.document, {'$set': {'b': 'c'}})
def migrateOne(db: database.Database):
old = db.get_collection("instances_old")
new = db.get_collection("instances")
# if already migrated return
if old.count() == new.count():
return
# if previous migration failed restore state
if old.count() > 0 and new.count() == 0:
new.drop()
old = db.get_collection("instances_old")
old.rename("instances")
# migrate
new.rename("instances_old")
old = db.get_collection("instances_old",
CodecOptions(uuid_representation=JAVA_LEGACY))
new = db.get_collection("instances",
CodecOptions(uuid_representation=STANDARD))
# don't overwrite data
if new.count() > 0:
return
for record in old.find():
new.insert(record)
def children(self):
"""Children."""
# Do not decode a BSONObj with an invalid size, or that is considered
# invalid by pymongo.
if not bson or not self.is_valid or self.size < 5 or self.size > 17 * 1024 * 1024:
return
options = CodecOptions(document_class=collections.OrderedDict)
bsondoc = bson.decode(self.raw_memory, codec_options=options)
for key, val in list(bsondoc.items()):
yield 'key', key
yield 'value', bson.json_util.dumps(val)
def replace(self, collection_name: str, new_document: dict,
query_filter: dict) -> dict:
"""Replace a document in its collection, plus saving the old version in its "archive" collection."""
# Grabbing current (ie. database) version of the document
document = [
doc for doc in self.instance[collection_name].with_options(
codec_options=CodecOptions(tz_aware=self.tz_awareness,
tzinfo=pytz.timezone('Etc/GMT+0'))).
find({'_id': new_document['_id']})
]
# Back-it-up as a new version in archive collection
self.instance[collection_name +
"_archived"].with_options(codec_options=CodecOptions(
tz_aware=self.tz_awareness,
tzinfo=pytz.timezone('Etc/GMT+0'))).insert({
'action':
'update',
'author':
ObjectId(g.user['_id']),
'document':
document,
})
logging.info(f"Document archived within: {collection_name}_archive")
# Updating meta and inserting the document
new_document['common']['update'] = {
'date': datetime.now(),
'author': ObjectId(g.user['_id']),
}
newDocument = self.instance[collection_name].with_options(
codec_options=CodecOptions(tz_aware=self.tz_awareness,
tzinfo=pytz.timezone(
'Etc/GMT+0'))).find_one_and_replace(
query_filter, new_document)
logging.info(f"Document updated succesfully: {newDocument['_id']}")
# Return the whole new document
return newDocument
def run_test(doc_cls):
db = self.db
input_docs = [
{'x': Int64(k)} for k in [1, 2, 3]]
for doc in input_docs:
db.test.insert_one(doc)
test = db.get_collection('test', codec_options=CodecOptions(
type_registry=TypeRegistry([UndecipherableIntDecoder()]),
document_class=doc_cls))
for doc in test.find({}, batch_size=1):
self.assertIsInstance(doc, doc_cls)
self.assertIsInstance(doc['x'], UndecipherableInt64Type)
def test_legacy_csharp_uuid(self):
data = self.csharp_data
# Test decoding
docs = bson.decode_all(data, CodecOptions(SON, False, PYTHON_LEGACY))
for d in docs:
self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring']))
docs = bson.decode_all(data, CodecOptions(SON, False, STANDARD))
for d in docs:
self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring']))
docs = bson.decode_all(data, CodecOptions(SON, False, JAVA_LEGACY))
for d in docs:
self.assertNotEqual(d['newguid'], uuid.UUID(d['newguidstring']))
docs = bson.decode_all(data, CodecOptions(SON, False, CSHARP_LEGACY))
for d in docs:
self.assertEqual(d['newguid'], uuid.UUID(d['newguidstring']))
# Test encoding
encoded = b''.join([
bson.BSON.encode(doc, False,
CodecOptions(uuid_representation=PYTHON_LEGACY))
for doc in docs
])
self.assertNotEqual(data, encoded)
encoded = b''.join([
bson.BSON.encode(doc, False,
CodecOptions(uuid_representation=STANDARD))
for doc in docs
])
self.assertNotEqual(data, encoded)
encoded = b''.join([
bson.BSON.encode(doc, False,
CodecOptions(uuid_representation=JAVA_LEGACY))
for doc in docs
])
self.assertNotEqual(data, encoded)
encoded = b''.join([
bson.BSON.encode(doc, False,
CodecOptions(uuid_representation=CSHARP_LEGACY))
for doc in docs
])
self.assertEqual(data, encoded)
def create_data_encryption_key(self, kms_providers):
# create data encryption key and store in DB
client_encryption = ClientEncryption(
# pass in the kms_providers variable from the previous step
kms_providers,
self.key_vault_namespace,
self.client,
CodecOptions(uuid_representation=STANDARD))
data_key_id = client_encryption.create_data_key("local")
uuid_data_key_id = UUID(bytes=data_key_id)
print(f'data key created using KMS provider: {uuid_data_key_id} \n')
base_64_data_key_id = base64.b64encode(data_key_id)
return data_key_id
def test_with_codec_options(self):
# {u'date': datetime.datetime(2015, 6, 3, 18, 40, 50, 826000),
# u'_id': UUID('026fab8f-975f-4965-9fbf-85ad874c60ff')}
# encoded with JAVA_LEGACY uuid representation.
bson_string = (
b'-\x00\x00\x00\x05_id\x00\x10\x00\x00\x00\x03eI_\x97\x8f\xabo\x02'
b'\xff`L\x87\xad\x85\xbf\x9f\tdate\x00\x8a\xd6\xb9\xbaM'
b'\x01\x00\x00\x00')
document = RawBSONDocument(
bson_string,
codec_options=CodecOptions(uuid_representation=JAVA_LEGACY))
self.assertEqual(uuid.UUID('026fab8f-975f-4965-9fbf-85ad874c60ff'),
document['_id'])
def get_recent_historical_data(self, symbol, start_date):
"""
Returns data for a symbol traded after start_date (exclusive)
:param str symbol:
:param start_date:
:return:
"""
collection = self.db[HISTORICAL_DATA_COLLECTION].with_options(CodecOptions(tz_aware=True, tzinfo=TIMEZONE))
res = list(collection.find({"symbol": symbol, "datetime": {"$gt": start_date}}).sort("datetime", 1))
for data in res:
self._modify_historical_data(data)
logger.info("{0} data returned for {1}, traded after {2}".format(len(res), symbol, start_date))
return res
def _bson_to_dict(data: Any, opts: CodecOptions) -> Any:
"""Decode a BSON string to document_class."""
data, view = get_data_and_view(data)
try:
if _raw_document_class(opts.document_class):
return opts.document_class(data, opts)
_, end = _get_object_size(data, 0, len(data))
return _elements_to_dict(data, view, 4, end, opts)
except InvalidBSON:
raise
except Exception:
# Change exception type to InvalidBSON but preserve traceback.
_, exc_value, exc_tb = sys.exc_info()
raise InvalidBSON(str(exc_value)).with_traceback(exc_tb)
def fromfile(cls, filename, zipped=False):
options = CodecOptions(document_class=OrderedDict)
with open(filename, 'rb') as f:
abson = f.read()
if zipped:
abson = gzip.decompress(abson)
dict_bson = bson.BSON.decode(abson, codec_options=options)
bsonimagelist = BsonImageList()
for _, data in dict_bson.items(): # key ignored
bsonimage = BsonImage()
bsonimage.set_campos(data['filename'], data['content'],
**data['metadata'])
bsonimagelist.addBsonImage(bsonimage)
return bsonimagelist
def remove(self, collection_name: str, document_id):
"""Move a document from its collection to its "archived" version."""
# Ducktyping -> oid & string becomes oid
document_id = ObjectId(document_id) if isinstance(document_id,
str) else document_id
# Grabbing current (ie. database) version of the document
document = [
item for item in self.instance[collection_name].with_options(
codec_options=CodecOptions(tz_aware=self.tz_awareness,
tzinfo=pytz.timezone('Etc/GMT+0'))).
find({'_id': document_id})
]
# Backing-it-up as a new version in archive collection
self.instance[collection_name +
"_archived"].with_options(codec_options=CodecOptions(
tz_aware=self.tz_awareness,
tzinfo=pytz.timezone('Etc/GMT+0'))).insert({
'action':
'remove',
'author':
ObjectId(g.user['_id']),
'document':
document,
})
# Remove record from database
self.instance[collection_name].with_options(codec_options=CodecOptions(
tz_aware=self.tz_awareness, tzinfo=pytz.timezone(
'Etc/GMT+0'))).delete_one({'_id': document_id})
logging.info(f"Document archived within: {collection_name}_archive")
# DONE:50 Remove related from Relations documents
# Cleaning relations involving the deleted document
return entity_tools.clean_relations_of(ObjectId(document_id))
def _command(self, sock_info, command, slave_ok=False, value=1, check=True,
allowable_errors=None, read_preference=ReadPreference.PRIMARY,
codec_options=CodecOptions(), **kwargs):
"""Internal command helper."""
if isinstance(command, string_type):
command = SON([(command, value)])
command.update(kwargs)
return sock_info.command(self.__name,
command,
slave_ok,
read_preference,
codec_options,
check,
allowable_errors)
def create_app(loop):
app = web.Application(middlewares=settings.MIDDLEWARES,
client_max_size=(1024**2) * 10)
db_client = AsyncIOMotorClient(settings.MONGODB_ADDR,
serverSelectionTimeoutMS=3000)
db = db_client.get_database(codec_options=CodecOptions(tz_aware=True))
app['db'] = db
sio = socketio.AsyncServer(engineio_logger=False)
sio.attach(app)
app['sio'] = sio
loop.run_until_complete(setup_app(app))
return app
def get_mongodb_connection(object):
objectid_codec = ObjectIdCodec()
type_registry = TypeRegistry([objectid_codec])
codec_options = CodecOptions(type_registry=type_registry)
username = urllib.parse.quote_plus(object.config['user'])
password = urllib.parse.quote_plus(object.config['psw'])
client = pymongo.MongoClient(
'%s://%s:%s@%s:%s/%s' %
(object.config['type'], username, password, object.config['host'],
object.config['port'], object.config['dbname']))
db = client[object.config['dbname']]
collection = db.get_collection(object.config['collection'],
codec_options=codec_options)
return collection
def run_test(doc_cls):
codecopts = CodecOptions(type_registry=TypeRegistry([
UppercaseTextDecoder(), UndecipherableIntEncoder()]),
document_class=doc_cls)
self.create_targets(codec_options=codecopts)
change_stream = self.change_stream()
doc = {'a': UndecipherableInt64Type(101), 'b': 'xyz'}
self.input_target.insert_one(doc)
change = next(change_stream)
self.assertIsInstance(change, doc_cls)
self.assertEqual(change['fullDocument']['a'], 101)
self.assertEqual(change['fullDocument']['b'], 'XYZ')
def _unpack_response(response, cursor_id=None, codec_options=CodecOptions()):
"""Unpack a response from the database.
Check the response for errors and unpack, returning a dictionary
containing the response data.
Can raise CursorNotFound, NotMasterError, ExecutionTimeout, or
OperationFailure.
:Parameters:
- `response`: byte string as returned from the database
- `cursor_id` (optional): cursor_id we sent to get this response -
used for raising an informative exception when we get cursor id not
valid at server response
- `codec_options` (optional): an instance of
:class:`~bson.codec_options.CodecOptions`
"""
response_flag = struct.unpack("<i", response[:4])[0]
if response_flag & 1:
# Shouldn't get this response if we aren't doing a getMore
if cursor_id is None:
raise ProtocolError("No cursor id for getMore operation")
# Fake a getMore command response. OP_GET_MORE provides no document.
msg = "Cursor not found, cursor id: %d" % (cursor_id, )
errobj = {"ok": 0, "errmsg": msg, "code": 43}
raise CursorNotFound(msg, 43, errobj)
elif response_flag & 2:
error_object = bson.BSON(response[20:]).decode()
# Fake the ok field if it doesn't exist.
error_object.setdefault("ok", 0)
if error_object["$err"].startswith("not master"):
raise NotMasterError(error_object["$err"], error_object)
elif error_object.get("code") == 50:
raise ExecutionTimeout(error_object.get("$err"),
error_object.get("code"), error_object)
raise OperationFailure("database error: %s" % error_object.get("$err"),
error_object.get("code"), error_object)
result = {
"cursor_id": struct.unpack("<q", response[4:12])[0],
"starting_from": struct.unpack("<i", response[12:16])[0],
"number_returned": struct.unpack("<i", response[16:20])[0],
"data": bson.decode_all(response[20:], codec_options)
}
assert len(result["data"]) == result["number_returned"]
return result
def test_pymongo():
response = b"%s%s%s%s%s" % (struct.pack("<i", 4), struct.pack(
"<q", 1), struct.pack("<i", 1), struct.pack(
"<i", 1), bson.BSON.encode({"hello": "world"}))
assert asynmongo._unpack_response(response, 1) == {
'starting_from': 1,
'number_returned': 1,
'cursor_id': 1,
'data': [{
'hello': 'world'
}]
}
payload = bson.BSON.encode({"hello": "world"})
response = b"%s%s%s" % (struct.pack(
"<B", 0), struct.pack("<i", len(payload)), payload)
assert asynmongo._unpack_response(response, 2013) == {
'first_payload_type': 0,
'data': [{
'hello': 'world'
}],
'first_payload_size': 22
}
opts = CodecOptions(SON)
assert auth._auth_key(1, "a", "b") == "90b38d5dbfabd0b883e17ae67847220a"
assert message.query(0, "%s.$cmd" % "mydb", 0, 1, SON({
'getnonce': 1
}), SON({}), opts) == (
1804289383,
b'>\x00\x00\x00gE\x8bk\x00\x00\x00\x00\xd4\x07\x00\x00\x00\x00\x00\x00mydb.$cmd\x00\x00\x00\x00\x00\x01\x00\x00\x00\x13\x00\x00\x00\x10getnonce\x00\x01\x00\x00\x00\x00\x05\x00\x00\x00\x00',
19)
assert message.query(0, "col.a", 0, 1, {"_id": 1}, None, opts) == (
846930886,
b'0\x00\x00\x00\xc6#{2\x00\x00\x00\x00\xd4\x07\x00\x00\x00\x00\x00\x00col.a\x00\x00\x00\x00\x00\x01\x00\x00\x00\x0e\x00\x00\x00\x10_id\x00\x01\x00\x00\x00\x00',
14)
assert message.update("col.a", 0, 0, {"_id": 1}, {
"a": 1
}, 1, (), False, opts) == (
1681692777,
b'8\x00\x00\x00i\x98<d\x00\x00\x00\x00\xd1\x07\x00\x00\x00\x00\x00\x00col.a\x00\x00\x00\x00\x00\x0e\x00\x00\x00\x10_id\x00\x01\x00\x00\x00\x00\x0c\x00\x00\x00\x10a\x00\x01\x00\x00\x00\x00<\x00\x00\x00i\x98<d\x00\x00\x00\x00\xd4\x07\x00\x00\x00\x00\x00\x00col.$cmd\x00\x00\x00\x00\x00\xff\xff\xff\xff\x17\x00\x00\x00\x10getlasterror\x00\x01\x00\x00\x00\x00',
14)
msg = message.delete("col.a", {"_id": 1}, 1, (), opts, 0)
assert len(msg) == 3 and msg[0] == 1714636915 and msg[-1] == 14
msg = message.insert("col.a", [{"a": 1}], False, 1, (), 0, opts)
assert len(msg) == 3 and msg[0] == 1957747793 and msg[-1] == 12
def test_all_bson_types(self):
# Because we can't round-trip all BSON types (see _DEPRECATED_BSON_TYPES
# above for how these are handled), make this test a special case,
# instead of mangling our create_test function below.
with open(os.path.join(_TEST_PATH, 'multi-type.json')) as spec_file:
case_spec = json.load(spec_file)
for valid_case in case_spec.get('valid', []):
B = binascii.unhexlify(b(valid_case['bson']))
E = valid_case['extjson']
# Make sure that the BSON and JSON decode to the same document.
self.assertEqual(
json_util.loads(E,
json_options=json_util.CANONICAL_JSON_OPTIONS),
BSON(B).decode(codec_options=CodecOptions(document_class=SON,
tz_aware=True)))
def __init__(self, mongo_uri, db_name, worker_id, signing_key):
# noinspection PyArgumentList
options = CodecOptions(tz_aware=True, tzinfo=pytz.UTC)
self.db = pymongo.MongoClient(mongo_uri).get_database(
db_name, codec_options=options)
self.users = self.db.get_collection('users')
self.users.create_index([('username', pymongo.ASCENDING)], unique=True)
self.entries = self.db.get_collection('entries')
self.entries.create_index([('author_id', pymongo.ASCENDING)])
self.entries.create_index([('timestamp', pymongo.DESCENDING)])
self.argon2 = argon2.PasswordHasher()
self.id_gen = IDGenerator(int(worker_id))
self.jwt = JWTEncoder(signing_key)
def get_last_n_historical_data(self, symbol, days=1):
"""
Returns the data related to a symbol for the last number of `n` days (if available)
:param str symbol:
:param int days:
:return: list of dict
"""
collection = self.db[HISTORICAL_DATA_COLLECTION].with_options(CodecOptions(tz_aware=True, tzinfo=TIMEZONE))
res = list(collection.find({"symbol": symbol}).sort("datetime", -1).limit(days))
res.reverse()
for data in res:
self._modify_historical_data(data)
logger.info("Getting last {0} data for {1}. (available: {2})".format(days, symbol, len(res)))
return res
def __init__(self):
threading.Thread.__init__(self)
with open("mongostring", 'r') as mfile:
mstring = mfile.read()
self.client = MongoClient(mstring)
self.db = self.client.hk_data
options = CodecOptions(tz_aware=True)
self.settingsdb = self.db.settings.with_options(codec_options=options)
self.settings = {}
self.keepGoing = True
#The keepgoing variables don't have locks. I am not confident that this is a good
#decision, but I think it's reasonable since they are all modified in only one place,
#the logicclass's stop method. I think that means that all of the operations on keepGoing are
#atomic. the only things that ever access keepGoing are "while keepgoing:" and keepgoing=False
self.settingsLock = threading.Lock()
self.get_settings()
def dbConnect(self):
mongoConf = self.mongoConf
db = pymongo.MongoClient(mongoConf['host'],
mongoConf['port'])[mongoConf['dbn']]
if mongoConf.get('username') is not None:
# 登陆授权
db.authenticate(mongoConf['username'], mongoConf['password'])
# 确认登陆
db.client.server_info()
# 设置 collection 的时区生效
self.collection = db[mongoConf['collection']].with_options(
codec_options=CodecOptions(tz_aware=True, tzinfo=LOCAL_TIMEZONE))
def find(self,
collection_name: str,
query_filter: dict = None,
projector: dict = None,
skip: int = 0,
limit: int = 0) -> list:
"""Run a find query within global database."""
logging.debug("Query filter for find query on " + collection_name)
logging.debug(query_filter)
return [
item for item in self.instance[collection_name].with_options(
codec_options=CodecOptions(
tz_aware=self.tz_awareness,
tzinfo=pytz.timezone('Etc/GMT+0'))).find(
query_filter, projector).skip(skip).limit(limit)
]
def read_from_db(col: Collection, pbo: PlotBrowseOptions) -> None:
try:
print_func(f"Waching {col.database.name}->{col.name}")
last_doc = next(col.find({'_id': -1}).limit(1), None)
latest_ts = last_doc['meta']['ts'] if last_doc else time()
plots_col = col.database.get_collection("plots")
col = col.database.get_collection(
col.name,
codec_options=CodecOptions(document_class=RawBSONDocument))
while True:
for rdoc in col.find({'meta.ts': {'$gt': latest_ts}}):
print_func("*" * 20)
print_func(
f"Loaded doc size: {round(len(rdoc.raw)/1024/1024, 2)} MBytes"
)
doc = bson.decode(rdoc.raw)
if doc['meta']['ts'] > latest_ts:
latest_ts = doc['meta']['ts']
scol = SVGCollection()
plots: list[dict] = []
walk_and_process_dict(arg=doc,
callback=_gather_plots,
match=['plot'],
plots=plots,
debug=pbo.debug)
_handle_json(plots if plots else doc,
scol,
plots_col,
pbo,
concatenate=False)
gc.collect()
sleep(1)
except KeyboardInterrupt:
import sys
print_func("\nUntil next time...\n")
sys.exit(0)
def __init__(self, db_name):
loop = asyncio.get_event_loop()
client = motor_asyncio.AsyncIOMotorClient(
"mongodb://127.0.0.1:27017/", io_loop=loop
)
log.info("Connecting to mongodb database")
self.db = client.get_database(
db_name,
codec_options=CodecOptions(
type_registry=TypeRegistry(
fallback_encoder=lambda obj: Binary(bytes(obj))
if isinstance(obj, Streamable)
else obj
)
),
)
log.info("Connected to mongodb database")
_TEST_PATH = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'bson_corpus')
_DEPRECATED_BSON_TYPES = {
# Symbol
'0x0E': text_type,
# Undefined
'0x06': type(None),
# DBPointer
'0x0C': DBRef
}
# Need to set tz_aware=True in order to use "strict" dates in extended JSON.
codec_options = CodecOptions(tz_aware=True, document_class=SON)
# We normally encode UUID as binary subtype 0x03,
# but we'll need to encode to subtype 0x04 for one of the tests.
codec_options_uuid_04 = codec_options._replace(uuid_representation=STANDARD)
json_options_uuid_04 = json_util.JSONOptions(
strict_number_long=True,
strict_uuid=True,
datetime_representation=json_util.DatetimeRepresentation.NUMBERLONG,
uuid_representation=STANDARD)
json_options_iso8601 = json_util.JSONOptions(
datetime_representation=json_util.DatetimeRepresentation.ISO8601)
to_extjson = functools.partial(
json_util.dumps, json_options=json_util.CANONICAL_JSON_OPTIONS)
to_extjson_uuid_04 = functools.partial(json_util.dumps,
json_options=json_options_uuid_04)
to_extjson_iso8601 = functools.partial(json_util.dumps,
