def test_simple(self): codec_instances = [codec() for codec in self.codecs] def assert_proper_initialization(type_registry, codec_instances): self.assertEqual( type_registry._encoder_map, { self.types[0]: codec_instances[0].transform_python, self.types[1]: codec_instances[1].transform_python }) self.assertEqual( type_registry._decoder_map, { int: codec_instances[0].transform_bson, str: codec_instances[1].transform_bson }) self.assertEqual(type_registry._fallback_encoder, self.fallback_encoder) type_registry = TypeRegistry(codec_instances, self.fallback_encoder) assert_proper_initialization(type_registry, codec_instances) type_registry = TypeRegistry(fallback_encoder=self.fallback_encoder, type_codecs=codec_instances) assert_proper_initialization(type_registry, codec_instances) # Ensure codec list held by the type registry doesn't change if we # mutate the initial list. codec_instances_copy = list(codec_instances) codec_instances.pop(0) self.assertListEqual(type_registry._TypeRegistry__type_codecs, codec_instances_copy)
def test_type_registry_eq(self): codec_instances = [codec() for codec in self.codecs] self.assertEqual(TypeRegistry(codec_instances), TypeRegistry(codec_instances)) codec_instances_2 = [codec() for codec in self.codecs] self.assertNotEqual(TypeRegistry(codec_instances), TypeRegistry(codec_instances_2))
def test_initialize_fail(self): err_msg = "Expected an instance of TypeCodecBase, got .* instead" with self.assertRaisesRegex(TypeError, err_msg): TypeRegistry(self.codecs) with self.assertRaisesRegex(TypeError, err_msg): TypeRegistry([type('AnyType', (object,), {})()]) err_msg = "fallback_encoder %r is not a callable" % (True,) with self.assertRaisesRegex(TypeError, err_msg): TypeRegistry([], True) err_msg = "fallback_encoder %r is not a callable" % ('hello',) with self.assertRaisesRegex(TypeError, err_msg): TypeRegistry(fallback_encoder='hello')
def test_infinite_loop_exceeds_max_recursion_depth(self): codecopts = CodecOptions(type_registry=TypeRegistry( [self.B2A()], fallback_encoder=self.fallback_encoder_A2B)) # Raises max recursion depth exceeded error with self.assertRaises(RuntimeError): encode({'x': self.TypeA(100)}, codec_options=codecopts)
def test_break_resume_token(self): # Get one document from a change stream to determine resumeToken type. self.create_targets() change_stream = self.change_stream() self.input_target.insert_one({"data": "test"}) change = next(change_stream) resume_token_decoder = type_obfuscating_decoder_factory( type(change['_id']['_data'])) # Custom-decoding the resumeToken type breaks resume tokens. codecopts = CodecOptions(type_registry=TypeRegistry( [resume_token_decoder(), UndecipherableIntEncoder()])) # Re-create targets, change stream and proceed. self.create_targets(codec_options=codecopts) docs = [{'_id': 1}, {'_id': 2}, {'_id': 3}] change_stream = self.change_stream() self.insert_and_check(change_stream, docs[0], docs[0]) self.kill_change_stream_cursor(change_stream) self.insert_and_check(change_stream, docs[1], docs[1]) self.kill_change_stream_cursor(change_stream) self.insert_and_check(change_stream, docs[2], docs[2])
def test_custom_type_in_pipeline(self): codecopts = CodecOptions(type_registry=TypeRegistry( [UndecipherableIntEncoder(), UppercaseTextDecoder()])) self.create_targets(codec_options=codecopts) input_docs = [{ '_id': UndecipherableInt64Type(1), 'data': 'hello' }, { '_id': 2, 'data': 'world' }, { '_id': UndecipherableInt64Type(3), 'data': '!' }] expected_docs = [{'_id': 2, 'data': 'WORLD'}, {'_id': 3, 'data': '!'}] # UndecipherableInt64Type should be encoded with the TypeRegistry. change_stream = self.change_stream([{ '$match': { 'documentKey._id': { '$gte': UndecipherableInt64Type(2) } } }]) self.input_target.insert_one(input_docs[0]) self.insert_and_check(change_stream, input_docs[1], expected_docs[0]) self.kill_change_stream_cursor(change_stream) self.insert_and_check(change_stream, input_docs[2], expected_docs[1])
def __init__(self, collection: str, model_cls: Type[BaseModel]): self._db = get_db() type_registry = TypeRegistry([DecimalCodec()]) codec_options = CodecOptions(type_registry=type_registry) self._collection = self._db.get_collection(collection, codec_options=codec_options) self._collection.create_index("id", unique=True) self._model_cls = model_cls
def test_encode_custom_then_fallback(self): codecopts = CodecOptions(type_registry=TypeRegistry( [self.B2A()], fallback_encoder=self.fallback_encoder_A2BSON)) testdoc = {'x': self.TypeB(123)} expected_bytes = encode({'x': 123}) self.assertEqual(encode(testdoc, codec_options=codecopts), expected_bytes)
def __init__(self, mongodb_uri): codec_options = CodecOptions(type_registry=TypeRegistry( [PickledBinaryDecoder()], fallback_encoder=fallback_pickle_encoder)) self.client = MongoClient(mongodb_uri) self.db = self.client['trivicord'] self.collection = self.db.get_collection('games', codec_options=codec_options)
def _init_collection( self, col_name: str, indexes: Optional[list[IndexModel]] = None) -> Collection: codecs = CodecOptions( type_registry=TypeRegistry([c() for c in [DecimalCodec]])) col = self._database.get_collection(col_name, codecs) if indexes: col.create_indexes(indexes) return col
def test_encode_decode_numpy_data(self): data = {'array': np.array([1, 2, 3])} type_registry = TypeRegistry([NumpyArrayCodec()]) codec_options = CodecOptions(type_registry=type_registry) encoded = BSON.encode(data, codec_options=codec_options) decoded = BSON.decode(encoded, codec_options=codec_options) self.assertIsInstance(decoded, type(data)) self.assertIn('array', decoded) np.testing.assert_array_equal(data['array'], decoded['array'])
def Mongo_init(): decimal_c = DecimalCodec() date_c = DateCodec() type_registry = TypeRegistry([decimal_c, date_c]) codec_options = CodecOptions(type_registry=type_registry) man = MongoManager(codec_options=codec_options, user="******", host="cluster0.xjgrr.mongodb.net", passwd="GWSgnYU4pu7zs2S", db="DataTracker") DBManager.store_data("mongo", man)
def get_db_collection(cls, type_codecs: [CoreEntityCodec], document_class: MutableMapping = AttributeDict): """ Provide a collection with codec options :return: Collection for the current database connection """ db: Database = cls.get_database() codec_options = CodecOptions(document_class=document_class, type_registry=TypeRegistry(type_codecs)) return db.get_collection(name=cls._collection_name, codec_options=codec_options)
def run_test(doc_cls): db = self.db input_docs = [ {'x': Int64(k)} for k in [1, 2, 3]] for doc in input_docs: db.test.insert_one(doc) test = db.get_collection('test', codec_options=CodecOptions( type_registry=TypeRegistry([UndecipherableIntDecoder()]), document_class=doc_cls)) for doc in test.find({}, batch_size=1): self.assertIsInstance(doc, doc_cls) self.assertIsInstance(doc['x'], UndecipherableInt64Type)
def run_test(base, attrs): msg = (r"TypeEncoders cannot change how built-in types " r"are encoded \(encoder .* transforms type .*\)") for pytype in _BUILT_IN_TYPES: attrs.update({'python_type': pytype, 'transform_python': lambda x: x}) codec = type('testcodec', (base, ), attrs) codec_instance = codec() with self.assertRaisesRegex(TypeError, msg): TypeRegistry([codec_instance,]) # Test only some subtypes as not all can be subclassed. if pytype in [bool, type(None), RE_TYPE,]: continue class MyType(pytype): pass attrs.update({'python_type': MyType, 'transform_python': lambda x: x}) codec = type('testcodec', (base, ), attrs) codec_instance = codec() with self.assertRaisesRegex(TypeError, msg): TypeRegistry([codec_instance,])
def get_mongodb_connection(object): objectid_codec = ObjectIdCodec() type_registry = TypeRegistry([objectid_codec]) codec_options = CodecOptions(type_registry=type_registry) username = urllib.parse.quote_plus(object.config['user']) password = urllib.parse.quote_plus(object.config['psw']) client = pymongo.MongoClient( '%s://%s:%s@%s:%s/%s' % (object.config['type'], username, password, object.config['host'], object.config['port'], object.config['dbname'])) db = client[object.config['dbname']] collection = db.get_collection(object.config['collection'], codec_options=codec_options) return collection
def run_test(doc_cls): codecopts = CodecOptions(type_registry=TypeRegistry([ UppercaseTextDecoder(), UndecipherableIntEncoder()]), document_class=doc_cls) self.create_targets(codec_options=codecopts) change_stream = self.change_stream() doc = {'a': UndecipherableInt64Type(101), 'b': 'xyz'} self.input_target.insert_one(doc) change = next(change_stream) self.assertIsInstance(change, doc_cls) self.assertEqual(change['fullDocument']['a'], 101) self.assertEqual(change['fullDocument']['b'], 'XYZ')
def __init__(self, db_name): loop = asyncio.get_event_loop() client = motor_asyncio.AsyncIOMotorClient( "mongodb://127.0.0.1:27017/", io_loop=loop ) log.info("Connecting to mongodb database") self.db = client.get_database( db_name, codec_options=CodecOptions( type_registry=TypeRegistry( fallback_encoder=lambda obj: Binary(bytes(obj)) if isinstance(obj, Streamable) else obj ) ), ) log.info("Connected to mongodb database")
def test_simple(self): codecopts = CodecOptions(type_registry=TypeRegistry( [UndecipherableIntEncoder(), UppercaseTextDecoder()])) self.create_targets(codec_options=codecopts) input_docs = [ { '_id': UndecipherableInt64Type(1), 'data': 'hello' }, { '_id': 2, 'data': 'world' }, { '_id': UndecipherableInt64Type(3), 'data': '!' }, ] expected_docs = [ { '_id': 1, 'data': 'HELLO' }, { '_id': 2, 'data': 'WORLD' }, { '_id': 3, 'data': '!' }, ] change_stream = self.change_stream() self.insert_and_check(change_stream, input_docs[0], expected_docs[0]) self.kill_change_stream_cursor(change_stream) self.insert_and_check(change_stream, input_docs[1], expected_docs[1]) self.kill_change_stream_cursor(change_stream) self.insert_and_check(change_stream, input_docs[2], expected_docs[2])
def __init__( self, model_class: Type[T], database: Database, col_name: str, indexes: Optional[list[Union[IndexModel, str]]] = None, wrap_object_str_id=True, ): codecs = CodecOptions( type_registry=TypeRegistry([c() for c in [DecimalCodec]])) self.collection = database.get_collection(col_name, codecs) if indexes: indexes = [ parse_str_index_model(i) if isinstance(i, str) else i for i in indexes ] self.collection.create_indexes(indexes) self.model_class = model_class self.wrap_object_id = model_class.__fields__[ "id"].type_ == ObjectIdStr and wrap_object_str_id
def test_simple_separate_codecs(self): class MyIntEncoder(TypeEncoder): python_type = self.types[0] def transform_python(self, value): return value.x class MyIntDecoder(TypeDecoder): bson_type = int def transform_bson(self, value): return self.types[0](value) codec_instances = [MyIntDecoder(), MyIntEncoder()] type_registry = TypeRegistry(codec_instances) self.assertEqual( type_registry._encoder_map, {MyIntEncoder.python_type: codec_instances[1].transform_python}) self.assertEqual( type_registry._decoder_map, {MyIntDecoder.bson_type: codec_instances[0].transform_bson})
def get_names(): ## Initializing the mongo connection connection_string = 'mongodb+srv://Zeevtest:[email protected]/test?retryWrites=true&w=majority' from bson.decimal128 import Decimal128 from bson.codec_options import TypeCodec class DecimalCodec(TypeCodec): python_type = Decimal # the Python type acted upon by this type codec bson_type = Decimal128 # the BSON type acted upon by this type codec def transform_python(self, value): """Function that transforms a custom type value into a type that BSON can encode.""" return Decimal128(value) def transform_bson(self, value): """Function that transforms a vanilla BSON type value into our custom type.""" return value.to_decimal() decimal_codec = DecimalCodec() today = date.today() cutoff_date = today - timedelta(days=70) from bson.codec_options import TypeRegistry type_registry = TypeRegistry([decimal_codec]) from bson.codec_options import CodecOptions codec_options = CodecOptions(type_registry=type_registry) mongo_client = pymongo.MongoClient(connection_string) mydb = mongo_client["patents"] db = mongo_client["patents"] examiners_collection = mydb.get_collection("examiners_new", codec_options=codec_options) return jsonify([(i['examiner']) for i in examiners_collection.find()])
def __init__(self, name: str, host: str = 'localhost', port: int = 27017, database: str = '', serializer: Union[Serializer, None] = None, connection_timeout: float = 30000) -> None: """MongoDB implementation of storage class See also: `StorageInterface` Args: name: Symbolic name for the storage instance. host: MongoDB host port: MongoDB port database: The database to use, if empty the name of the storage is used connection_timeout: How long to try to connect to database before raising an error in milliseconds Raises: StorageTimeoutError: If connection to database has not been established before connection_timeout is reached """ super().__init__(name) type_registry = TypeRegistry([NumpyArrayCodec()]) codec_options = CodecOptions(type_registry=type_registry) self._client = MongoClient(host, port, serverSelectionTimeoutMS=connection_timeout) self._check_server_connection(connection_timeout) self._db = self._client.get_database(database or name, codec_options=codec_options) self._collection = self._db.get_collection('storage') if serializer is None: serializer = _serializer self._serialize = serializer.encode_data self._unserialize = serializer.decode_data
from binance.websockets import BinanceSocketManager from bson import CodecOptions from bson.codec_options import TypeRegistry from kucoin.exceptions import KucoinAPIException from pymongo.errors import PyMongoError from library import get_binance_klines, get_binance_interval_unit, setup_logger, binance_obj, kucoin_client, \ DecimalCodec, try_get_klines, TradeMsg, get_last_db_record, \ get_time_from_binance_tmstmp, from_binance_socket_kline from mongodb import mongo_client logger = setup_logger("Kline-Crawl-Manager-LTF") db = mongo_client.klines decimal_codec = DecimalCodec() type_registry = TypeRegistry([decimal_codec]) codec_options = CodecOptions(type_registry=type_registry) trades = {} klines_socket = {} def to_mongo_binance(_kline): _data = to_mongo(_kline) _data['buy_btc_volume'] = _kline.buy_btc_volume _data['buy_quantity'] = _kline.buy_quantity _data['sell_btc_volume'] = _kline.sell_btc_volume _data['sell_quantity'] = _kline.sell_quantity return _data
def test_chaining_encoders_fails(self): codecopts = CodecOptions( type_registry=TypeRegistry([self.A2B(), self.B2BSON()])) with self.assertRaises(InvalidDocument): encode({'x': self.TypeA(123)}, codec_options=codecopts)
return value.isoformat() def transform_bson(self, value: str): return arrow.get(value).datetime class DateCodec(TypeCodec): python_type = datetime.date bson_type = string_type def transform_python(self, value: datetime.date): return value.isoformat() def transform_bson(self, value: str): return value class TimeCodec(TypeCodec): python_type = datetime.time bson_type = string_type def transform_python(self, value: datetime.time): return value.isoformat() def transform_bson(self, value: str): return value type_registry = TypeRegistry([DateCodec(), TimeCodec(), ArrowCodec()]) codec_options = CodecOptions(type_registry=type_registry)
class NumpyCodec(TypeCodec): sub_type = USER_DEFINED_SUBTYPE + 1 python_type = np.ndarray bson_type = Binary def transform_python(self, a: np.ndarray): data = encode_ndarray(a) return Binary(data, NumpyCodec.sub_type) def transform_bson(self, data: Binary): if data.subtype == NumpyCodec.sub_type: return decode_ndarray(data) return data CODEC_OPTIONS = CodecOptions(type_registry=TypeRegistry([NumpyCodec()])) class BSONTarWriter(object): def __init__(self, path: str): self._path = path self._impl = TarWriter(path) self._meta_doc = {} def close(self): if self._meta_doc is not None: data = BSON.encode(self._meta_doc, codec_options=CODEC_OPTIONS) self._impl.write('meta.bson', data) self._meta_doc = None self._impl.close()
def hello_world(examiner_name): ## Initializing the mongo connection connection_string = 'mongodb+srv://Zeevtest:[email protected]/test?retryWrites=true&w=majority' from bson.decimal128 import Decimal128 from bson.codec_options import TypeCodec class DecimalCodec(TypeCodec): python_type = Decimal # the Python type acted upon by this type codec bson_type = Decimal128 # the BSON type acted upon by this type codec def transform_python(self, value): """Function that transforms a custom type value into a type that BSON can encode.""" return Decimal128(value) def transform_bson(self, value): """Function that transforms a vanilla BSON type value into our custom type.""" return value.to_decimal() decimal_codec = DecimalCodec() today = date.today() cutoff_date = today - timedelta(days=70) from bson.codec_options import TypeRegistry type_registry = TypeRegistry([decimal_codec]) from bson.codec_options import CodecOptions codec_options = CodecOptions(type_registry=type_registry) mongo_client = pymongo.MongoClient(connection_string) mydb = mongo_client["patents"] db = mongo_client["patents"] examiners_collection = mydb.get_collection("examiners_new", codec_options=codec_options) print(examiner_name) examiner_record = examiners_collection.find_one( {'examiner': examiner_name}) if examiner_record is None: print("no examiner record") else: if "total_refused" in examiner_record: examiner_apps_refused = examiner_record.get("total_refused") else: examiner_apps_refused = 0 if "total_refused_with_interview" in examiner_record: examiner_apps_refused_with_interview = examiner_record.get( "total_refused_with_interview") else: examiner_apps_refused_with_interview = 0 if "total_refused_without_interview" in examiner_record: examiner_apps_refused_without_interview = examiner_record.get( "total_refused_without_interview") else: examiner_apps_refused_without_interview = 0 if "total_granted" in examiner_record: examiner_apps_granted = examiner_record.get("total_granted") else: examiner_apps_granted = 0 if "total_granted_with_interview" in examiner_record: examiner_apps_granted_with_interview = examiner_record.get( "total_granted_with_interview") else: examiner_apps_granted_with_interview = 0 if "total_granted_without_interview" in examiner_record: examiner_apps_granted_without_interview = examiner_record.get( "total_granted_without_interview") else: examiner_apps_granted_without_interview = 0 examiner_apps_we_have = examiner_apps_granted + examiner_apps_refused examiner_grant_rate = "%.0f%%" % (100 * examiner_apps_granted / examiner_apps_we_have) examiner_grant_rate_with_interview = "%.0f%%" % ( 100 * examiner_apps_granted_with_interview / (examiner_apps_granted_with_interview + examiner_apps_refused_with_interview)) examiner_grant_rate_without_interview = "%.0f%%" % ( 100 * examiner_apps_granted_without_interview / (examiner_apps_granted_without_interview + examiner_apps_refused_without_interview)) interview_improvement_rate = "%.0f%%" % ( 100 * (examiner_apps_granted_with_interview / (examiner_apps_granted_with_interview + examiner_apps_refused_with_interview)) / (examiner_apps_granted_without_interview / (examiner_apps_granted_without_interview + examiner_apps_refused_without_interview)) - 100) # Here comes the monthly data: successful_month_retrieved = examiner_record.get("successful_month") failed_month_retrieved = examiner_record.get("failed_month") successful_month = [] failed_month = [] month_stat = [] #possible_response_month = [] total_successful_responses = 0 total_failed_responses = 0 for i in range(1, 14): successful_month.append(0) failed_month.append(0) month_stat.append(0) #possible_response_month.append(0) try: for key, value in successful_month_retrieved.items(): key = int(key) successful_month[key] = value total_successful_responses = total_successful_responses + value except AttributeError: print("no successful month data") total_successful_responses = 0.00000000000000000000000001 try: for key, value in failed_month_retrieved.items(): key = int(key) failed_month[key] = value total_failed_responses = total_failed_responses + value except AttributeError: print("no failed month data") #print ("monthly statistics, month by month") for i in range(0, 13): if (failed_month[i] != 0): month_stat[i] = successful_month[i] / (successful_month[i] + failed_month[i]) #month_stat[i] = "%.0f%%" % (100 * month_stat[i]) #if (i !=0): #print ("Month",str(i),": ",month_stat[i], " ") response_success_rate = total_successful_responses / ( total_successful_responses + total_failed_responses) # Run through the 6 months including and after the one where the office action was issued: recommended_month_stat = 0 recommended_month_stat = "%.0f%%" % (100 * recommended_month_stat) response_success_rate = "%.0f%%" % (100 * response_success_rate) # Here ends the monthly data. reporting_text = "<br><br>Hi there! The examiner's name is: " reporting_text = reporting_text + examiner_name reporting_text = reporting_text + "<br><br>We crunched through " reporting_text = reporting_text + str(examiner_apps_we_have) reporting_text = reporting_text + " applications for this examiner and here is what we can tell you:<br><br>" reporting_text = reporting_text + "Grant rate (chances to eventually reach a grant): " reporting_text = reporting_text + str( examiner_grant_rate) + "with interview: " + str( examiner_grant_rate_with_interview ) + " / without interview: " + str( examiner_grant_rate_without_interview ) + " improvement rate: " + str(interview_improvement_rate) reporting_text = reporting_text + "<br>Response success rate (chances to overcome one office action): " reporting_text = reporting_text + str(response_success_rate) reporting_text = reporting_text + "<br><br>Full monthly stats: " months = {} for i in range(0, 13): if (failed_month[i] != 0): month_stat[i] = successful_month[i] / (successful_month[i] + failed_month[i]) month_stat[i] = "%.0f%%" % (100 * month_stat[i]) if (i != 0): reporting_text = reporting_text + str( get_month_text(i)) + ": " + month_stat[i] + " | " months[get_month_text(i)] = month_stat[i] #print (reporting_text) # Now is the time to check if the office action matches any of our pre-tracked queries # and if yes, to fire it up. # starting from checking if the applicant is in our tracked applicant's list. sender_email = "*****@*****.**" receiver_email = "*****@*****.**" password = "******" # Create the plain-text and HTML version of your message text = reporting_text html = "<html><body>" + reporting_text + "</body></html>" result = {} result['examiner_name'] = examiner_name result['examiner_apps_we_have'] = examiner_apps_we_have result['examiner_grant_rate'] = examiner_grant_rate result[ 'examiner_grant_rate_with_interview'] = examiner_grant_rate_with_interview result[ 'examiner_grant_rate_without_interview'] = examiner_grant_rate_without_interview result['response_success_rate'] = response_success_rate result['interview_improvement_rate'] = interview_improvement_rate result['months'] = months return (result)
def get_codec_options(): type_registry = tr_flags type_registry.append(ColorMongoEncoder()) return CodecOptions(type_registry=TypeRegistry(type_registry))
def test_type_registry_repr(self): codec_instances = [codec() for codec in self.codecs] type_registry = TypeRegistry(codec_instances) r = ("TypeRegistry(type_codecs=%r, fallback_encoder=%r)" % (codec_instances, None)) self.assertEqual(r, repr(type_registry))