def test_simple(self):
codec_instances = [codec() for codec in self.codecs]
def assert_proper_initialization(type_registry, codec_instances):
self.assertEqual(
type_registry._encoder_map, {
self.types[0]: codec_instances[0].transform_python,
self.types[1]: codec_instances[1].transform_python
})
self.assertEqual(
type_registry._decoder_map, {
int: codec_instances[0].transform_bson,
str: codec_instances[1].transform_bson
})
self.assertEqual(type_registry._fallback_encoder,
self.fallback_encoder)
type_registry = TypeRegistry(codec_instances, self.fallback_encoder)
assert_proper_initialization(type_registry, codec_instances)
type_registry = TypeRegistry(fallback_encoder=self.fallback_encoder,
type_codecs=codec_instances)
assert_proper_initialization(type_registry, codec_instances)
# Ensure codec list held by the type registry doesn't change if we
# mutate the initial list.
codec_instances_copy = list(codec_instances)
codec_instances.pop(0)
self.assertListEqual(type_registry._TypeRegistry__type_codecs,
codec_instances_copy)
def test_type_registry_eq(self):
codec_instances = [codec() for codec in self.codecs]
self.assertEqual(TypeRegistry(codec_instances),
TypeRegistry(codec_instances))
codec_instances_2 = [codec() for codec in self.codecs]
self.assertNotEqual(TypeRegistry(codec_instances),
TypeRegistry(codec_instances_2))
def test_initialize_fail(self):
err_msg = "Expected an instance of TypeCodecBase, got .* instead"
with self.assertRaisesRegex(TypeError, err_msg):
TypeRegistry(self.codecs)
with self.assertRaisesRegex(TypeError, err_msg):
TypeRegistry([type('AnyType', (object,), {})()])
err_msg = "fallback_encoder %r is not a callable" % (True,)
with self.assertRaisesRegex(TypeError, err_msg):
TypeRegistry([], True)
err_msg = "fallback_encoder %r is not a callable" % ('hello',)
with self.assertRaisesRegex(TypeError, err_msg):
TypeRegistry(fallback_encoder='hello')
def test_infinite_loop_exceeds_max_recursion_depth(self):
codecopts = CodecOptions(type_registry=TypeRegistry(
[self.B2A()], fallback_encoder=self.fallback_encoder_A2B))
# Raises max recursion depth exceeded error
with self.assertRaises(RuntimeError):
encode({'x': self.TypeA(100)}, codec_options=codecopts)
def test_break_resume_token(self):
# Get one document from a change stream to determine resumeToken type.
self.create_targets()
change_stream = self.change_stream()
self.input_target.insert_one({"data": "test"})
change = next(change_stream)
resume_token_decoder = type_obfuscating_decoder_factory(
type(change['_id']['_data']))
# Custom-decoding the resumeToken type breaks resume tokens.
codecopts = CodecOptions(type_registry=TypeRegistry(
[resume_token_decoder(),
UndecipherableIntEncoder()]))
# Re-create targets, change stream and proceed.
self.create_targets(codec_options=codecopts)
docs = [{'_id': 1}, {'_id': 2}, {'_id': 3}]
change_stream = self.change_stream()
self.insert_and_check(change_stream, docs[0], docs[0])
self.kill_change_stream_cursor(change_stream)
self.insert_and_check(change_stream, docs[1], docs[1])
self.kill_change_stream_cursor(change_stream)
self.insert_and_check(change_stream, docs[2], docs[2])
def test_custom_type_in_pipeline(self):
codecopts = CodecOptions(type_registry=TypeRegistry(
[UndecipherableIntEncoder(),
UppercaseTextDecoder()]))
self.create_targets(codec_options=codecopts)
input_docs = [{
'_id': UndecipherableInt64Type(1),
'data': 'hello'
}, {
'_id': 2,
'data': 'world'
}, {
'_id': UndecipherableInt64Type(3),
'data': '!'
}]
expected_docs = [{'_id': 2, 'data': 'WORLD'}, {'_id': 3, 'data': '!'}]
# UndecipherableInt64Type should be encoded with the TypeRegistry.
change_stream = self.change_stream([{
'$match': {
'documentKey._id': {
'$gte': UndecipherableInt64Type(2)
}
}
}])
self.input_target.insert_one(input_docs[0])
self.insert_and_check(change_stream, input_docs[1], expected_docs[0])
self.kill_change_stream_cursor(change_stream)
self.insert_and_check(change_stream, input_docs[2], expected_docs[1])
def __init__(self, collection: str, model_cls: Type[BaseModel]):
self._db = get_db()
type_registry = TypeRegistry([DecimalCodec()])
codec_options = CodecOptions(type_registry=type_registry)
self._collection = self._db.get_collection(collection, codec_options=codec_options)
self._collection.create_index("id", unique=True)
self._model_cls = model_cls
def test_encode_custom_then_fallback(self):
codecopts = CodecOptions(type_registry=TypeRegistry(
[self.B2A()], fallback_encoder=self.fallback_encoder_A2BSON))
testdoc = {'x': self.TypeB(123)}
expected_bytes = encode({'x': 123})
self.assertEqual(encode(testdoc, codec_options=codecopts),
expected_bytes)
def __init__(self, mongodb_uri):
codec_options = CodecOptions(type_registry=TypeRegistry(
[PickledBinaryDecoder()],
fallback_encoder=fallback_pickle_encoder))
self.client = MongoClient(mongodb_uri)
self.db = self.client['trivicord']
self.collection = self.db.get_collection('games',
codec_options=codec_options)
def _init_collection(
self,
col_name: str,
indexes: Optional[list[IndexModel]] = None) -> Collection:
codecs = CodecOptions(
type_registry=TypeRegistry([c() for c in [DecimalCodec]]))
col = self._database.get_collection(col_name, codecs)
if indexes:
col.create_indexes(indexes)
return col
def test_encode_decode_numpy_data(self):
data = {'array': np.array([1, 2, 3])}
type_registry = TypeRegistry([NumpyArrayCodec()])
codec_options = CodecOptions(type_registry=type_registry)
encoded = BSON.encode(data, codec_options=codec_options)
decoded = BSON.decode(encoded, codec_options=codec_options)
self.assertIsInstance(decoded, type(data))
self.assertIn('array', decoded)
np.testing.assert_array_equal(data['array'], decoded['array'])
def Mongo_init():
decimal_c = DecimalCodec()
date_c = DateCodec()
type_registry = TypeRegistry([decimal_c, date_c])
codec_options = CodecOptions(type_registry=type_registry)
man = MongoManager(codec_options=codec_options,
user="******",
host="cluster0.xjgrr.mongodb.net",
passwd="GWSgnYU4pu7zs2S",
db="DataTracker")
DBManager.store_data("mongo", man)
def get_db_collection(cls,
type_codecs: [CoreEntityCodec],
document_class: MutableMapping = AttributeDict):
"""
Provide a collection with codec options
:return: Collection for the current database connection
"""
db: Database = cls.get_database()
codec_options = CodecOptions(document_class=document_class,
type_registry=TypeRegistry(type_codecs))
return db.get_collection(name=cls._collection_name,
codec_options=codec_options)
def run_test(doc_cls):
db = self.db
input_docs = [
{'x': Int64(k)} for k in [1, 2, 3]]
for doc in input_docs:
db.test.insert_one(doc)
test = db.get_collection('test', codec_options=CodecOptions(
type_registry=TypeRegistry([UndecipherableIntDecoder()]),
document_class=doc_cls))
for doc in test.find({}, batch_size=1):
self.assertIsInstance(doc, doc_cls)
self.assertIsInstance(doc['x'], UndecipherableInt64Type)
def run_test(base, attrs):
msg = (r"TypeEncoders cannot change how built-in types "
r"are encoded \(encoder .* transforms type .*\)")
for pytype in _BUILT_IN_TYPES:
attrs.update({'python_type': pytype,
'transform_python': lambda x: x})
codec = type('testcodec', (base, ), attrs)
codec_instance = codec()
with self.assertRaisesRegex(TypeError, msg):
TypeRegistry([codec_instance,])
# Test only some subtypes as not all can be subclassed.
if pytype in [bool, type(None), RE_TYPE,]:
continue
class MyType(pytype):
pass
attrs.update({'python_type': MyType,
'transform_python': lambda x: x})
codec = type('testcodec', (base, ), attrs)
codec_instance = codec()
with self.assertRaisesRegex(TypeError, msg):
TypeRegistry([codec_instance,])
def get_mongodb_connection(object):
objectid_codec = ObjectIdCodec()
type_registry = TypeRegistry([objectid_codec])
codec_options = CodecOptions(type_registry=type_registry)
username = urllib.parse.quote_plus(object.config['user'])
password = urllib.parse.quote_plus(object.config['psw'])
client = pymongo.MongoClient(
'%s://%s:%s@%s:%s/%s' %
(object.config['type'], username, password, object.config['host'],
object.config['port'], object.config['dbname']))
db = client[object.config['dbname']]
collection = db.get_collection(object.config['collection'],
codec_options=codec_options)
return collection
def run_test(doc_cls):
codecopts = CodecOptions(type_registry=TypeRegistry([
UppercaseTextDecoder(), UndecipherableIntEncoder()]),
document_class=doc_cls)
self.create_targets(codec_options=codecopts)
change_stream = self.change_stream()
doc = {'a': UndecipherableInt64Type(101), 'b': 'xyz'}
self.input_target.insert_one(doc)
change = next(change_stream)
self.assertIsInstance(change, doc_cls)
self.assertEqual(change['fullDocument']['a'], 101)
self.assertEqual(change['fullDocument']['b'], 'XYZ')
def __init__(self, db_name):
loop = asyncio.get_event_loop()
client = motor_asyncio.AsyncIOMotorClient(
"mongodb://127.0.0.1:27017/", io_loop=loop
)
log.info("Connecting to mongodb database")
self.db = client.get_database(
db_name,
codec_options=CodecOptions(
type_registry=TypeRegistry(
fallback_encoder=lambda obj: Binary(bytes(obj))
if isinstance(obj, Streamable)
else obj
)
),
)
log.info("Connected to mongodb database")
def test_simple(self):
codecopts = CodecOptions(type_registry=TypeRegistry(
[UndecipherableIntEncoder(),
UppercaseTextDecoder()]))
self.create_targets(codec_options=codecopts)
input_docs = [
{
'_id': UndecipherableInt64Type(1),
'data': 'hello'
},
{
'_id': 2,
'data': 'world'
},
{
'_id': UndecipherableInt64Type(3),
'data': '!'
},
]
expected_docs = [
{
'_id': 1,
'data': 'HELLO'
},
{
'_id': 2,
'data': 'WORLD'
},
{
'_id': 3,
'data': '!'
},
]
change_stream = self.change_stream()
self.insert_and_check(change_stream, input_docs[0], expected_docs[0])
self.kill_change_stream_cursor(change_stream)
self.insert_and_check(change_stream, input_docs[1], expected_docs[1])
self.kill_change_stream_cursor(change_stream)
self.insert_and_check(change_stream, input_docs[2], expected_docs[2])
def __init__(
self,
model_class: Type[T],
database: Database,
col_name: str,
indexes: Optional[list[Union[IndexModel, str]]] = None,
wrap_object_str_id=True,
):
codecs = CodecOptions(
type_registry=TypeRegistry([c() for c in [DecimalCodec]]))
self.collection = database.get_collection(col_name, codecs)
if indexes:
indexes = [
parse_str_index_model(i) if isinstance(i, str) else i
for i in indexes
]
self.collection.create_indexes(indexes)
self.model_class = model_class
self.wrap_object_id = model_class.__fields__[
"id"].type_ == ObjectIdStr and wrap_object_str_id
def test_simple_separate_codecs(self):
class MyIntEncoder(TypeEncoder):
python_type = self.types[0]
def transform_python(self, value):
return value.x
class MyIntDecoder(TypeDecoder):
bson_type = int
def transform_bson(self, value):
return self.types[0](value)
codec_instances = [MyIntDecoder(), MyIntEncoder()]
type_registry = TypeRegistry(codec_instances)
self.assertEqual(
type_registry._encoder_map,
{MyIntEncoder.python_type: codec_instances[1].transform_python})
self.assertEqual(
type_registry._decoder_map,
{MyIntDecoder.bson_type: codec_instances[0].transform_bson})
def get_names():
## Initializing the mongo connection
connection_string = 'mongodb+srv://Zeevtest:[email protected]/test?retryWrites=true&w=majority'
from bson.decimal128 import Decimal128
from bson.codec_options import TypeCodec
class DecimalCodec(TypeCodec):
python_type = Decimal # the Python type acted upon by this type codec
bson_type = Decimal128 # the BSON type acted upon by this type codec
def transform_python(self, value):
"""Function that transforms a custom type value into a type
that BSON can encode."""
return Decimal128(value)
def transform_bson(self, value):
"""Function that transforms a vanilla BSON type value into our
custom type."""
return value.to_decimal()
decimal_codec = DecimalCodec()
today = date.today()
cutoff_date = today - timedelta(days=70)
from bson.codec_options import TypeRegistry
type_registry = TypeRegistry([decimal_codec])
from bson.codec_options import CodecOptions
codec_options = CodecOptions(type_registry=type_registry)
mongo_client = pymongo.MongoClient(connection_string)
mydb = mongo_client["patents"]
db = mongo_client["patents"]
examiners_collection = mydb.get_collection("examiners_new",
codec_options=codec_options)
return jsonify([(i['examiner']) for i in examiners_collection.find()])
def __init__(self,
name: str,
host: str = 'localhost',
port: int = 27017,
database: str = '',
serializer: Union[Serializer, None] = None,
connection_timeout: float = 30000) -> None:
"""MongoDB implementation of storage class
See also: `StorageInterface`
Args:
name: Symbolic name for the storage instance.
host: MongoDB host
port: MongoDB port
database: The database to use, if empty the name of the storage is used
connection_timeout: How long to try to connect to database before raising an error in milliseconds
Raises:
StorageTimeoutError: If connection to database has not been established before connection_timeout is reached
"""
super().__init__(name)
type_registry = TypeRegistry([NumpyArrayCodec()])
codec_options = CodecOptions(type_registry=type_registry)
self._client = MongoClient(host,
port,
serverSelectionTimeoutMS=connection_timeout)
self._check_server_connection(connection_timeout)
self._db = self._client.get_database(database or name,
codec_options=codec_options)
self._collection = self._db.get_collection('storage')
if serializer is None:
serializer = _serializer
self._serialize = serializer.encode_data
self._unserialize = serializer.decode_data
from binance.websockets import BinanceSocketManager
from bson import CodecOptions
from bson.codec_options import TypeRegistry
from kucoin.exceptions import KucoinAPIException
from pymongo.errors import PyMongoError
from library import get_binance_klines, get_binance_interval_unit, setup_logger, binance_obj, kucoin_client, \
DecimalCodec, try_get_klines, TradeMsg, get_last_db_record, \
get_time_from_binance_tmstmp, from_binance_socket_kline
from mongodb import mongo_client
logger = setup_logger("Kline-Crawl-Manager-LTF")
db = mongo_client.klines
decimal_codec = DecimalCodec()
type_registry = TypeRegistry([decimal_codec])
codec_options = CodecOptions(type_registry=type_registry)
trades = {}
klines_socket = {}
def to_mongo_binance(_kline):
_data = to_mongo(_kline)
_data['buy_btc_volume'] = _kline.buy_btc_volume
_data['buy_quantity'] = _kline.buy_quantity
_data['sell_btc_volume'] = _kline.sell_btc_volume
_data['sell_quantity'] = _kline.sell_quantity
return _data
def test_chaining_encoders_fails(self):
codecopts = CodecOptions(
type_registry=TypeRegistry([self.A2B(), self.B2BSON()]))
with self.assertRaises(InvalidDocument):
encode({'x': self.TypeA(123)}, codec_options=codecopts)
return value.isoformat()
def transform_bson(self, value: str):
return arrow.get(value).datetime
class DateCodec(TypeCodec):
python_type = datetime.date
bson_type = string_type
def transform_python(self, value: datetime.date):
return value.isoformat()
def transform_bson(self, value: str):
return value
class TimeCodec(TypeCodec):
python_type = datetime.time
bson_type = string_type
def transform_python(self, value: datetime.time):
return value.isoformat()
def transform_bson(self, value: str):
return value
type_registry = TypeRegistry([DateCodec(), TimeCodec(), ArrowCodec()])
codec_options = CodecOptions(type_registry=type_registry)
class NumpyCodec(TypeCodec):
sub_type = USER_DEFINED_SUBTYPE + 1
python_type = np.ndarray
bson_type = Binary
def transform_python(self, a: np.ndarray):
data = encode_ndarray(a)
return Binary(data, NumpyCodec.sub_type)
def transform_bson(self, data: Binary):
if data.subtype == NumpyCodec.sub_type:
return decode_ndarray(data)
return data
CODEC_OPTIONS = CodecOptions(type_registry=TypeRegistry([NumpyCodec()]))
class BSONTarWriter(object):
def __init__(self, path: str):
self._path = path
self._impl = TarWriter(path)
self._meta_doc = {}
def close(self):
if self._meta_doc is not None:
data = BSON.encode(self._meta_doc, codec_options=CODEC_OPTIONS)
self._impl.write('meta.bson', data)
self._meta_doc = None
self._impl.close()
def hello_world(examiner_name):
## Initializing the mongo connection
connection_string = 'mongodb+srv://Zeevtest:[email protected]/test?retryWrites=true&w=majority'
from bson.decimal128 import Decimal128
from bson.codec_options import TypeCodec
class DecimalCodec(TypeCodec):
python_type = Decimal # the Python type acted upon by this type codec
bson_type = Decimal128 # the BSON type acted upon by this type codec
def transform_python(self, value):
"""Function that transforms a custom type value into a type
that BSON can encode."""
return Decimal128(value)
def transform_bson(self, value):
"""Function that transforms a vanilla BSON type value into our
custom type."""
return value.to_decimal()
decimal_codec = DecimalCodec()
today = date.today()
cutoff_date = today - timedelta(days=70)
from bson.codec_options import TypeRegistry
type_registry = TypeRegistry([decimal_codec])
from bson.codec_options import CodecOptions
codec_options = CodecOptions(type_registry=type_registry)
mongo_client = pymongo.MongoClient(connection_string)
mydb = mongo_client["patents"]
db = mongo_client["patents"]
examiners_collection = mydb.get_collection("examiners_new",
codec_options=codec_options)
print(examiner_name)
examiner_record = examiners_collection.find_one(
{'examiner': examiner_name})
if examiner_record is None:
print("no examiner record")
else:
if "total_refused" in examiner_record:
examiner_apps_refused = examiner_record.get("total_refused")
else:
examiner_apps_refused = 0
if "total_refused_with_interview" in examiner_record:
examiner_apps_refused_with_interview = examiner_record.get(
"total_refused_with_interview")
else:
examiner_apps_refused_with_interview = 0
if "total_refused_without_interview" in examiner_record:
examiner_apps_refused_without_interview = examiner_record.get(
"total_refused_without_interview")
else:
examiner_apps_refused_without_interview = 0
if "total_granted" in examiner_record:
examiner_apps_granted = examiner_record.get("total_granted")
else:
examiner_apps_granted = 0
if "total_granted_with_interview" in examiner_record:
examiner_apps_granted_with_interview = examiner_record.get(
"total_granted_with_interview")
else:
examiner_apps_granted_with_interview = 0
if "total_granted_without_interview" in examiner_record:
examiner_apps_granted_without_interview = examiner_record.get(
"total_granted_without_interview")
else:
examiner_apps_granted_without_interview = 0
examiner_apps_we_have = examiner_apps_granted + examiner_apps_refused
examiner_grant_rate = "%.0f%%" % (100 * examiner_apps_granted /
examiner_apps_we_have)
examiner_grant_rate_with_interview = "%.0f%%" % (
100 * examiner_apps_granted_with_interview /
(examiner_apps_granted_with_interview +
examiner_apps_refused_with_interview))
examiner_grant_rate_without_interview = "%.0f%%" % (
100 * examiner_apps_granted_without_interview /
(examiner_apps_granted_without_interview +
examiner_apps_refused_without_interview))
interview_improvement_rate = "%.0f%%" % (
100 * (examiner_apps_granted_with_interview /
(examiner_apps_granted_with_interview +
examiner_apps_refused_with_interview)) /
(examiner_apps_granted_without_interview /
(examiner_apps_granted_without_interview +
examiner_apps_refused_without_interview)) - 100)
# Here comes the monthly data:
successful_month_retrieved = examiner_record.get("successful_month")
failed_month_retrieved = examiner_record.get("failed_month")
successful_month = []
failed_month = []
month_stat = []
#possible_response_month = []
total_successful_responses = 0
total_failed_responses = 0
for i in range(1, 14):
successful_month.append(0)
failed_month.append(0)
month_stat.append(0)
#possible_response_month.append(0)
try:
for key, value in successful_month_retrieved.items():
key = int(key)
successful_month[key] = value
total_successful_responses = total_successful_responses + value
except AttributeError:
print("no successful month data")
total_successful_responses = 0.00000000000000000000000001
try:
for key, value in failed_month_retrieved.items():
key = int(key)
failed_month[key] = value
total_failed_responses = total_failed_responses + value
except AttributeError:
print("no failed month data")
#print ("monthly statistics, month by month")
for i in range(0, 13):
if (failed_month[i] != 0):
month_stat[i] = successful_month[i] / (successful_month[i] +
failed_month[i])
#month_stat[i] = "%.0f%%" % (100 * month_stat[i])
#if (i !=0):
#print ("Month",str(i),": ",month_stat[i], " ")
response_success_rate = total_successful_responses / (
total_successful_responses + total_failed_responses)
# Run through the 6 months including and after the one where the office action was issued:
recommended_month_stat = 0
recommended_month_stat = "%.0f%%" % (100 * recommended_month_stat)
response_success_rate = "%.0f%%" % (100 * response_success_rate)
# Here ends the monthly data.
reporting_text = "<br><br>Hi there! The examiner's name is: "
reporting_text = reporting_text + examiner_name
reporting_text = reporting_text + "<br><br>We crunched through "
reporting_text = reporting_text + str(examiner_apps_we_have)
reporting_text = reporting_text + " applications for this examiner and here is what we can tell you:<br><br>"
reporting_text = reporting_text + "Grant rate (chances to eventually reach a grant): "
reporting_text = reporting_text + str(
examiner_grant_rate) + "with interview: " + str(
examiner_grant_rate_with_interview
) + " / without interview: " + str(
examiner_grant_rate_without_interview
) + " improvement rate: " + str(interview_improvement_rate)
reporting_text = reporting_text + "<br>Response success rate (chances to overcome one office action): "
reporting_text = reporting_text + str(response_success_rate)
reporting_text = reporting_text + "<br><br>Full monthly stats: "
months = {}
for i in range(0, 13):
if (failed_month[i] != 0):
month_stat[i] = successful_month[i] / (successful_month[i] +
failed_month[i])
month_stat[i] = "%.0f%%" % (100 * month_stat[i])
if (i != 0):
reporting_text = reporting_text + str(
get_month_text(i)) + ": " + month_stat[i] + " | "
months[get_month_text(i)] = month_stat[i]
#print (reporting_text)
# Now is the time to check if the office action matches any of our pre-tracked queries
# and if yes, to fire it up.
# starting from checking if the applicant is in our tracked applicant's list.
sender_email = "*****@*****.**"
receiver_email = "*****@*****.**"
password = "******"
# Create the plain-text and HTML version of your message
text = reporting_text
html = "<html><body>" + reporting_text + "</body></html>"
result = {}
result['examiner_name'] = examiner_name
result['examiner_apps_we_have'] = examiner_apps_we_have
result['examiner_grant_rate'] = examiner_grant_rate
result[
'examiner_grant_rate_with_interview'] = examiner_grant_rate_with_interview
result[
'examiner_grant_rate_without_interview'] = examiner_grant_rate_without_interview
result['response_success_rate'] = response_success_rate
result['interview_improvement_rate'] = interview_improvement_rate
result['months'] = months
return (result)
def get_codec_options():
type_registry = tr_flags
type_registry.append(ColorMongoEncoder())
return CodecOptions(type_registry=TypeRegistry(type_registry))
def test_type_registry_repr(self):
codec_instances = [codec() for codec in self.codecs]
type_registry = TypeRegistry(codec_instances)
r = ("TypeRegistry(type_codecs=%r, fallback_encoder=%r)" %
(codec_instances, None))
self.assertEqual(r, repr(type_registry))
