def walk(self, **kwargs):
address = kwargs["address"]
profile = kwargs.get("profile", [])
if profile:
profile = [profile]
mongo_client = pymongo.MongoClient(MONGO_URI)
mongo_db = mongo_client[MONGO_DB]
mongo_inventory = mongo_db.inventory
mongo_targets = mongo_db.targets
walked_first_time = mongo_targets.find_one({"address": address})
logger.warning(walked_first_time)
lock = MongoLock(client=mongo_client, db="sc4snmp")
with lock(address, self.request.id, expire=300, timeout=300):
ir = get_inventory(mongo_inventory, address)
retry = True
while retry:
retry, result = self.do_work(ir,
walk=True,
profiles=profile,
walked_first_time=walked_first_time)
# After a Walk tell schedule to recalc
work = {}
work["time"] = time.time()
work["address"] = address
work["result"] = result
return work
def test_delete_birthdays_should_do_nothing_when_no_birthdays(self):
db = test_utils.get_fake_db()
lock = MongoLock(client=db.client, db='db')
patch_citizen_handler._delete_birthdays_data(
0, {'birth_date': datetime(2019, 1, 1)}, lock, db, None)
count = db['birthdays'].count_documents({'import_id': 0})
self.assertEqual(0, count)
def test_decorator_should_not_call_func_when_cached_data_present(self):
self.db['cache'].insert_one({'import_id': 0, 'test': 'aaa'})
f = MagicMock()
lock = MongoLock(client=self.db.client, db=self.db.name)
decorator = response_cacher.cache_response('cache', self.db, lock)
wrap = decorator(f)
wrap(import_id=0)
f.assert_not_called()
def test_delete_percentile_age_should_delete_when_town_in_patch(self):
db = test_utils.get_fake_db()
lock = MongoLock(client=db.client, db='db')
db['percentile_age'].insert_one({'import_id': 0})
patch_citizen_handler._delete_percentile_age_data(
0, {'town': 'A'}, lock, db, None)
count = db['percentile_age'].count_documents({'import_id': 0})
self.assertEqual(0, count)
def test_delete_birthdays_should_delete_when_relatives_in_patch(self):
db = test_utils.get_fake_db()
lock = MongoLock(client=db.client, db='db')
db['birthdays'].insert_one({'import_id': 0})
patch_citizen_handler._delete_birthdays_data(0, {'relatives': []},
lock, db, None)
count = db['birthdays'].count_documents({'import_id': 0})
self.assertEqual(0, count)
def test_decorator_should_return_cached_data_when_present(self):
lock = MongoLock(client=self.db.client, db=self.db.name)
@response_cacher.cache_response('cache', self.db, lock)
def f(import_id: int):
pass
self.db['cache'].insert_one({'import_id': 0, 'test': 'aaa'})
response: Response = f(import_id=0)
self.assertEqual(201, response.status_code)
self.assertEqual({'test': 'aaa'}, response.json)
def test_decorator_should_not_cache_when_exception_in_func(self):
lock = MongoLock(client=self.db.client, db=self.db.name)
@response_cacher.cache_response('cache', self.db, lock)
def f(import_id: int):
raise ValueError()
with mock.patch('application.decorators.response_cacher._cache_data'
) as cache_mock:
try:
f(import_id=0)
except ValueError:
pass
cache_mock.assert_not_called()
def test_decorator_should_cache_data_when_not_cache_not_present(self):
lock = MongoLock(client=self.db.client, db=self.db.name)
@response_cacher.cache_response('cache', self.db, lock)
def f(import_id: int):
return Response(json.dumps({
'import_id': import_id,
'test': 'aaa'
},
ensure_ascii=False),
201,
mimetype='application/json; charset=utf-8')
with mock.patch('application.decorators.response_cacher._cache_data'
) as cache_mock:
f(import_id=0)
cache_mock.assert_called()
def test_decorator_should_return_function_response_not_cache_not_present(
self):
lock = MongoLock(client=self.db.client, db=self.db.name)
@response_cacher.cache_response('cache', self.db, lock)
def f(import_id: int):
return Response(json.dumps({
'import_id': import_id,
'test': 'aaa'
},
ensure_ascii=False),
201,
mimetype='application/json; charset=utf-8')
response = f(import_id=0)
self.assertEqual(201, response.status_code)
self.assertEqual({'import_id': 0, 'test': 'aaa'}, response.json)
def poll(self, **kwargs):
address = kwargs["address"]
profiles = kwargs["profiles"]
mongo_client = pymongo.MongoClient(MONGO_URI)
mongo_db = mongo_client[MONGO_DB]
mongo_inventory = mongo_db.inventory
lock = MongoLock(client=mongo_client, db="sc4snmp")
with lock(kwargs["address"], self.request.id, expire=90, timeout=20):
ir = get_inventory(mongo_inventory, address)
_, result = self.do_work(ir, profiles=profiles)
# After a Walk tell schedule to recalc
work = {}
work["time"] = time.time()
work["address"] = address
work["result"] = result
work["detectchange"] = False
work["frequency"] = kwargs["frequency"]
return work
def test_create_lock_by_collection():
connection[db_name][col_name].remove()
collection = connection[db_name][col_name]
assert MongoLock(collection=collection).lock('key', 'owner')
def lock():
connection[db_name][col_name].remove()
return MongoLock(client=connection, db=db_name, collection=col_name)
import os
from mongolock import MongoLock
from application.data_validator import DataValidator
from application.custom_mongo_client import CustomMongoClient
from application.service import make_app
db_uri = os.environ['DATABASE_URI']
port = int(os.environ['DATABASE_PORT'])
db_name = os.environ['DATABASE_NAME']
replica_set = os.environ['REPLICA_SET']
client = CustomMongoClient(db_uri, port, replica_set)
lock = MongoLock(client=client, db=db_name)
with lock('indexes', str(os.getpid()), timeout=60, expire=10):
client.create_db_indexes(db_name)
db = client[db_name]
data_validator = DataValidator()
app = make_app(db, data_validator, lock)
if __name__ == '__main__':
app.run()
def _configure_mongolock(app):
from . import metrics
from . import models
from mongolock import MongoLock
metrics.lock = MongoLock(client=models.MessageStore._get_db().connection)