def test_ItemCache(self):
db_context = DbContext(connection)
m = ModelContext(foo, db_context)
x = ItemCache(m, TestOperations())
for i in range(998):
x.insert({'name': 'John Doe #%s' % i})
self.assertEqual(x.item_counter, 998)
for i in range(5):
x.insert({'name': 'John Doe #%s' % i})
self.assertEqual(x.item_counter, 4)
def main():
feeds_file_input = os.path.abspath('config/feed_list.json')
db_config_file = os.path.abspath('config/db_config.json')
config_handler = ConfigHandler(feeds_file_input, db_config_file)
uri, db_name, feeds_name_collection, feeds_collection, image_collection = config_handler.get_db_config(
)
db_context = DbContext(uri, db_name, feeds_name_collection,
feeds_collection, image_collection)
Logger.log('reading {0} ...'.format(feeds_file_input))
feed_list, feed_list_jsondata = config_handler.load_feed_list()
Logger.log('collecting from {0} feeds'.format(len(feed_list)))
Logger.log('inserting the feed list into the database...')
for feed in feed_list_jsondata:
db_context.feeds_name_collection.update_one({'url': feed['url']},
{'$set': feed},
upsert=True)
images_path_file = 'config/image_collection.json'
images_path = config_handler.load_image_collection_path(images_path_file)
scraper = Scraper(feed_list, images_path)
entries = scraper.get_entries()
# get the metadata in interest and the images
with Pool() as pool:
metadata = pool.map(scraper.get_metadata, entries)
Logger.log('inserting metadata into the database...')
for feed_data in metadata:
db_context.feeds_collection.update_one({'link': feed_data['link']},
{'$set': feed_data},
upsert=True)
#db_context.feeds_collection.update_many(metadata, {'$set': metadata}, upsert=True)
metadata_number = db_context.feeds_collection.find({}).count()
Logger.log('{0} metadata inserted'.format(metadata_number))
Logger.log('creating indexes...')
# compound index
db_context.feeds_collection.create_index([('title', pymongo.TEXT),
('summary', pymongo.TEXT)],
default_language='english',
name='title_summary_index')
db_context.feeds_collection.create_index([("image_path", pymongo.ASCENDING)
])
Logger.log('downloading the images...')
scraper.download_images(metadata, images_path)
Logger.log('inserting image collection path into the database...')
full_img_path = scraper.download_dir
data = {'path': full_img_path}
db_context.image_collection.update_one(data, {'$set': data}, upsert=True)
Logger.log('all done.\n')
class testReception(unittest.TestCase):
DBCONTEXT = DbContext("reception_test.db")
def setUp(self):
if self.get_user("user1") is None:
self.DBCONTEXT.create_table(self.DBCONTEXT.sql_create_users_table)
self.DBCONTEXT.insert_into_table(
User.REGISTER_USER_SQL,
("user1", "11111111", "User1", "Test1", "*****@*****.**"))
self.DBCONTEXT.insert_into_table(
User.REGISTER_USER_SQL,
("user2", "22222222", "User2", "Test2", "*****@*****.**"))
self.DBCONTEXT.insert_into_table(
User.REGISTER_USER_SQL,
("user3", "33333333", "User3", "Test3", "*****@*****.**"))
def get_user(self, username):
return self.DBCONTEXT.query_from_table(User.GET_USER_SQL, (username, ))
def test_user_exist(self):
self.assertTrue(self.get_user("user1") is not None)
self.assertTrue(self.get_user("user2") is not None)
self.assertTrue(self.get_user("user3") is not None)
def test_login(self):
self.assertTrue(User.login("user1", reception.ADDRESS, "11111111"))
self.assertTrue(User.login("user2", reception.ADDRESS, "22222222"))
self.assertFalse(User.login("user2", reception.ADDRESS, "11111111"))
self.assertFalse(User.login("user1", reception.ADDRESS, "22222222"))
def test_register_user(self):
if self.get_user("user4") is None:
User.register("user4", "44444444", "User4", "Test4",
"*****@*****.**")
User.register("user5", "55555555", "User5", "Test5",
"*****@*****.**")
self.assertTrue(self.get_user("user4") is not None)
self.assertTrue(self.get_user("user5") is not None)
def test_values_factory_for_model(self):
db_context = DbContext(connection)
model_cnxt = ModelContext(foo, db_context)
self.assertTrue(model_cnxt.default_values['sex'] == "M")
self.assertTrue(model_cnxt.default_values['age'] == 20)
self.assertTrue(model_cnxt.default_values.get('id', None) == None)
def test_model_context1(self):
db_context = DbContext(connection)
m = ModelContext(foo, db_context)
def __init__(self, feeds_file_input, db_config_file):
self.config_handler = ConfigHandler(feeds_file_input, db_config_file)
URI, db_name, feeds_name_collection, feeds_collection, image_collection = self.config_handler.get_db_config(
)
self.db_context = DbContext(URI, db_name, feeds_name_collection,
feeds_collection, image_collection)
class User:
EMAIL_REGEX = re.compile(r"[^@]+@[^@]+\.[^@]+")
NAME_REGEX = re.compile(r"[a-zA-Z0-9]+")
PASSWORD_REGEX = re.compile(r"[A-Za-z0-9@#$%^&+=]{8,}")
REGISTER_USER_SQL = 'INSERT INTO Users (username, password, first_name, last_name, email) VALUES(?,?,?,?,?);'
GET_USER_SQL = 'SELECT * FROM Users WHERE username = ?'
DBCONTEXT = DbContext("reception.db")
def __init__(self) -> None:
self._username = ''
self._password = ''
self._first_name = ''
self._last_name = ''
self._email = ''
username = property(operator.attrgetter('_username'))
email = property(operator.attrgetter('_email'))
last_name = property(operator.attrgetter('_last_name'))
first_name = property(operator.attrgetter('_first_name'))
password = property(operator.attrgetter('_password'))
@username.setter
def username(self, username: str):
if not username:
raise Exception("Username cannot be empty")
if not self.NAME_REGEX.fullmatch(username):
raise Exception("Username can only contain alphanumeric characters")
self._username = username
@password.setter
def password(self, password: str):
if not password:
raise Exception("Password cannot be empty")
if not self.PASSWORD_REGEX.fullmatch(password):
raise Exception("Password has to be at-least 8 characters and can contain alphabets, "
"numbers and special characters")
self._password = sha256_crypt.hash(password)
@first_name.setter
def first_name(self, first_name: str):
if not first_name:
raise Exception("First name cannot be empty")
if not self.NAME_REGEX.fullmatch(first_name):
raise Exception("First name can only contain alphanumeric characters")
self._first_name = first_name
@last_name.setter
def last_name(self, last_name: str):
if not last_name:
raise Exception("Last name cannot be empty")
if not self.NAME_REGEX.fullmatch(last_name):
raise Exception("Last name can only contain alphanumeric characters")
self._last_name = last_name
@email.setter
def email(self, email: str):
if not email:
raise Exception("Email cannot be empty")
if not self.EMAIL_REGEX.fullmatch(email):
raise Exception("Invalid email address")
self._email = email
def register(self, username: str, password: str, first_name: str, last_name: str, email: str):
self.last_name = last_name
self.first_name = first_name
self.password = password
self.email = email
self.username = username
try:
self.DBCONTEXT.insert_into_table(self.REGISTER_USER_SQL,
(self.username, self.password, self.first_name, self.last_name, self.email))
except Error:
raise Exception("The username exists and cannot be created")
def login(self, username: str, address, face_login: bool = False, password: str = "") -> bool:
self.username = username
try:
users = self.DBCONTEXT.query_from_table(self.GET_USER_SQL, (self.username,))
if len(users) == 0:
return False
if not sha256_crypt.verify(password, users[0][1]) and not face_login:
return False
self.first_name = users[0][2]
self.email = users[0][4]
self.last_name = users[0][3]
self.connect_to_master_pi(address)
return True
except Error as e:
print(str(e))
return False
def to_json(self):
return json.dumps(self, default=lambda o: o.__dict__,
sort_keys=True, indent=4)
def connect_to_master_pi(self, address):
user = self.to_json()
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
print("Connecting to {}...".format(address))
s.connect(address)
print("Connected.")
print("Logging in as {}".format(user))
socket_utils.send_json(s, user)
print("Waiting for Master Pi...")
while True:
object_response = socket_utils.recv_json(s)
if "logout" in object_response:
print("Master Pi logged out.")
print()
break