def test_env_resources():
dev_resources: SectionProxy = ResourceManager(env="dev").get_resources(
section="ENVIRONMENT")
prod_resources: SectionProxy = ResourceManager(env="prod").get_resources(
section="ENVIRONMENT")
assert dev_resources["ENV"] != prod_resources["ENV"]
def main_compute_insert_md5_and_gray(env: str,
loglevel: str = "DEBUG",
overwrite: bool = True):
"""
This function will create the database if not exists and two collection in a mongoDB database. Then it will
get all images from the original images collection and it will compute the md5 and the gray level of each image.
All information about transformation state will be stored in one monitoring collection containing url, state and
timestamp, and a second target collection containing md5, new image, size and insertion timestamp.
:param env: Environment that will load the correct property file
:param loglevel: Logging level for message generation
:param overwrite: Truncate the collection or not
"""
resources: SectionProxy = ResourceManager(env=env).get_resources()
db_resources: SectionProxy = ResourceManager(env=env).get_resources(
section="MONGODB")
logger: Logger = LoggerManager(logger_name="IMAGE",
log_folder=resources["LOG_DIR"],
log_level=loglevel).logger
db_logger: Logger = LoggerManager(logger_name="MONGO",
log_folder=resources["LOG_DIR"],
log_level=loglevel).logger
main_logger: Logger = LoggerManager(logger_name="MAIN",
log_folder=resources["LOG_DIR"],
log_level=loglevel).logger
image_manager = ImageManager(resources=resources, logger=logger)
mongo_manager = MongoManager(resources=db_resources, logger=db_logger)
mongo_manager.init_database()
original_collection: Collection = mongo_manager.get_collection(
collection_name="original_images")
target_collection: Collection = mongo_manager.get_collection(
collection_name="target_images",
index_collection="md5",
overwrite=overwrite)
monitoring_collection: Collection = mongo_manager.get_collection(
collection_name="monitoring", overwrite=overwrite)
image_documents = original_collection.find()
documents, errors = image_manager.collect_gray_and_md5(
documents=image_documents)
main_logger.info("Inserting {} documents...".format(len(documents)))
for document in documents:
main_logger.debug("Inserting document md5: {}, size:{}...".format(
document["md5"], document["size"]))
mongo_manager.insert_one_document(collection=target_collection,
document=document)
main_logger.info("Inserting {} monitoring messages...".format(len(errors)))
state, _ = mongo_manager.insert_many_documents(
collection=monitoring_collection, documents=errors)
main_logger.info("Final state for monitoring insert: {}".format(state))
def main_collect_insert_images(env: str,
imgfile: str,
loglevel: str = "DEBUG",
overwrite: bool = True):
"""
This function will create the database if not exists and a collection in a mongoDB database. Then it will
download all images from the image file provided, they will be wrote on file system of the Docker.
Finally all images will be stored inside the collection without index, with their url, image shape,
the state of download and the insertion timestamp.
:param env: Environment that will load the correct property file
:param imgfile: Path to the image file containing URLs
:param loglevel: Logging level for message generation
:param overwrite: Truncate the collection or not
"""
resources: SectionProxy = ResourceManager(env=env).get_resources()
db_resources: SectionProxy = ResourceManager(env=env).get_resources(
section="MONGODB")
logger: Logger = LoggerManager(logger_name="IMAGE",
log_folder=resources["LOG_DIR"],
log_level=loglevel).logger
db_logger: Logger = LoggerManager(logger_name="MONGO",
log_folder=resources["LOG_DIR"],
log_level=loglevel).logger
main_logger: Logger = LoggerManager(logger_name="MAIN",
log_folder=resources["LOG_DIR"],
log_level=loglevel).logger
image_file_path: str = imgfile if imgfile != "" else os.path.join(
resources["DATA_DIR"], "urls.txt")
image_manager = ImageManager(resources=resources,
img_url_file=image_file_path,
logger=logger)
mongo_manager = MongoManager(resources=db_resources, logger=db_logger)
mongo_manager.init_database()
collection: Collection = mongo_manager.get_collection(
collection_name="original_images", overwrite=overwrite)
documents: list = image_manager.collect_images()
state, result_ids = mongo_manager.insert_many_documents(
collection=collection, documents=documents)
main_logger.info("Insertion state: {} with {} insertion".format(
state, len(result_ids)))
main_logger.debug("Ids inserted: {}".format(result_ids))
def test_read_image():
resources: SectionProxy = ResourceManager(env="dev").get_resources(
section="DEFAULT")
logger: Logger = LoggerManager(logger_name="TEST",
log_level="DEBUG").logger
image_manager = ImageManager(resources=resources, logger=logger)
image = image_manager.read_image(
os.path.join(resources["TEST_DIR"], "original_0.jpg"))
assert image is not None
def main_init_directory_structure(env: str):
"""
This function will create all directory needed for the project
:param env: Environment that will load the correct property file
"""
resources: SectionProxy = ResourceManager(env=env).get_resources()
logger: Logger = LoggerManager(logger_name="DIRECTORY",
log_folder=resources["LOG_DIR"],
log_level="DEBUG").logger
init_directory_structure(resources=resources, logger=logger)
def test_mongodb():
resources: SectionProxy = ResourceManager(env="dev").get_resources(
section="MONGODB")
logger: Logger = LoggerManager(logger_name="TEST",
log_folder=resources["LOG_DIR"],
log_level="DEBUG").logger
mongo_manager = MongoManager(resources=resources, logger=logger)
database = mongo_manager.init_database()
assert database == resources["NAME"]
def test_gray():
resources: SectionProxy = ResourceManager(env="dev").get_resources(
section="DEFAULT")
logger: Logger = LoggerManager(logger_name="TEST",
log_folder=resources["LOG_DIR"],
log_level="DEBUG").logger
image_manager = ImageManager(resources=resources, logger=logger)
image: ndarray = image_manager.read_image(
os.path.join(resources["TEST_DIR"], "original_0.jpg"))
_, gray_image = image_manager.compute_gray_level(image)
assert (gray_image == divide(image, 3)).any()
def test_md5():
resources: SectionProxy = ResourceManager(env="dev").get_resources(
section="DEFAULT")
logger: Logger = LoggerManager(logger_name="TEST",
log_folder=resources["LOG_DIR"],
log_level="DEBUG").logger
image_manager = ImageManager(resources=resources, logger=logger)
image: ndarray = image_manager.read_image(
os.path.join(resources["TEST_DIR"], "original_0.jpg"))
encoded_image = image_manager.encode_image(image)
_, md5 = image_manager.compute_md5(encoded_image)
assert len(md5) == 32
def test_find():
resources: SectionProxy = ResourceManager(env="dev").get_resources(
section="MONGODB")
logger: Logger = LoggerManager(logger_name="TEST",
log_folder=resources["LOG_DIR"],
log_level="INFO").logger
mongo_manager = MongoManager(resources=resources, logger=logger)
mongo_manager.init_database()
collection: Collection = mongo_manager.get_collection(
collection_name="unit_test")
inserted_object = collection.find_one({"name": "unit_test"})
assert inserted_object["number"] == 35
def test_download_image():
url = "https://picsum.photos/259/260"
resources: SectionProxy = ResourceManager(env="dev").get_resources(
section="DEFAULT")
logger: Logger = LoggerManager(logger_name="TEST",
log_folder=resources["LOG_DIR"],
log_level="DEBUG").logger
image_manager = ImageManager(resources=resources, logger=logger)
_, _, shape = image_manager.download_image(image_link=url,
index=0,
key_folder="TEST_DIR")
assert shape == (260, 259, 3)
def test_insertion():
resources: SectionProxy = ResourceManager(env="dev").get_resources(
section="MONGODB")
logger: Logger = LoggerManager(logger_name="TEST",
log_folder=resources["LOG_DIR"],
log_level="INFO").logger
mongo_manager = MongoManager(resources=resources, logger=logger)
mongo_manager.init_database()
collection: Collection = mongo_manager.get_collection(
collection_name="unit_test", overwrite=True)
document = {"name": "unit_test", "number": 35}
state, _ = mongo_manager.insert_one_document(collection=collection,
document=document)
assert state == "success"
def test_empty_resources():
resources: SectionProxy = ResourceManager(env="dev").get_resources(
section="ENVIRONMENT")
assert resources["ENV"] is not None