def __init__(self, source=None, destination=None, file_name=''):
self.source = f"/{source}/"
self.file_name = ''
self.destination = f"/{destination}/"
self.file = "test.tiff"
self.config = Config()
self.watcher = OnWatchFile()
self.watcher.connect_directory_monitoring()
basedir = os.path.abspath(os.path.dirname(__file__))
self.basedir_files_tiff = os.path.join(basedir, "files_test/file.tiff")
self.basedir_files_xml = os.path.join(basedir, "files_test/file.xml")
def test_connect_directory_monitoring_exception(self):
"""Test connect directory monitoring Exception """
watcher = OnWatchFile()
watcher.smb_server_name = 'localhost'
watcher.password = ''
watcher.connected = False
with self.assertRaises(Exception) as context:
watcher.connect_directory_monitoring()
self.assertTrue(Exception in context.expected)
def test_connect_directory_monitoring_value_error(self):
"""Test connect directory monitoring ValueError"""
watcher = OnWatchFile()
watcher.smb_user_name = 'tests_user'
watcher.smb_server_name = 'gdafsrvdesa01.ar.bsch'
watcher.connect_directory_monitoring()
def test_connect_directory_monitoring(self, mocked_get, mock_conn):
mock_conn.return_value = True
mocked_get.return_value = '127.0.0.1'
watcher = OnWatchFile()
watcher.smb_server_name = '127.0.0.1'
watcher.smb_user_name = 'tests_user'
watcher.smb_server_name = 'gdafsrvdesa01.ar.bsch'
smb = watcher.connect_directory_monitoring()
self.assertTrue(smb)
import signal
import time
from datetime import timedelta
from app.job import Job, ProgramKilled, signal_handler
from app.on_watch_file import OnWatchFile
from app.settings import get_logger
logger = get_logger('Main Watcher File')
if __name__ == "__main__":
logger.info('Start Watcher File')
watch = OnWatchFile()
watch.connect_directory_monitoring()
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
job = Job(interval=timedelta(seconds=watch.wait_time_seconds_job), execute=watch.periodically)
job.start()
while True:
try:
time.sleep(1)
except ProgramKilled:
print("Program killed: running cleanup code")
job.stop()
break
class TestMonitoring:
"""
Test monitoring
"""
file_name = ''
source = None
destination = None
def __init__(self, source=None, destination=None, file_name=''):
self.source = f"/{source}/"
self.file_name = ''
self.destination = f"/{destination}/"
self.file = "test.tiff"
self.config = Config()
self.watcher = OnWatchFile()
self.watcher.connect_directory_monitoring()
basedir = os.path.abspath(os.path.dirname(__file__))
self.basedir_files_tiff = os.path.join(basedir, "files_test/file.tiff")
self.basedir_files_xml = os.path.join(basedir, "files_test/file.xml")
def move_file_test(self):
try:
for element in range(3):
print(element)
uid = uuid.uuid4()
self.file_name = f"{uid.hex}.tiff"
file_tiff = f"/{self.source}/{self.file_name}"
print(file_tiff)
with open(self.basedir_files_tiff, "rb") as file:
self.watcher.conn.storeFile(
self.config.smb_watch_directory,
file_tiff,
file,
)
"""file_xml = f"/{self.destination}/prueba4_{uid.hex}.xml"
print(file_xml)
with open(self.basedir_files_tiff, "rb") as file:
self.watcher.conn.storeFile(
self.config.smb_watch_directory,
file_xml,
file,
)
"""
except Exception as error:
print(f"STORE FILE {error}")
def move_file_test_2(self):
try:
for element in range(1):
print(element)
uid = uuid.uuid4()
file_tiff = f"/DNI_NO_KOFAX/prueba4_{uid.hex}.tiff"
print(file_tiff)
with open(self.basedir_files_tiff, "rb") as file:
self.watcher.conn.storeFile(
self.config.smb_watch_directory,
file_tiff,
file,
)
file_xml = f"/DNI_NO_KOFAX/prueba4_{uid.hex}.xml"
print(file_xml)
with open(self.basedir_files_tiff, "rb") as file:
self.watcher.conn.storeFile(
self.config.smb_watch_directory,
file_xml,
file,
)
except Exception as error:
print(f"STORE FILE {error}")
def copy_file_samba_share(self):
logger.info(f"Start file {self.file_name} transfer: {self.file_name} "
f" to destination {self.destination}")
contents = self.file_contents(f"{self.source}/{self.file_name}")
if contents:
successful_transfer = self.write_file_transfer(
f"{self.destination}/{self.file_name}", contents)
if successful_transfer:
self.delete_remote_file()
else:
logger.warning(f"Filename: {self.file_name} is not read")
def write_file_transfer(self, path, contents):
successful_transfer = True
try:
with tempfile.NamedTemporaryFile() as file_obj:
file_obj.write(contents)
file_obj.seek(0)
self.watcher.conn.storeFile(self.config.smb_watch_directory,
path, file_obj)
logger.info(f"File {self.file_name} transfer: {self.file_name} "
f" to destination {self.destination}")
except NotReadyError as error:
logger.warning(f"Filename: {self.file_name} is {error}")
successful_transfer = False
except Exception as error:
logger.error(f"Filename: {self.file_name} read error: {error} ")
successful_transfer = False
return successful_transfer
def file_contents(self, path):
# optengo los atributos del archivo
content = None
try:
attribute = self.watcher.conn.getAttributes(
self.config.smb_watch_directory, path)
logger.info(f"Attributes Filename: {attribute}")
if attribute.file_size > 0:
with tempfile.NamedTemporaryFile() as file_obj:
self.watcher.conn.retrieveFile(
self.config.smb_watch_directory, path, file_obj)
file_obj.seek(0)
content = file_obj.read()
logger.info(f"Content Filename: {attribute.filename}")
else:
logger.warning(f"Filename: {attribute.filename} is not size")
except NotReadyError as error:
logger.warning(f"Filename: {self.file_name} is {error}")
except Exception as error:
logger.error(f"Filename: {self.file_name} read error: {error} ")
return content
def delete_remote_file(self):
try:
self.watcher.conn.deleteFiles(self.config.smb_watch_directory,
f"{self.source}/{self.file_name}")
logger.info(
f"Delete file {self.file_name} to source {self.source}")
except Exception as error:
logger.error(f"Filename: {self.file_name} read error: {error} ")
def setUp(self):
self.file = "test.tiff"
self.setup_module()
self.config = Config()
self.config.schema = ''
self.on_watch_file = OnWatchFile()
self.on_watch_file.session = self.session
basedir = os.path.abspath(os.path.dirname(__file__))
self.basedir_monitoring = os.path.join(basedir, "monitoring")
self.basedir_files_tiff = os.path.join(basedir, "files_test/file.tiff")
self.basedir_files_xml = os.path.join(basedir, "files_test/file.xml")
list_file = [
{
"alloc_size": 0,
"create_time": 1606743144.2058148,
"file_attributes": 16,
"file_id": None,
"file_size": 0,
"filename": ".",
"isDirectory": True,
"isNormal": False,
"isReadOnly": False,
"last_access_time": 1608919592.1831565,
"last_attr_change_time": 1608919592.1831565,
"last_write_time": 1608919592.1831565,
"short_name": "",
},
{
"alloc_size": 0,
"create_time": 1606743144.2058148,
"file_attributes": 16,
"file_id": None,
"file_size": 0,
"filename": "",
"isDirectory": True,
"isNormal": False,
"isReadOnly": False,
"last_access_time": 1608919592.1831565,
"last_attr_change_time": 1608919592.1831565,
"last_write_time": 1608919592.1831565,
"short_name": "",
},
{
"alloc_size": 1798144,
"create_time": float(time.time()),
"file_attributes": 32,
"file_id": None,
"file_size": 1797347,
"filename": "file.tiff",
"isDirectory": False,
"isNormal": False,
"isReadOnly": False,
"last_access_time": float(time.time()),
"last_attr_change_time": float(time.time()),
"last_write_time": float(time.time()),
"short_name": "",
},
]
self.list_file = []
for key, element in enumerate(list_file):
self.list_file.append(
namedtuple("SharedFile", element.keys())(*element.values())
)
self.id_type_import = self.type_import_id(self.session, "new_file")
def test_connect_directory_monitoring_not_access(self):
"""Test connect directory monitoring Exception """
watcher = OnWatchFile()
watcher.smb_server_name = 'localhost'
watcher.connect_directory_monitoring()
class TesOnWatchFile(unittest.TestCase, TestConfig):
"""
Test OnWatchFile
"""
def setUp(self):
self.file = "test.tiff"
self.setup_module()
self.config = Config()
self.config.schema = ''
self.on_watch_file = OnWatchFile()
self.on_watch_file.session = self.session
basedir = os.path.abspath(os.path.dirname(__file__))
self.basedir_monitoring = os.path.join(basedir, "monitoring")
self.basedir_files_tiff = os.path.join(basedir, "files_test/file.tiff")
self.basedir_files_xml = os.path.join(basedir, "files_test/file.xml")
list_file = [
{
"alloc_size": 0,
"create_time": 1606743144.2058148,
"file_attributes": 16,
"file_id": None,
"file_size": 0,
"filename": ".",
"isDirectory": True,
"isNormal": False,
"isReadOnly": False,
"last_access_time": 1608919592.1831565,
"last_attr_change_time": 1608919592.1831565,
"last_write_time": 1608919592.1831565,
"short_name": "",
},
{
"alloc_size": 0,
"create_time": 1606743144.2058148,
"file_attributes": 16,
"file_id": None,
"file_size": 0,
"filename": "",
"isDirectory": True,
"isNormal": False,
"isReadOnly": False,
"last_access_time": 1608919592.1831565,
"last_attr_change_time": 1608919592.1831565,
"last_write_time": 1608919592.1831565,
"short_name": "",
},
{
"alloc_size": 1798144,
"create_time": float(time.time()),
"file_attributes": 32,
"file_id": None,
"file_size": 1797347,
"filename": "file.tiff",
"isDirectory": False,
"isNormal": False,
"isReadOnly": False,
"last_access_time": float(time.time()),
"last_attr_change_time": float(time.time()),
"last_write_time": float(time.time()),
"short_name": "",
},
]
self.list_file = []
for key, element in enumerate(list_file):
self.list_file.append(
namedtuple("SharedFile", element.keys())(*element.values())
)
self.id_type_import = self.type_import_id(self.session, "new_file")
def test_new_file_detection_monitoring(self):
"""Test new file detection monitoring"""
self.on_watch_file.source_folder = "TestDirectory"
self.on_watch_file.new_file_detection_monitoring(self.list_file)
self.assertTrue(self.on_watch_file.file in self.list_file)
def test_new_file_detection_monitoring_exception(self):
"""Test new file detection monitoring exception"""
self.on_watch_file.source_folder = []
with self.assertRaises(Exception) as context:
self.on_watch_file.new_file_detection_monitoring([1, 2])
self.assertTrue(Exception in context.expected)
def test_insert_new_file_in_database(self):
"""Test new file detection monitoring"""
self.on_watch_file.source_folder = "TestDirectory"
self.on_watch_file.file = self.list_file[2]
self.on_watch_file.insert_new_file_in_database()
self.assertTrue(self.on_watch_file.file in self.list_file)
def test_insert_new_file_in_database_exception(self):
"""Test insert new file in database Exception"""
self.on_watch_file.source_folder = "TestDirectory"
self.on_watch_file.file = []
setattr(self.on_watch_file, "file.create_time", '')
with self.assertRaises(Exception) as context:
self.on_watch_file.insert_new_file_in_database()
self.assertTrue(Exception in context.expected)
@should_raise(ValueError('Error database'))
def test_insert_new_file_in_database_exception_error_database(self):
"""Test insert new file in database SQLAlchemyError """
self.on_watch_file.source_folder = "TestDirectory"
files = [
{
"alloc_size": 1798144,
"create_time": float(time.time()),
"file_attributes": 32,
"file_id": None,
"file_size": 1797347,
"filename": None,
"isDirectory": False,
"isNormal": False,
"isReadOnly": False,
"last_access_time": float(time.time()),
"last_attr_change_time": float(time.time()),
"last_write_time": float(time.time()),
"short_name": "",
},
]
list_file = []
for key, element in enumerate(files):
list_file.append(
namedtuple("SharedFile", element.keys())(*element.values())
)
self.on_watch_file.file = list_file[0]
self.on_watch_file.insert_new_file_in_database()
self.on_watch_file.insert_new_file_in_database()
def test_connect_directory_monitoring_exception(self):
"""Test connect directory monitoring Exception """
watcher = OnWatchFile()
watcher.smb_server_name = 'localhost'
watcher.password = ''
watcher.connected = False
with self.assertRaises(Exception) as context:
watcher.connect_directory_monitoring()
self.assertTrue(Exception in context.expected)
@should_raise(ValueError("Can not access the system"))
def test_connect_directory_monitoring_not_access(self):
"""Test connect directory monitoring Exception """
watcher = OnWatchFile()
watcher.smb_server_name = 'localhost'
watcher.connect_directory_monitoring()
@mock.patch('app.on_watch_file.OnWatchFile.connect_directory_monitoring')
@mock.patch('socket.gethostbyname')
def test_connect_directory_monitoring(self, mocked_get, mock_conn):
mock_conn.return_value = True
mocked_get.return_value = '127.0.0.1'
watcher = OnWatchFile()
watcher.smb_server_name = '127.0.0.1'
watcher.smb_user_name = 'tests_user'
watcher.smb_server_name = 'gdafsrvdesa01.ar.bsch'
smb = watcher.connect_directory_monitoring()
self.assertTrue(smb)
@should_raise(ValueError("Can not access the system"))
def test_connect_directory_monitoring_value_error(self):
"""Test connect directory monitoring ValueError"""
watcher = OnWatchFile()
watcher.smb_user_name = 'tests_user'
watcher.smb_server_name = 'gdafsrvdesa01.ar.bsch'
watcher.connect_directory_monitoring()
def test_periodically(self):
"""Test Periodically"""
self.on_watch_file.smb_directory_list = 'OTROS_NO_KOFAX, DNI_NO_KOFAX'
watcher = self.on_watch_file.periodically()
self.assertFalse(watcher)
def test_periodically_exception(self):
"""Test Periodically Exception"""
self.on_watch_file.smb_directory_list = ''
with self.assertRaises(Exception) as context:
self.on_watch_file.periodically()
self.assertTrue(Exception in context.expected)
def test_check_date_format_return_now(self):
result_date = self.on_watch_file.check_date_format()
self.assertTrue(datetime.now(), result_date)
def test_check_date_format_return_microsecond(self):
result_date = self.on_watch_file.check_date_format(datetime.now())
self.assertTrue(datetime.now(), result_date)
def test_check_date_format_return_not_microsecond(self):
date = datetime.strptime('2021-01-14 09:44:42', '%Y-%m-%d %H:%M:%S')
result_date = self.on_watch_file.check_date_format(date)
self.assertTrue(result_date, date)
def test_date_format_from_times_tamp(self):
date = 1610628721.6383548
result_date = self.on_watch_file.date_format_from_times_tamp(date)
self.assertTrue(result_date, date)
def tearDown(self):
self.session.rollback()
self.session.close()