def _manage_all_imported_files(self):
"""
File management at the end of correct import:
- Save files in their final location on the NAS
- Delete files from their import location on the NAS
Done by Michael Sandoz 02.2016
"""
share_nas = self.env.ref('sbc_switzerland.share_on_nas').value
# to delete after treatment
list_zip_to_delete = []
imported_letter_path = ""
if self.manual_import:
imported_letter_path = self.env.ref(
'sbc_switzerland.scan_letter_imported').value
else:
imported_letter_path = self.check_path(self.import_folder_path)
smb_conn = self._get_smb_connection()
if smb_conn and smb_conn.connect(
SmbConfig.smb_ip, SmbConfig.smb_port):
list_paths = smb_conn.listPath(share_nas, imported_letter_path)
for shared_file in list_paths:
if func.check_file(shared_file.filename) == 1:
# when this is manual import we don't have to copy all
# files, web letters are stock in the same folder...
if not self.manual_import or self.is_in_list_letter(
shared_file.filename):
file_obj = BytesIO()
smb_conn.retrieveFile(
share_nas,
imported_letter_path +
shared_file.filename,
file_obj)
file_obj.seek(0)
self._copy_imported_to_done_letter(
shared_file.filename, file_obj, True)
elif func.is_zip(shared_file.filename):
zip_file = BytesIO()
smb_conn.retrieveFile(
share_nas,
imported_letter_path +
shared_file.filename,
zip_file)
zip_file.seek(0)
zip_ = zipfile.ZipFile(
zip_file, 'r')
zip_to_remove = False
for f in zip_.namelist():
# when this is manual import we are not sure that this
# zip contains current letters treated
if not self.manual_import or self.is_in_list_letter(f):
self._copy_imported_to_done_letter(
f, BytesIO(zip_.read(f)), False)
zip_to_remove = True
if zip_to_remove:
list_zip_to_delete.append(shared_file.filename)
# delete zip file from origin import folder on the NAS
for filename in list_zip_to_delete:
try:
smb_conn.delete_files(
share_nas,
imported_letter_path +
filename)
except Exception as inst:
logger.info('Failed to delete zip file on NAS: {}'
.format(inst))
smb_conn.close()
def _compute_nber_letters(self):
"""
Counts the number of scans. If a zip file is given, the number of
scans inside is counted.
"""
for letter in self:
if letter.manual_import or (
letter.state and letter.state != 'draft'):
super(ImportLettersHistory, letter)._compute_nber_letters()
else:
# files are not selected by user so we find them on NAS
# folder 'Imports' counter
tmp = 0
smb_conn = letter._get_smb_connection()
share_nas = letter.env.ref(
'sbc_switzerland.share_on_nas').value
imported_letter_path = letter.import_folder_path
if smb_conn and smb_conn.connect(
SmbConfig.smb_ip, SmbConfig.smb_port) and \
imported_letter_path:
imported_letter_path = letter.check_path(
imported_letter_path)
try:
list_paths = smb_conn.listPath(
share_nas,
imported_letter_path)
except OperationFailure:
logger.info('--------------- PATH NO CORRECT ------'
'-----')
list_paths = []
for shared_file in list_paths:
if func.check_file(shared_file.filename) == 1:
tmp += 1
elif func.is_zip(shared_file.filename):
logger.info(
'File to retrieve: {}'.format(
imported_letter_path +
shared_file.filename))
file_obj = BytesIO()
smb_conn.retrieveFile(
share_nas,
imported_letter_path +
shared_file.filename,
file_obj)
try:
zip_ = zipfile.ZipFile(file_obj, 'r')
list_file = zip_.namelist()
# loop over all files in zip
for tmp_file in list_file:
tmp += (func.check_file(
tmp_file) == 1)
except zipfile.BadZipfile:
raise exceptions.UserError(
_('Zip file corrupted (' +
shared_file.filename + ')'))
except zipfile.LargeZipFile:
raise exceptions.UserError(
_('Zip64 is not supported(' +
shared_file.filename + ')'))
smb_conn.close()
else:
logger.info("""Failed to list files in imported \
folder Imports oh the NAS in emplacement: {}""".format(
imported_letter_path))
letter.nber_letters = tmp
def _run_analyze(self):
"""
Analyze each attachment:
- check for duplicate file names and skip them
- decompress zip file if necessary
- call _analyze_attachment for every resulting file
"""
self.ensure_one()
# keep track of file names to detect duplicates
# file_name_history = []
logger.info("Imported letters analysis started...")
progress = 1
share_nas = self.env.ref('sbc_switzerland.share_on_nas').value
smb_conn = self._get_smb_connection()
if not self.manual_import:
imported_letter_path = self.check_path(self.import_folder_path)
if smb_conn and smb_conn.connect(
SmbConfig.smb_ip, SmbConfig.smb_port):
list_paths = smb_conn.listPath(share_nas, imported_letter_path)
for shared_file in list_paths:
if func.check_file(shared_file.filename) == 1:
logger.info("Analyzing letter {}/{}".format(
progress, self.nber_letters))
with NamedTemporaryFile() as file_obj:
smb_conn.retrieveFile(
share_nas,
imported_letter_path +
shared_file.filename,
file_obj)
self._analyze_attachment(
self._convert_pdf(file_obj),
shared_file.filename)
progress += 1
elif func.is_zip(shared_file.filename):
zip_file = BytesIO()
# retrieve zip file from imports letters stored on
# NAS
smb_conn.retrieveFile(
share_nas,
imported_letter_path +
shared_file.filename,
zip_file)
zip_file.seek(0)
zip_ = zipfile.ZipFile(
zip_file, 'r')
# loop over files inside zip
for f in zip_.namelist():
logger.info(
"Analyzing letter {}/{}".format(
progress, self.nber_letters))
self._analyze_attachment(
self._convert_pdf(zip_.open(f)), f)
progress += 1
smb_conn.close()
else:
logger.info('Failed to list files in Imports on the NAS in \
emplacement: {}'.format(
imported_letter_path))
else:
super(ImportLettersHistory, self)._run_analyze()
# remove all the files (now they are inside import_line_ids)
self.data.unlink()
if not self.manual_import:
self._manage_all_imported_files()
self.import_completed = True
logger.info("Imported letters analysis completed.")
def _compute_nber_letters(self):
"""
Counts the number of scans. If a zip file is given, the number of
scans inside is counted.
"""
for letter in self:
if letter.manual_import or (
letter.state and letter.state != 'draft'):
super(ImportLettersHistory, letter)._compute_nber_letters()
else:
# files are not selected by user so we find them on NAS
# folder 'Imports' counter
tmp = 0
smb_conn = letter._get_smb_connection()
share_nas = letter.env.ref(
'sbc_switzerland.share_on_nas').value
imported_letter_path = letter.import_folder_path
if smb_conn and smb_conn.connect(
SmbConfig.smb_ip, SmbConfig.smb_port) and \
imported_letter_path:
imported_letter_path = letter.check_path(
imported_letter_path)
try:
listPaths = smb_conn.listPath(
share_nas,
imported_letter_path)
except OperationFailure:
logger.info('--------------- PATH NO CORRECT ------'
'-----')
listPaths = []
for sharedFile in listPaths:
if func.check_file(sharedFile.filename) == 1:
tmp += 1
elif func.is_zip(sharedFile.filename):
logger.info(
'File to retrieve: {}'.format(
imported_letter_path +
sharedFile.filename))
file_obj = BytesIO()
smb_conn.retrieveFile(
share_nas,
imported_letter_path +
sharedFile.filename,
file_obj)
try:
zip_ = zipfile.ZipFile(file_obj, 'r')
list_file = zip_.namelist()
# loop over all files in zip
for tmp_file in list_file:
tmp += (func.check_file(
tmp_file) == 1)
except zipfile.BadZipfile:
raise exceptions.UserError(
_('Zip file corrupted (' +
sharedFile.filename + ')'))
except zipfile.LargeZipFile:
raise exceptions.UserError(
_('Zip64 is not supported(' +
sharedFile.filename + ')'))
smb_conn.close()
else:
logger.info("""Failed to list files in imported \
folder Imports oh the NAS in emplacement: {}""".format(
imported_letter_path))
letter.nber_letters = tmp
def _manage_all_imported_files(self):
"""
File management at the end of correct import:
- Save files in their final location on the NAS
- Delete files from their import location on the NAS
Done by Michael Sandoz 02.2016
"""
share_nas = self.env.ref('sbc_switzerland.share_on_nas').value
# to delete after treatment
list_zip_to_delete = []
imported_letter_path = ""
if self.manual_import:
imported_letter_path = self.env.ref(
'sbc_switzerland.scan_letter_imported').value
else:
imported_letter_path = self.check_path(self.import_folder_path)
smb_conn = self._get_smb_connection()
if smb_conn and smb_conn.connect(
SmbConfig.smb_ip, SmbConfig.smb_port):
listPaths = smb_conn.listPath(share_nas, imported_letter_path)
for sharedFile in listPaths:
if func.check_file(sharedFile.filename) == 1:
# when this is manual import we don't have to copy all
# files, web letters are stock in the same folder...
if not self.manual_import or self.is_in_list_letter(
sharedFile.filename):
file_obj = BytesIO()
smb_conn.retrieveFile(
share_nas,
imported_letter_path +
sharedFile.filename,
file_obj)
file_obj.seek(0)
self._copy_imported_to_done_letter(
sharedFile.filename, file_obj, True)
elif func.is_zip(sharedFile.filename):
zip_file = BytesIO()
smb_conn.retrieveFile(
share_nas,
imported_letter_path +
sharedFile.filename,
zip_file)
zip_file.seek(0)
zip_ = zipfile.ZipFile(
zip_file, 'r')
zip_to_remove = False
for f in zip_.namelist():
# when this is manual import we are not sure that this
# zip contains current letters treated
if not self.manual_import or self.is_in_list_letter(f):
self._copy_imported_to_done_letter(
f, BytesIO(zip_.read(f)), False)
zip_to_remove = True
if zip_to_remove:
list_zip_to_delete.append(sharedFile.filename)
# delete zip file from origin import folder on the NAS
for filename in list_zip_to_delete:
try:
smb_conn.deleteFiles(
share_nas,
imported_letter_path +
filename)
except Exception as inst:
logger.info('Failed to delete zip file on NAS: {}'
.format(inst))
smb_conn.close()
def _run_analyze(self):
"""
Analyze each attachment:
- check for duplicate file names and skip them
- decompress zip file if necessary
- call _analyze_attachment for every resulting file
"""
self.ensure_one()
# keep track of file names to detect duplicates
# file_name_history = []
logger.info("Imported letters analysis started...")
progress = 1
share_nas = self.env.ref('sbc_switzerland.share_on_nas').value
smb_conn = self._get_smb_connection()
if not self.manual_import:
imported_letter_path = self.check_path(self.import_folder_path)
if smb_conn and smb_conn.connect(
SmbConfig.smb_ip, SmbConfig.smb_port):
listPaths = smb_conn.listPath(share_nas, imported_letter_path)
for sharedFile in listPaths:
if func.check_file(sharedFile.filename) == 1:
logger.info("Analyzing letter {}/{}".format(
progress, self.nber_letters))
with NamedTemporaryFile() as file_obj:
smb_conn.retrieveFile(
share_nas,
imported_letter_path +
sharedFile.filename,
file_obj)
self._analyze_attachment(
self._convert_pdf(file_obj),
sharedFile.filename)
progress += 1
elif func.is_zip(sharedFile.filename):
zip_file = BytesIO()
# retrieve zip file from imports letters stored on
# NAS
smb_conn.retrieveFile(
share_nas,
imported_letter_path +
sharedFile.filename,
zip_file)
zip_file.seek(0)
zip_ = zipfile.ZipFile(
zip_file, 'r')
# loop over files inside zip
for f in zip_.namelist():
logger.info(
"Analyzing letter {}/{}".format(
progress, self.nber_letters))
self._analyze_attachment(
self._convert_pdf(zip_.open(f)), f)
progress += 1
smb_conn.close()
else:
logger.info('Failed to list files in Imports on the NAS in \
emplacement: {}'.format(
imported_letter_path))
else:
super(ImportLettersHistory, self)._run_analyze()
# remove all the files (now they are inside import_line_ids)
self.data.unlink()
if not self.manual_import:
self._manage_all_imported_files()
self.import_completed = True
logger.info("Imported letters analysis completed.")
def _compute_nber_letters(self):
"""
Counts the number of scans. If a zip file is given, the number of
scans inside is counted.
"""
for letter in self:
if letter.manual_import or (letter.state
and letter.state != "draft"):
super(ImportLettersHistory, letter)._compute_nber_letters()
else:
# files are not selected by user so we find them on NAS
# folder 'Imports' counter
tmp = 0
smb_conn = letter._get_smb_connection()
share_nas = letter.env.ref(
"sbc_switzerland.share_on_nas").value
imported_letter_path = letter.import_folder_path
if (smb_conn and smb_conn.connect(SmbConfig.smb_ip,
SmbConfig.smb_port)
and imported_letter_path):
imported_letter_path = letter.check_path(
imported_letter_path)
try:
list_paths = smb_conn.listPath(share_nas,
imported_letter_path)
except OperationFailure:
logger.error(
"--------------- PATH NOT CORRECT ---------------")
list_paths = []
for shared_file in list_paths:
if func.check_file(shared_file.filename) == 1:
tmp += 1
elif func.is_zip(shared_file.filename):
logger.debug(
"File to retrieve:"
f" {imported_letter_path + shared_file.filename}"
)
file_obj = BytesIO()
smb_conn.retrieveFile(
share_nas,
imported_letter_path + shared_file.filename,
file_obj,
)
try:
zip_ = zipfile.ZipFile(file_obj, "r")
list_file = zip_.namelist()
# loop over all files in zip
for tmp_file in list_file:
tmp += func.check_file(tmp_file) == 1
except zipfile.BadZipfile:
raise exceptions.UserError(
_("Zip file corrupted (" +
shared_file.filename + ")"))
except zipfile.LargeZipFile:
raise exceptions.UserError(
_("Zip64 is not supported(" +
shared_file.filename + ")"))
smb_conn.close()
else:
logger.error(f"""Failed to list files in imported folder \
Imports oh the NAS in emplacement: {imported_letter_path}"""
)
letter.nber_letters = tmp
