def _manage_all_imported_files(self): """ File management at the end of correct import: - Save files in their final location on the NAS - Delete files from their import location on the NAS Done by Michael Sandoz 02.2016 """ share_nas = self.env.ref('sbc_switzerland.share_on_nas').value # to delete after treatment list_zip_to_delete = [] imported_letter_path = "" if self.manual_import: imported_letter_path = self.env.ref( 'sbc_switzerland.scan_letter_imported').value else: imported_letter_path = self.check_path(self.import_folder_path) smb_conn = self._get_smb_connection() if smb_conn and smb_conn.connect( SmbConfig.smb_ip, SmbConfig.smb_port): list_paths = smb_conn.listPath(share_nas, imported_letter_path) for shared_file in list_paths: if func.check_file(shared_file.filename) == 1: # when this is manual import we don't have to copy all # files, web letters are stock in the same folder... if not self.manual_import or self.is_in_list_letter( shared_file.filename): file_obj = BytesIO() smb_conn.retrieveFile( share_nas, imported_letter_path + shared_file.filename, file_obj) file_obj.seek(0) self._copy_imported_to_done_letter( shared_file.filename, file_obj, True) elif func.is_zip(shared_file.filename): zip_file = BytesIO() smb_conn.retrieveFile( share_nas, imported_letter_path + shared_file.filename, zip_file) zip_file.seek(0) zip_ = zipfile.ZipFile( zip_file, 'r') zip_to_remove = False for f in zip_.namelist(): # when this is manual import we are not sure that this # zip contains current letters treated if not self.manual_import or self.is_in_list_letter(f): self._copy_imported_to_done_letter( f, BytesIO(zip_.read(f)), False) zip_to_remove = True if zip_to_remove: list_zip_to_delete.append(shared_file.filename) # delete zip file from origin import folder on the NAS for filename in list_zip_to_delete: try: smb_conn.delete_files( share_nas, imported_letter_path + filename) except Exception as inst: logger.info('Failed to delete zip file on NAS: {}' .format(inst)) smb_conn.close()
def _compute_nber_letters(self): """ Counts the number of scans. If a zip file is given, the number of scans inside is counted. """ for letter in self: if letter.manual_import or ( letter.state and letter.state != 'draft'): super(ImportLettersHistory, letter)._compute_nber_letters() else: # files are not selected by user so we find them on NAS # folder 'Imports' counter tmp = 0 smb_conn = letter._get_smb_connection() share_nas = letter.env.ref( 'sbc_switzerland.share_on_nas').value imported_letter_path = letter.import_folder_path if smb_conn and smb_conn.connect( SmbConfig.smb_ip, SmbConfig.smb_port) and \ imported_letter_path: imported_letter_path = letter.check_path( imported_letter_path) try: list_paths = smb_conn.listPath( share_nas, imported_letter_path) except OperationFailure: logger.info('--------------- PATH NO CORRECT ------' '-----') list_paths = [] for shared_file in list_paths: if func.check_file(shared_file.filename) == 1: tmp += 1 elif func.is_zip(shared_file.filename): logger.info( 'File to retrieve: {}'.format( imported_letter_path + shared_file.filename)) file_obj = BytesIO() smb_conn.retrieveFile( share_nas, imported_letter_path + shared_file.filename, file_obj) try: zip_ = zipfile.ZipFile(file_obj, 'r') list_file = zip_.namelist() # loop over all files in zip for tmp_file in list_file: tmp += (func.check_file( tmp_file) == 1) except zipfile.BadZipfile: raise exceptions.UserError( _('Zip file corrupted (' + shared_file.filename + ')')) except zipfile.LargeZipFile: raise exceptions.UserError( _('Zip64 is not supported(' + shared_file.filename + ')')) smb_conn.close() else: logger.info("""Failed to list files in imported \ folder Imports oh the NAS in emplacement: {}""".format( imported_letter_path)) letter.nber_letters = tmp
def _run_analyze(self): """ Analyze each attachment: - check for duplicate file names and skip them - decompress zip file if necessary - call _analyze_attachment for every resulting file """ self.ensure_one() # keep track of file names to detect duplicates # file_name_history = [] logger.info("Imported letters analysis started...") progress = 1 share_nas = self.env.ref('sbc_switzerland.share_on_nas').value smb_conn = self._get_smb_connection() if not self.manual_import: imported_letter_path = self.check_path(self.import_folder_path) if smb_conn and smb_conn.connect( SmbConfig.smb_ip, SmbConfig.smb_port): list_paths = smb_conn.listPath(share_nas, imported_letter_path) for shared_file in list_paths: if func.check_file(shared_file.filename) == 1: logger.info("Analyzing letter {}/{}".format( progress, self.nber_letters)) with NamedTemporaryFile() as file_obj: smb_conn.retrieveFile( share_nas, imported_letter_path + shared_file.filename, file_obj) self._analyze_attachment( self._convert_pdf(file_obj), shared_file.filename) progress += 1 elif func.is_zip(shared_file.filename): zip_file = BytesIO() # retrieve zip file from imports letters stored on # NAS smb_conn.retrieveFile( share_nas, imported_letter_path + shared_file.filename, zip_file) zip_file.seek(0) zip_ = zipfile.ZipFile( zip_file, 'r') # loop over files inside zip for f in zip_.namelist(): logger.info( "Analyzing letter {}/{}".format( progress, self.nber_letters)) self._analyze_attachment( self._convert_pdf(zip_.open(f)), f) progress += 1 smb_conn.close() else: logger.info('Failed to list files in Imports on the NAS in \ emplacement: {}'.format( imported_letter_path)) else: super(ImportLettersHistory, self)._run_analyze() # remove all the files (now they are inside import_line_ids) self.data.unlink() if not self.manual_import: self._manage_all_imported_files() self.import_completed = True logger.info("Imported letters analysis completed.")
def _compute_nber_letters(self): """ Counts the number of scans. If a zip file is given, the number of scans inside is counted. """ for letter in self: if letter.manual_import or ( letter.state and letter.state != 'draft'): super(ImportLettersHistory, letter)._compute_nber_letters() else: # files are not selected by user so we find them on NAS # folder 'Imports' counter tmp = 0 smb_conn = letter._get_smb_connection() share_nas = letter.env.ref( 'sbc_switzerland.share_on_nas').value imported_letter_path = letter.import_folder_path if smb_conn and smb_conn.connect( SmbConfig.smb_ip, SmbConfig.smb_port) and \ imported_letter_path: imported_letter_path = letter.check_path( imported_letter_path) try: listPaths = smb_conn.listPath( share_nas, imported_letter_path) except OperationFailure: logger.info('--------------- PATH NO CORRECT ------' '-----') listPaths = [] for sharedFile in listPaths: if func.check_file(sharedFile.filename) == 1: tmp += 1 elif func.is_zip(sharedFile.filename): logger.info( 'File to retrieve: {}'.format( imported_letter_path + sharedFile.filename)) file_obj = BytesIO() smb_conn.retrieveFile( share_nas, imported_letter_path + sharedFile.filename, file_obj) try: zip_ = zipfile.ZipFile(file_obj, 'r') list_file = zip_.namelist() # loop over all files in zip for tmp_file in list_file: tmp += (func.check_file( tmp_file) == 1) except zipfile.BadZipfile: raise exceptions.UserError( _('Zip file corrupted (' + sharedFile.filename + ')')) except zipfile.LargeZipFile: raise exceptions.UserError( _('Zip64 is not supported(' + sharedFile.filename + ')')) smb_conn.close() else: logger.info("""Failed to list files in imported \ folder Imports oh the NAS in emplacement: {}""".format( imported_letter_path)) letter.nber_letters = tmp
def _manage_all_imported_files(self): """ File management at the end of correct import: - Save files in their final location on the NAS - Delete files from their import location on the NAS Done by Michael Sandoz 02.2016 """ share_nas = self.env.ref('sbc_switzerland.share_on_nas').value # to delete after treatment list_zip_to_delete = [] imported_letter_path = "" if self.manual_import: imported_letter_path = self.env.ref( 'sbc_switzerland.scan_letter_imported').value else: imported_letter_path = self.check_path(self.import_folder_path) smb_conn = self._get_smb_connection() if smb_conn and smb_conn.connect( SmbConfig.smb_ip, SmbConfig.smb_port): listPaths = smb_conn.listPath(share_nas, imported_letter_path) for sharedFile in listPaths: if func.check_file(sharedFile.filename) == 1: # when this is manual import we don't have to copy all # files, web letters are stock in the same folder... if not self.manual_import or self.is_in_list_letter( sharedFile.filename): file_obj = BytesIO() smb_conn.retrieveFile( share_nas, imported_letter_path + sharedFile.filename, file_obj) file_obj.seek(0) self._copy_imported_to_done_letter( sharedFile.filename, file_obj, True) elif func.is_zip(sharedFile.filename): zip_file = BytesIO() smb_conn.retrieveFile( share_nas, imported_letter_path + sharedFile.filename, zip_file) zip_file.seek(0) zip_ = zipfile.ZipFile( zip_file, 'r') zip_to_remove = False for f in zip_.namelist(): # when this is manual import we are not sure that this # zip contains current letters treated if not self.manual_import or self.is_in_list_letter(f): self._copy_imported_to_done_letter( f, BytesIO(zip_.read(f)), False) zip_to_remove = True if zip_to_remove: list_zip_to_delete.append(sharedFile.filename) # delete zip file from origin import folder on the NAS for filename in list_zip_to_delete: try: smb_conn.deleteFiles( share_nas, imported_letter_path + filename) except Exception as inst: logger.info('Failed to delete zip file on NAS: {}' .format(inst)) smb_conn.close()
def _run_analyze(self): """ Analyze each attachment: - check for duplicate file names and skip them - decompress zip file if necessary - call _analyze_attachment for every resulting file """ self.ensure_one() # keep track of file names to detect duplicates # file_name_history = [] logger.info("Imported letters analysis started...") progress = 1 share_nas = self.env.ref('sbc_switzerland.share_on_nas').value smb_conn = self._get_smb_connection() if not self.manual_import: imported_letter_path = self.check_path(self.import_folder_path) if smb_conn and smb_conn.connect( SmbConfig.smb_ip, SmbConfig.smb_port): listPaths = smb_conn.listPath(share_nas, imported_letter_path) for sharedFile in listPaths: if func.check_file(sharedFile.filename) == 1: logger.info("Analyzing letter {}/{}".format( progress, self.nber_letters)) with NamedTemporaryFile() as file_obj: smb_conn.retrieveFile( share_nas, imported_letter_path + sharedFile.filename, file_obj) self._analyze_attachment( self._convert_pdf(file_obj), sharedFile.filename) progress += 1 elif func.is_zip(sharedFile.filename): zip_file = BytesIO() # retrieve zip file from imports letters stored on # NAS smb_conn.retrieveFile( share_nas, imported_letter_path + sharedFile.filename, zip_file) zip_file.seek(0) zip_ = zipfile.ZipFile( zip_file, 'r') # loop over files inside zip for f in zip_.namelist(): logger.info( "Analyzing letter {}/{}".format( progress, self.nber_letters)) self._analyze_attachment( self._convert_pdf(zip_.open(f)), f) progress += 1 smb_conn.close() else: logger.info('Failed to list files in Imports on the NAS in \ emplacement: {}'.format( imported_letter_path)) else: super(ImportLettersHistory, self)._run_analyze() # remove all the files (now they are inside import_line_ids) self.data.unlink() if not self.manual_import: self._manage_all_imported_files() self.import_completed = True logger.info("Imported letters analysis completed.")
def _compute_nber_letters(self): """ Counts the number of scans. If a zip file is given, the number of scans inside is counted. """ for letter in self: if letter.manual_import or (letter.state and letter.state != "draft"): super(ImportLettersHistory, letter)._compute_nber_letters() else: # files are not selected by user so we find them on NAS # folder 'Imports' counter tmp = 0 smb_conn = letter._get_smb_connection() share_nas = letter.env.ref( "sbc_switzerland.share_on_nas").value imported_letter_path = letter.import_folder_path if (smb_conn and smb_conn.connect(SmbConfig.smb_ip, SmbConfig.smb_port) and imported_letter_path): imported_letter_path = letter.check_path( imported_letter_path) try: list_paths = smb_conn.listPath(share_nas, imported_letter_path) except OperationFailure: logger.error( "--------------- PATH NOT CORRECT ---------------") list_paths = [] for shared_file in list_paths: if func.check_file(shared_file.filename) == 1: tmp += 1 elif func.is_zip(shared_file.filename): logger.debug( "File to retrieve:" f" {imported_letter_path + shared_file.filename}" ) file_obj = BytesIO() smb_conn.retrieveFile( share_nas, imported_letter_path + shared_file.filename, file_obj, ) try: zip_ = zipfile.ZipFile(file_obj, "r") list_file = zip_.namelist() # loop over all files in zip for tmp_file in list_file: tmp += func.check_file(tmp_file) == 1 except zipfile.BadZipfile: raise exceptions.UserError( _("Zip file corrupted (" + shared_file.filename + ")")) except zipfile.LargeZipFile: raise exceptions.UserError( _("Zip64 is not supported(" + shared_file.filename + ")")) smb_conn.close() else: logger.error(f"""Failed to list files in imported folder \ Imports oh the NAS in emplacement: {imported_letter_path}""" ) letter.nber_letters = tmp