def create_unrar_instance(self): """ Start the unrar instance using the user's options """ # Generate extraction path and save for post-proc if not self.unpack_dir_info: try: self.unpack_dir_info = prepare_extraction_path(self.nzo) except: # Prevent fatal crash if directory creation fails self.abort() return # Get the information extraction_path, _, _, one_folder, _ = self.unpack_dir_info # Set options if self.nzo.password: password_command = '-p%s' % self.nzo.password else: password_command = '-p-' if one_folder or cfg.flat_unpack(): action = 'e' else: action = 'x' # The first NZF self.rarfile_nzf = self.have_next_volume() # Generate command rarfile_path = os.path.join(self.nzo.downpath, self.rarfile_nzf.filename) if sabnzbd.WIN32: # For Unrar to support long-path, we need to cricumvent Python's list2cmdline # See: https://github.com/sabnzbd/sabnzbd/issues/1043 command = ['%s' % sabnzbd.newsunpack.RAR_COMMAND, action, '-vp', '-idp', '-o+', '-ai', password_command, '%s' % clip_path(rarfile_path), '%s\\' % long_path(extraction_path)] else: # Don't use "-ai" (not needed for non-Windows) command = ['%s' % sabnzbd.newsunpack.RAR_COMMAND, action, '-vp', '-idp', '-o+', password_command, '%s' % rarfile_path, '%s/' % extraction_path] if cfg.ignore_unrar_dates(): command.insert(3, '-tsm-') # Let's start from the first one! self.cur_volume = 1 stup, need_shell, command, creationflags = build_command(command, flatten_command=True) logging.debug('Running unrar for DirectUnpack %s', command) self.active_instance = Popen(command, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, startupinfo=stup, creationflags=creationflags) # Add to runners ACTIVE_UNPACKERS.append(self) # Doing the first logging.info('DirectUnpacked volume %s for %s', self.cur_volume, self.cur_setname)
def create_unrar_instance(self): """ Start the unrar instance using the user's options """ # Generate extraction path and save for post-proc if not self.unpack_dir_info: try: self.unpack_dir_info = prepare_extraction_path(self.nzo) except: # Prevent fatal crash if directory creation fails self.abort() return # Get the information extraction_path, _, _, one_folder, _ = self.unpack_dir_info # Set options if self.nzo.password: password_command = "-p%s" % self.nzo.password else: password_command = "-p-" if one_folder or cfg.flat_unpack(): action = "e" else: action = "x" # The first NZF self.rarfile_nzf = self.have_next_volume() # Ignore if maybe this set is not there any more # This can happen due to race/timing issues when creating the sets if not self.rarfile_nzf: return # Generate command rarfile_path = os.path.join(self.nzo.downpath, self.rarfile_nzf.filename) if sabnzbd.WIN32: # For Unrar to support long-path, we need to cricumvent Python's list2cmdline # See: https://github.com/sabnzbd/sabnzbd/issues/1043 command = [ "%s" % sabnzbd.newsunpack.RAR_COMMAND, action, "-vp", "-idp", "-o+", "-ai", password_command, "%s" % clip_path(rarfile_path), "%s\\" % long_path(extraction_path), ] else: # Don't use "-ai" (not needed for non-Windows) command = [ "%s" % sabnzbd.newsunpack.RAR_COMMAND, action, "-vp", "-idp", "-o+", password_command, "%s" % rarfile_path, "%s/" % extraction_path, ] if cfg.ignore_unrar_dates(): command.insert(3, "-tsm-") # Let's start from the first one! self.cur_volume = 1 # Need to disable buffer to have direct feedback self.active_instance = build_and_run_command(command, flatten_command=True, bufsize=0) # Add to runners ACTIVE_UNPACKERS.append(self) # Doing the first logging.info("DirectUnpacked volume %s for %s", self.cur_volume, self.cur_setname)
def run(self): # Input and output linebuf = "" last_volume_linebuf = "" unrar_log = [] rarfiles = [] extracted = [] start_time = time.time() # Need to read char-by-char because there's no newline after new-disk message while 1: # We need to lock, so we don't crash if unpacker is deleted while we read with START_STOP_LOCK: if not self.active_instance or not self.active_instance.stdout: break char = platform_btou(self.active_instance.stdout.read(1)) if not char: # End of program break linebuf += char # Error? Let PP-handle it if linebuf.endswith(( "ERROR: ", "Cannot create", "in the encrypted file", "CRC failed", "checksum failed", "You need to start extraction from a previous volume", "password is incorrect", "Incorrect password", "Write error", "checksum error", "Cannot open", "start extraction from a previous volume", "Unexpected end of archive", )): logging.info("Error in DirectUnpack of %s: %s", self.cur_setname, linebuf.strip()) self.abort() if linebuf.endswith("\n"): # List files we used if linebuf.startswith("Extracting from"): filename = re.search(EXTRACTFROM_RE, linebuf.strip()).group(1) if filename not in rarfiles: rarfiles.append(filename) # List files we extracted m = re.search(EXTRACTED_RE, linebuf) if m: # In case of flat-unpack, UnRar still prints the whole path (?!) unpacked_file = m.group(2) if cfg.flat_unpack(): unpacked_file = os.path.basename(unpacked_file) extracted.append( real_path(self.unpack_dir_info[0], unpacked_file)) # Did we reach the end? if linebuf.endswith("All OK"): # Stop timer and finish self.unpack_time += time.time() - start_time ACTIVE_UNPACKERS.remove(self) # Add to success rarfile_path = os.path.join(self.nzo.downpath, self.rarfile_nzf.filename) self.success_sets[self.cur_setname] = ( rar_volumelist(rarfile_path, self.nzo.password, rarfiles), extracted, ) logging.info("DirectUnpack completed for %s", self.cur_setname) self.nzo.set_action_line(T("Direct Unpack"), T("Completed")) # List success in history-info msg = T("Unpacked %s files/folders in %s") % ( len(extracted), format_time_string(self.unpack_time)) msg = "%s - %s" % (T("Direct Unpack"), msg) self.nzo.set_unpack_info("Unpack", msg, self.cur_setname) # Write current log and clear unrar_log.append(linebuf.strip()) linebuf = "" last_volume_linebuf = "" logging.debug("DirectUnpack Unrar output %s", "\n".join(unrar_log)) unrar_log = [] rarfiles = [] extracted = [] # Are there more files left? while self.nzo.files and not self.next_sets: with self.next_file_lock: self.next_file_lock.wait() # Is there another set to do? if self.next_sets: # Start new instance nzf = self.next_sets.pop(0) self.reset_active() self.cur_setname = nzf.setname # Wait for the 1st volume to appear self.wait_for_next_volume() self.create_unrar_instance() start_time = time.time() else: self.killed = True break if linebuf.endswith("[C]ontinue, [Q]uit "): # Stop timer self.unpack_time += time.time() - start_time # Wait for the next one.. self.wait_for_next_volume() # Possible that the instance was deleted while locked if not self.killed: # If unrar stopped or is killed somehow, writing will cause a crash try: # Give unrar some time to do it's thing self.active_instance.stdin.write(b"C\n") start_time = time.time() time.sleep(0.1) except IOError: self.abort() break # Did we unpack a new volume? Sometimes UnRar hangs on 1 volume if not last_volume_linebuf or last_volume_linebuf != linebuf: # Next volume self.cur_volume += 1 self.nzo.set_action_line(T("Direct Unpack"), self.get_formatted_stats()) logging.info("DirectUnpacked volume %s for %s", self.cur_volume, self.cur_setname) # If lines did not change and we don't have the next volume, this download is missing files! # In rare occasions we can get stuck forever with repeating lines if last_volume_linebuf == linebuf: if not self.have_next_volume( ) or self.duplicate_lines > 10: logging.info( "DirectUnpack failed due to missing files %s", self.cur_setname) self.abort() else: logging.debug( 'Duplicate output line detected: "%s"', last_volume_linebuf) self.duplicate_lines += 1 else: self.duplicate_lines = 0 last_volume_linebuf = linebuf # Show the log if linebuf.endswith("\n"): unrar_log.append(linebuf.strip()) linebuf = "" # Add last line unrar_log.append(linebuf.strip()) logging.debug("DirectUnpack Unrar output %s", "\n".join(unrar_log)) # Make more space self.reset_active() if self in ACTIVE_UNPACKERS: ACTIVE_UNPACKERS.remove(self) # Set the thread to killed so it never gets restarted by accident self.killed = True
def run(self): # Input and output linebuf = '' last_volume_linebuf = '' unrar_log = [] rarfiles = [] extracted = [] start_time = time.time() # Need to read char-by-char because there's no newline after new-disk message while 1: if not self.active_instance: break char = self.active_instance.stdout.read(1) linebuf += char if not char: # End of program break # Error? Let PP-handle it if linebuf.endswith(('ERROR: ', 'Cannot create', 'in the encrypted file', 'CRC failed', 'checksum failed', 'You need to start extraction from a previous volume', 'password is incorrect', 'Write error', 'checksum error', 'start extraction from a previous volume')): logging.info('Error in DirectUnpack of %s', self.cur_setname) self.abort() if linebuf.endswith('\n'): # List files we used if linebuf.startswith('Extracting from'): filename = TRANS((re.search(EXTRACTFROM_RE, linebuf.strip()).group(1))) if filename not in rarfiles: rarfiles.append(filename) # List files we extracted m = re.search(EXTRACTED_RE, linebuf) if m: # In case of flat-unpack, UnRar still prints the whole path (?!) unpacked_file = TRANS(m.group(2)) if cfg.flat_unpack(): unpacked_file = os.path.basename(unpacked_file) extracted.append(real_path(self.unpack_dir_info[0], unpacked_file)) # Did we reach the end? if linebuf.endswith('All OK'): # Stop timer and finish self.unpack_time += time.time() - start_time ACTIVE_UNPACKERS.remove(self) # Add to success rarfile_path = os.path.join(self.nzo.downpath, self.rarfile_nzf.filename) self.success_sets[self.cur_setname] = (rar_volumelist(rarfile_path, self.nzo.password, rarfiles), extracted) logging.info('DirectUnpack completed for %s', self.cur_setname) self.nzo.set_action_line(T('Direct Unpack'), T('Completed')) # List success in history-info msg = T('Unpacked %s files/folders in %s') % (len(extracted), format_time_string(self.unpack_time)) msg = '%s - %s' % (T('Direct Unpack'), msg) self.nzo.set_unpack_info('Unpack', '[%s] %s' % (unicoder(self.cur_setname), msg)) # Write current log and clear unrar_log.append(linebuf.strip()) linebuf = '' last_volume_linebuf = '' logging.debug('DirectUnpack Unrar output %s', '\n'.join(unrar_log)) unrar_log = [] rarfiles = [] extracted = [] # Are there more files left? while self.nzo.files and not self.next_sets: with self.next_file_lock: self.next_file_lock.wait() # Is there another set to do? if self.next_sets: # Start new instance nzf = self.next_sets.pop(0) self.reset_active() self.cur_setname = nzf.setname # Wait for the 1st volume to appear self.wait_for_next_volume() self.create_unrar_instance() start_time = time.time() else: self.killed = True break if linebuf.endswith('[C]ontinue, [Q]uit '): # Stop timer self.unpack_time += time.time() - start_time # Wait for the next one.. self.wait_for_next_volume() # Possible that the instance was deleted while locked if not self.killed: # If unrar stopped or is killed somehow, writing will cause a crash try: # Give unrar some time to do it's thing self.active_instance.stdin.write('C\n') start_time = time.time() time.sleep(0.1) except IOError: self.abort() break # Did we unpack a new volume? Sometimes UnRar hangs on 1 volume if not last_volume_linebuf or last_volume_linebuf != linebuf: # Next volume self.cur_volume += 1 self.nzo.set_action_line(T('Direct Unpack'), self.get_formatted_stats()) logging.info('DirectUnpacked volume %s for %s', self.cur_volume, self.cur_setname) # If lines did not change and we don't have the next volume, this download is missing files! if last_volume_linebuf == linebuf: if not self.have_next_volume(): logging.info('DirectUnpack failed due to missing files %s', self.cur_setname) self.abort() else: logging.debug('Duplicate output line detected: "%s"', last_volume_linebuf) last_volume_linebuf = linebuf # Show the log if linebuf.endswith('\n'): unrar_log.append(linebuf.strip()) linebuf = '' # Add last line unrar_log.append(linebuf.strip()) logging.debug('DirectUnpack Unrar output %s', '\n'.join(unrar_log)) # Make more space self.reset_active() if self in ACTIVE_UNPACKERS: ACTIVE_UNPACKERS.remove(self) # Set the thread to killed so it never gets restarted by accident self.killed = True
def run(self): # Input and output linebuf = '' last_volume_linebuf = '' unrar_log = [] rarfiles = [] extracted = [] start_time = time.time() # Need to read char-by-char because there's no newline after new-disk message while 1: if not self.active_instance: break char = self.active_instance.stdout.read(1) linebuf += char if not char: # End of program break # Error? Let PP-handle it if linebuf.endswith(('ERROR: ', 'Cannot create', 'in the encrypted file', 'CRC failed', \ 'checksum failed', 'You need to start extraction from a previous volume', \ 'password is incorrect', 'Write error', 'checksum error', \ 'start extraction from a previous volume')): logging.info('Error in DirectUnpack of %s', self.cur_setname) self.abort() if linebuf.endswith('\n'): # List files we used if linebuf.startswith('Extracting from'): filename = TRANS((re.search(EXTRACTFROM_RE, linebuf.strip()).group(1))) if filename not in rarfiles: rarfiles.append(filename) # List files we extracted m = re.search(EXTRACTED_RE, linebuf) if m: # In case of flat-unpack, UnRar still prints the whole path (?!) unpacked_file = TRANS(m.group(2)) if cfg.flat_unpack(): unpacked_file = os.path.basename(unpacked_file) extracted.append( real_path(self.unpack_dir_info[0], unpacked_file)) # Did we reach the end? if linebuf.endswith('All OK'): # Stop timer and finish self.unpack_time += time.time() - start_time ACTIVE_UNPACKERS.remove(self) # Add to success rarfile_path = os.path.join(self.nzo.downpath, self.rarfile_nzf.filename) self.success_sets[self.cur_setname] = (rar_volumelist( rarfile_path, self.nzo.password, rarfiles), extracted) logging.info('DirectUnpack completed for %s', self.cur_setname) self.nzo.set_action_line(T('Direct Unpack'), T('Completed')) # List success in history-info msg = T('Unpacked %s files/folders in %s') % ( len(extracted), format_time_string(self.unpack_time)) msg = '%s - %s' % (T('Direct Unpack'), msg) self.nzo.set_unpack_info( 'Unpack', '[%s] %s' % (unicoder(self.cur_setname), msg)) # Write current log and clear unrar_log.append(linebuf.strip()) linebuf = '' last_volume_linebuf = '' logging.debug('DirectUnpack Unrar output %s', '\n'.join(unrar_log)) unrar_log = [] rarfiles = [] extracted = [] # Are there more files left? while self.nzo.files and not self.next_sets: with self.next_file_lock: self.next_file_lock.wait() # Is there another set to do? if self.next_sets: # Start new instance nzf = self.next_sets.pop(0) self.reset_active() self.cur_setname = nzf.setname # Wait for the 1st volume to appear self.wait_for_next_volume() self.create_unrar_instance() start_time = time.time() else: self.killed = True break if linebuf.endswith('[C]ontinue, [Q]uit '): # Stop timer self.unpack_time += time.time() - start_time # Wait for the next one.. self.wait_for_next_volume() # Possible that the instance was deleted while locked if not self.killed: # If unrar stopped or is killed somehow, writing will cause a crash try: # Give unrar some time to do it's thing self.active_instance.stdin.write('C\n') start_time = time.time() time.sleep(0.1) except IOError: self.abort() break # Did we unpack a new volume? Sometimes UnRar hangs on 1 volume if not last_volume_linebuf or last_volume_linebuf != linebuf: # Next volume self.cur_volume += 1 self.nzo.set_action_line(T('Direct Unpack'), self.get_formatted_stats()) logging.info('DirectUnpacked volume %s for %s', self.cur_volume, self.cur_setname) # If lines did not change and we don't have the next volume, this download is missing files! if last_volume_linebuf == linebuf: if not self.have_next_volume(): logging.info( 'DirectUnpack failed due to missing files %s', self.cur_setname) self.abort() else: logging.debug( 'Duplicate output line detected: "%s"', last_volume_linebuf) last_volume_linebuf = linebuf # Show the log if linebuf.endswith('\n'): unrar_log.append(linebuf.strip()) linebuf = '' # Add last line unrar_log.append(linebuf.strip()) logging.debug('DirectUnpack Unrar output %s', '\n'.join(unrar_log)) # Make more space self.reset_active() if self in ACTIVE_UNPACKERS: ACTIVE_UNPACKERS.remove(self) # Set the thread to killed so it never gets restarted by accident self.killed = True
def create_unrar_instance(self): """ Start the unrar instance using the user's options """ # Generate extraction path and save for post-proc if not self.unpack_dir_info: self.unpack_dir_info = prepare_extraction_path(self.nzo) extraction_path, _, _, one_folder, _ = self.unpack_dir_info # Set options if self.nzo.password: password_command = '-p%s' % self.nzo.password else: password_command = '-p-' if one_folder or cfg.flat_unpack(): action = 'e' else: action = 'x' # The first NZF self.rarfile_nzf = self.have_next_volume() # Generate command rarfile_path = os.path.join(self.nzo.downpath, self.rarfile_nzf.filename) if sabnzbd.WIN32: if not has_win_device(rarfile_path): command = [ '%s' % sabnzbd.newsunpack.RAR_COMMAND, action, '-vp', '-idp', '-o+', '-ai', password_command, '%s' % clip_path(rarfile_path), clip_path(extraction_path) ] else: # Need long-path notation in case of forbidden-names command = [ '%s' % sabnzbd.newsunpack.RAR_COMMAND, action, '-vp', '-idp', '-o+', '-ai', password_command, '%s' % clip_path(rarfile_path), '%s\\' % extraction_path ] else: # Don't use "-ai" (not needed for non-Windows) command = [ '%s' % sabnzbd.newsunpack.RAR_COMMAND, action, '-vp', '-idp', '-o+', password_command, '%s' % rarfile_path, '%s/' % extraction_path ] if cfg.ignore_unrar_dates(): command.insert(3, '-tsm-') # Let's start from the first one! self.cur_volume = 1 stup, need_shell, command, creationflags = build_command(command) logging.debug('Running unrar for DirectUnpack %s', command) self.active_instance = Popen(command, shell=need_shell, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, startupinfo=stup, creationflags=creationflags) # Add to runners ACTIVE_UNPACKERS.append(self) # Doing the first logging.info('DirectUnpacked volume %s for %s', self.cur_volume, self.cur_setname)