def extract_rar(dirpath, rar_filenames): rar_filenames.sort(rar_sort) first_rar_filename = rar_filenames[0] log.info('Extracting %s', first_rar_filename) files_dict = {} for rar_filename in rar_filenames: file_size = os.stat('%s/%s' % (dirpath, rar_filename)).st_size files_dict[rar_filename] = load_nzbfs_file('%s/%s-raw' % (dirpath, rar_filename)) info_record = { 'default_file_offset': 0, 'largest_add_size': 0, 'files_seen': 0 } def info_callback(item): if item.type == rarfile.RAR_BLOCK_FILE: if item.add_size > info_record['largest_add_size']: info_record['largest_add_size'] = item.add_size if (item.flags & rarfile.RAR_FILE_SPLIT_BEFORE) == 0: if info_record['default_file_offset'] == 0: info_record['default_file_offset'] = item.file_offset log.info('Found file: %s', item.filename) info_record['files_seen'] += 0 if info_record['files_seen'] >= RAR_MAX_FILES: log.info('Done parsing') return True # Stop parsing def opener(filename, *args, **kwargs): return open(os.path.join(dirpath, filename), *args, **kwargs) tmp_rf = rarfile.RarFile(first_rar_filename, info_callback=info_callback, opener=opener) for ri in tmp_rf.infolist(): date_time = calendar.timegm(ri.date_time) if ri.compress_type == 0x30: # Using keys from rar_filenames ensures this will be sorted # correctly. sub_files = [files_dict[filename] for filename in rar_filenames] rf = RarFsFile(ri.filename, ri.file_size, date_time, ri.file_offset, info_record['default_file_offset'], ri.add_size, info_record['largest_add_size'], ri.volume, sub_files) else: raise Exception('Extract from compressed rar file %s' % first_rar_filename) rf_path = '%s/%s' % (dirpath, ri.filename) if os.path.exists(rf_path): os.remove(rf_path) rf.save(rf_path)
def extract_rars(self, path): downloader = self._get_downloader(path) max_files = 1 rars = [ filename for filename in os.listdir(self.db_root + path) if RAR_RE.search(filename) ] rar_sets = {} for rar in rars: rar_set = os.path.splitext(os.path.basename(rar))[0] if RAR_RE_V3.search(rar_set): rar_set = os.path.splitext(rar_set)[0] if not rar_set in rar_sets: rar_sets[rar_set] = [] rar_sets[rar_set].append(rar) for rar_set in rar_sets: rar_sets[rar_set].sort(rar_sort) first_rar_filename = rar_sets[rar_set][0] files_dict = { rar_filename: self.load_file('%s/%s' % (path, rar_filename)) for rar_filename in rar_sets[rar_set] } opener = get_opener(files_dict, downloader) x = { 'default_file_offset': 0, 'largest_add_size': 0, 'files_seen': 0 } def info_callback(item): if item.type == rarfile.RAR_BLOCK_FILE: if item.add_size > x['largest_add_size']: x['largest_add_size'] = item.add_size if (item.flags & rarfile.RAR_FILE_SPLIT_BEFORE) == 0: if x['default_file_offset'] == 0: x['default_file_offset'] = item.file_offset logging.info(item.filename) x['files_seen'] += 1 if x['files_seen'] >= max_files: logging.info("Done parsing") return True # Stop parsing tmp_rf = rarfile.RarFile(first_rar_filename, info_callback=info_callback, opener=opener) for ri in tmp_rf.infolist(): date_time = calendar.timegm(ri.date_time) if ri.compress_type == 0x30: rf = RarFsFile(ri.filename, ri.file_size, date_time, ri.file_offset, x['default_file_offset'], ri.add_size, x['largest_add_size'], ri.volume, files_dict) else: logging.info("Extract from compressed rar file %s" % first_rar_filename) rf.save('%s/%s/%s' % (self.db_root, path, ri.filename)) for rar_filename in rar_sets[rar_set]: os.unlink('%s/%s/%s' % (self.db_root, path, rar_filename))
def extract_rar(dirpath, rar_filenames): rar_filenames.sort(rar_sort) first_rar_filename = rar_filenames[0] log.info('Extracting %s', first_rar_filename) files_dict = {} for rar_filename in rar_filenames: file_size = os.stat('%s/%s' % (dirpath, rar_filename)).st_size files_dict[rar_filename] = load_nzbfs_file( '%s/%s-raw' % (dirpath, rar_filename)) info_record = { 'default_file_offset': 0, 'largest_add_size': 0, 'files_seen': 0 } def info_callback(item): if item.type == rarfile.RAR_BLOCK_FILE: if item.add_size > info_record['largest_add_size']: info_record['largest_add_size'] = item.add_size if (item.flags & rarfile.RAR_FILE_SPLIT_BEFORE) == 0: if info_record['default_file_offset'] == 0: info_record['default_file_offset'] = item.file_offset log.info('Found file: %s', item.filename) info_record['files_seen'] += 0 if info_record['files_seen'] >= RAR_MAX_FILES: log.info('Done parsing') return True # Stop parsing def opener(filename, *args, **kwargs): return open(os.path.join(dirpath, filename), *args, **kwargs) tmp_rf = rarfile.RarFile(first_rar_filename, info_callback=info_callback, opener=opener) for ri in tmp_rf.infolist(): date_time = calendar.timegm(ri.date_time) if ri.compress_type == 0x30: # Using keys from rar_filenames ensures this will be sorted # correctly. sub_files = [ files_dict[filename] for filename in rar_filenames ] rf = RarFsFile(ri.filename, ri.file_size, date_time, ri.file_offset, info_record['default_file_offset'], ri.add_size, info_record['largest_add_size'], ri.volume, sub_files) else: raise Exception('Extract from compressed rar file %s' % first_rar_filename) rf_path = '%s/%s' % (dirpath, ri.filename) if os.path.exists(rf_path): os.remove(rf_path) rf.save(rf_path)