def sync_pos_files_lib_with_db(monitor_name, lib_path, extension): db_file = '\\'.join([paths.DB_LOG_PATH, monitor_name + ".txt"]) db_json = File.get_json_data(db_file) # Get all files in the directory files_list = File.list_files_sub(lib_path) for element in files_list: if element["filename"].endswith(extension): abs_path = element["path"] + "\\" + element["filename"] metadata = Ffmpeg.metadata_to_json(Ffmpeg.get_metadata(abs_path)) file_id = metadata.get("DISC", None) if file_id is None: raise ValueError("File has no ID: " + abs_path) json_info = db_json.get(file_id, None) position = json_info[YoutubeVideo.NUMBER] if int(metadata.get("TRACK")) != position: if json_info is None: raise ValueError("File has no JSON: " + abs_path) tags = {"track": str(position)} Ffmpeg.add_tags(abs_path, tags) new_name = element["path"] + "\\" + json_info[ YoutubeVideo.FILE_NAME] + "." + extension os.rename(abs_path, new_name)
def update_extra_credits(): db_file = '\\'.join([paths.DB_LOG_PATH, "ExtraCreditz.txt"]) db_json = File.get_json_data(db_file) check_path = "G:\\Filme\\ExtraCreditz" files_list = list_files_sub(check_path) for element in files_list: abs_path = element["path"] + "\\" + element["filename"] if abs_path.endswith(constants.MP4): metadata = Ffmpeg.metadata_to_json(Ffmpeg.get_metadata(abs_path)) print(metadata) track = int(metadata.get("TRACK", -1)) episode_id = metadata.get("EPISODE_ID", None) if (track > 0) and (episode_id is None): for key, value in db_json.items(): if value["NUMBER"] == track and value["STATUS"] == "UNABLE": print(key, track) tags = { "episode_id": key } if File.exists(abs_path): Ffmpeg.add_tags(abs_path, tags) value["STATUS"] = "DOWNLOADED" File.write_json_data(db_file, db_json)
def add_to_db(): name = "ThePrimeThanatos" check_path = "G:\\Music\\" + name extension = constants.MP3 result_path = "D:\\Automatica\\Python\\PyCharmProjects\\Kevolehc\\Kevolehc\\youtube\\files\\_db" dbfile = result_path + "\\" + name + ".txt" files_list = list_files_sub(check_path) result = {} for element in files_list: abs_path = element["path"] + "\\" + element["filename"] if extension == constants.MP4 or extension == constants.MKV: if abs_path.endswith(extension): metadata = Ffmpeg.metadata_to_json(Ffmpeg.get_metadata(abs_path)) episode_id = metadata.get("EPISODE_ID", None) if episode_id: title = metadata.get("TITLE") track = metadata.get("TRACK") result[episode_id] = {"TITLE": title, "STATUS": "DOWNLOADED", "NUMBER": track, "CHANNEL_NAME": name, "FILE_NAME": element["filename"], "FORMAT": extension} if extension == constants.MP3: if abs_path.endswith(extension): metadata = Ffmpeg.metadata_to_json(Ffmpeg.get_metadata(abs_path)) episode_id = metadata.get("DISC", None) if episode_id: title = metadata.get("TITLE") track = metadata.get("TRACK") result[episode_id] = {"TITLE": title, "STATUS": "DOWNLOADED", "NUMBER": track, "CHANNEL_NAME": name, "FILE_NAME": element["filename"], "FORMAT": extension} File.write_json_data(dbfile, result)
def delete_job(self, name): self.logger.debug("Supervisord: delete job %s Stop_job --> remove_process_group --> delete_config_file --> reload_config"%(name)) filee = File() conf_dir = SUPERVISORD["CONF_DIR"] + "/" + name + SUPERVISORD["CONF_EXTENSION"] filee.delete(conf_dir) time.sleep(1) self.sa.stop_process(name) self.sa.remove_process_group(name) return 0
def check_db_integrity(): monitors_db = paths.MAIN_MONITORS_PATH # monitors_db = paths.PGM_MONITORS_PATH # monitors_db = paths.SECONDARY_MONITORS_PATH dk_file = paths.API_KEY_PATH worker = YoutubeWorker(dk_file) manager = MonitorManager(monitors_db, worker, paths.YOUTUBE_API_LOG) for monitor in manager.monitors: default_reference_date = yt_datetime.get_default_ytdate() db_file = '\\'.join([paths.DB_LOG_PATH, monitor.name + ".txt"]) db_json = File.get_json_data(db_file) prepare_videos(manager, monitor, default_reference_date) for video in monitor.videos: db_video = db_json.get(video.id, None) # Check if video exists in db if db_video is None: db_json[video.id] = { "STATUS": "MISSING", "TITLE": video.title, "PUBLISHED_AT": video.publishedAt, "FILE_NAME": video.file_name, "SAVE_LOCATION": video.save_location, "NUMBER": video.number, "CHANNEL_NAME": video.channel_name, "FORMAT": monitor.format } print(db_json[video.id]) else: # Add timestamp if missing if db_video.get(YoutubeVideo.PUBLISHED_AT, None) is None: db_json[video.id][ YoutubeVideo.PUBLISHED_AT] = video.publishedAt # Compare timestamp if compare_yt_dates(db_video.get(YoutubeVideo.PUBLISHED_AT), video.publishedAt) != 0: print(str(db_video.get(YoutubeVideo.NUMBER)), "API: ", video.publishedAt, "DB: ", db_video.get(YoutubeVideo.PUBLISHED_AT), sep=" | ") db_json[video.id][ YoutubeVideo.PUBLISHED_AT] = video.publishedAt # Check and update video title in db if changed if not db_video.get(YoutubeVideo.TITLE) == video.title: print(str(db_video.get(YoutubeVideo.NUMBER)), video.title, db_video.get(YoutubeVideo.TITLE), sep=" | ") db_json[video.id][YoutubeVideo.TITLE] = video.title File.write_json_data(db_file, db_json)
def print_titles(output_filename, data): path = join(paths.TITLES_OUTPUT_PATH, output_filename) lines = [] for element in data: # print(element) lines.append(element["title"][0]) lines = sorted(set(lines), key=lambda s: s.lower()) File.write_lines_to_file_utf8(path, lines)
def set_supervisord_schedule(self,host=None, jid=None, name=None, ip=None): if not (host and jid and name and ip): print 'Missing options: host=%s, jid=%d, name=%s, ip=%s'%(host, jid, name, ip) self.logger.warning('Missing options: host=%s, jid=%d, name=%s, ip=%s'%(host, jid, name, ip)) supervisord_config = self.create_config_file(host=host, jid=jid, name=name, ip=ip) full_dir = SUPERVISORD["CONF_DIR"] + '/' + name + SUPERVISORD["CONF_EXTENSION"] filee = File() print filee.write(dir = full_dir, text = supervisord_config) self.logger.info("config file: %s, content: %s"%(full_dir, supervisord_config)) return 0
def finish(self, output_data): self.output_data = output_data File.write(self.output_file, self.output_data) self.time_finish = time.time() self.time_execution = self.time_finish - self.time_start if self.debug: print("------====== RESULT ======------") for line in self.output_data: print(line) print("------====== STATS ======------") print("Program ran for: ", self.time_execution, "seconds.")
def create_config_file(self,host=None, jid=None, name=None, ip=None): try: filee = File() supervisord_config_template = filee.read(SUPERVISORD["CONF_TEMPLATE_DIR"]) except Exception as e: self.logger.error(str(e)) base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) supervisord_config = supervisord_config_template.replace('{name}', name) supervisord_config = supervisord_config.replace('{base_dir}', base_dir) supervisord_config = supervisord_config.replace('{host}', host) supervisord_config = supervisord_config.replace('{jid}', str(jid)) supervisord_config = supervisord_config.replace('{ip}', ip) return supervisord_config
def shift_db_at_position(monitor_name, position, step): db_file = '\\'.join([paths.DB_LOG_PATH, monitor_name + ".txt"]) db_json = File.get_json_data(db_file) for key, value in db_json.items(): if value["NUMBER"] >= position: number = value["NUMBER"] next_number = number + step value["FILE_NAME"] = value["FILE_NAME"].replace( str(number) + " - ", str(next_number) + " - ") value["NUMBER"] = next_number File.write_json_data(db_file, db_json)
def shift_playlist_at_position(monitor_name, position, step): playlist_location = "F:\\Google Drive\\Mu\\plist\\" playlist_file = '\\'.join([playlist_location, monitor_name + ".txt"]) data = File.get_file_lines(playlist_file, "8") for i in range(len(data)): line = data[i] if "[" in line[1:2]: track_nr = int(line[167:]) if track_nr >= position: next_track_nr = track_nr + step data[i] = line[:167] + str(next_track_nr) File.write_lines_to_file_utf8(playlist_file, data)
def test_tags(): file_abs_path = "D:\\Automatica\\Python\\PyCharmProjects\\Kevolehc\\Kevolehc\\youtube\\" \ "files\\monitors\\GameChops" \ "\\605 - GameChops - Undertale Remix - Arcien - Heartache (from _Hopes & Dreams_) - GameChops.mp3" # "\\1053 - Extra Credits - ♫ Policing London - _Alleyways & Truncheons_ - Extra History Music.mp4" # "\\677 - ThePrimeThanatos - 'Back To The 80's' _ Best of Synthwave And Retro Electro Music Mix _ Vol. 21.mp3" if file_abs_path.endswith(constants.MP3): tags = { "genre": "GameChops", # "title": "'Back To The 80's | Best of Synthwave And Retro Electro Music Mix | Vol. 21", "title": "Undertale & Remix - Arcien - & Heartache (from \"Hopes & ss & Dreams\") & \" - GameChops \" zz \" &", "track": "605", "comment": "by Mefodii" } else: tags = { "author": "ExtraCreditz", "title": "♫ Policing London - \"Alleyways & Truncheons\" - Extra History Music", "track": str(1053), "copyright": "ExtraCreditz", "episode_id": "EO_16t_Fe5s", "comment": "by Mefodii" } print(file_abs_path) print(tags) print(File.exists(file_abs_path)) Ffmpeg.add_tags(file_abs_path, tags)
def __init__(self, data_file, is_open_stream = False): """Extracts meta data. If is_open_stream is True data_file is treated as a readable binary stream.""" self.data_stream = File(data_file, is_open_stream) magic_number = self.data_stream.read(len(MAGIC_NUMBER)) if magic_number != MAGIC_NUMBER.encode("utf-8"): raise NotCorrectMagicNumber("Compressed file should start with\ {}".format(MAGIC_NUMBER)) meta = self._get_meta(self.data_stream) self.orig_length = meta["orig_length"] self.word_length = meta["word_length"] self.encode_dict = meta["encode_dict"] self.data_index = meta["data_index"]
def download_video(self, queue: YoutubeQueue): # Download audio part default_audio_name = "audio" audio_file = queue.save_location + "\\" + default_audio_name output_file_path = audio_file + '.%(ext)s' ydl_opts = self.build_audio_fragment_download_options(output_file_path) with youtube_dl.YoutubeDL(ydl_opts) as ydl: ydl.download([queue.link]) result = ydl.extract_info("{}".format(queue.link)) queue.audio_dl_stats = copy.deepcopy(self.download_stats) # Download video part (has no sound) default_video_name = "video" video_file = queue.save_location + "\\" + default_video_name output_file_path = video_file + '.%(ext)s' ydl_opts = self.build_video_fragment_download_options(output_file_path) with youtube_dl.YoutubeDL(ydl_opts) as ydl: ydl.download([queue.link]) result = ydl.extract_info("{}".format(queue.link)) queue.video_dl_stats = copy.deepcopy(self.download_stats) # Replace tags from file_name by its values from queue.video_dl_stats queue.replace_file_name_tags() # Merge audio and video parts to single file audio_file = default_audio_name + "." + constants.M4A video_file_extension = File.get_file_name_with_extension(queue.save_location, default_video_name).split(".")[-1] video_file = default_video_name + "." + video_file_extension merged_file = queue.file_name + "." + queue.save_format Ffmpeg.merge_audio_and_video(queue.save_location, audio_file, video_file, merged_file)
def download_db_missing(): monitors_db = paths.MAIN_MONITORS_PATH # monitors_db = paths.SIDE_MONITORS_PATH # monitors_db = paths.SECONDARY_MONITORS_PATH dk_file = paths.API_RESERVE_KEY_PATH worker = YoutubeWorker(dk_file) manager = MonitorManager(monitors_db, worker, paths.YOUTUBE_API_LOG) for monitor in manager.monitors: db_file = '\\'.join([paths.DB_LOG_PATH, monitor.name + ".txt"]) db_json = File.get_json_data(db_file) for key, value in db_json.items(): if value["STATUS"] == "MISSING": # if value["STATUS"] == "UNABLE": yt_video = YoutubeVideo(key, value[YoutubeVideo.TITLE], value[YoutubeVideo.PUBLISHED_AT], value[YoutubeVideo.CHANNEL_NAME], value[YoutubeVideo.NUMBER]) monitor.append_video(yt_video) queue_manager = YoutubeQueueManager(paths.YOUTUBE_API_LOG) queue_manager.process_monitor_manager(manager) manager.append_tags() manager.update_track_list_log() manager.update_db_log()
def touch(filename, folder): ''' 创建文件 ''' if len(filename): file = File(filename) folder.fileList.append(file) else: print("文件名为空")
def __init__(self, data_file, word_len=1, is_open_stream=False): """ Initializes the compressor object and sets up everything, for compressing the file. The data_file is the path to the file, except if is_open_stream is True, in which case it is treated as a readable binary stream.""" self.file = File(data_file, word_len, is_open_stream) self.data_length = len(self.file) self.initialize() self.word_len = word_len
class CompressedData: def __init__(self, data_file, is_open_stream = False): """Extracts meta data. If is_open_stream is True data_file is treated as a readable binary stream.""" self.data_stream = File(data_file, is_open_stream) magic_number = self.data_stream.read(len(MAGIC_NUMBER)) if magic_number != MAGIC_NUMBER.encode("utf-8"): raise NotCorrectMagicNumber("Compressed file should start with\ {}".format(MAGIC_NUMBER)) meta = self._get_meta(self.data_stream) self.orig_length = meta["orig_length"] self.word_length = meta["word_length"] self.encode_dict = meta["encode_dict"] self.data_index = meta["data_index"] def _get_meta(self, data_stream): """Returns a dict containing original data length, the encode table and the compressed data""" meta = dict() orig_length = int.from_bytes(self.data_stream.read(8), "big") word_length = int.from_bytes(self.data_stream.read(8), "big") encode_dict_temp = BytesIO() while True: c = self.data_stream.read(1) encode_dict_temp.write(c) if c == b"}": break data_start = self.data_stream.tell() + len("DICT_END") encode_dict_temp.seek(0) encode_dict = encode_dict_temp.read() encode_dict = encode_dict[encode_dict.index(b"{") + 1: encode_dict.index(b"}")] meta["orig_length"] = orig_length meta["word_length"] = word_length meta["encode_dict"] = encode_dict meta["data_index"] = data_start return meta
def __init__(self, input_file, output_file, debug=False): self.time_start = time.time() self.time_finish = None self.time_execution = None self.input_file = input_file self.output_file = output_file self.input_data = File.read(input_file) self.output_data = [] self.debug = debug
def pull(cls, tokenizer, limit=None): dout = File.new_dir(os.path.join('dataset', cls.__name__.lower()), ensure_dir=True) draw = os.path.join(dout, 'raw') dann = os.path.join(dout, 'ann') if not os.path.isdir(draw): os.makedirs(draw) cls.download(draw) if not os.path.isdir(dann): os.makedirs(dann) anns = cls.annotate(tokenizer, draw, dann, limit=limit) return anns
def get_channel_uploads_from_date2(self, yt_id, yt_date): items = [] next_page = True token = "" while next_page: request = self.youtube.search().list(part="snippet,id", channelId=yt_id, maxResults=MAX_RESULTS, order="date", publishedAfter=yt_date, pageToken=token) response = request.execute() File.append_to_file("debug.txt", response.get('items')) File.append_to_file("debug.txt", "") items += response.get('items') token = response.get('nextPageToken') if not response.get('items') or not token: next_page = False return items
def cp(filename, pwd, folder, user): ''' 复制文件 ''' if pwd == '/': print("不能将文件复制到根目录") return folderList = pwd.split("/") oldFolder = folder newname = folderList.pop(-1) # 最后一个默认是新文件名 if not len(newname): print("目标文件名不能为空") return if not len(folderList): # 当前文件夹下 # 判断重名 for each in folder.fileList: if each.name == newname: print("文件重名") return # 找到要复制的文件 for each in oldFolder.fileList: if each.name == filename: # 新建文件 file = File(newname) file.content = each.content folder.fileList.append(file) break else: print("文件不存在") return # 其他情况, # 先切换到目标目录 pwd = pwd[:pwd.rfind(newname) - 1] if not len(pwd): # 此时说明在根目录下 print("不能将文件复制到根目录下") return folder = cd(pwd, folder, user) # 再复制文件 # 判断重名 for each in folder.fileList: if each.name == newname: print("文件重名") return # 找到要复制的文件 for each in oldFolder.fileList: if each.name == filename: # 新建文件 file = File(newname) file.content = each.content folder.fileList.append(file) break else: print("文件不存在") return
def create_supervisord_config(ip, name, jid, thomson_host): file = File() supervisord_config_template = file.read_template() base_dir = os.path.abspath(os.path.dirname(sys.argv[0])) supervisord_config = supervisord_config_template.replace('{name}', name) supervisord_config = supervisord_config.replace('{ip}', ip) supervisord_config = supervisord_config.replace('{jid}', str(jid)) supervisord_config = supervisord_config.replace('{host}', thomson_host) supervisord_config = supervisord_config.replace('{base_dir}', base_dir) full_dir = SUPERVISORD["CONF_DIR"] + '/' + name + SUPERVISORD["CONF_EXTENSION"] file.write_conf_file(dir = full_dir, text = supervisord_config) logger.info("config file: %s, content: %s"%(full_dir, supervisord_config)) return 0
def get_metadata(file_abs_path): file_format = file_abs_path.split(".")[-1] file_path = "\\".join(file_abs_path.split("\\")[:-1]) temp_abs_path = file_path + "\\tempmeta." + file_format temp_metadata_file = file_path + "\\tempmetadata.txt" os.rename(file_abs_path, temp_abs_path) read_metadata_command = "ffmpeg" \ " -i " + temp_abs_path + \ " -f ffmetadata " + temp_metadata_file os.system(read_metadata_command) os.rename(temp_abs_path, file_abs_path) metadata = File.get_file_lines(temp_metadata_file, "8") os.remove(temp_metadata_file) return metadata
def __init__(self, dk_file): self.dk = File.get_file_lines(dk_file)[0] self.youtube = googleapiclient.discovery.build(api_service_name, api_version, developerKey=self.dk)
def test_download_videos(links_json): video_list = [] q_list = [] name = "CriticalRole" format = constants.MKV output_directory = paths.YOUTUBE_RESULT_PATH data = {} for item in links_json: item_id = item["LINK"][32:] item_nr = item["TRACK"] data[item_id] = item_nr dk_file = '/'.join([paths.INPUT_FILES_PATH, paths.YOUTUBE_DK_FILE]) worker = YoutubeWorker(dk_file) videos = worker.get_videos(list(data.keys())) for item in videos: print(item) yt_video_params = YoutubeVideo.parse_json_yt_response(item) yt_video = YoutubeVideo(yt_video_params[YoutubeVideo.ID], yt_video_params[YoutubeVideo.TITLE], yt_video_params[YoutubeVideo.PUBLISHED_AT], yt_video_params[YoutubeVideo.CHANNEL_NAME], data.get(yt_video_params[YoutubeVideo.ID])) video_list.append(yt_video) for video in video_list: file_name = " - ".join([str(video.number), name, str(video.title)]) save_location = output_directory queue = YoutubeQueue(video.id, file_name, save_location, format) print(repr(queue)) video.file_name = queue.file_name video.save_location = queue.save_location q_list.append(queue) q_len = str(len(q_list)) i = 0 for queue in q_list: i += 1 q_progress = str(i) + "/" + q_len result_file = queue.save_location + "\\" + queue.file_name + "." + queue.save_format if File.exists(result_file): print("Queue ignored, file exist: " + q_progress) else: print("Process queue: " + q_progress + " - " + result_file) try: YoutubeDownloader.download(queue) except DownloadError: print("Unable to download - " + queue.link) for video in video_list: file_extension = format file_abs_path = "\\".join([video.save_location, video.file_name]) + "." + file_extension tags = { "author": name, "title": video.title, "track": str(video.number), "copyright": name, "episode_id": video.id, "comment": "by Mefodii" } if File.exists(file_abs_path): Ffmpeg.add_tags(file_abs_path, tags) else: print("Not found: " + file_abs_path)
def _load_file(self): self.file = File(self.__path) self.file.load()
class CompileFile(object): ''' class for loading file text and compiling markstache tags within markdown and JSON files. accepts a file cache list of File classes. ''' def __init__(self, file_path, root, file_extensions, valid_context_types): assert isinstance(file_path, str), \ 'expected file_path as str but got {}' \ .format(type(file_path)) assert isinstance(root, str), \ 'expected root as str but got {}' \ .format(type(file_path)) assert isinstance(file_extensions, dict), \ 'expected file_extension as dict but got {}' \ .format(type(file_extensions)) assert all([isinstance(val, list) for val in file_extensions.itervalues()]), \ 'file_extensions contains non-list elements: {}' \ .format([type(e) for e in file_extensions]) all_extensions = [ ext for val in file_extensions.itervalues() for ext in val ] assert len(all_extensions) == len(set(all_extensions)), \ 'file_extensions contains shared extensions across file types' assert isinstance(valid_context_types, tuple), \ 'expected valid_context_types as tuple but got {}' \ .format(type(valid_context_types)) assert all([isinstance(type_element, type) for type_element in valid_context_types]), \ 'non-type type passed to valid_context_types' # self._file_cache = file_cache if file_cache else [] self.file = None self.__path = os.path.abspath(os.path.join(root, file_path)) self.context = {} self.staches = {'matches': [], 'renders': []} self.__re = { 'stache': re.compile(r'{{.*}}'), 'ref': re.compile( r'^{{[\s\t]*_ref[\s\t]*=[\s\t]*(\(.*\)|[\'\"].*[\'\"])[\s\t]*}}$' ) } self.__file_extensions = file_extensions self.__valid_context_types = valid_context_types def process_file(self): ''' processes File class based on whether it is a markdown file, JSON file, or other. In the first two cases, files are compiles based on markstache tags in file ''' cache_paths = [f.path for f in FILE_CACHE.cache] if self.__path in cache_paths: return self._load_file() self._stache_search() if len(self.staches['matches']) == 0: self._finally() return self._compile() self._finally() def set_context(self, context=None): ''' set the context dict for compilation ''' assert isinstance(context, dict), \ 'expected context as dict but got {}' \ .format(type(context)) def _is_valid_type(var): ''' check context dict recursively for invalid type in values ''' if isinstance(var, dict): for key, val in var.iteritems(): if isinstance(val, dict): _is_valid_type(val) else: assert isinstance(val, self.__valid_context_types), \ 'invalid type passed in context:\n {}: {}'.format(key, type(val)) try: _is_valid_type(context) self.context = context except AssertionError as error: raise AssertionError, str(error) def _set_re(self, re_name, re_regex): assert isinstance(re_name, str), \ 'expected re_name as str but got {}' \ .format(type(re_name)) assert isinstance(re_regex, re._pattern_type), \ 'expected re_regex as regex object but got {}' \ .format(type(re_regex)) self.__re[re_name] = re_regex def _stache_search(self): self.staches['matches'] = self.__re['stache'].findall( self.file.contents) def _compile(self): for stache in self.staches['matches']: if re.match(self.__re['ref'], stache): self.staches['renders'].append('s') else: to_eval = stache.strip().strip('{}') compile_context = self.context.copy() try: eval_val = str(eval(to_eval, compile_context)) except NameError as error: eval_val = '' raise NameError, '{} in context dict'.format(str(error)) self.staches['renders'].append(eval_val) for i in range(len(self.staches['matches'])): # if re.match(self.__re['ref'], (self.staches['matches'][i]): self.file.contents = re.sub(self.__re['stache'], self.staches['renders'][i], self.file.contents) def _load_file(self): self.file = File(self.__path) self.file.load() def _finally(self): FILE_CACHE.add_to_cache(self.file)
def __init__(self, filename, bounds=Bounds(0,-1)): self._file = File(filename) self._bounds = bounds
class RawChunk(): def __init__(self, filename, bounds=Bounds(0,-1)): self._file = File(filename) self._bounds = bounds def close(self): self._file.close() @property def file(self): return self._file @property def bounds(self): return self._bounds @property def valid(self): return self._bounds.apply(self._file.length).size > 0 @property def size(self): return self._bounds.apply(self._file.length).size def __repr__(self): return '<RawChunk %s%s>' % (self._file.name, self._bounds) def read(self, limit=256): return self._file.read(self._bounds, limit) def readhex(self, limit=256): return self._file.readhex(self._bounds, limit) def readhexs(self, bounds, bytesperline=16): return self._file.readhexs(self._bounds, bytesperline) def crc32(self): return self._file.crc32(self._bounds) def adler32(self): return self._file.adler32(self._bounds) def md5(self): return self._file.md5(self._bounds) def _split(self, position): # print 'D: cutting %s at %d' % (self._bounds, position) if not self.bounds.contains(position): before, after = (RawChunk(self._file.name, self._bounds), None) else: before = RawChunk(self._file.name, Bounds(self._bounds.start, position - 1)) after = RawChunk(self._file.name, Bounds(position, self._bounds.end)) before = before.valid and before or None after = after.valid and after or None self.close() return (before, after) def chop(self, bounds): before, remains = self._split(bounds.start) if remains is None: chopped, after = (None, None) else: chopped, after = remains._split(bounds.end + 1) return (before, chopped, after) def cutat(self, position): return self._split(position) def debit(self, count, chopsize): chops = [] remains = self for i in range(0, count): if remains is not None and remains.valid: head, remains = remains._split(remains.bounds.start + chopsize) chops.append(head) chops.append(remains) return chops def divide(self, count): chopsize = max(self.size / count, 1) return self.debit(count, chopsize) # r = RawChunk("test/data") # print r # print r.bounds.size # print r.size # print [ r.read() for r in r.debit(1, -1) ]