def sync_pos_files_lib_with_db(monitor_name, lib_path, extension):
db_file = '\\'.join([paths.DB_LOG_PATH, monitor_name + ".txt"])
db_json = File.get_json_data(db_file)
# Get all files in the directory
files_list = File.list_files_sub(lib_path)
for element in files_list:
if element["filename"].endswith(extension):
abs_path = element["path"] + "\\" + element["filename"]
metadata = Ffmpeg.metadata_to_json(Ffmpeg.get_metadata(abs_path))
file_id = metadata.get("DISC", None)
if file_id is None:
raise ValueError("File has no ID: " + abs_path)
json_info = db_json.get(file_id, None)
position = json_info[YoutubeVideo.NUMBER]
if int(metadata.get("TRACK")) != position:
if json_info is None:
raise ValueError("File has no JSON: " + abs_path)
tags = {"track": str(position)}
Ffmpeg.add_tags(abs_path, tags)
new_name = element["path"] + "\\" + json_info[
YoutubeVideo.FILE_NAME] + "." + extension
os.rename(abs_path, new_name)
def update_extra_credits():
db_file = '\\'.join([paths.DB_LOG_PATH, "ExtraCreditz.txt"])
db_json = File.get_json_data(db_file)
check_path = "G:\\Filme\\ExtraCreditz"
files_list = list_files_sub(check_path)
for element in files_list:
abs_path = element["path"] + "\\" + element["filename"]
if abs_path.endswith(constants.MP4):
metadata = Ffmpeg.metadata_to_json(Ffmpeg.get_metadata(abs_path))
print(metadata)
track = int(metadata.get("TRACK", -1))
episode_id = metadata.get("EPISODE_ID", None)
if (track > 0) and (episode_id is None):
for key, value in db_json.items():
if value["NUMBER"] == track and value["STATUS"] == "UNABLE":
print(key, track)
tags = {
"episode_id": key
}
if File.exists(abs_path):
Ffmpeg.add_tags(abs_path, tags)
value["STATUS"] = "DOWNLOADED"
File.write_json_data(db_file, db_json)
def add_to_db():
name = "ThePrimeThanatos"
check_path = "G:\\Music\\" + name
extension = constants.MP3
result_path = "D:\\Automatica\\Python\\PyCharmProjects\\Kevolehc\\Kevolehc\\youtube\\files\\_db"
dbfile = result_path + "\\" + name + ".txt"
files_list = list_files_sub(check_path)
result = {}
for element in files_list:
abs_path = element["path"] + "\\" + element["filename"]
if extension == constants.MP4 or extension == constants.MKV:
if abs_path.endswith(extension):
metadata = Ffmpeg.metadata_to_json(Ffmpeg.get_metadata(abs_path))
episode_id = metadata.get("EPISODE_ID", None)
if episode_id:
title = metadata.get("TITLE")
track = metadata.get("TRACK")
result[episode_id] = {"TITLE": title, "STATUS": "DOWNLOADED", "NUMBER": track,
"CHANNEL_NAME": name, "FILE_NAME": element["filename"], "FORMAT": extension}
if extension == constants.MP3:
if abs_path.endswith(extension):
metadata = Ffmpeg.metadata_to_json(Ffmpeg.get_metadata(abs_path))
episode_id = metadata.get("DISC", None)
if episode_id:
title = metadata.get("TITLE")
track = metadata.get("TRACK")
result[episode_id] = {"TITLE": title, "STATUS": "DOWNLOADED", "NUMBER": track,
"CHANNEL_NAME": name, "FILE_NAME": element["filename"], "FORMAT": extension}
File.write_json_data(dbfile, result)
def delete_job(self, name):
self.logger.debug("Supervisord: delete job %s Stop_job --> remove_process_group --> delete_config_file --> reload_config"%(name))
filee = File()
conf_dir = SUPERVISORD["CONF_DIR"] + "/" + name + SUPERVISORD["CONF_EXTENSION"]
filee.delete(conf_dir)
time.sleep(1)
self.sa.stop_process(name)
self.sa.remove_process_group(name)
return 0
def check_db_integrity():
monitors_db = paths.MAIN_MONITORS_PATH
# monitors_db = paths.PGM_MONITORS_PATH
# monitors_db = paths.SECONDARY_MONITORS_PATH
dk_file = paths.API_KEY_PATH
worker = YoutubeWorker(dk_file)
manager = MonitorManager(monitors_db, worker, paths.YOUTUBE_API_LOG)
for monitor in manager.monitors:
default_reference_date = yt_datetime.get_default_ytdate()
db_file = '\\'.join([paths.DB_LOG_PATH, monitor.name + ".txt"])
db_json = File.get_json_data(db_file)
prepare_videos(manager, monitor, default_reference_date)
for video in monitor.videos:
db_video = db_json.get(video.id, None)
# Check if video exists in db
if db_video is None:
db_json[video.id] = {
"STATUS": "MISSING",
"TITLE": video.title,
"PUBLISHED_AT": video.publishedAt,
"FILE_NAME": video.file_name,
"SAVE_LOCATION": video.save_location,
"NUMBER": video.number,
"CHANNEL_NAME": video.channel_name,
"FORMAT": monitor.format
}
print(db_json[video.id])
else:
# Add timestamp if missing
if db_video.get(YoutubeVideo.PUBLISHED_AT, None) is None:
db_json[video.id][
YoutubeVideo.PUBLISHED_AT] = video.publishedAt
# Compare timestamp
if compare_yt_dates(db_video.get(YoutubeVideo.PUBLISHED_AT),
video.publishedAt) != 0:
print(str(db_video.get(YoutubeVideo.NUMBER)),
"API: ",
video.publishedAt,
"DB: ",
db_video.get(YoutubeVideo.PUBLISHED_AT),
sep=" | ")
db_json[video.id][
YoutubeVideo.PUBLISHED_AT] = video.publishedAt
# Check and update video title in db if changed
if not db_video.get(YoutubeVideo.TITLE) == video.title:
print(str(db_video.get(YoutubeVideo.NUMBER)),
video.title,
db_video.get(YoutubeVideo.TITLE),
sep=" | ")
db_json[video.id][YoutubeVideo.TITLE] = video.title
File.write_json_data(db_file, db_json)
def print_titles(output_filename, data):
path = join(paths.TITLES_OUTPUT_PATH, output_filename)
lines = []
for element in data:
# print(element)
lines.append(element["title"][0])
lines = sorted(set(lines), key=lambda s: s.lower())
File.write_lines_to_file_utf8(path, lines)
def set_supervisord_schedule(self,host=None, jid=None, name=None, ip=None):
if not (host and jid and name and ip):
print 'Missing options: host=%s, jid=%d, name=%s, ip=%s'%(host, jid, name, ip)
self.logger.warning('Missing options: host=%s, jid=%d, name=%s, ip=%s'%(host, jid, name, ip))
supervisord_config = self.create_config_file(host=host, jid=jid, name=name, ip=ip)
full_dir = SUPERVISORD["CONF_DIR"] + '/' + name + SUPERVISORD["CONF_EXTENSION"]
filee = File()
print filee.write(dir = full_dir, text = supervisord_config)
self.logger.info("config file: %s, content: %s"%(full_dir, supervisord_config))
return 0
def finish(self, output_data):
self.output_data = output_data
File.write(self.output_file, self.output_data)
self.time_finish = time.time()
self.time_execution = self.time_finish - self.time_start
if self.debug:
print("------====== RESULT ======------")
for line in self.output_data:
print(line)
print("------====== STATS ======------")
print("Program ran for: ", self.time_execution, "seconds.")
def create_config_file(self,host=None, jid=None, name=None, ip=None):
try:
filee = File()
supervisord_config_template = filee.read(SUPERVISORD["CONF_TEMPLATE_DIR"])
except Exception as e:
self.logger.error(str(e))
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
supervisord_config = supervisord_config_template.replace('{name}', name)
supervisord_config = supervisord_config.replace('{base_dir}', base_dir)
supervisord_config = supervisord_config.replace('{host}', host)
supervisord_config = supervisord_config.replace('{jid}', str(jid))
supervisord_config = supervisord_config.replace('{ip}', ip)
return supervisord_config
def shift_db_at_position(monitor_name, position, step):
db_file = '\\'.join([paths.DB_LOG_PATH, monitor_name + ".txt"])
db_json = File.get_json_data(db_file)
for key, value in db_json.items():
if value["NUMBER"] >= position:
number = value["NUMBER"]
next_number = number + step
value["FILE_NAME"] = value["FILE_NAME"].replace(
str(number) + " - ",
str(next_number) + " - ")
value["NUMBER"] = next_number
File.write_json_data(db_file, db_json)
def shift_playlist_at_position(monitor_name, position, step):
playlist_location = "F:\\Google Drive\\Mu\\plist\\"
playlist_file = '\\'.join([playlist_location, monitor_name + ".txt"])
data = File.get_file_lines(playlist_file, "8")
for i in range(len(data)):
line = data[i]
if "[" in line[1:2]:
track_nr = int(line[167:])
if track_nr >= position:
next_track_nr = track_nr + step
data[i] = line[:167] + str(next_track_nr)
File.write_lines_to_file_utf8(playlist_file, data)
def test_tags():
file_abs_path = "D:\\Automatica\\Python\\PyCharmProjects\\Kevolehc\\Kevolehc\\youtube\\" \
"files\\monitors\\GameChops" \
"\\605 - GameChops - Undertale Remix - Arcien - Heartache (from _Hopes & Dreams_) - GameChops.mp3"
# "\\1053 - Extra Credits - ♫ Policing London - _Alleyways & Truncheons_ - Extra History Music.mp4"
# "\\677 - ThePrimeThanatos - 'Back To The 80's' _ Best of Synthwave And Retro Electro Music Mix _ Vol. 21.mp3"
if file_abs_path.endswith(constants.MP3):
tags = {
"genre": "GameChops",
# "title": "'Back To The 80's | Best of Synthwave And Retro Electro Music Mix | Vol. 21",
"title": "Undertale & Remix - Arcien - & Heartache (from \"Hopes & ss & Dreams\") & \" - GameChops \" zz \" &",
"track": "605",
"comment": "by Mefodii"
}
else:
tags = {
"author": "ExtraCreditz",
"title": "♫ Policing London - \"Alleyways & Truncheons\" - Extra History Music",
"track": str(1053),
"copyright": "ExtraCreditz",
"episode_id": "EO_16t_Fe5s",
"comment": "by Mefodii"
}
print(file_abs_path)
print(tags)
print(File.exists(file_abs_path))
Ffmpeg.add_tags(file_abs_path, tags)
def __init__(self, data_file, is_open_stream = False):
"""Extracts meta data.
If is_open_stream is True data_file is treated as a readable
binary stream."""
self.data_stream = File(data_file, is_open_stream)
magic_number = self.data_stream.read(len(MAGIC_NUMBER))
if magic_number != MAGIC_NUMBER.encode("utf-8"):
raise NotCorrectMagicNumber("Compressed file should start with\
{}".format(MAGIC_NUMBER))
meta = self._get_meta(self.data_stream)
self.orig_length = meta["orig_length"]
self.word_length = meta["word_length"]
self.encode_dict = meta["encode_dict"]
self.data_index = meta["data_index"]
def download_video(self, queue: YoutubeQueue):
# Download audio part
default_audio_name = "audio"
audio_file = queue.save_location + "\\" + default_audio_name
output_file_path = audio_file + '.%(ext)s'
ydl_opts = self.build_audio_fragment_download_options(output_file_path)
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
ydl.download([queue.link])
result = ydl.extract_info("{}".format(queue.link))
queue.audio_dl_stats = copy.deepcopy(self.download_stats)
# Download video part (has no sound)
default_video_name = "video"
video_file = queue.save_location + "\\" + default_video_name
output_file_path = video_file + '.%(ext)s'
ydl_opts = self.build_video_fragment_download_options(output_file_path)
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
ydl.download([queue.link])
result = ydl.extract_info("{}".format(queue.link))
queue.video_dl_stats = copy.deepcopy(self.download_stats)
# Replace tags from file_name by its values from queue.video_dl_stats
queue.replace_file_name_tags()
# Merge audio and video parts to single file
audio_file = default_audio_name + "." + constants.M4A
video_file_extension = File.get_file_name_with_extension(queue.save_location, default_video_name).split(".")[-1]
video_file = default_video_name + "." + video_file_extension
merged_file = queue.file_name + "." + queue.save_format
Ffmpeg.merge_audio_and_video(queue.save_location, audio_file, video_file, merged_file)
def download_db_missing():
monitors_db = paths.MAIN_MONITORS_PATH
# monitors_db = paths.SIDE_MONITORS_PATH
# monitors_db = paths.SECONDARY_MONITORS_PATH
dk_file = paths.API_RESERVE_KEY_PATH
worker = YoutubeWorker(dk_file)
manager = MonitorManager(monitors_db, worker, paths.YOUTUBE_API_LOG)
for monitor in manager.monitors:
db_file = '\\'.join([paths.DB_LOG_PATH, monitor.name + ".txt"])
db_json = File.get_json_data(db_file)
for key, value in db_json.items():
if value["STATUS"] == "MISSING":
# if value["STATUS"] == "UNABLE":
yt_video = YoutubeVideo(key, value[YoutubeVideo.TITLE],
value[YoutubeVideo.PUBLISHED_AT],
value[YoutubeVideo.CHANNEL_NAME],
value[YoutubeVideo.NUMBER])
monitor.append_video(yt_video)
queue_manager = YoutubeQueueManager(paths.YOUTUBE_API_LOG)
queue_manager.process_monitor_manager(manager)
manager.append_tags()
manager.update_track_list_log()
manager.update_db_log()
def touch(filename, folder):
'''
创建文件
'''
if len(filename):
file = File(filename)
folder.fileList.append(file)
else:
print("文件名为空")
def __init__(self, data_file, word_len=1, is_open_stream=False):
""" Initializes the compressor object and sets up everything,
for compressing the file.
The data_file is the path to the file, except if is_open_stream is True,
in which case it is treated as a readable binary stream."""
self.file = File(data_file, word_len, is_open_stream)
self.data_length = len(self.file)
self.initialize()
self.word_len = word_len
class CompressedData:
def __init__(self, data_file, is_open_stream = False):
"""Extracts meta data.
If is_open_stream is True data_file is treated as a readable
binary stream."""
self.data_stream = File(data_file, is_open_stream)
magic_number = self.data_stream.read(len(MAGIC_NUMBER))
if magic_number != MAGIC_NUMBER.encode("utf-8"):
raise NotCorrectMagicNumber("Compressed file should start with\
{}".format(MAGIC_NUMBER))
meta = self._get_meta(self.data_stream)
self.orig_length = meta["orig_length"]
self.word_length = meta["word_length"]
self.encode_dict = meta["encode_dict"]
self.data_index = meta["data_index"]
def _get_meta(self, data_stream):
"""Returns a dict containing original data length,
the encode table and the compressed data"""
meta = dict()
orig_length = int.from_bytes(self.data_stream.read(8), "big")
word_length = int.from_bytes(self.data_stream.read(8), "big")
encode_dict_temp = BytesIO()
while True:
c = self.data_stream.read(1)
encode_dict_temp.write(c)
if c == b"}":
break
data_start = self.data_stream.tell() + len("DICT_END")
encode_dict_temp.seek(0)
encode_dict = encode_dict_temp.read()
encode_dict = encode_dict[encode_dict.index(b"{") + 1: encode_dict.index(b"}")]
meta["orig_length"] = orig_length
meta["word_length"] = word_length
meta["encode_dict"] = encode_dict
meta["data_index"] = data_start
return meta
def __init__(self, input_file, output_file, debug=False):
self.time_start = time.time()
self.time_finish = None
self.time_execution = None
self.input_file = input_file
self.output_file = output_file
self.input_data = File.read(input_file)
self.output_data = []
self.debug = debug
def pull(cls, tokenizer, limit=None):
dout = File.new_dir(os.path.join('dataset', cls.__name__.lower()),
ensure_dir=True)
draw = os.path.join(dout, 'raw')
dann = os.path.join(dout, 'ann')
if not os.path.isdir(draw):
os.makedirs(draw)
cls.download(draw)
if not os.path.isdir(dann):
os.makedirs(dann)
anns = cls.annotate(tokenizer, draw, dann, limit=limit)
return anns
def get_channel_uploads_from_date2(self, yt_id, yt_date):
items = []
next_page = True
token = ""
while next_page:
request = self.youtube.search().list(part="snippet,id",
channelId=yt_id,
maxResults=MAX_RESULTS,
order="date",
publishedAfter=yt_date,
pageToken=token)
response = request.execute()
File.append_to_file("debug.txt", response.get('items'))
File.append_to_file("debug.txt", "")
items += response.get('items')
token = response.get('nextPageToken')
if not response.get('items') or not token:
next_page = False
return items
def cp(filename, pwd, folder, user):
'''
复制文件
'''
if pwd == '/':
print("不能将文件复制到根目录")
return
folderList = pwd.split("/")
oldFolder = folder
newname = folderList.pop(-1) # 最后一个默认是新文件名
if not len(newname):
print("目标文件名不能为空")
return
if not len(folderList): # 当前文件夹下
# 判断重名
for each in folder.fileList:
if each.name == newname:
print("文件重名")
return
# 找到要复制的文件
for each in oldFolder.fileList:
if each.name == filename:
# 新建文件
file = File(newname)
file.content = each.content
folder.fileList.append(file)
break
else:
print("文件不存在")
return
# 其他情况，
# 先切换到目标目录
pwd = pwd[:pwd.rfind(newname) - 1]
if not len(pwd): # 此时说明在根目录下
print("不能将文件复制到根目录下")
return
folder = cd(pwd, folder, user)
# 再复制文件
# 判断重名
for each in folder.fileList:
if each.name == newname:
print("文件重名")
return
# 找到要复制的文件
for each in oldFolder.fileList:
if each.name == filename:
# 新建文件
file = File(newname)
file.content = each.content
folder.fileList.append(file)
break
else:
print("文件不存在")
return
def create_supervisord_config(ip, name, jid, thomson_host):
file = File()
supervisord_config_template = file.read_template()
base_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
supervisord_config = supervisord_config_template.replace('{name}', name)
supervisord_config = supervisord_config.replace('{ip}', ip)
supervisord_config = supervisord_config.replace('{jid}', str(jid))
supervisord_config = supervisord_config.replace('{host}', thomson_host)
supervisord_config = supervisord_config.replace('{base_dir}', base_dir)
full_dir = SUPERVISORD["CONF_DIR"] + '/' + name + SUPERVISORD["CONF_EXTENSION"]
file.write_conf_file(dir = full_dir, text = supervisord_config)
logger.info("config file: %s, content: %s"%(full_dir, supervisord_config))
return 0
def get_metadata(file_abs_path):
file_format = file_abs_path.split(".")[-1]
file_path = "\\".join(file_abs_path.split("\\")[:-1])
temp_abs_path = file_path + "\\tempmeta." + file_format
temp_metadata_file = file_path + "\\tempmetadata.txt"
os.rename(file_abs_path, temp_abs_path)
read_metadata_command = "ffmpeg" \
" -i " + temp_abs_path + \
" -f ffmetadata " + temp_metadata_file
os.system(read_metadata_command)
os.rename(temp_abs_path, file_abs_path)
metadata = File.get_file_lines(temp_metadata_file, "8")
os.remove(temp_metadata_file)
return metadata
def __init__(self, dk_file):
self.dk = File.get_file_lines(dk_file)[0]
self.youtube = googleapiclient.discovery.build(api_service_name,
api_version,
developerKey=self.dk)
def test_download_videos(links_json):
video_list = []
q_list = []
name = "CriticalRole"
format = constants.MKV
output_directory = paths.YOUTUBE_RESULT_PATH
data = {}
for item in links_json:
item_id = item["LINK"][32:]
item_nr = item["TRACK"]
data[item_id] = item_nr
dk_file = '/'.join([paths.INPUT_FILES_PATH, paths.YOUTUBE_DK_FILE])
worker = YoutubeWorker(dk_file)
videos = worker.get_videos(list(data.keys()))
for item in videos:
print(item)
yt_video_params = YoutubeVideo.parse_json_yt_response(item)
yt_video = YoutubeVideo(yt_video_params[YoutubeVideo.ID], yt_video_params[YoutubeVideo.TITLE],
yt_video_params[YoutubeVideo.PUBLISHED_AT],
yt_video_params[YoutubeVideo.CHANNEL_NAME], data.get(yt_video_params[YoutubeVideo.ID]))
video_list.append(yt_video)
for video in video_list:
file_name = " - ".join([str(video.number), name, str(video.title)])
save_location = output_directory
queue = YoutubeQueue(video.id, file_name, save_location, format)
print(repr(queue))
video.file_name = queue.file_name
video.save_location = queue.save_location
q_list.append(queue)
q_len = str(len(q_list))
i = 0
for queue in q_list:
i += 1
q_progress = str(i) + "/" + q_len
result_file = queue.save_location + "\\" + queue.file_name + "." + queue.save_format
if File.exists(result_file):
print("Queue ignored, file exist: " + q_progress)
else:
print("Process queue: " + q_progress + " - " + result_file)
try:
YoutubeDownloader.download(queue)
except DownloadError:
print("Unable to download - " + queue.link)
for video in video_list:
file_extension = format
file_abs_path = "\\".join([video.save_location, video.file_name]) + "." + file_extension
tags = {
"author": name,
"title": video.title,
"track": str(video.number),
"copyright": name,
"episode_id": video.id,
"comment": "by Mefodii"
}
if File.exists(file_abs_path):
Ffmpeg.add_tags(file_abs_path, tags)
else:
print("Not found: " + file_abs_path)
def _load_file(self):
self.file = File(self.__path)
self.file.load()
class CompileFile(object):
'''
class for loading file text and compiling markstache tags within markdown and JSON files.
accepts a file cache list of File classes.
'''
def __init__(self, file_path, root, file_extensions, valid_context_types):
assert isinstance(file_path, str), \
'expected file_path as str but got {}' \
.format(type(file_path))
assert isinstance(root, str), \
'expected root as str but got {}' \
.format(type(file_path))
assert isinstance(file_extensions, dict), \
'expected file_extension as dict but got {}' \
.format(type(file_extensions))
assert all([isinstance(val, list) for val in file_extensions.itervalues()]), \
'file_extensions contains non-list elements: {}' \
.format([type(e) for e in file_extensions])
all_extensions = [
ext for val in file_extensions.itervalues() for ext in val
]
assert len(all_extensions) == len(set(all_extensions)), \
'file_extensions contains shared extensions across file types'
assert isinstance(valid_context_types, tuple), \
'expected valid_context_types as tuple but got {}' \
.format(type(valid_context_types))
assert all([isinstance(type_element, type) for type_element in valid_context_types]), \
'non-type type passed to valid_context_types'
# self._file_cache = file_cache if file_cache else []
self.file = None
self.__path = os.path.abspath(os.path.join(root, file_path))
self.context = {}
self.staches = {'matches': [], 'renders': []}
self.__re = {
'stache':
re.compile(r'{{.*}}'),
'ref':
re.compile(
r'^{{[\s\t]*_ref[\s\t]*=[\s\t]*(\(.*\)|[\'\"].*[\'\"])[\s\t]*}}$'
)
}
self.__file_extensions = file_extensions
self.__valid_context_types = valid_context_types
def process_file(self):
'''
processes File class based on whether it is a markdown file, JSON file, or other. In the
first two cases, files are compiles based on markstache tags in file
'''
cache_paths = [f.path for f in FILE_CACHE.cache]
if self.__path in cache_paths:
return
self._load_file()
self._stache_search()
if len(self.staches['matches']) == 0:
self._finally()
return
self._compile()
self._finally()
def set_context(self, context=None):
'''
set the context dict for compilation
'''
assert isinstance(context, dict), \
'expected context as dict but got {}' \
.format(type(context))
def _is_valid_type(var):
'''
check context dict recursively for invalid type in values
'''
if isinstance(var, dict):
for key, val in var.iteritems():
if isinstance(val, dict):
_is_valid_type(val)
else:
assert isinstance(val, self.__valid_context_types), \
'invalid type passed in context:\n {}: {}'.format(key, type(val))
try:
_is_valid_type(context)
self.context = context
except AssertionError as error:
raise AssertionError, str(error)
def _set_re(self, re_name, re_regex):
assert isinstance(re_name, str), \
'expected re_name as str but got {}' \
.format(type(re_name))
assert isinstance(re_regex, re._pattern_type), \
'expected re_regex as regex object but got {}' \
.format(type(re_regex))
self.__re[re_name] = re_regex
def _stache_search(self):
self.staches['matches'] = self.__re['stache'].findall(
self.file.contents)
def _compile(self):
for stache in self.staches['matches']:
if re.match(self.__re['ref'], stache):
self.staches['renders'].append('s')
else:
to_eval = stache.strip().strip('{}')
compile_context = self.context.copy()
try:
eval_val = str(eval(to_eval, compile_context))
except NameError as error:
eval_val = ''
raise NameError, '{} in context dict'.format(str(error))
self.staches['renders'].append(eval_val)
for i in range(len(self.staches['matches'])):
# if re.match(self.__re['ref'], (self.staches['matches'][i]):
self.file.contents = re.sub(self.__re['stache'],
self.staches['renders'][i],
self.file.contents)
def _load_file(self):
self.file = File(self.__path)
self.file.load()
def _finally(self):
FILE_CACHE.add_to_cache(self.file)
def __init__(self, filename, bounds=Bounds(0,-1)): self._file = File(filename) self._bounds = bounds
class RawChunk():
def __init__(self, filename, bounds=Bounds(0,-1)):
self._file = File(filename)
self._bounds = bounds
def close(self):
self._file.close()
@property
def file(self):
return self._file
@property
def bounds(self):
return self._bounds
@property
def valid(self):
return self._bounds.apply(self._file.length).size > 0
@property
def size(self):
return self._bounds.apply(self._file.length).size
def __repr__(self):
return '<RawChunk %s%s>' % (self._file.name, self._bounds)
def read(self, limit=256):
return self._file.read(self._bounds, limit)
def readhex(self, limit=256):
return self._file.readhex(self._bounds, limit)
def readhexs(self, bounds, bytesperline=16):
return self._file.readhexs(self._bounds, bytesperline)
def crc32(self):
return self._file.crc32(self._bounds)
def adler32(self):
return self._file.adler32(self._bounds)
def md5(self):
return self._file.md5(self._bounds)
def _split(self, position):
# print 'D: cutting %s at %d' % (self._bounds, position)
if not self.bounds.contains(position):
before, after = (RawChunk(self._file.name, self._bounds), None)
else:
before = RawChunk(self._file.name, Bounds(self._bounds.start, position - 1))
after = RawChunk(self._file.name, Bounds(position, self._bounds.end))
before = before.valid and before or None
after = after.valid and after or None
self.close()
return (before, after)
def chop(self, bounds):
before, remains = self._split(bounds.start)
if remains is None:
chopped, after = (None, None)
else:
chopped, after = remains._split(bounds.end + 1)
return (before, chopped, after)
def cutat(self, position):
return self._split(position)
def debit(self, count, chopsize):
chops = []
remains = self
for i in range(0, count):
if remains is not None and remains.valid:
head, remains = remains._split(remains.bounds.start + chopsize)
chops.append(head)
chops.append(remains)
return chops
def divide(self, count):
chopsize = max(self.size / count, 1)
return self.debit(count, chopsize)
# r = RawChunk("test/data")
# print r
# print r.bounds.size
# print r.size
# print [ r.read() for r in r.debit(1, -1) ]
