class Scheduler:
def __init__(self):
self.downloader = Downloader()
self.hasher = Hasher()
self.storage = Storage()
def _wrap_path(self, md5):
filename = "{}.jpg".format(md5)
STORAGE_PATH = os.path.join(".", "images")
path = os.path.join(STORAGE_PATH, filename)
return path
def set_calc_type(self, type_):
self.downloader.set_calc_type(type_)
self.hasher.set_calc_type(type_)
self.storage.set_calc_type(type_)
def process(self):
# 1.加载图片列表
url_list = utils.urllist()
# 2.调度下载模块
content_list = self.downloader.process(url_list)
# 3. 计算哈希
md5_list = self.hasher.process(content_list)
for md5 in md5_list:
print(md5)
# 4.存储模块
item_list = []
for content, md5 in zip(content_list, md5_list):
path = self._wrap_path(md5)
item = (content, path)
item_list.append(item)
self.storage.process(item_list)
def statictics(self, log):
table = prettytable.PrettyTable("类型", "线程总耗时")
network_row = ["network"]
cpu_row = ["cpu"]
disk_row = ["disk"]
network_row.append(log["network_time"][0])
cpu_row.append(log["cpu_time"][0])
disk_row.append(log["disk_time"][0])
table.add_row(network_row)
table.add_row(cpu_row)
table.add_row(disk_row)
print(table)
def __init__(self,
Downloader=Downloader(),
Uploader=LocalFileSystemUploader()):
self.downloader = Downloader
self.uploader = Uploader
self.cdc_handler = None
if USE_CDC_HANDLER:
self.cdc_handler = CDCHandler()
def main():
static_path = os.path.join(os.getcwd(), 'Static')
blizzard_addons_directory_path = f'{args.blizz_addon_path}'
addons = os.listdir(blizzard_addons_directory_path)
extracted_elvui_directory = os.path.join(static_path, args.extracted_elvui)
if not os.path.exists(static_path):
logger.info(f"Creating static path: {static_path}.")
os.mkdir(static_path)
# Download handling.
downloader = Downloader(static_directory_path=static_path)
downloader.run()
# Zipping Handling.
zip_file = UnZipFile(zip_path=downloader.zip_file_path,
current_version=downloader.current_elvui_version,
extracted_dir_name=args.extracted_elvui,
static_directory_path=static_path)
zip_file.unzip()
# ElvUI addons backuping.
elvui_addons = list(
filter(lambda dir: (str(dir).lower().find('elvui') > -1), addons))
elvui_directories = list(
map(
lambda elvui_directory: os.path.join(
blizzard_addons_directory_path, elvui_directory),
elvui_addons))
backup_file_handler = ZipFile(static_directory_path=static_path)
for elvui_directory in elvui_directories:
backup_file_handler.add_directory_to_be_compressed(elvui_directory)
backup_file_handler.compress()
# ElvUI file replacement.
replace_handler = ReplaceDirs(
blizzard_addons_path=args.blizz_addon_path,
extracted_version_path=f'{extracted_elvui_directory}')
replace_handler.replace()
def main():
if terminal.get_argument_count() < 3:
terminal.print_help()
exit()
else:
firstArg = terminal.get_argument(1)
secondArg = terminal.get_argument(2)
if firstArg == "help":
terminal.print_help()
elif firstArg == "edit":
MP3(secondArg).edit()
else:
downloader = Downloader(secondArg)
new_videos = downloader.find_new_videos(firstArg)
terminal.print_green(
str(len(new_videos)) + " new videos found in playlist")
downloader.download_videos(new_videos)
for fail in fails.fails:
terminal.print_red("failed: " + fail)
class Scheduler:
def __init__(self):
self.downloader = Downloader()
self.hasher = Hasher()
self.storager = Storager()
self.md5_map = {}
def set_calc_type(self, type_):
self.downloader.set_calc_type(type_)
self.hasher.set_calc_type(type_)
self.storager.set_calc_type(type_)
def _wrap_path(self, md5):
if md5 in self.md5_map:
self.md5_map[md5] += 1
else:
self.md5_map[md5] = 1
number = self.md5_map[md5]
filename = f'{md5}.jpg' if number == 1 else f'{md5}_{number}.jpg'
storage_path = os.path.join(".", "images")
path = os.path.join(storage_path, filename)
return path
def process(self):
time_statictics = {
"network_time": [],
"cpu_time": [],
"disk_time": [],
}
timer = utils.Timer()
url_list = utils.url_list()
timer.tick()
content_list = self.downloader.process(url_list)
time_cost = timer.tock()
time_statictics["network_time"].append(time_cost)
timer.tick()
md5_list = self.hasher.process(content_list)
time_cost = timer.tock()
time_statictics["cpu_time"].append(time_cost)
item_list = []
for content, md5 in zip(content_list, md5_list):
path = self._wrap_path(md5)
item = (content, path)
item_list.append(item)
timer.tick()
self.storager.process(item_list)
time_cost = timer.tock()
time_statictics["disk_time"].append(time_cost)
return time_statictics
def create_row(self, type_, times):
return [
type_,
times[0],
times[1],
'%.4f%%' % ((times[0] - times[1]) / times[0] * 100),
times[2],
'%.4f%%' % ((times[0] - times[2]) / times[0] * 100),
]
def statictics(self, single_log, multi_thread_log, multi_process_log):
# table = prettytable.PrettyTable(["类型", "单线程耗时", "多线程耗时", "多线程提升率", "多进程耗时", "多进程提升率"])
table = prettytable.PrettyTable([
"type", "single thread", "multi thread", "thread perform",
"multi process", "process perform"
])
network_time = (
single_log["network_time"][0],
multi_thread_log["network_time"][0],
multi_process_log["network_time"][0],
)
network_row = self.create_row('network', network_time)
cpu_time = (
single_log["cpu_time"][0],
multi_thread_log["cpu_time"][0],
multi_process_log["cpu_time"][0],
)
cpu_row = self.create_row('cpu', cpu_time)
disk_time = (
single_log["disk_time"][0],
multi_thread_log["disk_time"][0],
multi_process_log["disk_time"][0],
)
disk_row = self.create_row('disk', disk_time)
table.add_row(network_row)
table.add_row(cpu_row)
table.add_row(disk_row)
print(table)
def __init__(self):
self.downloader = Downloader()
self.hasher = Hasher()
self.storager = Storager()
self.md5_map = {}
#paths
P_TOKENS = Path('tokens').absolute()
# paths that needs to be checked inside clipper
P_PACKAGES = Path('packages').absolute()
P_CLIPS = Path('clips').absolute()
P_OUTPUTS = Path('outputs').absolute()
P_VIDEOS_MEDIA = Path('videos').absolute()
dirs_paths = list([P_PACKAGES, P_CLIPS, P_OUTPUTS, P_VIDEOS_MEDIA])
check_paths()
modules = list()
modules.append(
Packager(ERROR_MESSAGES, P_PACKAGES, P_CLIPS, P_OUTPUTS, P_VIDEOS_MEDIA))
modules.append(Downloader(ERROR_MESSAGES, modules[0], P_TOKENS_FILE))
modules.append(Encoder(ERROR_MESSAGES, modules[0]))
modules.append(Editor(ERROR_MESSAGES, modules[0], P_VIDEOS_MEDIA))
modules.append(Uploader(ERROR_MESSAGES, modules[0], P_TOKENS, P_VIDEOS_MEDIA))
modules.append(Tweeter(ERROR_MESSAGES, P_TOKENS_FILE, modules[0], P_TWEETS))
modules.append(Wrapper(ERROR_MESSAGES, modules, P_SCHEDULE))
modules.append(Helper())
user_input('clear', [])
welcome()
while True:
user_inp = input('>> ')
if user_inp:
inp = user_inp.split(' ')
class Scheduler:
"""调度模块
"""
def __init__(self):
self.downloader = Downloader()
self.hasher = Hasher()
self.storager = Storager()
def _wrap_path(self, md5):
filename = '{}.jpg'.format(md5)
STORAGE_PATH = os.path.join('.', 'images')
path = os.path.join(STORAGE_PATH, filename)
return path
def set_calc_type(self, type_):
self.downloader.set_calc_type(type_)
self.hasher.set_calc_type(type_)
self.storager.set_calc_type(type_)
def process(self):
time_statictics = {}
time_statictics['network_time'] = []
time_statictics['cpu_time'] = []
time_statictics['disk_time'] = []
timer = utils.Timer()
# 1. 加载图片url列表
url_list = utils.urllist()
# 2. 调度下载模块
timer.tick()
content_list = self.downloader.process(url_list)
time_cost = timer.tock()
time_statictics['network_time'].append(time_cost)
# 3. 调度哈希模块
timer.tick()
md5_list = self.hasher.process(content_list)
time_cost = timer.tock()
time_statictics['cpu_time'].append(time_cost)
# 4. 调度存储模块
item_list = []
for content, md5 in zip(content_list, md5_list):
path = self._wrap_path(md5)
item = (content, path)
item_list.append(item)
timer.tick()
self.storager.process(item_list)
time_cost = timer.tock()
time_statictics['disk_time'].append(time_cost)
return time_statictics
def statictics(self, single_log, multi_log, multiprocess_log,
pycoroutine_log):
table = prettytable.PrettyTable([
'类型', '单线程总耗时', '多线程总耗时', '多线程提升率', '多进程总耗时', '多进程提升率', '协程总耗时',
'协程提升率'
])
network_row = ['network']
cpu_row = ['cpu']
disk_row = ['disk']
# 单线程数据
network_row.append(single_log['network_time'][0])
cpu_row.append(single_log['cpu_time'][0])
disk_row.append(single_log['disk_time'][0])
# 多线程数据
network_row.append(multi_log['network_time'][0])
cpu_row.append(multi_log['cpu_time'][0])
disk_row.append(multi_log['disk_time'][0])
# 多线程提升率
time_ = single_log['network_time'][0] - multi_log['network_time'][0]
lift_rate = '%.4f%%' % ((time_ / single_log['network_time'][0]) * 100)
network_row.append(lift_rate)
time_ = single_log['cpu_time'][0] - multi_log['cpu_time'][0]
lift_rate = '%.4f%%' % ((time_ / single_log['cpu_time'][0]) * 100)
cpu_row.append(lift_rate)
time_ = single_log['disk_time'][0] - multi_log['disk_time'][0]
lift_rate = '%.4f%%' % ((time_ / single_log['disk_time'][0]) * 100)
disk_row.append(lift_rate)
# 多进程数据
network_row.append(multiprocess_log['network_time'][0])
cpu_row.append(multiprocess_log['cpu_time'][0])
disk_row.append(multiprocess_log['disk_time'][0])
# 多进程提升率
time_ = single_log['network_time'][0] - multiprocess_log[
'network_time'][0]
lift_rate = '%.4f%%' % ((time_ / single_log['network_time'][0]) * 100)
network_row.append(lift_rate)
time_ = single_log['cpu_time'][0] - multiprocess_log['cpu_time'][0]
lift_rate = '%.4f%%' % ((time_ / single_log['cpu_time'][0]) * 100)
cpu_row.append(lift_rate)
time_ = single_log['disk_time'][0] - multiprocess_log['disk_time'][0]
lift_rate = '%.4f%%' % ((time_ / single_log['disk_time'][0]) * 100)
disk_row.append(lift_rate)
# 协程运行数据
network_row.append(pycoroutine_log['network_time'][0])
cpu_row.append(pycoroutine_log['cpu_time'][0])
disk_row.append(pycoroutine_log['disk_time'][0])
# 协程运行提升率
time_ = single_log['network_time'][0] - pycoroutine_log[
'network_time'][0]
lift_rate = '%.4f%%' % ((time_ / single_log['network_time'][0]) * 100)
network_row.append(lift_rate)
time_ = single_log['cpu_time'][0] - pycoroutine_log['cpu_time'][0]
lift_rate = '%.4f%%' % ((time_ / single_log['cpu_time'][0]) * 100)
cpu_row.append(lift_rate)
time_ = single_log['disk_time'][0] - pycoroutine_log['disk_time'][0]
lift_rate = '%.4f%%' % ((time_ / single_log['disk_time'][0]) * 100)
disk_row.append(lift_rate)
table.add_row(network_row)
table.add_row(cpu_row)
table.add_row(disk_row)
print(table)
def my_thread(myself, song: Song):
myself.thread_access.acquire()
logs = []
downloader = Downloader(DOWNLOAD_PATH, logger=logs.append)
log = logs.append
try:
log("Searching: "+song.get_display_name())
msg = myself.interact(text="Searching: "+song.get_display_name(),action=ChatAction.TYPING)
# Update YouTube data
downloader.retrieve_youtube_id(song)
log("Retreived YouTube ID: "+song.youtube_id)
# Check Index for pre-downloaded files
if myself.storage.find_file(song):
log("Indexed file found: "+str(myself.storage.get_index(song)))
else:
# Initiate Download
if song.retry_count < downloader.max_retries:
log("Downloader try: "+str(song.retry_count))
downloader.download(song)
# Verify Initiation
if song.message == "Download couldn't start":
log("Download couldn't start: "+song.get_display_name())
msg = myself.interact(msg=msg,text="Download couldn't start: "+song.get_display_name(),action=ChatAction.TYPING)
raise Exception("Download couldn't start")
elif song.message == "Download started":
log("Download started: "+song.get_display_name())
msg = myself.interact(msg=msg,text="Download started: "+song.get_display_name(),action=ChatAction.TYPING)
# Download Timeout
start_time = time.time()
while not myself.storage.find_file(song):
if time.time() - start_time >= 300: # Timeout of 5 mins
break
msg = myself.interact(msg=msg,text="Downloading (🙅♂️) : "+song.get_display_name(),action=ChatAction.TYPING)
time.sleep(1)
msg = myself.interact(msg=msg,text="Downloading (🙆♂️) : "+song.get_display_name(),action=ChatAction.TYPING)
time.sleep(1)
msg = myself.interact(msg=msg,text="Downloading (🤷♂️) : "+song.get_display_name(),action=ChatAction.TYPING)
time.sleep(1)
# Verify completion
if not myself.storage.find_file(song):
song.message = "Download couldn't complete"
log("Download couldn't complete: "+song.get_display_name())
msg = myself.interact(msg=msg,text="Download couldn't complete: "+song.get_display_name(),action=ChatAction.TYPING)
return
# Succesful Download
song.message = 'Download completed'
log("Download completed: "+song.get_display_name())
myself.storage.update_index(song)
myself.storage.save_index()
log("Index Updated: "+song.get_display_name())
msg = myself.interact(msg=msg,text="Download completed! Now Uploading: "+song.get_display_name(),action=ChatAction.UPLOAD_AUDIO,filename=myself.storage.get_filepath(song))
# myself.context.bot.send_document(chat_id=myself.update.effective_chat.id, document=open(filename, 'rb'))
if myself.upload_to_chat:
song.message = 'Upload completed'
log("Uploaded: "+song.get_display_name())
msg.delete()
except Exception as e :
print("Exception:",e)
print("Thread failed for Song: {}\n Logs: {}".format(song.get_display_name(),'\n\t'.join(logs)))
myself.thread_access.release()
return
def __init__(self):
self.downloader = Downloader()
self.hasher = Hasher()
self.storage = Storage()