def prepare_train(self,
train_loader,
n_train,
val_loader,
n_val,
loss_name,
optimizer,
lr,
batch_size,
max_epoches,
save_directory,
reduce_factor=None,
epoches_limit=0,
metrics=None,
early_stoping=None,
**kwargs):
self.loss_function = get_loss(loss_name, **kwargs)
self.lr = lr
self.optimizer = get_optimizer(optimizer)(learning_rate=lr)
self.train_loader = train_loader
self.val_loader = val_loader
self.train_examples = n_train
self.val_examples = n_val
self.max_epoches = max_epoches
self.batch_size = batch_size
self.save_directory = save_directory
self.reduce_factor = reduce_factor
self.epoches_limit = epoches_limit
self.early_stoping = early_stoping
self.metric_list = None
if metrics is not None:
self.metric_list = []
for metric in metrics:
self.metric_list.append(metric)
make_dir(save_directory)
def album_rip(album_id):
"""
:param id: String (Album ID)
"""
meta = client.get_meta(type="album", id=int(album_id))
# Check for multiple artists.
if meta['list'][0]['album_info']['result']['multi_artist_yn'] == "Y":
# Label V.A. if amount of artists > 2
if meta['list'][0]['album_info']['result']['artist_disp_nm'].count(",") > 2:
meta['list'][0]['album_info']['result']['artist_disp_nm'] = "Various Artists"
album_directory_name = "{} - {} [{}]".format(meta['list'][0]['album_info']['result']['artist_disp_nm'],
meta['list'][0]['album_info']['result']['title'].strip(),
utils.get_date(meta['list'][0]['album_info']['result']['release_ymd']))
# Check for availability.
if meta['list'][0]['album_info']['result']['is_album_str_noright']:
logger_bugs.warning('No streaming rights for {}.'.format(album_directory_name))
else:
logger_bugs.info("Album: {}.".format(album_directory_name))
if config.prefs['artist_folders']:
album_path = utils.sanitize(os.path.join(config.prefs['downloads_directory'],
meta['list'][0]['album_info']['result']['artist_disp_nm'], album_directory_name))
else:
album_path = utils.sanitize(os.path.join(config.prefs['downloads_directory'], album_directory_name))
utils.make_dir(album_path)
cover_path = os.path.join(album_path, config.prefs['cover_name'])
try:
download_cover(meta['list'][0]['album_info']['result']['img_urls'], cover_path)
# If we're unable to request from the url we'll set the cover_path to False
except requests.exceptions.HTTPError:
cover_path = False
for track in meta['list'][0]['album_info']['result']['tracks']:
# Check for availability.
if not track['track_str_rights']:
logger_bugs.warning('No streaming rights for #{} - {}.'.format(track['track_no'], track['track_title']))
else:
track_quality = utils.determine_quality(track=track)
pre_path = os.path.join(album_path, "{}. .BugsPy".format(track['track_no']))
post_path = utils.sanitize(os.path.join(album_path, "{}. {}.{}".format(track['track_no'], track['track_title'],
track_quality)))
if utils.exist_check(post_path):
logger_bugs.info("{} already exists.".format(post_path))
else:
download_track(pre_path=pre_path, track_id=track['track_id'], track_title=track['track_title'],
track_number=track['track_no'])
os.rename(pre_path, post_path)
try:
tag(album=meta, track=track, file_path=post_path, cover_path=cover_path)
# TODO: Come back to this exception and implement a better solution on f_file.save() within tag()
except error:
logger_bugs.warning("_writeblock error, falling back to a smaller cover artwork.")
config.prefs['cover_size'] = "500"
cover_path = os.path.join(album_path, "fallback_cover.jpg")
download_cover(meta['list'][0]['album_info']['result']['img_urls'], cover_path)
tag(album=meta, track=track, file_path=post_path, cover_path=cover_path)
def log_setup():
filename = '{:%H.%M.%S}.log'.format(datetime.now())
folder_name = os.path.join(prefs['log_directory'],
'{:%Y-%m-%d}'.format(datetime.now()))
make_dir(folder_name)
log_path = os.path.join(folder_name, filename)
logging.basicConfig(
level=logging.DEBUG,
handlers=[logging.FileHandler(log_path, 'w', 'utf-8')],
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
console.setFormatter(formatter)
logging.getLogger("").addHandler(console)
# Suppress requests module if level < WARNING
logging.getLogger("requests").setLevel(logging.WARNING)
def video_to_frame(root_path, save_path, data_list, ver):
save_img_path_list = []
make_dir(save_path)
for file_name in data_list:
save_dir_path, save_img_path = return_save_img_path(
file_name, save_path)
make_dir(save_dir_path)
make_dir(save_img_path)
save_img_path_list.append(save_img_path)
video_path = root_path + file_name
cap = cv2.VideoCapture(video_path)
if not cap.isOpened():
break
n = 1
while True:
ret, frame = cap.read()
if ret:
cv2.imwrite('{}{}.{}'.format(save_img_path, n, 'jpg'), frame)
n += 1
else:
break
return save_img_path_list
def __init__(self, config_file):
self.config_file = config_file
self.debug = False
self.log_file = None
# self.log_file = 'aaaa.log'
self.original_log_file = None
# self.log_level = 'Debug'
self.log_level = 'Info'
self.console_level = 'Info'
self.output_format = 'epub'
self.kindle_compression_level = 1
self.no_dropcaps_symbols = '\'"-.…0123456789‒–—«»'
self.transliterate = False
self.transliterate_author_and_title = False
self.noMOBIoptimization = False
self.screen_height = 800
self.screen_width = 573
self.default_profile = 'default'
self.input_dir = None
self.output_dir = None
self.output_pattern = None
self.save_structure = None
self.delete_source_file = None
self.apnx = None
self.characters_per_page = 2300
self.log = None
self.profiles = {}
self.profiles['default'] = {}
self.profiles['default']['name'] = 'default'
self.profiles['default']['description'] = 'Default profile'
self.profiles['default']['outputFormat'] = None
self.profiles['default']['transliterate'] = None
self.profiles['default']['screenHeight'] = None
self.profiles['default']['screenWidth'] = None
self.profiles['default']['transliterateAuthorAndTitle'] = None
self.profiles['default']['hyphens'] = True
self.profiles['default']['hyphensReplaceNBSP'] = True
self.profiles['default']['dropcaps'] = 'None'
self.profiles['default']['tocMaxLevel'] = 1000
self.profiles['default']['tocKindleLevel'] = 2
self.profiles['default']['tocBeforeBody'] = False
self.profiles['default']['tocType'] = 'Normal'
self.profiles['default']['originalcss'] = 'default.css'
self.profiles['default']['css'] = 'default.css'
self.profiles['default']['parse_css'] = True
self.profiles['default']['chapterOnNewPage'] = True
self.profiles['default']['chapterLevel'] = 100
self.profiles['default']['authorFormat'] = '#l #f #m'
self.profiles['default'][
'bookTitleFormat'] = '(#abbrseries #number) #title'
self.profiles['default']['annotationTitle'] = 'Annotation'
self.profiles['default']['tocTitle'] = 'Content'
self.profiles['default']['notesMode'] = 'default'
self.profiles['default']['notesBodies'] = 'notes'
self.profiles['default']['vignettes'] = {}
self.profiles['default']['vignettes_save'] = {}
self.profiles['default']['generateTOCPage'] = True
self.profiles['default']['generateAnnotationPage'] = True
self.profiles['default']['generateOPFGuide'] = True
self.profiles['default']['kindleRemovePersonalLabel'] = True
self.profiles['default']['removePngTransparency'] = False
self.profiles['default']['generateAPNX'] = None
self.profiles['default']['charactersPerPage'] = 2300
self.profiles['default']['seriesPositions'] = 2
self.profiles['default']['openBookFromCover'] = False
self.profiles['default']['scaleImages'] = 0.0
self.profiles['default']['coverDefault'] = 'default_cover.jpg'
self.profiles['default']['coverStamp'] = 'None'
self.profiles['default']['outputPattern'] = None
self.current_profile = {}
self.mhl = False
self.recursive = False
self.send_to_kindle = {}
self.send_to_kindle['send'] = False
self.send_to_kindle['deleteSendedBook'] = True
self.send_to_kindle['smtpServer'] = 'smtp.gmail.com'
self.send_to_kindle['smtpPort'] = 465
self.send_to_kindle['smtpLogin'] = '******'
self.send_to_kindle['smtpPassword'] = '******'
self.send_to_kindle['fromUserEmail'] = '[Your Google Email]'
self.send_to_kindle['toKindleEmail'] = '[Your Kindle Email]'
# if no configuration - provide minimum defaults
if not os.path.exists(self.config_file):
# configuration
self.write()
# default css
default_css = import_module('modules.default_css')
default_css_path = os.path.abspath(
os.path.join(os.path.dirname(self.config_file), 'default.css'))
make_dir(default_css_path)
with codecs.open(default_css_path, "w", 'utf-8') as f:
f.write(default_css.default_css)
f.close()
# default cover
default_cover_path = os.path.join(get_executable_path(),
'default_cover.jpg')
if os.path.exists(default_cover_path):
copy_file(
default_cover_path,
os.path.abspath(
os.path.join(os.path.dirname(self.config_file),
'default_cover.jpg')))
self._load()
def main():
"""
Main function which will control the flow of our script when called.
"""
total = len(args.u)
for n, url in enumerate(args.u, 1):
logger_genie.info("Album {} of {}".format(n, total))
album_id = utils.check_url(url)
if not album_id:
return
meta = client.get_meta(album_id)
album_fol = "{} - {}".format(
parse.unquote(meta['DATA0']['DATA'][0]['ARTIST_NAME']),
parse.unquote(meta['DATA0']['DATA'][0]['ALBUM_NAME']))
if prefs['artist_folders']:
album_fol_abs = os.path.join(
os.path.dirname(__file__), prefs['download_directory'],
parse.unquote(
utils.sanitize(meta['DATA0']['DATA'][0]['ARTIST_NAME'])),
utils.sanitize(album_fol))
else:
album_fol_abs = os.path.join(os.path.dirname(__file__),
prefs['download_directory'],
utils.sanitize(album_fol))
logger_genie.info("Album found: " + album_fol)
utils.make_dir(album_fol_abs)
cover_url = parse.unquote(
meta['DATA0']['DATA'][0]['ALBUM_IMG_PATH_600'])
download_cover(cover_url, album_fol_abs)
f_meta = {
"track_total": len(meta['DATA1']['DATA']),
"album_artist":
parse.unquote(meta['DATA0']['DATA'][0]['ARTIST_NAME']),
"release_date": meta['DATA0']['DATA'][0]['ALBUM_RELEASE_DT'],
"planning":
parse.unquote(meta['DATA0']['DATA'][0]['ALBUM_PLANNER'])
}
f_meta['disc_total'] = meta['DATA1']['DATA'][f_meta['track_total'] -
1]['ALBUM_CD_NO']
for track in meta['DATA1']['DATA']:
try:
s_meta = client.get_stream_meta(track['SONG_ID'], args.f)
except HTTPError:
logger_genie.warning("Could not get stream info for {}".format(
track['SONG_ID']))
continue
except exceptions.StreamMetadataError:
continue
cur = track['ROWNUM']
track_title = parse.unquote(track['SONG_NAME'])
f_meta['track_artist'] = parse.unquote(track['ARTIST_NAME'])
ext = utils.get_ext(s_meta['FILE_EXT'])
post_abs = os.path.join(
album_fol_abs, "{}. {}.{}".format(cur.zfill(2),
utils.sanitize(track_title),
ext))
if utils.exist_check(post_abs):
continue
if not utils.allowed_check(s_meta['STREAMING_LICENSE_YN']):
continue
pre_abs = os.path.join(album_fol_abs, cur + ".genie-dl")
specs = utils.parse_specs(s_meta['FILE_EXT'], s_meta['FILE_BIT'])
download_track(s_meta['STREAMING_MP3_URL'], track_title, pre_abs,
cur, f_meta['track_total'], specs)
try:
fix_tags(pre_abs, ext, f_meta)
logger_genie.debug("Tags updated: {}".format(f_meta))
except Exception as e:
raise e
try:
os.rename(pre_abs, post_abs)
logger_genie.debug("{} has been renamed".format(post_abs))
except OSError:
raise exceptions.TrackRenameError(
"Could not rename {}".format(pre_abs))
def model_train(X_train, y_train, depth, config_dict, flags):
batch_size = config_dict['batch_size']
epochs = config_dict['epoch']
lr = config_dict['lr']
num_classes = config_dict['num_classes']
model_mode = config_dict['model_mode']
img_shape = config_dict['img_shape']
channel = 1
if flags['color']:
channel = 3
print(f'ndim: {X_train.ndim}\
\nshape: {X_train.shape}\
\nimages: {len(X_train)}\
\nlabels: {len(y_train)}')
X_train = X_train.reshape(
(X_train.shape[0], depth, img_shape[0], img_shape[1], channel))
X_train = X_train.astype("float32")
# データのロード
X_train, X_valid, y_train, y_valid = train_test_split(X_train,
y_train,
test_size=0.2)
print(X_train.ndim, X_train.shape, type(X_train))
# モデル作成, model_modeに合わせて構築を変える
model = ModelCreate(X_train, lr, num_classes)
if model_mode == '3DCNN':
model = model.CNN3D_model()
elif model_mode == 'C3D':
model = model.C3D_model()
elif model_mode == 'LSTM':
model = model.LSTM_model()
history = model.fit(X_train,
y_train,
validation_data=(X_valid, y_valid),
batch_size=batch_size,
epochs=epochs,
verbose=1)
# 保存名
save_name = 'Batch{0}_Epoch{1}_LR{2}'.format(batch_size, epochs, lr)
extends = ['.hdf5', '.h5', '_']
save_name_list = [save_name + e for e in extends]
# 保存先
subs = ['./weight/', './model/', './Result/']
path_sub = config_dict["version"] + '/' + model_mode + '/'
save_path_list = [s + path_sub for s in subs]
for path_name in save_path_list:
make_dir(path_name)
# モデルの重み、モデル本体を保存
model.save_weights(save_path_list[0] + save_name_list[0])
model.save(save_path_list[1] + save_name_list[1], include_optimizer=False)
# 学習履歴をプロット
plot_history(history, save_path_list[2], save_name_list[2])