def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('bangumi',
sa.Column('cover_color', sa.String(), nullable=True))
op.add_column('episodes',
sa.Column('thumbnail_color', sa.String(), nullable=True))
# ### end Alembic commands ###
connection = op.get_bind()
result = connection.execute(
sa.text(
'SELECT bangumi.id, bangumi.image FROM bangumi WHERE bangumi.image NOTNULL'
))
for row in result:
bangumi_id = row[0]
bangumi_image = row[1]
base_path = get_base_path()
path = urlparse(bangumi_image).path
extname = os.path.splitext(path)[1]
bangumi_path = base_path + '/' + str(bangumi_id)
cover_path = bangumi_path + '/cover' + extname
if not os.path.exists(bangumi_path):
print 'cover not found for {0}'.format(bangumi_id)
continue
try:
cover_color = get_dominant_color(cover_path, quality=5)
connection.execute(
sa.text(
"UPDATE bangumi SET cover_color = '{0}' WHERE id = '{1}'".
format(cover_color, bangumi_id)))
except Exception as error:
print error
# query episodes
episode_result = connection.execute(
sa.text(
"SELECT e.id, e.episode_no FROM episodes e WHERE e.status = 2 AND e.bangumi_id = '{0}'"
.format(bangumi_id)))
for eps in episode_result:
episode_id = eps[0]
episode_no = eps[1]
thumbnail_path = u'{0}/thumbnails/{1}.png'.format(
bangumi_path, episode_no)
if not os.path.exists(thumbnail_path):
print 'thumbnail not found for {0}'.format(episode_id)
continue
try:
thumbnail_color = get_dominant_color(thumbnail_path, quality=5)
connection.execute(
sa.text(
"UPDATE episodes SET thumbnail_color = '{0}' WHERE id = '{1}'"
.format(thumbnail_color, episode_id)))
except Exception as error:
print error
print 'Finish for bangumi #{0}'.format(bangumi_id)
print 'All done'
def __create_thumbnail(self, episode, file_path):
time = '00:00:01.000'
video_manager.create_episode_thumbnail(episode, file_path, time)
thumbnail_path = '{0}/thumbnails/{1}.png'.format(str(episode.bangumi_id), episode.episode_no)
thumbnail_file_path = '{0}/{1}'.format(self.base_path, thumbnail_path)
color = get_dominant_color(thumbnail_file_path)
width, height = get_dimension(thumbnail_file_path)
episode.thumbnail_image = Image(file_path=thumbnail_path,
dominant_color=color,
width=width,
height=height)
episode.thumbnail_color = color
def __create_thumbnail(self, episode, file_path):
time = '00:00:01.000'
video_manager.create_episode_thumbnail(episode, file_path, time)
thumbnail_path = '{0}/thumbnails/{1}.png'.format(str(episode.bangumi_id), episode.episode_no)
thumbnail_file_path = '{0}/{1}'.format(self.base_path, thumbnail_path)
color = get_dominant_color(thumbnail_file_path)
width, height = get_dimension(thumbnail_file_path)
episode.thumbnail_image = Image(file_path=thumbnail_path,
dominant_color=color,
width=width,
height=height)
episode.thumbnail_color = color
def update_bangumi(self, bangumi_id=None):
fr = open('./config/config.yml', 'r')
config = yaml.load(fr)
download_dir = config['download']['location'] + '/' + str(bangumi_id)
files = self.__list_file_recursively(download_dir)
session = SessionManager.Session()
eps_list = session.query(Episode).\
filter(Episode.bangumi_id == bangumi_id).all()
existed_video_files = session.query(VideoFile).filter(
VideoFile.bangumi_id == bangumi_id).all()
episodes = {}
video_files = []
for eps in eps_list:
if self.__episode_has_video_file(existed_video_files, eps):
continue
episodes[eps.episode_no] = eps
for f in files:
if self.__parse_episode_number(f) == eps.episode_no:
eps.status = Episode.STATUS_DOWNLOADED
video_files.append(
VideoFile(bangumi_id=bangumi_id,
episode_id=eps.id,
file_path=f.decode('utf-8'),
status=VideoFile.STATUS_DOWNLOADED))
break
while True:
for eps in episodes.values():
if not eps:
continue
episode_num = str(eps.episode_no)
file_name = "None"
for video_file in video_files:
if video_file.episode_id == eps.id:
file_name = video_file.file_path
break
print(episode_num + ": \t" + file_name)
print("Right? Y/N")
x = raw_input(">>> Input: ")
if x == "Y":
video_manager = VideoManager()
video_manager.set_base_path(config['download']['location'])
for video_file in video_files:
for eps in episodes.values():
if eps.id == video_file.episode_id:
video_manager.create_episode_thumbnail(
eps, video_file.file_path, '00:00:01.000')
thumbnail_path = '{0}/thumbnails/{1}.png'.format(
str(bangumi_id), eps.episode_no)
thumbnail_file_path = '{0}/thumbnails/{1}.png'.format(
download_dir, eps.episode_no)
width, height = get_dimension(thumbnail_file_path)
eps.thumbnail_image = Image(
file_path=thumbnail_path,
dominant_color=get_dominant_color(
thumbnail_file_path),
width=width,
height=height)
meta_dict = video_manager.get_video_meta(
u'{0}/{1}/{2}'.format(
video_manager.base_path,
bangumi_id.encode('utf-8'),
video_file.file_path))
if meta_dict is not None:
video_file.resolution_w = meta_dict['width']
video_file.resolution_h = meta_dict['height']
video_file.duration = meta_dict['duration']
session.add(video_file)
break
session.commit()
return
else:
video_files = []
for f in files:
print f
x = raw_input(">>> Episode Num")
if not x:
continue
x = int(x)
eps = episodes[x]
if not eps:
continue
eps.status = Episode.STATUS_DOWNLOADED
video_files.append(
VideoFile(bangumi_id=bangumi_id,
episode_id=eps.id,
file_path=f.decode('utf-8'),
status=VideoFile.STATUS_DOWNLOADED))
def add_bangumi(self, content, uid):
try:
bangumi_data = json.loads(content)
bangumi = Bangumi(bgm_id=bangumi_data.get('bgm_id'),
name=bangumi_data.get('name'),
name_cn=bangumi_data.get('name_cn'),
type=bangumi_data.get('type'),
summary=bangumi_data.get('summary'),
eps=bangumi_data.get('eps'),
image=bangumi_data.get('image'),
air_date=bangumi_data.get('air_date'),
air_weekday=bangumi_data.get('air_weekday'),
status=self.__get_bangumi_status(bangumi_data.get('air_date')),
created_by_uid=uid,
maintained_by_uid=uid)
# bangumi.dmhy = bangumi_data.get('dmhy')
# bangumi.acg_rip = bangumi_data.get('acg_rip')
# bangumi.libyk_so = bangumi_data.get('libyk_so')
bangumi.eps_no_offset = bangumi_data.get('eps_no_offset')
session = SessionManager.Session()
session.add(bangumi)
bangumi.episodes = []
for eps_item in bangumi_data['episodes']:
eps = Episode(bgm_eps_id=eps_item.get('bgm_eps_id'),
episode_no=eps_item.get('episode_no'),
name=eps_item.get('name'),
name_cn=eps_item.get('name_cn'),
duration=eps_item.get('duration'),
status=Episode.STATUS_NOT_DOWNLOADED)
if is_valid_date(eps_item.get('airdate')):
eps.airdate = eps_item.get('airdate')
eps.bangumi = bangumi
bangumi.episodes.append(eps)
session.commit()
bangumi_id = str(bangumi.id)
try:
(cover_file_path, cover_path) = self.__save_bangumi_cover(bangumi)
# get dominant color
bangumi.cover_color = get_dominant_color(cover_file_path)
(width, height) = get_dimension(cover_file_path)
bangumi.cover_image = Image(file_path=cover_path,
dominant_color=bangumi.cover_color,
width=width,
height=height)
session.commit()
except Exception as error:
sentry_wrapper.sentry_middleware.captureException()
logger.warn(error)
# delete bangumi for download error
session.delete(bangumi)
session.commit()
raise ServerError('Fail to Download Image')
return json_resp({'data': {'id': bangumi_id}})
finally:
SessionManager.Session.remove()
if not os.path.exists(bangumi_dir):
os.makedirs(bangumi_dir)
print 'bangumi %s folder created' % (str(bangumi.id), )
path = urlparse(bangumi.image).path
extname = os.path.splitext(path)[1]
bangumi_cover_path = bangumi_dir + '/cover' + extname
if not os.path.exists(bangumi_cover_path):
# download bangumi image
print 'start to download bangumi cover of %s (%s)' % (
bangumi.name, str(bangumi.id))
file_downloader.download_file(bangumi.image,
bangumi_cover_path)
if bangumi.cover_color is None:
try:
bangumi.cover_color = get_dominant_color(
bangumi_cover_path, 5)
session.commit()
except Exception as err:
print err
if bangumi.cover_image_id is None:
try:
width, height = get_dimension(bangumi_cover_path)
cover_image = Image(file_path='{0}/cover{1}'.format(
str(bangumi.id), extname),
dominant_color=bangumi.cover_color,
width=width,
height=height)
bangumi.cover_image = cover_image
session.commit()
except Exception as err:
print err
def add_bangumi(self, content, uid):
try:
bangumi_data = json.loads(content)
bangumi = Bangumi(bgm_id=bangumi_data.get('bgm_id'),
name=bangumi_data.get('name'),
name_cn=bangumi_data.get('name_cn'),
type=bangumi_data.get('type'),
summary=bangumi_data.get('summary'),
eps=bangumi_data.get('eps'),
image=bangumi_data.get('image'),
air_date=bangumi_data.get('air_date'),
air_weekday=bangumi_data.get('air_weekday'),
status=self.__get_bangumi_status(
bangumi_data.get('air_date')),
created_by_uid=uid,
maintained_by_uid=uid)
# bangumi.dmhy = bangumi_data.get('dmhy')
# bangumi.acg_rip = bangumi_data.get('acg_rip')
# bangumi.libyk_so = bangumi_data.get('libyk_so')
bangumi.eps_no_offset = bangumi_data.get('eps_no_offset')
session = SessionManager.Session()
session.add(bangumi)
bangumi.episodes = []
for eps_item in bangumi_data['episodes']:
eps = Episode(bgm_eps_id=eps_item.get('bgm_eps_id'),
episode_no=eps_item.get('episode_no'),
name=eps_item.get('name'),
name_cn=eps_item.get('name_cn'),
duration=eps_item.get('duration'),
status=Episode.STATUS_NOT_DOWNLOADED)
if eps_item.get('airdate') != '':
eps.airdate = eps_item.get('airdate')
eps.bangumi = bangumi
bangumi.episodes.append(eps)
session.commit()
bangumi_id = str(bangumi.id)
try:
(cover_file_path,
cover_path) = self.__save_bangumi_cover(bangumi)
# get dominant color
bangumi.cover_color = get_dominant_color(cover_file_path)
(width, height) = get_dimension(cover_file_path)
bangumi.cover_image = Image(file_path=cover_path,
dominant_color=bangumi.cover_color,
width=width,
height=height)
session.commit()
except Exception as error:
sentry_wrapper.sentry_middleware.captureException()
logger.warn(error)
# delete bangumi for download error
session.delete(bangumi)
session.commit()
raise ServerError('Fail to Download Image')
return json_resp({'data': {'id': bangumi_id}})
finally:
SessionManager.Session.remove()
def update_bangumi(self, bangumi_id=None):
fr = open('./config/config.yml', 'r')
config = yaml.load(fr)
download_dir = config['download']['location'] + '/' + str(bangumi_id)
files = self.__list_file_recursively(download_dir)
session = SessionManager.Session()
try:
eps_list = session.query(Episode).\
filter(Episode.bangumi_id == bangumi_id).all()
bangumi = session.query(Bangumi).\
filter(Bangumi.id == bangumi_id).one()
eps_no_offset = 0
if bangumi.eps_no_offset is not None:
eps_no_offset = bangumi.eps_no_offset
existed_video_files = session.query(VideoFile).filter(
VideoFile.bangumi_id == bangumi_id).all()
episodes = {}
video_files = []
for eps in eps_list:
# if self.__episode_has_video_file(existed_video_files, eps):
# continue
episodes[eps.episode_no] = eps
for f in files:
if self.__video_path_already_added(existed_video_files,
f.decode('utf-8')):
continue
if self.__parse_episode_number(
f) + eps_no_offset == eps.episode_no:
eps.status = Episode.STATUS_DOWNLOADED
video_files.append(
VideoFile(bangumi_id=bangumi_id,
episode_id=eps.id,
file_path=f.decode('utf-8'),
status=VideoFile.STATUS_DOWNLOADED))
break
while True:
for eps in episodes.values():
if not eps:
continue
episode_num = str(eps.episode_no)
line = episode_num + ": \t"
file_name = None
for video_file in video_files:
if video_file.episode_id == eps.id:
if file_name is None:
line = line + video_file.file_path
else:
line = line + "\n \t" + video_file.file_path
if video_file.label is not None:
line = line + "\t" + video_file.label.decode(
'utf-8')
file_name = video_file.file_path
if file_name is None:
line = line + "None"
print(line)
print("Right? Y/N")
x = raw_input(">>> Input: ")
if x == "Y":
video_manager = VideoManager()
video_manager.set_base_path(config['download']['location'])
for video_file in video_files:
for eps in episodes.values():
if eps.id == video_file.episode_id:
video_manager.create_episode_thumbnail(
eps, video_file.file_path, '00:00:01.000')
thumbnail_path = '{0}/thumbnails/{1}.png'.format(
str(bangumi_id), eps.episode_no)
thumbnail_file_path = '{0}/thumbnails/{1}.png'.format(
download_dir, eps.episode_no)
width, height = get_dimension(
thumbnail_file_path)
eps.thumbnail_image = Image(
file_path=thumbnail_path,
dominant_color=get_dominant_color(
thumbnail_file_path),
width=width,
height=height)
meta_dict = video_manager.get_video_meta(
u'{0}/{1}/{2}'.format(
video_manager.base_path,
bangumi_id.encode('utf-8'),
video_file.file_path))
if meta_dict is not None:
video_file.resolution_w = meta_dict[
'width']
video_file.resolution_h = meta_dict[
'height']
video_file.duration = meta_dict['duration']
session.add(video_file)
# break
session.commit()
return
else:
video_files = []
for f in files:
print f
x = raw_input(
">>> Episode Num and Label (separated by comma)")
if not x:
continue
arguments = x.split(',')
x = int(arguments[0])
label = None
if len(arguments) > 1:
label = arguments[1]
eps = episodes[x]
if not eps:
continue
eps.status = Episode.STATUS_DOWNLOADED
video_files.append(
VideoFile(bangumi_id=bangumi_id,
episode_id=eps.id,
file_path=f.decode('utf-8'),
label=label,
status=VideoFile.STATUS_DOWNLOADED))
finally:
SessionManager.Session.remove()
def update_bangumi(self, bangumi_id=None):
fr = open('./config/config.yml', 'r')
config = yaml.load(fr)
download_dir = config['download']['location'] + '/' + str(bangumi_id)
files = self.__list_file_recursively(download_dir)
session = SessionManager.Session()
try:
eps_list = session.query(Episode).\
filter(Episode.bangumi_id == bangumi_id).all()
bangumi = session.query(Bangumi).\
filter(Bangumi.id == bangumi_id).one()
eps_no_offset = 0
if bangumi.eps_no_offset is not None:
eps_no_offset = bangumi.eps_no_offset
existed_video_files = session.query(VideoFile).filter(VideoFile.bangumi_id == bangumi_id).all()
episodes = {}
video_files = []
for eps in eps_list:
# if self.__episode_has_video_file(existed_video_files, eps):
# continue
episodes[eps.episode_no] = eps
for f in files:
if self.__video_path_already_added(existed_video_files, f.decode('utf-8')):
continue
if self.__parse_episode_number(f) + eps_no_offset == eps.episode_no:
eps.status = Episode.STATUS_DOWNLOADED
video_files.append(VideoFile(bangumi_id=bangumi_id,
episode_id=eps.id,
file_path=f.decode('utf-8'),
status=VideoFile.STATUS_DOWNLOADED))
break
while True:
for eps in episodes.values():
if not eps:
continue
episode_num = str(eps.episode_no)
line = episode_num + ": \t"
file_name = None
for video_file in video_files:
if video_file.episode_id == eps.id:
if file_name is None:
line = line + video_file.file_path
else:
line = line + "\n \t" + video_file.file_path
if video_file.label is not None:
line = line + "\t" + video_file.label.decode('utf-8')
file_name = video_file.file_path
if file_name is None:
line = line + "None"
print (line)
print("Right? Y/N")
x = raw_input(">>> Input: ")
if x == "Y":
video_manager = VideoManager()
video_manager.set_base_path(config['download']['location'])
for video_file in video_files:
for eps in episodes.values():
if eps.id == video_file.episode_id:
video_manager.create_episode_thumbnail(eps, video_file.file_path, '00:00:01.000')
thumbnail_path = '{0}/thumbnails/{1}.png'.format(str(bangumi_id), eps.episode_no)
thumbnail_file_path = '{0}/thumbnails/{1}.png'.format(download_dir, eps.episode_no)
width, height = get_dimension(thumbnail_file_path)
eps.thumbnail_image = Image(file_path=thumbnail_path,
dominant_color=get_dominant_color(thumbnail_file_path),
width=width,
height=height)
meta_dict = video_manager.get_video_meta(u'{0}/{1}/{2}'.format(video_manager.base_path, bangumi_id.encode('utf-8'), video_file.file_path))
if meta_dict is not None:
video_file.resolution_w = meta_dict['width']
video_file.resolution_h = meta_dict['height']
video_file.duration = meta_dict['duration']
session.add(video_file)
# break
session.commit()
return
else:
video_files = []
for f in files:
print f
x = raw_input(">>> Episode Num and Label (separated by comma)")
if not x:
continue
arguments = x.split(',')
x = int(arguments[0])
label = None
if len(arguments) > 1:
label = arguments[1]
eps = episodes[x]
if not eps:
continue
eps.status = Episode.STATUS_DOWNLOADED
video_files.append(VideoFile(bangumi_id=bangumi_id,
episode_id=eps.id,
file_path=f.decode('utf-8'),
label=label,
status=VideoFile.STATUS_DOWNLOADED))
finally:
SessionManager.Session.remove()
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('image',
sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
sa.Column('file_path', sa.TEXT(), nullable=False),
sa.Column('dominant_color', sa.String(), nullable=True),
sa.Column('width', sa.Integer(), nullable=True),
sa.Column('height', sa.Integer(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.add_column(u'bangumi', sa.Column('cover_image_id', postgresql.UUID(as_uuid=True), nullable=True))
op.add_column(u'episodes', sa.Column('thumbnail_image_id', postgresql.UUID(as_uuid=True), nullable=True))
# ### end Alembic commands ###
# import data into image table.
connection = op.get_bind()
result = connection.execute(sa.text(
'SELECT bangumi.id, bangumi.image, bangumi.cover_color FROM bangumi WHERE bangumi.image NOTNULL'))
base_path = get_base_path()
for row in result:
bangumi_id = row[0]
bangumi_image = row[1]
dominant_color = row[2]
path = urlparse(bangumi_image).path
extname = os.path.splitext(path)[1]
image_path = '{0}/cover{1}'.format(str(bangumi_id), extname)
if not os.path.exists('{0}/{1}'.format(base_path, image_path)):
print 'cover not found for {0}'.format(bangumi_id)
continue
try:
image_id = uuid4()
connection.execute(sa.text(
"INSERT INTO image (id, file_path, dominant_color) VALUES ('{0}', '{1}', '{2}')".format(
image_id, image_path, dominant_color)))
connection.execute(sa.text(
"UPDATE bangumi SET cover_image_id = '{0}' WHERE id = '{1}'".format(image_id, bangumi_id)))
except Exception as error:
print error
# query episodes
episode_result = connection.execute(sa.text(
"SELECT e.id, e.episode_no, e.thumbnail_color FROM episodes e WHERE e.status = 2 AND e.bangumi_id = '{0}'".format(
bangumi_id)))
for eps in episode_result:
episode_id = eps[0]
episode_no = eps[1]
episode_thumbnail_color = eps[2]
thumbnail_path = '{0}/thumbnails/{1}.png'.format(str(bangumi_id), episode_no)
if not os.path.exists(u'{0}/{1}'.format(base_path, thumbnail_path)):
print 'thumbnail not found for {0}'.format(episode_id)
continue
try:
image_id = uuid4()
connection.execute(sa.text(
"INSERT INTO image (id, file_path, dominant_color) VALUES ('{0}', '{1}', '{2}')".format(
image_id, thumbnail_path, episode_thumbnail_color)))
connection.execute(sa.text(
"UPDATE episodes SET thumbnail_image_id = '{0}' WHERE id = '{1}'".format(image_id, episode_id)))
except Exception as error:
print error
print 'image table import completed. Start reading image dimension and fix color'
image_result = connection.execute(sa.text('SELECT image.id, image.file_path, image.dominant_color FROM image'))
for image in image_result:
image_id = image[0]
image_file_path = image[1]
image_dominant_color = image[2]
try:
im = Image.open('{0}/{1}'.format(base_path, image_file_path))
(width, height) = im.size
if image_dominant_color is None:
image_dominant_color = get_dominant_color('{0}/{1}'.format(base_path, image_file_path), 5)
connection.execute(sa.text(
"UPDATE image SET width = '{0}', height = '{1}', dominant_color = '{2}' WHERE id = '{3}'".format(
width, height, image_dominant_color, image_id)))
except Exception as error:
print error
print 'All done'
