def upgrade(): # ### commands auto generated by Alembic - please adjust! ### op.add_column('bangumi', sa.Column('cover_color', sa.String(), nullable=True)) op.add_column('episodes', sa.Column('thumbnail_color', sa.String(), nullable=True)) # ### end Alembic commands ### connection = op.get_bind() result = connection.execute( sa.text( 'SELECT bangumi.id, bangumi.image FROM bangumi WHERE bangumi.image NOTNULL' )) for row in result: bangumi_id = row[0] bangumi_image = row[1] base_path = get_base_path() path = urlparse(bangumi_image).path extname = os.path.splitext(path)[1] bangumi_path = base_path + '/' + str(bangumi_id) cover_path = bangumi_path + '/cover' + extname if not os.path.exists(bangumi_path): print 'cover not found for {0}'.format(bangumi_id) continue try: cover_color = get_dominant_color(cover_path, quality=5) connection.execute( sa.text( "UPDATE bangumi SET cover_color = '{0}' WHERE id = '{1}'". format(cover_color, bangumi_id))) except Exception as error: print error # query episodes episode_result = connection.execute( sa.text( "SELECT e.id, e.episode_no FROM episodes e WHERE e.status = 2 AND e.bangumi_id = '{0}'" .format(bangumi_id))) for eps in episode_result: episode_id = eps[0] episode_no = eps[1] thumbnail_path = u'{0}/thumbnails/{1}.png'.format( bangumi_path, episode_no) if not os.path.exists(thumbnail_path): print 'thumbnail not found for {0}'.format(episode_id) continue try: thumbnail_color = get_dominant_color(thumbnail_path, quality=5) connection.execute( sa.text( "UPDATE episodes SET thumbnail_color = '{0}' WHERE id = '{1}'" .format(thumbnail_color, episode_id))) except Exception as error: print error print 'Finish for bangumi #{0}'.format(bangumi_id) print 'All done'
def __create_thumbnail(self, episode, file_path): time = '00:00:01.000' video_manager.create_episode_thumbnail(episode, file_path, time) thumbnail_path = '{0}/thumbnails/{1}.png'.format(str(episode.bangumi_id), episode.episode_no) thumbnail_file_path = '{0}/{1}'.format(self.base_path, thumbnail_path) color = get_dominant_color(thumbnail_file_path) width, height = get_dimension(thumbnail_file_path) episode.thumbnail_image = Image(file_path=thumbnail_path, dominant_color=color, width=width, height=height) episode.thumbnail_color = color
def __create_thumbnail(self, episode, file_path): time = '00:00:01.000' video_manager.create_episode_thumbnail(episode, file_path, time) thumbnail_path = '{0}/thumbnails/{1}.png'.format(str(episode.bangumi_id), episode.episode_no) thumbnail_file_path = '{0}/{1}'.format(self.base_path, thumbnail_path) color = get_dominant_color(thumbnail_file_path) width, height = get_dimension(thumbnail_file_path) episode.thumbnail_image = Image(file_path=thumbnail_path, dominant_color=color, width=width, height=height) episode.thumbnail_color = color
def update_bangumi(self, bangumi_id=None): fr = open('./config/config.yml', 'r') config = yaml.load(fr) download_dir = config['download']['location'] + '/' + str(bangumi_id) files = self.__list_file_recursively(download_dir) session = SessionManager.Session() eps_list = session.query(Episode).\ filter(Episode.bangumi_id == bangumi_id).all() existed_video_files = session.query(VideoFile).filter( VideoFile.bangumi_id == bangumi_id).all() episodes = {} video_files = [] for eps in eps_list: if self.__episode_has_video_file(existed_video_files, eps): continue episodes[eps.episode_no] = eps for f in files: if self.__parse_episode_number(f) == eps.episode_no: eps.status = Episode.STATUS_DOWNLOADED video_files.append( VideoFile(bangumi_id=bangumi_id, episode_id=eps.id, file_path=f.decode('utf-8'), status=VideoFile.STATUS_DOWNLOADED)) break while True: for eps in episodes.values(): if not eps: continue episode_num = str(eps.episode_no) file_name = "None" for video_file in video_files: if video_file.episode_id == eps.id: file_name = video_file.file_path break print(episode_num + ": \t" + file_name) print("Right? Y/N") x = raw_input(">>> Input: ") if x == "Y": video_manager = VideoManager() video_manager.set_base_path(config['download']['location']) for video_file in video_files: for eps in episodes.values(): if eps.id == video_file.episode_id: video_manager.create_episode_thumbnail( eps, video_file.file_path, '00:00:01.000') thumbnail_path = '{0}/thumbnails/{1}.png'.format( str(bangumi_id), eps.episode_no) thumbnail_file_path = '{0}/thumbnails/{1}.png'.format( download_dir, eps.episode_no) width, height = get_dimension(thumbnail_file_path) eps.thumbnail_image = Image( file_path=thumbnail_path, dominant_color=get_dominant_color( thumbnail_file_path), width=width, height=height) meta_dict = video_manager.get_video_meta( u'{0}/{1}/{2}'.format( video_manager.base_path, bangumi_id.encode('utf-8'), video_file.file_path)) if meta_dict is not None: video_file.resolution_w = meta_dict['width'] video_file.resolution_h = meta_dict['height'] video_file.duration = meta_dict['duration'] session.add(video_file) break session.commit() return else: video_files = [] for f in files: print f x = raw_input(">>> Episode Num") if not x: continue x = int(x) eps = episodes[x] if not eps: continue eps.status = Episode.STATUS_DOWNLOADED video_files.append( VideoFile(bangumi_id=bangumi_id, episode_id=eps.id, file_path=f.decode('utf-8'), status=VideoFile.STATUS_DOWNLOADED))
def add_bangumi(self, content, uid): try: bangumi_data = json.loads(content) bangumi = Bangumi(bgm_id=bangumi_data.get('bgm_id'), name=bangumi_data.get('name'), name_cn=bangumi_data.get('name_cn'), type=bangumi_data.get('type'), summary=bangumi_data.get('summary'), eps=bangumi_data.get('eps'), image=bangumi_data.get('image'), air_date=bangumi_data.get('air_date'), air_weekday=bangumi_data.get('air_weekday'), status=self.__get_bangumi_status(bangumi_data.get('air_date')), created_by_uid=uid, maintained_by_uid=uid) # bangumi.dmhy = bangumi_data.get('dmhy') # bangumi.acg_rip = bangumi_data.get('acg_rip') # bangumi.libyk_so = bangumi_data.get('libyk_so') bangumi.eps_no_offset = bangumi_data.get('eps_no_offset') session = SessionManager.Session() session.add(bangumi) bangumi.episodes = [] for eps_item in bangumi_data['episodes']: eps = Episode(bgm_eps_id=eps_item.get('bgm_eps_id'), episode_no=eps_item.get('episode_no'), name=eps_item.get('name'), name_cn=eps_item.get('name_cn'), duration=eps_item.get('duration'), status=Episode.STATUS_NOT_DOWNLOADED) if is_valid_date(eps_item.get('airdate')): eps.airdate = eps_item.get('airdate') eps.bangumi = bangumi bangumi.episodes.append(eps) session.commit() bangumi_id = str(bangumi.id) try: (cover_file_path, cover_path) = self.__save_bangumi_cover(bangumi) # get dominant color bangumi.cover_color = get_dominant_color(cover_file_path) (width, height) = get_dimension(cover_file_path) bangumi.cover_image = Image(file_path=cover_path, dominant_color=bangumi.cover_color, width=width, height=height) session.commit() except Exception as error: sentry_wrapper.sentry_middleware.captureException() logger.warn(error) # delete bangumi for download error session.delete(bangumi) session.commit() raise ServerError('Fail to Download Image') return json_resp({'data': {'id': bangumi_id}}) finally: SessionManager.Session.remove()
if not os.path.exists(bangumi_dir): os.makedirs(bangumi_dir) print 'bangumi %s folder created' % (str(bangumi.id), ) path = urlparse(bangumi.image).path extname = os.path.splitext(path)[1] bangumi_cover_path = bangumi_dir + '/cover' + extname if not os.path.exists(bangumi_cover_path): # download bangumi image print 'start to download bangumi cover of %s (%s)' % ( bangumi.name, str(bangumi.id)) file_downloader.download_file(bangumi.image, bangumi_cover_path) if bangumi.cover_color is None: try: bangumi.cover_color = get_dominant_color( bangumi_cover_path, 5) session.commit() except Exception as err: print err if bangumi.cover_image_id is None: try: width, height = get_dimension(bangumi_cover_path) cover_image = Image(file_path='{0}/cover{1}'.format( str(bangumi.id), extname), dominant_color=bangumi.cover_color, width=width, height=height) bangumi.cover_image = cover_image session.commit() except Exception as err: print err
def add_bangumi(self, content, uid): try: bangumi_data = json.loads(content) bangumi = Bangumi(bgm_id=bangumi_data.get('bgm_id'), name=bangumi_data.get('name'), name_cn=bangumi_data.get('name_cn'), type=bangumi_data.get('type'), summary=bangumi_data.get('summary'), eps=bangumi_data.get('eps'), image=bangumi_data.get('image'), air_date=bangumi_data.get('air_date'), air_weekday=bangumi_data.get('air_weekday'), status=self.__get_bangumi_status( bangumi_data.get('air_date')), created_by_uid=uid, maintained_by_uid=uid) # bangumi.dmhy = bangumi_data.get('dmhy') # bangumi.acg_rip = bangumi_data.get('acg_rip') # bangumi.libyk_so = bangumi_data.get('libyk_so') bangumi.eps_no_offset = bangumi_data.get('eps_no_offset') session = SessionManager.Session() session.add(bangumi) bangumi.episodes = [] for eps_item in bangumi_data['episodes']: eps = Episode(bgm_eps_id=eps_item.get('bgm_eps_id'), episode_no=eps_item.get('episode_no'), name=eps_item.get('name'), name_cn=eps_item.get('name_cn'), duration=eps_item.get('duration'), status=Episode.STATUS_NOT_DOWNLOADED) if eps_item.get('airdate') != '': eps.airdate = eps_item.get('airdate') eps.bangumi = bangumi bangumi.episodes.append(eps) session.commit() bangumi_id = str(bangumi.id) try: (cover_file_path, cover_path) = self.__save_bangumi_cover(bangumi) # get dominant color bangumi.cover_color = get_dominant_color(cover_file_path) (width, height) = get_dimension(cover_file_path) bangumi.cover_image = Image(file_path=cover_path, dominant_color=bangumi.cover_color, width=width, height=height) session.commit() except Exception as error: sentry_wrapper.sentry_middleware.captureException() logger.warn(error) # delete bangumi for download error session.delete(bangumi) session.commit() raise ServerError('Fail to Download Image') return json_resp({'data': {'id': bangumi_id}}) finally: SessionManager.Session.remove()
def update_bangumi(self, bangumi_id=None): fr = open('./config/config.yml', 'r') config = yaml.load(fr) download_dir = config['download']['location'] + '/' + str(bangumi_id) files = self.__list_file_recursively(download_dir) session = SessionManager.Session() try: eps_list = session.query(Episode).\ filter(Episode.bangumi_id == bangumi_id).all() bangumi = session.query(Bangumi).\ filter(Bangumi.id == bangumi_id).one() eps_no_offset = 0 if bangumi.eps_no_offset is not None: eps_no_offset = bangumi.eps_no_offset existed_video_files = session.query(VideoFile).filter( VideoFile.bangumi_id == bangumi_id).all() episodes = {} video_files = [] for eps in eps_list: # if self.__episode_has_video_file(existed_video_files, eps): # continue episodes[eps.episode_no] = eps for f in files: if self.__video_path_already_added(existed_video_files, f.decode('utf-8')): continue if self.__parse_episode_number( f) + eps_no_offset == eps.episode_no: eps.status = Episode.STATUS_DOWNLOADED video_files.append( VideoFile(bangumi_id=bangumi_id, episode_id=eps.id, file_path=f.decode('utf-8'), status=VideoFile.STATUS_DOWNLOADED)) break while True: for eps in episodes.values(): if not eps: continue episode_num = str(eps.episode_no) line = episode_num + ": \t" file_name = None for video_file in video_files: if video_file.episode_id == eps.id: if file_name is None: line = line + video_file.file_path else: line = line + "\n \t" + video_file.file_path if video_file.label is not None: line = line + "\t" + video_file.label.decode( 'utf-8') file_name = video_file.file_path if file_name is None: line = line + "None" print(line) print("Right? Y/N") x = raw_input(">>> Input: ") if x == "Y": video_manager = VideoManager() video_manager.set_base_path(config['download']['location']) for video_file in video_files: for eps in episodes.values(): if eps.id == video_file.episode_id: video_manager.create_episode_thumbnail( eps, video_file.file_path, '00:00:01.000') thumbnail_path = '{0}/thumbnails/{1}.png'.format( str(bangumi_id), eps.episode_no) thumbnail_file_path = '{0}/thumbnails/{1}.png'.format( download_dir, eps.episode_no) width, height = get_dimension( thumbnail_file_path) eps.thumbnail_image = Image( file_path=thumbnail_path, dominant_color=get_dominant_color( thumbnail_file_path), width=width, height=height) meta_dict = video_manager.get_video_meta( u'{0}/{1}/{2}'.format( video_manager.base_path, bangumi_id.encode('utf-8'), video_file.file_path)) if meta_dict is not None: video_file.resolution_w = meta_dict[ 'width'] video_file.resolution_h = meta_dict[ 'height'] video_file.duration = meta_dict['duration'] session.add(video_file) # break session.commit() return else: video_files = [] for f in files: print f x = raw_input( ">>> Episode Num and Label (separated by comma)") if not x: continue arguments = x.split(',') x = int(arguments[0]) label = None if len(arguments) > 1: label = arguments[1] eps = episodes[x] if not eps: continue eps.status = Episode.STATUS_DOWNLOADED video_files.append( VideoFile(bangumi_id=bangumi_id, episode_id=eps.id, file_path=f.decode('utf-8'), label=label, status=VideoFile.STATUS_DOWNLOADED)) finally: SessionManager.Session.remove()
def update_bangumi(self, bangumi_id=None): fr = open('./config/config.yml', 'r') config = yaml.load(fr) download_dir = config['download']['location'] + '/' + str(bangumi_id) files = self.__list_file_recursively(download_dir) session = SessionManager.Session() try: eps_list = session.query(Episode).\ filter(Episode.bangumi_id == bangumi_id).all() bangumi = session.query(Bangumi).\ filter(Bangumi.id == bangumi_id).one() eps_no_offset = 0 if bangumi.eps_no_offset is not None: eps_no_offset = bangumi.eps_no_offset existed_video_files = session.query(VideoFile).filter(VideoFile.bangumi_id == bangumi_id).all() episodes = {} video_files = [] for eps in eps_list: # if self.__episode_has_video_file(existed_video_files, eps): # continue episodes[eps.episode_no] = eps for f in files: if self.__video_path_already_added(existed_video_files, f.decode('utf-8')): continue if self.__parse_episode_number(f) + eps_no_offset == eps.episode_no: eps.status = Episode.STATUS_DOWNLOADED video_files.append(VideoFile(bangumi_id=bangumi_id, episode_id=eps.id, file_path=f.decode('utf-8'), status=VideoFile.STATUS_DOWNLOADED)) break while True: for eps in episodes.values(): if not eps: continue episode_num = str(eps.episode_no) line = episode_num + ": \t" file_name = None for video_file in video_files: if video_file.episode_id == eps.id: if file_name is None: line = line + video_file.file_path else: line = line + "\n \t" + video_file.file_path if video_file.label is not None: line = line + "\t" + video_file.label.decode('utf-8') file_name = video_file.file_path if file_name is None: line = line + "None" print (line) print("Right? Y/N") x = raw_input(">>> Input: ") if x == "Y": video_manager = VideoManager() video_manager.set_base_path(config['download']['location']) for video_file in video_files: for eps in episodes.values(): if eps.id == video_file.episode_id: video_manager.create_episode_thumbnail(eps, video_file.file_path, '00:00:01.000') thumbnail_path = '{0}/thumbnails/{1}.png'.format(str(bangumi_id), eps.episode_no) thumbnail_file_path = '{0}/thumbnails/{1}.png'.format(download_dir, eps.episode_no) width, height = get_dimension(thumbnail_file_path) eps.thumbnail_image = Image(file_path=thumbnail_path, dominant_color=get_dominant_color(thumbnail_file_path), width=width, height=height) meta_dict = video_manager.get_video_meta(u'{0}/{1}/{2}'.format(video_manager.base_path, bangumi_id.encode('utf-8'), video_file.file_path)) if meta_dict is not None: video_file.resolution_w = meta_dict['width'] video_file.resolution_h = meta_dict['height'] video_file.duration = meta_dict['duration'] session.add(video_file) # break session.commit() return else: video_files = [] for f in files: print f x = raw_input(">>> Episode Num and Label (separated by comma)") if not x: continue arguments = x.split(',') x = int(arguments[0]) label = None if len(arguments) > 1: label = arguments[1] eps = episodes[x] if not eps: continue eps.status = Episode.STATUS_DOWNLOADED video_files.append(VideoFile(bangumi_id=bangumi_id, episode_id=eps.id, file_path=f.decode('utf-8'), label=label, status=VideoFile.STATUS_DOWNLOADED)) finally: SessionManager.Session.remove()
def upgrade(): # ### commands auto generated by Alembic - please adjust! ### op.create_table('image', sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), sa.Column('file_path', sa.TEXT(), nullable=False), sa.Column('dominant_color', sa.String(), nullable=True), sa.Column('width', sa.Integer(), nullable=True), sa.Column('height', sa.Integer(), nullable=True), sa.PrimaryKeyConstraint('id') ) op.add_column(u'bangumi', sa.Column('cover_image_id', postgresql.UUID(as_uuid=True), nullable=True)) op.add_column(u'episodes', sa.Column('thumbnail_image_id', postgresql.UUID(as_uuid=True), nullable=True)) # ### end Alembic commands ### # import data into image table. connection = op.get_bind() result = connection.execute(sa.text( 'SELECT bangumi.id, bangumi.image, bangumi.cover_color FROM bangumi WHERE bangumi.image NOTNULL')) base_path = get_base_path() for row in result: bangumi_id = row[0] bangumi_image = row[1] dominant_color = row[2] path = urlparse(bangumi_image).path extname = os.path.splitext(path)[1] image_path = '{0}/cover{1}'.format(str(bangumi_id), extname) if not os.path.exists('{0}/{1}'.format(base_path, image_path)): print 'cover not found for {0}'.format(bangumi_id) continue try: image_id = uuid4() connection.execute(sa.text( "INSERT INTO image (id, file_path, dominant_color) VALUES ('{0}', '{1}', '{2}')".format( image_id, image_path, dominant_color))) connection.execute(sa.text( "UPDATE bangumi SET cover_image_id = '{0}' WHERE id = '{1}'".format(image_id, bangumi_id))) except Exception as error: print error # query episodes episode_result = connection.execute(sa.text( "SELECT e.id, e.episode_no, e.thumbnail_color FROM episodes e WHERE e.status = 2 AND e.bangumi_id = '{0}'".format( bangumi_id))) for eps in episode_result: episode_id = eps[0] episode_no = eps[1] episode_thumbnail_color = eps[2] thumbnail_path = '{0}/thumbnails/{1}.png'.format(str(bangumi_id), episode_no) if not os.path.exists(u'{0}/{1}'.format(base_path, thumbnail_path)): print 'thumbnail not found for {0}'.format(episode_id) continue try: image_id = uuid4() connection.execute(sa.text( "INSERT INTO image (id, file_path, dominant_color) VALUES ('{0}', '{1}', '{2}')".format( image_id, thumbnail_path, episode_thumbnail_color))) connection.execute(sa.text( "UPDATE episodes SET thumbnail_image_id = '{0}' WHERE id = '{1}'".format(image_id, episode_id))) except Exception as error: print error print 'image table import completed. Start reading image dimension and fix color' image_result = connection.execute(sa.text('SELECT image.id, image.file_path, image.dominant_color FROM image')) for image in image_result: image_id = image[0] image_file_path = image[1] image_dominant_color = image[2] try: im = Image.open('{0}/{1}'.format(base_path, image_file_path)) (width, height) = im.size if image_dominant_color is None: image_dominant_color = get_dominant_color('{0}/{1}'.format(base_path, image_file_path), 5) connection.execute(sa.text( "UPDATE image SET width = '{0}', height = '{1}', dominant_color = '{2}' WHERE id = '{3}'".format( width, height, image_dominant_color, image_id))) except Exception as error: print error print 'All done'