def convert_webp(path_class, src_root, webp_cache_root):
if not yutils.is_gif(path_class.ext) and not yutils.is_photo(path_class.ext) and yutils.is_webp(path_class.path):
# 如果是webp需要转换一下.然后把webp文件放到另外目录.
src_root = ypath.join(path_class.pic_root, str(src_root))
webp_cache_root = ypath.join(path_class.pic_root, str(webp_cache_root))
move_target = ypath.decompose_path(path_class.path, src_root, webp_cache_root)
if os.path.exists(move_target):
os.remove(move_target)
convert_target = None
try:
im = Image.open(path_class.path)
ext = '.png' if im.format == 'PNG' or im.mode == 'RGBA' else '.jpg'
convert_target = ypath.del_exten(path_class.path) + ext
if os.path.exists(convert_target):
os.remove(convert_target)
im.save(convert_target)
ypath.create_dirs(move_target)
shutil.move(path_class.path, move_target)
path_class.path = convert_target
return True
except Exception as e:
if convert_target and os.path.exists(convert_target):
os.remove(convert_target)
pass
return False
return False
def delete_not_exist():
if not os.path.exists(middle):
logger.info('middle not exist!')
return
print('[delete_not_exist] begin')
right_map = {}
for root, dirs, files in os.walk(src):
for file in files:
if not yutils.is_gif(file) and not yutils.is_photo(file):
continue
source_file = ypath.join(root, file)
rename_file = middle_out_file(source_file)
if os.path.exists(rename_file):
right_map[rename_file] = source_file
for root, dirs, files in os.walk(middle):
for file in files:
if not yutils.is_gif(file) and not yutils.is_photo(file):
continue
middle_path = ypath.join(root, file)
if middle_path in right_map:
pass # print('' + source_path + " src:" + right_map[source_path])
else:
thum_path = thum + middle_path[len(middle):]
if os.path.exists(middle_path):
os.remove(middle_path)
if os.path.exists(thum_path):
os.remove(thum_path)
print('删除文件:', middle_path, thum_path)
ypath.del_none_dir(middle)
ypath.del_none_dir(thum)
print('[delete_not_exist] end')
def search_by_dir_id(p_id):
dinfos = Dir.objects.annotate(p_id=F('parent_dir__id')).filter(
type=yutils.M_FTYPE_MOIVE, parent_dir_id=p_id).values('id', 'name')
minfos = Media.objects.filter(src_dir_id=p_id,
state=DBHelper.end_media_state()).annotate(
mpath=F('desc_mpath__param1')).values(
'desc_path', 'file_name', 'duration',
'size', 'width', 'height',
'r_frame_rate', 'mpath')
# ress = json.dumps(list(minfos))
# ress = json.loads(ress)
res_infos = []
for minfo in list(minfos):
tmp_dict = {}
tmp_dict['file_name'] = ypath.del_exten(minfo['file_name'])
tmp_dict['nginx_path'] = ypath.join(minfo['mpath'],
movie_config.dir_root,
minfo['desc_path'])
tmp_dict['duration'] = minfo['duration']
tmp_dict['size'] = minfo['size']
tmp_dict['width'] = minfo['width']
tmp_dict['height'] = minfo['height']
tmp_dict['r_frame_rate'] = minfo['r_frame_rate']
tmp_dict['img'] = ypath.join(minfo['mpath'],
movie_config.img_info.img_root,
minfo['desc_path'],
movie_config.img_info.thum)
res_infos.append(tmp_dict)
return (list(dinfos), res_infos)
def __init__(self):
pic_config = XMLBase.list_cfg_infos('pic_info') # XMLMedia.get_infos()
self.src_root = pic_config.dir_root # 源根目录.
self.webp_cache_root = pic_config.webp_cache # src中.有webp.需要转换成png.然后把该webp放置到这个路径.
self.desc_root = pic_config.dir_root # 输出根目录
self.desc_middle_root = ypath.join(self.desc_root,
pic_config.middle) # 放置放大图的地方
self.desc_thum_root = ypath.join(self.desc_root,
pic_config.thum) # 放置缩略图的地方
self.desc_webp_root = ypath.join(self.desc_root,
pic_config.webp) # 放置webp的地方
self.middle_area = int(pic_config.max_pic_size)**2
self.thum_size = int(pic_config.thum_size)
self.MULIT_THREAD_COUNT = 6 # 多线程转换尺寸.
self.watch = TimeWatch('PhotoService')
self.err_pic = []
self.src_dirs = None
self.desc_dirs = None
def local_list(path):
list = []
for root, dirs, files in os.walk(path):
for file in files:
if '.DS_Store' in file:
continue
temp = ypath.join(root, file)
size = os.path.getsize(temp)
list.append('%s|%d' % (temp.replace(path, ''), size))
return list
def create_thum(media_db: Media):
if MediaHelp.is_err(media_db.state):
return
desc_root = media_db.desc_mpath.path
media_tum_root = ypath.join(
desc_root, movie_config.img_info.img_root
) # TmpUtil.desc() / movie_config.img_info.img_root
# convert_root
# if media_db.state >= MediaHelp.STATE_VIDEO_THUM:
# logger.info('该文件已经转缩略图过了:' + media_db.abs_path)
# return
target_img_dir = ypath.join(media_tum_root, media_db.desc_path)
ypath.create_dirs(target_img_dir)
desc = ypath.join(target_img_dir, movie_config.img_info.img)
desc_thum = ypath.join(target_img_dir, movie_config.img_info.thum)
if os.path.exists(desc) and os.path.exists(desc_thum):
logger.info('该视频不需要做缩略图裁切,因为已有:%s' % desc)
return
# 裁切缩略图的比例
thum_percent = int(movie_config.base_info.thum_w) / int(
movie_config.base_info.thum_h)
max_thum_time = int(movie_config.base_info.max_thum_time)
min_thum_time = int(movie_config.base_info.min_thum_time)
ypath.create_dirs(desc)
r_time = random.randint(
min_thum_time if media_db.duration > min_thum_time else 0,
max_thum_time
if media_db.duration > max_thum_time else media_db.duration)
d_abs_path = ypath.join(media_root(desc_root), media_db.desc_path)
cmd = ffmpeg_tools + ' -i \"' + d_abs_path + '\" -y -vframes 1 -ss 00:00:' + str(
r_time) + ' -f image2 \"' + desc + '\"'
yutils.process_cmd(cmd)
if not os.path.exists(desc):
return
img = Image.open(desc)
w, h = img.size
crop_img = img.crop(yutils.crop_size(w, h, thum_percent))
crop_img.save(desc_thum)
def del_not_exist(middle_dir):
# 需要验证 还没验证呢
p_infos = Photo.objects.all()
with transaction.atomic():
for p_info in p_infos:
desc_abs_path = get_middle_abs_path(p_info.desc_mpath_id)
middle_path = ypath.join(desc_abs_path, middle_dir, p_info.desc_rela_path)
middle_exist = os.path.exists(middle_path)
if not os.path.exists(p_info.src_abs_path) or not middle_exist or p_info.state != PicHelp.STATE_FINISH:
p_info.delete()
if middle_exist or os.path.islink(middle_path):
os.remove(middle_path)
def create_ts(media_db: Media):
# d_abs_path = ypath.join(media_root(media_db), media_db.desc_path)
desc_root = media_db.desc_mpath.path
media_ts_dir = ypath.join(desc_root, movie_config.ts_info.ts_dir,
media_db.desc_path)
if media_db.state >= MediaHelp.STATE_VIDEO_TS and os.path.isdir(
media_ts_dir):
logger.info('该视频已经切片过了:' + media_db.abs_path)
return
if os.path.exists(media_ts_dir):
shutil.rmtree(media_ts_dir)
ypath.create_dirs(media_ts_dir, True)
media_desc_path = desc_path(
media_db) # ypath.join(desc_path(media_db), media_db.desc_path)
m3u8_file = ypath.join(media_ts_dir, movie_config.ts_info.u8name)
cmd = '\"' + ffmpeg_tools + '\" -i \"' + media_desc_path + \
'\" -codec copy -vbsf h264_mp4toannexb -map 0 -f segment -segment_list \"' + \
m3u8_file + '\" -segment_time 30 \"' + media_ts_dir + '/%05d.ts\"'
yutils.process_cmd(cmd)
modify_state(media_db, MediaHelp.STATE_VIDEO_TS)
# m3u8_path = ypath.join(media_ts_dir, movie_config.ts_info.u8name)
# media_db.m3u8_path = ypath.join(media_db.desc_path, movie_config.ts_info.u8name)
print(desc_root)
def begin_s2middle_by_threads(src_dir, desc_dir, delete_exist):
# , source_file, rename_path
for root, _, files in os.walk(src_dir):
for file in files:
source_file = ypath.join(root, file)
rename_file = middle_out_file(source_file, desc_dir)
if (not delete_exist) and os.path.exists(rename_file):
print('文件已存在!' + rename_file)
continue
if yutils.is_gif(source_file):
shutil.copy(source_file, rename_file)
continue
if not yutils.is_photo(source_file):
other_file.append(source_file)
continue
ypath.create_dirs(rename_file)
print('源：' + source_file)
img = Image.open(source_file)
# 压缩尺寸
w, h = img.size
pic_area = w * h
if pic_area > middle_area:
proportion = (middle_area / pic_area)**0.5
w = int(w * proportion)
h = int(h * proportion)
img.thumbnail((w, h), Image.ANTIALIAS)
# 处理旋转信息.
if 'exif' in img.info:
old_exif = piexif.load(img.info["exif"])
if '0th' in old_exif and piexif.ImageIFD.Orientation in old_exif[
'0th']:
orientation = old_exif['0th'][piexif.ImageIFD.Orientation]
if orientation == 6:
img = img.rotate(-90, expand=True)
if orientation == 3:
img = img.rotate(180)
if orientation == 8:
img = img.rotate(90, expand=True)
exif_bytes = piexif.dump({})
img.save(rename_file, 'JPEG', exif=exif_bytes)
else:
try:
img.save(rename_file)
except:
img = img.convert('RGB')
img.save(rename_file)
def read_thum():
Dir.objects.filter(type=yutils.M_FTYPE_PIC).delete()
desc = ypath.join(TmpUtil.desc(), pic_cfg.dir_root)
dict = ypath.path_result(desc, pic_cfg.thum, add_root=False)
dir_dict = {}
file_dict = {}
for key in dict:
if dict[key][ypath.KEYS.IS_DIR]:
dir_dict[key] = dict[key]
else:
file_dict[key] = dict[key]
dir_list = sorted(dir_dict.items(), key=lambda d: d[1][ypath.KEYS.LEVEL])
file_list = sorted(file_dict.items(), key=lambda d: d[1][ypath.KEYS.LEVEL])
file_db_list = insert_db(dir_list, file_list)
Dir.objects.bulk_create(file_db_list)
def gen_dir():
str_media_src = str(media_src_root.as_posix())
dir_db_paths = {}
for dir in os.listdir(str_media_src):
m_file_list = ypath.path_res(ypath.join(str_media_src, dir), parse_file=False)
all_media_dirs = Dir.objects.filter(tags=dir)
for dir_db in all_media_dirs:
if dir_db.abs_path not in m_file_list:
logger.info('被删除的路径:' + dir_db.abs_path)
dir_db.delete()
else:
dir_db_paths[dir_db.abs_path] = dir_db
for local_dir in m_file_list:
if local_dir.path not in dir_db_paths:
dir_db_paths[local_dir.path] = ServiceHelper.create_dir(dir_db_paths, local_dir, yutils.M_FTYPE_MOIVE,
dir) # create_dir(dir_db_paths, local_dir, dir)
logger.info('创建该文件夹:' + str(local_dir))
return dir_db_paths
def src2pc(delete_exist):
if not os.path.exists(src):
logger.info('src2pc:src not exist!')
return
ImageFile.LOAD_TRUNCATED_IMAGES = True
# cmd = 'for i in ' + src + '/*.jpg;do jpegoptim -m50 -d ' + desc + ' -p "$i";done'
# os.system(cmd)
from frames import ThreadingPool
tpool = ThreadingPool.ThreadingPool()
img_link_dic = {}
for root, dirs, files in os.walk(src):
for dir in dirs:
src_dir = ypath.join(root, dir)
out_dir, single_dir = middle_out_dir(src_dir)
img_link_dic[single_dir] = src_dir
tpool.append(begin_s2middle_by_threads, src_dir, out_dir,
delete_exist)
tpool.start()
print('end mulite thread!!!!!!!!!!!!!!')
return img_link_dic
def middle2thum(delete_exist):
if delete_exist and os.path.exists(thum):
shutil.rmtree(thum)
for root, dirs, files in os.walk(middle):
for file in files:
source_path = ypath.join(root, file)
print(source_path)
desc_path = thum + source_path[len(middle):]
if (not delete_exist) and os.path.exists(desc_path):
print('文件已存在!' + desc_path)
continue
dir = os.path.dirname(desc_path)
ypath.create_dirs(dir, is_dir=True)
# gif要走配置
if yutils.is_gif(file):
# gif_pic
shutil.copy(gif_space, desc_path)
pass
if not yutils.is_photo(file):
continue
img = Image.open(source_path)
w, h = img.size
crop_img = img.crop(yutils.crop_size(w, h)) # 保存裁切后的图片
crop_img.thumbnail((thum_width, thum_width), Image.ANTIALIAS)
if 'exif' in img.info:
exif_dict = piexif.load(crop_img.info["exif"])
exif_bytes = piexif.dump(exif_dict)
# crop_img.save(desc_path, 'JPEG') # 是否需要压缩质量,具体看情况而定.
crop_img.save(desc_path, 'JPEG',
exif=exif_bytes) # 是否需要压缩质量,具体看情况而定.
else:
try:
crop_img.save(desc_path)
except:
crop_img = crop_img.convert('RGB')
crop_img.save(desc_path)
print(desc_path)
sync_path.append(item.strip().split()[0])
download(sync_path)
def download_oncos():
yutils.process_cmd('coscmd list -ar ' + bucket_dir, done_call=download)
# 示例: python3 COSBrowser.py -l /Users/mr.yang/Documents/cache/ttt -b ttt
if __name__ == '__main__':
from frames.xml import XMLPic
pic_cfg = XMLPic.get_infos()
local_path = ypath.desc()
local_path = ypath.join(local_path, pic_cfg.dir_root)
bucket_dir = '/res/pic'
try:
opts, args = getopt.getopt(sys.argv[1:], 'l:b:c:',
['local=', 'bucket=', 'cmdf='])
except getopt.GetoptError as e:
print('参数错误!:' + e)
for o, a in opts:
if o in ('-l', '--local'):
# print('o:' + o + ' a:' + a)
if a.endswith('/') or a.endswith('\\'):
a = a[:-1]
local_path = a
if o in ('-b', '--bucket'):
print(a)
def media_root(dir_root):
return ypath.join(dir_root, src_root)
def desc_path(media_db: Media):
return ypath.join(media_root(media_db.desc_mpath.path), media_db.desc_path)
def src_list(src_root):
src_paths = {}
for src_dir in MediaPath.pdc().src_list:
src_paths[ypath.join(src_dir.path, src_root)] = src_dir.query
return src_paths
def begin_threads(self, create_db_list: list, f_list, err_list):
for link_item in f_list:
if not yutils.is_gif(link_item.ext) and not yutils.is_photo(
link_item.ext):
logger.info('这张不是图片:' + link_item.path)
continue
src_file = link_item.path
file_stat = os.stat(src_file)
if file_stat.st_size > PngImagePlugin.MAX_TEXT_MEMORY:
err_list.append(src_file)
continue
pi = Photo()
pi.src_abs_path = src_file
pi.src_name = link_item.relative
pi.src_mpath = MediaPath.pdc().search_by_abs_path(
link_item.pic_root)
try:
file_steam = open(src_file, 'rb')
pi.src_md5 = yutils.get_md5_steam(file_steam)
pi.ctime = int(file_stat.st_ctime)
pi.mtime = int(file_stat.st_mtime)
desc_rela_name, pi.src_size = PhotoHelper.file_desc_dir(
file_stat, pi.src_abs_path, pi.src_md5)
src_img = Image.open(file_steam)
if src_img.mode == 'RGB':
pi.desc_rela_path = desc_rela_name + '.jpg'
else:
pi.desc_rela_path = desc_rela_name + link_item.ext
pi.src_width, pi.src_height = src_img.size
except:
err_list.append(src_file)
logger.info('这张图有错误!!!!!!!!!!!!!!!!!!!!!!!:' + src_file)
continue
with lock:
desc_root = MediaPath.desc()
pi.desc_mpath = MediaPath.pdc().search_by_abs_path(
desc_root, False)
desc_middle_path = ypath.join(desc_root, self.desc_middle_root,
pi.desc_rela_path)
desc_thum_path = ypath.join(desc_root, self.desc_thum_root,
pi.desc_rela_path)
desc_webp_path = ypath.join(desc_root, self.desc_webp_root,
pi.desc_rela_path)
ypath.create_dirs(desc_middle_path)
ypath.create_dirs(desc_thum_path)
ypath.create_dirs(desc_webp_path)
m_img = PhotoHelper.convert_middle(src_img, link_item.path,
desc_middle_path,
self.middle_area)
w, h = m_img.size
pi.mid_width = w
pi.mid_height = h
pi.mid_size = os.path.getsize(desc_middle_path)
pi.state = PicHelp.STATE_FINISH
pi.is_gif = yutils.is_gif(link_item.ext)
# webp_file = ypath.join(desc_webp_root, mulit_file_list[middle_file][0] + '.webp')
# convert_webp(m_img, webp_file, middle_file)
t_img = PhotoHelper.cut_middle2thum(m_img, desc_thum_path,
self.thum_size)
if m_img is not t_img:
del m_img
del t_img
else:
del m_img
with lock:
create_db_list.append(pi)
if len(create_db_list) >= PhotoHelper.SYNC_PHOTO_DB_COUNT:
Photo.objects.bulk_create(create_db_list)
create_db_list.clear()
def movie_info_res(cmdlist, _):
if len(cmdlist) <= 0:
modify_state(media_db, MediaHelp.STATE_AUDIO_FINISH)
return
jsonbean = json.loads(''.join(cmdlist))
if 'streams' not in jsonbean.keys():
modify_state(media_db, MediaHelp.STATE_SRC_ERROR)
return
streamlist = jsonbean['streams']
format = jsonbean['format']
media_db.md5 = yutils.get_md5(media_db.abs_path)
media_db.duration = int(float(format['duration']))
# cur_file_info['duration'] =
media_db.size = int(format['size'])
if len(streamlist) <= 0:
modify_state(media_db, MediaHelp.STATE_AUDIO_FINISH)
return
audio_streams = []
decode_map = ''
for stream_item in streamlist:
if stream_item['codec_type'] == 'audio':
logger.info(str(stream_item))
audio_streams.append(stream_item)
else:
if stream_item['codec_type'] == 'video':
# 如果是视频.保存视频信息
media_db.codec_type = stream_item['codec_name']
media_db.codec_long_name = stream_item['codec_long_name']
media_db.width = int(stream_item['width'])
media_db.height = int(stream_item['height'])
try:
media_db.r_frame_rate = round(
eval(stream_item['r_frame_rate']))
media_db.avg_frame_rate = round(
eval(stream_item['avg_frame_rate']))
except ZeroDivisionError:
media_db.r_frame_rate = 0
media_db.avg_frame_rate = 0
decode_map += ' -map 0:' + str(stream_item['index'])
# 有多个语种
digout = False
if len(audio_streams) > 1:
for audio_stream in audio_streams:
if 'tags' in audio_stream and 'title' in audio_stream['tags'] \
and ('国语' == audio_stream['tags']['title'] or '中语' == audio_stream['tags']['title']):
decode_map += ' -map 0:' + str(audio_stream['index'])
media_db.audio_long_name = audio_stream['codec_long_name']
media_db.audio_name = audio_stream['codec_name']
digout = True
break
else:
if len(audio_streams) == 1:
media_db.audio_long_name = audio_streams[0]['codec_long_name']
media_db.audio_name = audio_streams[0]['codec_name']
logger.info('该视频音轨只有一个,不需要转换:' + media_db.abs_path)
modify_state(media_db, MediaHelp.STATE_AUDIO_FINISH)
return
if not digout:
if len(audio_streams) >= 1:
media_db.audio_long_name = audio_streams[0]['codec_long_name']
media_db.audio_name = audio_streams[0]['codec_name']
# media_db.audio_long_name = stream_item['codec_long_name']
# media_db.audio_name = stream_item['codec_name']
for audio_stream in audio_streams:
decode_map += ' -map 0:' + str(audio_stream['index'])
# out_content += str(index) + ':' + str(audio_stream) + '\n'
# index += 1
# select_audio = len(audio_streams)
# while len(audio_streams) <= select_audio or select_audio < 0:
# select_audio = int(input(out_content + '选择音轨:'))
with lock:
mulit_audio_path = ypath.join(MediaPath.src(), mulit_audio_dir)
desc_mulit_path = ypath.decompose_path(media_db.abs_path, src_db.path,
str(mulit_audio_path))
out_file = desc_mulit_path + '.chi' + ypath.file_exten(
media_db.abs_path)
ypath.create_dirs(desc_mulit_path)
if os.path.exists(out_file):
os.remove(out_file)
logger.info(out_file)
copy_cmd = ffmpeg_tools + ' -i \"' + media_db.abs_path + '\"' + decode_map + ' -vcodec copy -acodec copy \"' + out_file + '\"'
yutils.process_cmd(copy_cmd,
done_call=rm_on_audio_copy,
param=(media_db.abs_path, out_file,
desc_mulit_path))
def desc_list(desc_root):
desc_paths = {}
for desc_dir in MediaPath.pdc().desc_list:
desc_paths[ypath.join(desc_dir.path, desc_root)] = desc_dir.query
return desc_paths
pass # print('' + source_path + " src:" + right_map[source_path])
else:
thum_path = thum + middle_path[len(middle):]
if os.path.exists(middle_path):
os.remove(middle_path)
if os.path.exists(thum_path):
os.remove(thum_path)
print('删除文件:', middle_path, thum_path)
ypath.del_none_dir(middle)
ypath.del_none_dir(thum)
print('[delete_not_exist] end')
if __name__ == '__main__':
src = TmpUtil.src()
src = ypath.join(src, pic_cfg.dir_root)
desc = TmpUtil.desc()
desc = ypath.join(desc, pic_cfg.dir_root)
gif_space = TmpUtil.input_note(GIF_SPACE, '请指定gif的占位符的图片位置:\n')
logger.info('初始化成功src:', src, ',desc:', desc, 'gif_space:', gif_space)
middle = ypath.join(desc, pic_cfg.middle)
thum = ypath.join(desc, pic_cfg.thum)
other_file.clear()
# 去重
ypath.delrepeat_file(src)
# 去掉middle中的图.
def middle_out_dir(src_dir):
simple_path = src_dir[len(src):]
dir = yutils.md5_of_str(simple_path)
return ypath.join(middle, dir), dir
def middle_out_file(source_dir, desc_dir=None):
if not desc_dir:
desc_dir, _ = middle_out_dir(os.path.dirname(source_dir))
exten = ypath.file_exten(source_dir)
rename_file = ypath.join(desc_dir, yutils.get_md5(source_dir) + exten)
return rename_file
