def add_ffmpeg_state_to_redis(taskid, line):
# pdb.set_trace()
global process_step
global prev_dic
global dic
# state line
if line == 'all_complete\n':
dic['all_complete'] = 'all complete! Please download file in file list!'
process_step = 'all_complete'
#get line content
if process_step == 'unprocessed':
global process_total_time
reobj = re.compile(r'.*?Duration:.*?start.*?bitrate.*?')
if reobj.match(line):
head = line.find('Duration:')
process_time_list = line[int(head) + 10:int(head) + 20].split(':')
process_total_time = (float(process_time_list[0]) * 3600 +
float(process_time_list[1]) * 60 +
float(process_time_list[2]))
reobj = re.compile(r'.*?size.*?time.*?bitrate.*?')
if reobj.match(line):
present = line.find('time')
process_now_list = line[int(present) + 5:int(present) +
15].split(':')
process_now_time = (float(process_now_list[0]) * 3600 +
float(process_now_list[1]) * 60 +
float(process_now_list[2]))
dic[process_step] = 'Transcoding :' + str(
line[:-1]) + ' schedule: %0.1f %% ...' % (
float(process_now_time / process_total_time) * 100)
elif process_step == 'all_complete':
pass
if dic != prev_dic:
redistool.redis_set(
'_'.join(["task", "status", taskid]),
"Begin to process one task...\n" + '\n'.join(dic.values()))
prev_dic = copy.deepcopy(dic)
def H264toHevcProc(request):
if request.method == 'POST':
form = ftpFileForm(request.POST)
if form.is_valid():
filterparam = str(form.cleaned_data['filterparam'])
filterparam_json = json.loads(filterparam)
authcode = filterparam_json['before_file']['authcode']
filename = filterparam_json['before_file']['filename']
#add to file manage system
fileid = handleFtpFile(authcode, filename)
if (fileid == "error"):
return HttpResponse("Error")
#update user table
updateAuthStorage(authcode)
#add process(task) log
taskid = str(uuid.uuid1()).replace('-', '')
filterparam_json['taskid'] = taskid
filterparam_json['before_file']['fileid'] = fileid
filterparam_json['before_file']['filetype'] = filename.split(
'.')[-1]
oneProcesslog = processlog(taskid=taskid,
fileid=fileid,
dealmethod="transcode",
controljson=filterparam_json,
dealstate="processing",
dealtime=datetime.now())
oneProcesslog.save()
#add Redis state
# r.set('_'.join(["task","status",taskid]), "Copying file to workstation...")
redistool.redis_set('_'.join(["task", "status", taskid]),
"Waiting to be processed...")
# tasks.hd_to_4k.delay(filterparam_json)
tasks.ffmpeg_transcode.delay(filterparam_json)
# handleTranscodeTest(authcode, fileid)
return HttpResponse(taskid)
return HttpResponse("Error")
def ffmpeg_transcode(param_data):
try:
#begin
redistool.redis_set('_'.join(["task", "status", param_data['taskid']]),
"Copying file to workstation...")
# initializtion 1.14
parse_state.dic.clear()
parse_state.prev_dic.clear()
parse_state.process_step = "unprocessed"
# file path and name
file_work_basepath = ''.join(
[conf_dic['all']['work_path'], '\\', param_data['taskid']])
file_tmp_name = '1'
file_work_path = ''.join([file_work_basepath, '\\'])
before_file = ''.join([
file_work_path, file_tmp_name, '.',
param_data['before_file']['filetype']
])
after_file = ''.join([file_work_path, file_tmp_name, '_out.mp4'])
hd_to_4k_downloader(param_data, file_tmp_name)
bat_file_name = file_work_path + file_tmp_name + '_ffmpeg.bat'
#load the control file template,set the parameter
template_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(conf_dic['all']['template_path']))
bat_file_tmp_param = {
'before_file_name': before_file,
'after_file_name': after_file
}
bat_file_tmp_param.update(conf_dic['4k_tool'])
bat_file_tmp_param.update(
param_data['filters']['ffmpeg']['hevc_param'])
bat_file_tmp = template_env.get_template(
'ffmpeg_bat_template.bat').render(bat_file_tmp_param)
#write the new control file
if (os.path.exists(file_work_basepath) == False):
os.mkdir(file_work_basepath)
lines = "\r\n".join(bat_file_tmp.split('\n'))
bat_file = open(bat_file_name, 'w+')
bat_file.writelines(lines)
bat_file.close()
#download file and process and upload result
process2 = subprocess.Popen(bat_file_name,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True)
logtool.logger_main.info("Begin to Trancode, task: %s..." %
param_data['taskid'])
one_log = logtool.Pro_log(param_data['taskid'])
while True:
line = process2.stdout.readline()
print line
parse_state.add_ffmpeg_state_to_redis(param_data['taskid'], line)
one_log.info(line)
if not line:
break
logtool.logger_main.info("Trancode task: %s ok!" %
param_data['taskid'])
hd_to_4k_uploader(
param_data,
''.join([param_data['taskid'], '\\', file_tmp_name,
'_out.mp4']), "_265")
shutil.rmtree(file_work_basepath)
#update processlog
Session = sessionmaker(bind=sqltool.engine)
session = Session()
oneprocesslog = session.query(sqltool.Web_Task).filter(
sqltool.Web_Task.taskid == param_data['taskid'])
oneprocesslog.update({
sqltool.Web_Task.dealstate:
'succeed',
sqltool.Web_Task.completetime:
datetime.datetime.now()
})
session.commit()
session.close()
parse_state.add_ffmpeg_state_to_redis(param_data['taskid'],
'all_complete\n')
except EOFError:
redistool.redis_set(
'_'.join(["task", "status", param_data['taskid']]),
"Error, please try again or connect to [email protected]!")
def tctcreate(request):
# form = ftpFileForm(request.POST)
if (request.method != 'POST'):
return HttpResponse()
# filterparam = str( form.cleaned_data['filterparam'])
filterparam_json = json.loads(request.body.encode("utf-8"),
encoding="utf-8")
print filterparam_json
#new var
filePath = filterparam_json["task"]["in_uri"]
#new method add to file system
#add media file
fileid = str(uuid.uuid1()).replace('-', '')
filename = filePath.split('/')[-1] #filename in oursystem
filetype = filterparam_json["task"]["in_format"].split('.')[-1]
authcode = "38a43f8070e811e5ad0c90b11c94ab4d"
oneFile = models.mediafile(fileid=fileid,
filename=filename,
authcode=authcode,
filesize=0,
location=filePath,
filetype=filetype,
uploadtime=datetime.now(),
encodeinfo=' ')
oneFile.save()
#add task log
taskid = filterparam_json["task"][
"task_id"] #filterparam_json["task"]["task_id"]?or our own taskid
hevc_param = {
'keyframemin': '50',
'keyframemax': '50',
'vbvmaxbitrate': '30000',
'vbvbuffersize': '15000',
'bitrate': '15000'
}
json_tmp = {}
json_tmp['taskid'] = taskid
json_tmp['before_file'] = {}
json_tmp['before_file']['fileid'] = fileid
json_tmp['before_file']['authcode'] = authcode
json_tmp['before_file']['filetype'] = filetype
json_tmp['before_file']['location'] = str(filePath)
json_tmp['filters'] = {}
json_tmp['filters']['ffmpeg'] = {}
json_tmp['filters']['ffmpeg']['hevc_param'] = hevc_param
json_tmp['after_file'] = {}
json_tmp['after_file']['out_path'] = 'file://' + str(
filterparam_json["task"]["out_list"][0]["out_path"])
oneProcesslog = processlog(taskid=taskid,
fileid=fileid,
dealmethod="transcode",
controljson=json_tmp,
dealstate="processing",
dealtime=datetime.now())
oneProcesslog.save()
redistool.redis_set('_'.join(["task", "status", taskid]),
"Waiting to be processed...")
send_processlog(taskid, filterparam_json["task"]["out_list"][0])
tasks.ffmpeg_transcode.delay(json_tmp)
json_ret = {}
for name in ['cmd_code', 'from', 'to', 'timestamp']:
json_ret[name] = filterparam_json[name]
ts = datetime.now()
timeStr = ts.strftime("%Y%m%d_%H%M%S")
json_ret['re_timestamp'] = timeStr
json_ret['err_code'] = "0"
json_ret["err_info"] = ""
return HttpResponse(json.dumps(json_ret))
def add_state_to_redis(taskid, line):
# pdb.set_trace()
global process_step
global prev_dic
global dic
# state line
if line == 'demux_audio\n':
dic['demux_audio'] = 'Extracting Audio...'
process_step = 'demux_audio'
elif line == 'demux_sub\n':
dic['demux_sub'] = 'Extracting Subtitles...'
process_step = 'demux_sub'
elif line == 'h264_to_hevc\n':
dic['h264_to_hevc'] = 'Running h264_to_hevc...'
process_step = 'h264_to_hevc'
elif line == 'audio_speedup\n':
dic['audio_speedup'] = 'Audio speeding up...'
process_step = 'audio_speedup'
elif line == 'ac3_to_eac3\n':
dic['ac3_to_eac3'] = 'Turning ac3 to eac3...'
process_step = 'ac3_to_eac3'
elif line == 'get_audio_file\n':
dic['get_audio_file'] = 'Getting audio file...'
process_step = 'get_audio_file'
elif line == 'synthetic_mp4_file\n':
dic['synthetic_mp4_file'] = 'Synthetic mp4 file...'
process_step = 'synthetic_mp4_file'
elif line == 'synthetic_buffer_ts_file\n':
dic['synthetic_buffer_ts_file'] = 'Synthetic buffer ts file...'
process_step = 'synthetic_buffer_ts_file'
elif line == 'synthetic_ts_file\n':
dic['synthetic_ts_file'] = 'Synthetic ts file...'
process_step = 'synthetic_ts_file'
elif line == 'all_complete\n':
dic['all_complete'] = 'all_complete! Please download file in file list!'
process_step = 'all_complete'
#get line content
if process_step == 'demux_audio' or process_step == 'ac3_to_eac3' or process_step == 'get_audio_file' or process_step == 'synthetic_ts_file':
global process_total_time
reobj = re.compile(r'.*?Duration:.*?start.*?bitrate.*?')
if reobj.match(line):
head = line.find('Duration:')
process_time_list = line[int(head) + 10:int(head) + 20].split(':')
process_total_time = (float(process_time_list[0]) * 3600 +
float(process_time_list[1]) * 60 +
float(process_time_list[2]))
reobj = re.compile(r'size.*?time.*?bitrate.*?')
if reobj.match(line):
present = line.find('time')
process_now_list = line[int(present) + 5:int(present) +
15].split(':')
process_now_time = (float(process_now_list[0]) * 3600 +
float(process_now_list[1]) * 60 +
float(process_now_list[2]))
dic[process_step] = 'Running ' + process_step + ' :' + str(
line[:-1]) + ' schedule: %0.1f %% ...' % (
float(process_now_time / process_total_time) * 100)
elif process_step == 'demux_sub':
pass
elif process_step == 'h264_to_hevc':
reobj = re.compile(r".*?frames.*?fps.*?kb/s.*?eta.*?")
if reobj.match(line):
dic['h264_to_hevc'] = 'Running h264_to_hevc: %s...' % str(
line[:-1])
elif process_step == 'audio_speedup':
pass
elif process_step == 'synthetic_mp4_file':
reobj = re.compile(
'Importing HEVC:.*?|Importing ISO File:.*?|ISO File Writing:.*?')
if reobj.match(line):
dic['synthetic_mp4_file'] = 'Synthetic mp4 file: %s...' % str(
line[:-1])
elif process_step == 'synthetic_buffer_ts_file':
reobj = re.compile(r'Converting to MPEG-2 TS:.*?')
if reobj.match(line):
dic['synthetic_buffer_ts_file'] = 'Synthetic buffer ts file: %s...' % str(
line[:-1])
elif process_step == 'all_complete':
pass
if dic != prev_dic:
redistool.redis_set(
'_'.join(["task", "status", taskid]),
"Copying file to workstation...\n" + '\n'.join(dic.values()))
prev_dic = copy.deepcopy(dic)