def process_stream(streamstring):
streamjson = ast.literal_eval(streamstring)
pipeline1 = streamjson["pipeline"]+"/1"
stream = streamjson['source']['uri']
print("VA feeder: stream: "+stream, flush=True)
init_stream = None
zk_path = None
if 'uri-init' in streamjson['source']:
init_stream = streamjson['source']['uri-init']
print("VA feeder: init_stream: "+init_stream, flush=True)
zk_path = stream+"/"+pipeline1
m1 = re.search("(.*)/.*_([0-9]+.ts)$", stream)
if m1:
segment = stream.split('/')[-1].split('_')[-1]
zk_path = m1.group(1)+"/"+segment+"/"+pipeline1
print("VA feeder: zk_path "+zk_path, flush=True)
zk = ZKState(zk_path)
if zk.processed():
print("VA feeder: " + stream + " already complete", flush=True)
zk.close()
return
if zk.process_start():
merged_segment = None
if init_stream:
merged_segment = merge.create_merged_segment(init_stream, stream)
if merged_segment:
stream = "file://" + merged_segment
print("VA feeder: video-analytics merged segment: " +
stream, flush=True)
fps=va.loop({
"source": {
"uri": stream,
"type":"uri"
},
"destination": {
"type": "kafka",
"host": socket.gethostbyname("kafka-service")+":9092",
"topic": "seg_analytics_data"
},
"tags": streamjson["tags"],
"parameters": streamjson["parameters"],
}, streamjson["pipeline"])
if fps<0:
zk.process_abort()
else:
zk.process_end()
p.send(video_analytics_fps_topic, json.dumps({
"fps": fps,
"machine":machine_prefix+socket.gethostname()[0:3],
"time": datetime.datetime.utcnow().isoformat(),
}));
if merged_segment:
merge.delete_merged_segment(merged_segment)
zk.close()
def process_stream(stream):
# stream = "dash/video.mp4/index.mpd" or "hls/video.mp4/index.m3u8"
print("process stream: " + stream, flush=True)
stream_name = stream.split("/")[1]
if not isfile(ARCHIVE_ROOT + "/" + stream_name):
print("process aborted for stream name: " + stream_name, flush=True)
return
zk = ZKState("/content_provider_transcoder/" + ARCHIVE_ROOT + "/" + stream)
# "/content_provider_transcoder/" + ARCHIVE_ROOT + "/" + stream
# => "/content_provider_transcoder//var/www/archive/dash/video.mp4/index.mpd"
# or "/content_provider_transcoder//var/www/archive/hls/video.mp4/index.m3u8"
if zk.processed():
print("process already done", flush=True)
zk.close()
return
if stream.endswith(".mpd"):
print("it's a dash process", flush=True)
try:
mkdir(DASH_ROOT + "/" + stream_name)
except:
pass
if zk.process_start():
try:
cmd = GetABRCommand(ARCHIVE_ROOT + "/" + stream_name,
DASH_ROOT + "/" + stream_name, "dash")
r = call(cmd)
if r:
raise Exception("status code: " + str(r))
zk.process_end()
except:
print(traceback.format_exc(), flush=True)
zk.process_abort()
if stream.endswith(".m3u8"):
print("it's a hls process", flush=True)
try:
mkdir(HLS_ROOT + "/" + stream_name)
except:
pass
if zk.process_start():
try:
cmd = GetABRCommand(ARCHIVE_ROOT + "/" + stream_name,
HLS_ROOT + "/" + stream_name, "hls")
r = call(cmd)
if r:
raise Exception("status code: " + str(r))
zk.process_end()
except:
print(traceback.format_exc(), flush=True)
zk.process_abort()
zk.close()
def ADTranscode(kafkamsg, db):
zk = None
msg = KafkaMsgParser(kafkamsg)
# add zk state for each resolution file if we generate the ad clip each time for one solution
zk = ZKState(msg.target_path, msg.target_name)
if zk.processed():
print("AD transcoding finish the clip :", msg.target, flush=True)
zk.close()
return
if zk.process_start():
try:
print("mkdir -p " + msg.target_path, flush=True)
makedirs(msg.target_path)
except OSError as exc: # Python >2.5 (except OSError, exc: for Python <2.5)
if exc.errno == errno.EEXIST and isdir(msg.target_path):
pass
else:
raise
# copy static ADs to fill all resolutions
CopyADStatic(msg)
stream = ADClipDecision(msg, db)
if not stream:
print("Query AD clip failed and fall back to skipped ad clip!",
flush=True)
# mark zk as incomplete (so that the valid one can be generated next time)
zk.process_abort()
zk.close()
return
# try to re-generate resolution specific AD
SignalIncompletion(msg.target)
try:
# only generate one resolution for ad segment, if not generated, ad will fall back to skipped ad.
cmd = GetABRCommand(stream,
msg.target_path,
msg.streaming_type,
msg.GetRedition(),
duration=msg.segment_duration,
fade_type="audio",
content_type="ad")
process_id = subprocess.Popen(cmd, stdout=subprocess.PIPE)
# the `multiprocessing.Process` process will wait until
# the call to the `subprocess.Popen` object is completed
process_id.wait()
SignalCompletion(msg.target)
zk.process_end()
except Exception as e:
print(str(e))
CopyADStatic(msg)
zk.process_abort()
zk.close()
def process_stream_lives(msg):
stream_name = msg["name"]
stream_parameters = msg["parameters"]
codec = stream_parameters["codec_type"]
stream_type = msg["output"]["type"]
target = msg["output"]["target"]
loop = msg["loop"]
idx = msg["idx"] if "idx" in msg.keys() else int(random.random() * 10000)
stream = stream_type + "/" + stream_name
if not isfile(ARCHIVE_ROOT + "/" + stream_name):
return
target_root = VIDEO_ROOT + stream_type
try:
makedirs(target_root + "/" + stream_name)
except:
pass
if target != "file":
target_name = target + stream_type + "/media_" + str(idx) + "_"
else:
target_name = target_root + "/" + stream_name
print("LIVE transcode:", target_name, stream_type, flush=True)
zk = ZKState("/content_provider_transcoder/" + ARCHIVE_ROOT + "/lives/" +
str(idx) + "/" + stream)
if zk.processed():
zk.close()
return
if zk.process_start():
try:
if stream_parameters:
cmd = GetLiveCommand(ARCHIVE_ROOT + "/" + stream_name,
target_name,
stream_type,
params=stream_parameters,
loop=loop)
else:
cmd = GetLiveCommand(ARCHIVE_ROOT + "/" + stream_name,
target_name,
stream_type,
loop=loop)
print(cmd, flush=True)
r = execute(idx, stream_name, cmd)
if r:
raise Exception("status code: " + str(r))
zk.process_end()
except:
print(traceback.format_exc(), flush=True)
zk.process_abort()
zk.close()
def ADTranscode(kafkamsg, db):
msg=KafkaMsgParser(kafkamsg)
# path: /var/www/adinsert/hls/Content_seq7u10.mp4/adstream/u10/4, name: 360p.m3u8
zk_path="/ad-transcode/"+ ("/".join(msg.target_path.split("/")[-5:]))
print("zk_path: "+zk_path+"/"+msg.target_name, flush=True)
zks=ZKState(zk_path, msg.target_name)
start_time=time.time()
if zks.processed():
print("AD transcoding finish the clip :",msg.target, flush=True)
zks.close()
return
if zks.process_start():
try:
makedirs(msg.target_path)
except:
pass
stream = ADClipDecision(msg,db, True)
zkd_path="/".join(msg.target.replace(adinsert_archive_root+"/","").split("/")[:-1])
if not stream:
set_ad_path(zk_segment_prefix+"/"+zkd_path+"/link","/adstatic")
zks.process_abort()
else:
try:
stream_folder = msg.segment_path + "/" + stream.split("/")[-1]
print("Checking pre-transcoded stream: "+stream_folder, flush=True)
if isdir(stream_folder): # pre-transcoded AD exists
print("Prefetch the AD segment {} \n".format(stream_folder),flush=True)
CopyADSegment(msg,stream)
else:
print("Transcoding the AD segment {} \n".format(stream),flush=True)
# only generate one resolution for ad segment, if not generated, ad will fall back to skipped ad.
cmd = GetABRCommand(stream, msg.target_path, msg.streaming_type, msg.GetRedition(), duration=msg.segment_duration, fade_type="audio", content_type="ad")
print("Command for creating segments: {} from url {} \n".format(cmd, stream),flush=True)
process_id = subprocess.Popen(cmd,stdout=subprocess.PIPE)
# the `multiprocessing.Process` process will wait until
# the call to the `subprocess.Popen` object is completed
process_id.wait()
# signal that we are ready
set_ad_path(zk_segment_prefix+"/"+zkd_path+"/link","/adinsert/"+zkd_path)
zks.process_end()
print("Status transcode: Timing {0} {1} {2} {3} {4}".format(msg.start_time, start_time, time.time()-start_time, msg.user_name, msg.target), flush=True)
except Exception as e:
print(traceback.format_exc(), flush=True)
set_ad_path(zk_segment_prefix+"/"+zkd_path+"/link","/adstatic")
zks.process_abort()
zks.close()
def process_stream(stream):
stream_name = stream.split("/")[1]
if not isfile(ARCHIVE_ROOT + "/" + stream_name):
return
zk = ZKState("/content_provider_transcoder/" + ARCHIVE_ROOT + "/" + stream)
if zk.processed():
zk.close()
return
if stream.endswith(".mpd"):
try:
mkdir(DASH_ROOT + "/" + stream_name)
except:
pass
if zk.process_start():
try:
cmd = GetABRCommand(ARCHIVE_ROOT + "/" + stream_name,
DASH_ROOT + "/" + stream_name, "dash")
r = call(cmd)
if r:
raise Exception("status code: " + str(r))
zk.process_end()
except:
print(traceback.format_exc(), flush=True)
zk.process_abort()
if stream.endswith(".m3u8"):
try:
mkdir(HLS_ROOT + "/" + stream_name)
except:
pass
if zk.process_start():
try:
cmd = GetABRCommand(ARCHIVE_ROOT + "/" + stream_name,
HLS_ROOT + "/" + stream_name, "hls")
r = call(cmd)
if r:
raise Exception("status code: " + str(r))
zk.process_end()
except:
print(traceback.format_exc(), flush=True)
zk.process_abort()
zk.close()
def process_stream(stream):
stream_name = stream.split("/")[1]
if not isfile(archive_root + "/" + stream_name): return
zk = ZKState("/content_provider_transcoder/" + archive_root + "/" + stream)
if zk.processed():
zk.close()
return
if stream.endswith(".mpd"):
try:
mkdir(dash_root + "/" + stream_name)
except Exception as e:
print(str(e))
if zk.process_start():
try:
cmd = GetABRCommand(archive_root + "/" + stream_name,
dash_root + "/" + stream_name, "dash")
r = call(cmd)
if r: raise Exception("status code: " + str(r))
zk.process_end()
except Exception as e:
print(str(e))
zk.process_abort()
if stream.endswith(".m3u8"):
try:
mkdir(hls_root + "/" + stream_name)
except Exception as e:
print(str(e))
if zk.process_start():
try:
cmd = GetABRCommand(archive_root + "/" + stream_name,
hls_root + "/" + stream_name, "hls")
r = call(cmd)
if r: raise Exception("status code: " + str(r))
zk.process_end()
except Exception as e:
print(str(e))
zk.process_abort()
zk.close()
def process_stream_vods(msg):
stream_name = msg["name"]
stream_type = msg["output"]["type"]
stream_parameters = msg["parameters"]
loop = msg["loop"]
idx = msg["idx"] if "idx" in msg.keys() else int(random.random() * 10000)
stream = stream_type + "/" + stream_name
print("VOD transcode:", stream, flush=True)
if not isfile(ARCHIVE_ROOT + "/" + stream_name):
return
zk = ZKState("/content_provider_transcoder/" + ARCHIVE_ROOT + "/vods/" +
stream)
if zk.processed():
zk.close()
return
target_root = VIDEO_ROOT + stream_type
try:
makedirs(target_root + "/" + stream_name)
except:
pass
if zk.process_start():
try:
cmd = GetABRCommand(ARCHIVE_ROOT + "/" + stream_name,
target_root + "/" + stream_name,
stream_type,
params=stream_parameters,
loop=loop)
print(cmd, flush=True)
r = execute(idx, stream_name, cmd)
if r:
raise Exception("status code: " + str(r))
zk.process_end()
except:
print(traceback.format_exc(), flush=True)
zk.process_abort()
zk.close()
def process_stream(streamstring):
streamjson = ast.literal_eval(streamstring)
if 'source' not in streamjson:
print("VA feeder: missing source object in input ", flush=True)
return
if 'pipeline' not in streamjson:
print("VA feeder: missing pipeline in input", flush=True)
return
if 'uri' not in streamjson['source']:
print("VA feeder: missing uri in source", flush=True)
return
pipeline = streamjson["pipeline"]+"/1"
tags = {}
if 'tags' in streamjson:
tags = streamjson["tags"]
parameters = {}
if 'parameters' in streamjson:
parameters = streamjson["parameters"]
stream = streamjson['source']['uri']
print("VA feeder: stream: "+stream, flush=True)
if not stream:
print("VA feeder: empty uri", flush=True)
return
init_stream = None
zk_path = None
if 'uri-init' in streamjson['source']:
init_stream = streamjson['source']['uri-init']
print("VA feeder: init_stream: "+init_stream, flush=True)
zk_path = stream+"/"+pipeline
m1 = re.search("(.*)/.*_([0-9]+.ts)$", stream)
if m1:
segment = stream.split('/')[-1].split('_')[-1]
zk_path = m1.group(1)+"/"+segment+"/"+pipeline
print("VA feeder: zk_path "+zk_path, flush=True)
zk = ZKState(zk_path)
if zk.processed():
print("VA feeder: " + stream + " already complete", flush=True)
zk.close()
return
if zk.process_start():
merged_segment = None
if init_stream:
merged_segment = merge.create_merged_segment(init_stream, stream)
if merged_segment:
stream = "file://" + merged_segment
print("VA feeder: video-analytics merged segment: " +
stream, flush=True)
print("VA feeder: start analytic ", flush=True)
instanceid = start_analytic(stream, pipeline, tags, parameters)
if instanceid:
print("VA feeder: waiting for analytics to complete for stream: " +
stream + " analytics-instance-id: "+instanceid, flush=True)
while True:
time.sleep(sleep_for_status)
status, fps = get_analytic_status(instanceid.strip(), pipeline)
print("VA feeder: segment status : " + status, flush=True)
send_video_analytics_fps(fps)
if status == 'COMPLETED':
zk.process_end()
break
elif status == 'RUNNING':
continue
elif status == 'QUEUED':
continue
else:
print("VA feeder: segment processing failed", flush=True)
zk.process_abort()
break
if merged_segment:
merge.delete_merged_segment(merged_segment)
zk.close()
def process_stream(stream):
streams = []
stream_name = stream.split("/")[1]
config = configparser.ConfigParser()
config.read('config.ini')
src_mode = config.get('mode', 'srcMode')
src_path = config.get('path', 'srcPath')
src_protocol = ""
src_api = ""
if src_mode == "local":
streams = listdir(src_path)
elif src_mode == "live":
html = urlopen("http://" + src_path)
soup = BeautifulSoup(html, 'html.parser')
src_protocol = "rtmp://"
src_api = src_mode
for item in soup.findAll('a')[1:]:
streams.append(item.get('href'))
else:
return
if stream_name not in streams:
return
if src_mode == "live":
stream_name = stream_name[:stream_name.index('.')]
zk = ZKState(src_protocol + src_path + "/" + src_api + "/" +
stream.replace("/", "_"))
if zk.processed():
zk.close()
return
if stream.endswith(".mpd"):
try:
mkdir(DASH_ROOT + "/" + stream_name)
except Exception as e:
print(str(e))
if zk.process_start():
try:
cmd = GetABRCommand(
src_protocol + src_path + "/" + src_api + "/" +
stream_name, DASH_ROOT + "/" + stream_name, "dash")
r = call(cmd)
if r:
raise Exception("status code: " + str(r))
zk.process_end()
except Exception as e:
print(str(e))
zk.process_abort()
if stream.endswith(".m3u8"):
try:
mkdir(HLS_ROOT + "/" + stream_name)
except Exception as e:
print(str(e))
if zk.process_start():
try:
cmd = GetABRCommand(
src_protocol + src_path + "/" + src_api + "/" +
stream_name, HLS_ROOT + "/" + stream_name, "hls")
r = call(cmd)
if r:
raise Exception("status code: " + str(r))
zk.process_end()
except Exception as e:
print(str(e))
zk.process_abort()
zk.close()
vdhost = os.environ["VDHOST"]
dbhost = os.environ["DBHOST"]
while True:
try:
c = KafkaConsumer(topic,
bootstrap_servers=kkhost,
client_id=clientid,
group_id=groupid,
auto_offset_reset="earliest",
api_version=(0, 10))
for msg in c:
mode, clip_name = msg.value.decode('utf-8').split(",")
zk = ZKState("/state/" + clip_name, mode)
if not zk.processed():
if zk.process_start():
print("Processing " + clip_name + ":" + mode + "...",
flush=True)
while True:
print("Downloading " + clip_name, flush=True)
sts = call([
"/usr/bin/wget", "-O", clip_name,
vdhost + "/mp4/" + clip_name
])
if sts == 0: break
time.sleep(1)
call([
"/opt/gstreamer_gva/metaData_extract", "-i", clip_name,
def process_stream(streamstring):
streamjson = ast.literal_eval(streamstring)
pipeline1 = streamjson["pipeline"] + "/1"
stream = streamjson['source']['uri']
user = streamjson["user_info"]["name"]
elapsed_time = time.time() - streamjson["start_time"]
print("VA feeder: stream: " + stream + " " + user +
" elapsed-time on kafka queue:" + str(elapsed_time), flush=True)
zk_path = None
init_stream = None
if 'uri-init' in streamjson['source']:
init_stream = streamjson['source']['uri-init']
m1 = re.search(r'(dash/.*)/chunk-stream[0-9]*-([0-9]*.m4s)$', stream)
if m1:
zk_path = "/analytics/" + \
m1.group(1) + "/" + m1.group(2) + "/" + streamjson["pipeline"]
m1 = re.search("(hls/.*)/[0-9]*p_([0-9]*.ts)$", stream)
if m1:
zk_path = "/analytics/" + \
m1.group(1) + "/" + m1.group(2) + "/" + streamjson["pipeline"]
print("zk path: " + zk_path, flush=True)
zk = ZKState(zk_path)
if zk.processed():
print("VA feeder: " + user + " " + stream +
" already complete", flush=True)
zk.close()
return
if zk.process_start():
merged_segment = None
if init_stream:
merged_segment = merge.create_merged_segment(init_stream, stream)
if merged_segment:
stream = "file://" + merged_segment
print("VA feeder: video-analytics merged segment: " +
stream, flush=True)
fps = va.loop({
"source": {
"uri": stream,
"type": "uri"
},
"destination": {
"type": "kafka",
"host": socket.gethostbyname("kafka-service") + ":9092",
"topic": "seg_analytics_data"
},
"tags": streamjson["tags"],
"parameters": streamjson["parameters"],
"user": user,
"start_time": streamjson["start_time"],
}, streamjson["pipeline"])
if fps < 0:
zk.process_abort()
else:
zk.process_end()
if fps > 0:
global global_total_fps, global_seg_count
global_total_fps = global_total_fps + fps
global_seg_count = global_seg_count + 1
avg_fps = global_total_fps / global_seg_count
print("VA statistics : " + "avg_fps " + str(avg_fps) + " " +
str(global_total_fps) + " " + str(global_seg_count), flush=True)
if merged_segment:
merge.delete_merged_segment(merged_segment)
zk.close()