def launch_publisher_task(self, user_id, media_id, send_email, queue, callback_url):
if self.config.is_standalone:
user = self.get_user({u'_id': user_id}, {u'secret': 0})
if not user:
raise IndexError(to_bytes(u'No user with id {0}.'.format(user_id)))
media = self.get_media({u'_id': media_id})
if not media: # FIXME maybe a media access control here
raise IndexError(to_bytes(u'No media asset with id {0}.'.format(media_id)))
if not queue in self.config.publisher_queues:
raise IndexError(to_bytes(u'No publication queue with name {0}.'.format(queue)))
if media.status != Media.READY:
raise NotImplementedError(to_bytes(u"Cannot launch the task, input media asset's status is {0}.".format(
media.status)))
if len(media.public_uris) > 0:
raise NotImplementedError(to_bytes(u'Cannot launch the task, input media asset is already published.'))
other = self.get_publisher_task({u'media_id': media._id})
if other and other.status not in PublisherTask.FINAL_STATUS and other.status != PublisherTask.REVOKED:
raise NotImplementedError(to_bytes(u'Cannot launch the task, input media asset will be published by another'
' task with id {0}.'.format(other._id)))
# FIXME create a one-time password to avoid fixed secret authentication ...
callback = Callback(self.config.api_url + callback_url, u'node', self.config.node_secret)
if self.config.is_mock:
result_id = unicode(uuid.uuid4())
else:
result = PublisherWorker.publisher_task.apply_async(
args=(object2json(media, False), object2json(callback, False)), queue=queue)
result_id = result.id
if not result_id:
raise ValueError(to_bytes(u'Unable to transmit task to workers of queue {0}.'.format(queue)))
logging.info(u'New publication task {0} -> queue {1}.'.format(result_id, queue))
task = PublisherTask(user_id=user_id, media_id=media._id, send_email=send_email, _id=result_id)
task.statistic[u'add_date'] = int(time())
self._db.publisher_tasks.save(task.__dict__, safe=True)
return task
def launch_transform_task(
self, user_id, media_in_id, profile_id, filename, metadata, send_email, queue, callback_url
):
if self.is_standalone:
user = self.get_user({"_id": user_id}, {"secret": 0})
if not user:
raise IndexError(to_bytes("No user with id {0}.".format(user_id)))
media_in = self.get_media({"_id": media_in_id})
if not media_in: # FIXME maybe a media access control here
raise IndexError(to_bytes("No media asset with id {0}.".format(media_in_id)))
profile = self.get_transform_profile({"_id": profile_id})
if not profile: # FIXME maybe a profile access control here
raise IndexError(to_bytes("No transformation profile with id {0}.".format(profile_id)))
if not queue in self.config.transform_queues:
raise IndexError(to_bytes("No transformation queue with name {0}.".format(queue)))
media_out = Media(
user_id=user_id, parent_id=media_in_id, filename=filename, metadata=metadata, status=Media.PENDING
)
media_out.uri = self.config.storage_medias_uri(media_out)
TransformTask.validate_task(media_in, profile, media_out)
self.save_media(media_out) # Save pending output media
# FIXME create a one-time password to avoid fixed secret authentication ...
callback = Callback(self.config.api_url + callback_url, "node", self.config.node_secret)
if self.is_mock:
result_id = unicode(uuid.uuid4())
else:
result = TransformWorker.transform_task.apply_async(
args=(
object2json(media_in, False),
object2json(media_out, False),
object2json(profile, False),
object2json(callback, False),
),
queue=queue,
)
result_id = result.id
if not result_id:
raise ValueError(to_bytes("Unable to transmit task to workers of queue {0}.".format(queue)))
logging.info("New transformation task {0} -> queue {1}.".format(result_id, queue))
task = TransformTask(
user_id=user_id,
media_in_id=media_in._id,
media_out_id=media_out._id,
profile_id=profile._id,
send_email=send_email,
_id=result_id,
)
task.statistic["add_date"] = datetime_now()
self._db.transform_tasks.save(task.__dict__, safe=True)
return task
def revoke_publisher_task(self, task, callback_url, terminate=False, remove=False):
u"""
This do not delete tasks from tasks database (if remove=False) but set revoked attribute in tasks database and
broadcast revoke request to publication units with celery.
If the task is actually running it will be cancelled if terminated = True.
In any case, the output media asset will be deleted (task running or successfully finished).
"""
if valid_uuid(task, none_allowed=False):
task = self.get_publisher_task({u'_id': task})
task.is_valid(True)
if task.status in PublisherTask.CANCELED_STATUS:
raise ValueError(to_bytes(u'Cannot revoke a publication task with status {0}.'.format(task.status)))
if not self.config.is_mock:
revoke(task._id, terminate=terminate)
if task.status == PublisherTask.SUCCESS and not self.config.is_mock:
# Send revoke task to the worker that published the media
callback = Callback(self.config.api_url + callback_url, u'node', self.config.node_secret)
queue = task.get_hostname()
result = PublisherWorker.revoke_publisher_task.apply_async(
args=(task.publish_uri, object2json(callback, False)), queue=queue)
if not result.id:
raise ValueError(to_bytes(u'Unable to transmit task to queue {0}.'.format(queue)))
logging.info(u'New revoke publication task {0} -> queue {1}.'.format(result.id, queue))
self.update_publisher_task_and_media(task, revoke_task_id=result.id, status=PublisherTask.REVOKING)
else:
self.update_publisher_task_and_media(task, status=PublisherTask.REVOKED)
if remove:
self._db.publisher_tasks.remove({u'_id': task._id})
def transform_callback(status, measures):
data_json = object2json({u'task_id': request.id, u'status': status, u'measures': measures}, include_properties=False)
if callback is None:
print(u'{0} [ERROR] Unable to callback orchestrator: {1}'.format(request.id, data_json))
else:
r = callback.post(data_json)
print(u'{0} Code {1} {2} : {3}'.format(request.id, r.status_code, r.reason, r._content))
def dispatch_request(self, *args, **kwargs):
if not check_ip(request):
return
objResponse = {}
# Template information
objResponse['template_tag'] = ("" if self.action.pi_api_template == "" else
md5Checksum('templates/' + self.action.pi_api_template))
for attribute in (u'only_logged_user', u'only_member_user', u'only_admin_user',
u'only_orga_member_user', u'only_orga_admin_user', # User restrictions
u'cache_time', u'cache_by_user', # Cache information
u'user_info', u'json_only', 'no_template'): # Requested user infos + JSON-only
if hasattr(self.action, u'pi_api_' + attribute):
objResponse[attribute] = getattr(self.action, u'pi_api_' + attribute)
# Add the cache headers
response = make_response(object2json(objResponse, include_properties=False))
expires = datetime.utcnow() + timedelta(seconds=PI_META_CACHE)
expires = expires.strftime("%a, %d %b %Y %H:%M:%S GMT")
response.headers['Expire'] = expires
response.headers['Cache-Control'] = 'public, max-age=' + str(PI_META_CACHE)
# Return the final response
return response
def transform_callback(status):
data_json = object2json({"task_id": request.id, "status": status}, include_properties=False)
if callback is None:
print("{0} [ERROR] Unable to callback orchestrator: {1}".format(request.id, data_json))
else:
r = callback.post(data_json)
print("{0} Code {1} {2} : {3}".format(request.id, r.status_code, r.reason, r._content))
def revoke_publisher_task(publish_uri, callback_json):
def revoke_publish_callback(status, publish_uri):
data = {u'task_id': request.id, u'status': status}
if publish_uri:
data[u'publish_uri'] = publish_uri
data_json = object2json(data, False)
if callback is None:
print(u'{0} [ERROR] Unable to callback orchestrator: {1}'.format(request.id, data_json))
else:
r = callback.post(data_json)
print(u'{0} Code {1} {2} : {3}'.format(request.id, r.status_code, r.reason, r._content))
# ------------------------------------------------------------------------------------------------------------------
# Avoid 'referenced before assignment'
callback = None
request = current_task.request
try:
# Let's the task begin !
print(u'{0} Revoke publication task started'.format(request.id))
# Read current configuration to translate files URIs to local paths
local_config = PublisherLocalConfig.read(LOCAL_CONFIG_FILENAME, inspect_constructor=False)
print(object2json(local_config, True))
# Load and check task parameters
callback = Callback.from_json(callback_json, inspect_constructor=True)
callback.is_valid(True)
# Update callback socket according to configuration
if local_config.api_nat_socket and len(local_config.api_nat_socket) > 0:
callback.replace_netloc(local_config.api_nat_socket)
publish_root = dirname(local_config.publish_uri_to_path(publish_uri))
if not publish_root:
raise ValueError(to_bytes(u'Media asset is not hosted on this publication point.'))
# Remove publication directory
start_date, start_time = datetime_now(), time.time()
shutil.rmtree(publish_root, ignore_errors=True)
if valid_uri(publish_uri, check_404=True):
raise IOError(to_bytes(u'Media asset is reachable from publication URI {0}'.format(publish_uri)))
elapsed_time = time.time() - start_time
# Here all seem okay
print(u'{0} Revoke publication task successful, media asset unpublished from {1}'.format(
request.id, publish_uri))
revoke_publish_callback(PublisherTask.SUCCESS, publish_uri)
return {u'hostname': request.hostname, u'start_date': start_date, u'elapsed_time': elapsed_time, u'eta_time': 0,
u'percent': 100}
except Exception as error:
# Here something went wrong
print(u'{0} Revoke publication task failed'.format(request.id))
revoke_publish_callback(unicode(error), None)
raise
def list(self, head=False, **data):
values = []
response_dict = self.api_client.do_request(get, self.get_url(extra=(u'HEAD' if head else None)),
data=object2json(data, include_properties=False))
if self.cls is None:
return response_dict
for value_dict in response_dict:
values.append(dict2object(self.cls, value_dict, inspect_constructor=True))
return values
def publish_callback(status, publish_uri):
data = {u'task_id': request.id, u'status': status}
if publish_uri:
data[u'publish_uri'] = publish_uri
data_json = object2json(data, include_properties=False)
if callback is None:
print(u'{0} [ERROR] Unable to callback orchestrator: {1}'.format(request.id, data_json))
else:
r = callback.post(data_json)
print(u'{0} Code {1} {2} : {3}'.format(request.id, r.status_code, r.reason, r._content))
def add(self, *args, **kwargs):
if not(bool(args) ^ bool(kwargs)):
raise ValueError(to_bytes(u'You must set args OR kwargs.'))
if args and len(args) != 1:
raise ValueError(to_bytes(u'args should contain only 1 value.'))
value = args[0] if args else kwargs
response = self.api_client.do_request(post, self.get_url(), data=object2json(value, include_properties=False))
instance = dict2object(self.cls, response, inspect_constructor=True) if self.cls else response
# Recover user's secret
if isinstance(instance, User):
instance.secret = value.secret if args else kwargs[u'secret']
return instance
def dispatch_request(self, *args, **kwargs):
if not check_ip(request):
return
# Call the action
result = self.action(request, *args, **kwargs)
# Is it a redirect ?
if isinstance(result, PlugItRedirect):
response = make_response("")
response.headers['EbuIo-PlugIt-Redirect'] = result.url
if result.no_prefix:
response.headers['EbuIo-PlugIt-Redirect-NoPrefix'] = 'True'
return response
elif isinstance(result, PlugItSendFile):
response = send_file(result.filename, mimetype=result.mimetype, as_attachment=result.as_attachment, attachment_filename=result.attachment_filename)
response.headers['EbuIo-PlugIt-ItAFile'] = 'True'
return response
return object2json(result, include_properties=False)
def publisher_task(media_json, callback_json):
def copy_callback(start_date, elapsed_time, eta_time, src_size, dst_size, ratio):
publisher_task.update_state(state=PublisherTask.PROGRESS, meta={
u'hostname': request.hostname, u'start_date': start_date, u'elapsed_time': elapsed_time,
u'eta_time': eta_time, u'media_size': src_size, u'publish_size': dst_size, u'percent': int(100 * ratio)})
def publish_callback(status, publish_uri):
data = {u'task_id': request.id, u'status': status}
if publish_uri:
data[u'publish_uri'] = publish_uri
data_json = object2json(data, include_properties=False)
if callback is None:
print(u'{0} [ERROR] Unable to callback orchestrator: {1}'.format(request.id, data_json))
else:
r = callback.post(data_json)
print(u'{0} Code {1} {2} : {3}'.format(request.id, r.status_code, r.reason, r._content))
# ------------------------------------------------------------------------------------------------------------------
RATIO_DELTA = 0.01 # Update status if at least 1% of progress
TIME_DELTA = 1 # Update status if at least 1 second(s) elapsed
# Avoid 'referenced before assignment'
callback = publish_root = None
request = current_task.request
try:
# Let's the task begin !
print(u'{0} Publication task started'.format(request.id))
# Read current configuration to translate files URIs to local paths
local_config = PublisherLocalConfig.read(LOCAL_CONFIG_FILENAME, inspect_constructor=False)
print(object2json(local_config, include_properties=True))
# Load and check task parameters
callback = Callback.from_json(callback_json, inspect_constructor=True)
callback.is_valid(True)
# Update callback socket according to configuration
if local_config.api_nat_socket and len(local_config.api_nat_socket) > 0:
callback.replace_netloc(local_config.api_nat_socket)
media = Media.from_json(media_json, inspect_constructor=True)
media.is_valid(True)
# Verify that media file can be accessed
media_path = local_config.storage_medias_path(media, generate=False)
if not media_path:
raise NotImplementedError(to_bytes(u'Media asset will not be readed from shared storage : {0}'.format(
media.uri)))
publish_path, publish_uri = local_config.publish_point(media)
media_root, publish_root = dirname(media_path), dirname(publish_path)
infos = recursive_copy(media_root, publish_root, copy_callback, RATIO_DELTA, TIME_DELTA)
if not valid_uri(publish_uri, check_404=True):
raise IOError(to_bytes(u'Media asset is unreachable from publication URI {0}'.format(publish_uri)))
# Here all seem okay
print(u'{0} Publication task successful, media asset published as {1}'.format(request.id, publish_uri))
publish_callback(PublisherTask.SUCCESS, publish_uri)
return {u'hostname': request.hostname, u'start_date': infos[u'start_date'],
u'elapsed_time': infos[u'elapsed_time'], u'eta_time': 0, u'media_size': infos[u'src_size'],
u'publish_size': infos[u'src_size'], u'percent': 100}
except Exception as error:
# Here something went wrong
print(u'{0} Publication task failed'.format(request.id))
if publish_root:
shutil.rmtree(publish_root, ignore_errors=True)
publish_callback(unicode(error), None)
raise
def count(self, **data):
return self.api_client.do_request(get, self.get_url(extra=u'count'),
data=object2json(data, include_properties=False))
def __setitem__(self, index, value):
return self.api_client.do_request(patch, self.get_url(index), data=object2json(value, include_properties=True))
def transform_task(media_in_json, media_out_json, profile_json, callback_json):
def copy_callback(start_date, elapsed_time, eta_time, src_size, dst_size, ratio):
transform_task.update_state(state=TransformTask.PROGRESS, meta={
u'hostname': request.hostname, 'start_date': start_date, u'elapsed_time': elapsed_time,
u'eta_time': eta_time, u'media_in_size': src_size, u'media_out_size': dst_size,
u'percent': int(100 * ratio)})
def transform_callback(status, measures):
data_json = object2json({u'task_id': request.id, u'status': status, u'measures': measures}, include_properties=False)
if callback is None:
print(u'{0} [ERROR] Unable to callback orchestrator: {1}'.format(request.id, data_json))
else:
r = callback.post(data_json)
print(u'{0} Code {1} {2} : {3}'.format(request.id, r.status_code, r.reason, r._content))
# ------------------------------------------------------------------------------------------------------------------
RATIO_DELTA, TIME_DELTA = 0.01, 1 # Update status if at least 1% of progress and 1 second elapsed.
MAX_TIME_DELTA = 5 # Also ensure status update every 5 seconds.
DASHCAST_TIMEOUT_TIME = 10
try:
# Avoid 'referenced before assignment'
callback = dashcast_conf = None
encoder_out, request = u'', current_task.request
# Let's the task begin !
print(u'{0} Transformation task started'.format(request.id))
# Read current configuration to translate files uri to local paths
local_config = TransformLocalConfig.read(LOCAL_CONFIG_FILENAME, inspect_constructor=False)
print(object2json(local_config, include_properties=True))
# Load and check task parameters
callback = Callback.from_json(callback_json, inspect_constructor=True)
callback.is_valid(True)
# Update callback socket according to configuration
if local_config.api_nat_socket and len(local_config.api_nat_socket) > 0:
callback.replace_netloc(local_config.api_nat_socket)
media_in = Media.from_json(media_in_json, inspect_constructor=True)
media_out = Media.from_json(media_out_json, inspect_constructor=True)
profile = TransformProfile.from_json(profile_json, inspect_constructor=True)
media_in.is_valid(True)
media_out.is_valid(True)
profile.is_valid(True)
# Verify that media file can be accessed and create output path
media_in_path = local_config.storage_medias_path(media_in, generate=False)
if not media_in_path:
raise NotImplementedError(to_bytes(u'Input media asset will not be readed from shared storage : {0}'.format(
media_in.uri)))
media_out_path = local_config.storage_medias_path(media_out, generate=True)
if not media_out_path:
raise NotImplementedError(to_bytes(u'Output media asset will not be written to shared storage : {0}'.format(
media_out.uri)))
media_in_root = dirname(media_in_path)
media_out_root = dirname(media_out_path)
try_makedirs(media_out_root)
# Get input media duration and frames to be able to estimate ETA
media_in_duration = get_media_duration(media_in_path)
# Keep potential PSNR status
measures = {}
# NOT A REAL TRANSFORM : FILE COPY -----------------------------------------------------------------------------
if profile.encoder_name == u'copy':
infos = recursive_copy(media_in_root, media_out_root, copy_callback, RATIO_DELTA, TIME_DELTA)
media_out_tmp = media_in_path.replace(media_in_root, media_out_root)
os.rename(media_out_tmp, media_out_path)
start_date = infos[u'start_date']
elapsed_time = infos[u'elapsed_time']
media_in_size = infos[u'src_size']
# A REAL TRANSFORM : TRANSCODE WITH FFMPEG ---------------------------------------------------------------------
elif profile.encoder_name == u'ffmpeg':
start_date, start_time = datetime_now(), time.time()
prev_ratio = prev_time = 0
# Get input media size to be able to estimate ETA
media_in_size = get_size(media_in_root)
# Create FFmpeg subprocess
cmd = u'ffmpeg -y -i "{0}" {1} "{2}"'.format(media_in_path, profile.encoder_string, media_out_path)
print(cmd)
ffmpeg = Popen(shlex.split(cmd), stderr=PIPE, close_fds=True)
make_async(ffmpeg.stderr)
while True:
# Wait for data to become available
select.select([ffmpeg.stderr], [], [])
chunk = ffmpeg.stderr.read()
encoder_out += chunk
elapsed_time = time.time() - start_time
match = FFMPEG_REGEX.match(chunk)
if match:
stats = match.groupdict()
media_out_duration = stats[u'time']
try:
ratio = total_seconds(media_out_duration) / total_seconds(media_in_duration)
ratio = 0.0 if ratio < 0.0 else 1.0 if ratio > 1.0 else ratio
except ZeroDivisionError:
ratio = 1.0
delta_time = elapsed_time - prev_time
if (ratio - prev_ratio > RATIO_DELTA and delta_time > TIME_DELTA) or delta_time > MAX_TIME_DELTA:
prev_ratio, prev_time = ratio, elapsed_time
eta_time = int(elapsed_time * (1.0 - ratio) / ratio) if ratio > 0 else 0
transform_task.update_state(
state=TransformTask.PROGRESS,
meta={u'hostname': request.hostname,
u'start_date': start_date,
u'elapsed_time': elapsed_time,
u'eta_time': eta_time,
u'media_in_size': media_in_size,
u'media_in_duration': media_in_duration,
u'media_out_size': get_size(media_out_root),
u'media_out_duration': media_out_duration,
u'percent': int(100 * ratio),
u'encoding_frame': stats[u'frame'],
u'encoding_fps': stats[u'fps'],
u'encoding_bitrate': stats[u'bitrate'],
u'encoding_quality': stats[u'q']})
returncode = ffmpeg.poll()
if returncode is not None:
break
# FFmpeg output sanity check
if returncode != 0:
raise OSError(to_bytes(u'FFmpeg return code is {0}, encoding probably failed.'.format(returncode)))
# compute stats about the video
measures['psnr'] = get_media_psnr(media_in_path, media_out_path)
measures['ssim'] = get_media_ssim(media_in_path, media_out_path)
# measures of the data and its metadata
measures['bitrate'] = get_media_bitrate(media_out_path)
# FIXME: fake git url, commit
measures['git_url'] = 'https://github.com/videolan/x265'
measures['git_commit'] = 'd2051f9544434612a105d2f5267db23018cb3454'
# Output media file sanity check
# media_out_duration = get_media_duration(media_out_path)
# if total_seconds(media_out_duration) / total_seconds(media_in_duration) > 1.5 or < 0.8:
# salut
elif profile.encoder_name == u'from_git':
start_date, start_time = datetime_now(), time.time()
prev_ratio = prev_time = 0
# Get input media size to be able to estimate ETA
media_in_size = get_size(media_in_root)
metadata = media_out.metadata
dirpath = tempfile.mkdtemp()
prepare_cmd = u'git clone --depth=1 "{0}" "{1}" && cd "{1}" && git checkout "{2}" && {3}'.format(metadata['git_url'], dirpath, metadata['git_commit'], metadata['build_cmds'])
check_call(prepare_cmd, shell=True)
# Templated parameter
encoder_string = profile.encoder_string.replace(u"BITRATE", str(metadata['input_bitrate']))
cmd = u'cd "{0}" && ffmpeg -y -i "{1}" -f yuv4mpegpipe - | {2} "{3}"'.format(dirpath, media_in_path, encoder_string, media_out_path)
returncode = call(cmd, shell=True)
if returncode != 0:
raise OSError(to_bytes(u'Encoding return code is {0}, encoding probably failed.'.format(returncode)))
# compute stats about the video
measures['psnr'] = get_media_psnr(media_in_path, media_out_path)
measures['ssim'] = get_media_ssim(media_in_path, media_out_path)
# measures of the data and its metadata
measures['bitrate'] = get_media_bitrate(media_out_path)
# FIXME: don't put this in measures
measures['git_url'] = metadata['git_url']
measures['git_commit'] = metadata['git_commit']
# A REAL TRANSFORM : TRANSCODE WITH DASHCAST -------------------------------------------------------------------
elif profile.encoder_name == u'dashcast':
start_date, start_time = datetime_now(), time.time()
prev_ratio = prev_time = 0
# Get input media size and frames to be able to estimate ETA
media_in_size = get_size(media_in_root)
try:
media_in_frames = int(get_media_tracks(media_in_path)[u'video'][u'0:0'][u'estimated_frames'])
media_out_frames = 0
except:
raise ValueError(to_bytes(u'Unable to estimate # frames of input media asset'))
# Create DashCast configuration file and subprocess
dashcast_conf = u'dashcast_{0}.conf'.format(uuid.uuid4())
with open(dashcast_conf, u'w', u'utf-8') as f:
f.write(profile.dash_config)
cmd = u'DashCast -conf {0} -av "{1}" {2} -out "{3}" -mpd "{4}"'.format(
dashcast_conf, media_in_path, profile.dash_options, media_out_root, media_out.filename)
print(cmd)
dashcast = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE, close_fds=True)
make_async(dashcast.stdout.fileno())
make_async(dashcast.stderr.fileno())
while True:
# Wait for data to become available
select.select([dashcast.stdout.fileno()], [], [])
stdout, stderr = read_async(dashcast.stdout), read_async(dashcast.stderr)
elapsed_time = time.time() - start_time
match = DASHCAST_REGEX.match(stdout)
if match:
stats = match.groupdict()
media_out_frames = int(stats[u'frame'])
try:
ratio = float(media_out_frames) / media_in_frames
ratio = 0.0 if ratio < 0.0 else 1.0 if ratio > 1.0 else ratio
except ZeroDivisionError:
ratio = 1.0
delta_time = elapsed_time - prev_time
if (ratio - prev_ratio > RATIO_DELTA and delta_time > TIME_DELTA) or delta_time > MAX_TIME_DELTA:
prev_ratio, prev_time = ratio, elapsed_time
eta_time = int(elapsed_time * (1.0 - ratio) / ratio) if ratio > 0 else 0
transform_task.update_state(
state=TransformTask.PROGRESS,
meta={u'hostname': request.hostname,
u'start_date': start_date,
u'elapsed_time': elapsed_time,
u'eta_time': eta_time,
u'media_in_size': media_in_size,
u'media_in_duration': media_in_duration,
u'media_out_size': get_size(media_out_root),
u'percent': int(100 * ratio),
u'encoding_frame': media_out_frames})
match = DASHCAST_SUCCESS_REGEX.match(stdout)
returncode = dashcast.poll()
if returncode is not None or match:
encoder_out = u'stdout: {0}\nstderr: {1}'.format(stdout, stderr)
break
if media_out_frames == 0 and elapsed_time > DASHCAST_TIMEOUT_TIME:
encoder_out = u'stdout: {0}\nstderr: {1}'.format(stdout, stderr)
raise OSError(to_bytes(u'DashCast does not output frame number, encoding probably failed.'))
# DashCast output sanity check
if not exists(media_out_path):
raise OSError(to_bytes(u'Output media asset not found, DashCast encoding probably failed.'))
if returncode != 0:
raise OSError(to_bytes(u'DashCast return code is {0}, encoding probably failed.'.format(returncode)))
# FIXME check duration too !
# Here all seem okay -------------------------------------------------------------------------------------------
elapsed_time = time.time() - start_time
media_out_size = get_size(media_out_root)
media_out_duration = get_media_duration(media_out_path)
print(u'{0} Transformation task successful, output media asset {1}'.format(request.id, media_out.filename))
transform_callback(TransformTask.SUCCESS, measures)
return {u'hostname': request.hostname, u'start_date': start_date, u'elapsed_time': elapsed_time,
u'eta_time': 0, u'media_in_size': media_in_size, u'media_in_duration': media_in_duration,
u'media_out_size': media_out_size, u'media_out_duration': media_out_duration, u'percent': 100 }
except Exception as error:
# Here something went wrong
print(u'{0} Transformation task failed '.format(request.id))
transform_callback(u'ERROR\n{0}\n\nOUTPUT\n{1}'.format(unicode(error), encoder_out), {})
raise
finally:
if dashcast_conf:
try_remove(dashcast_conf)
def transform_task(media_in_json, media_out_json, profile_json, callback_json):
def copy_callback(start_date, elapsed_time, eta_time, src_size, dst_size, ratio):
transform_task.update_state(
state=TransformTask.PROGRESS,
meta={
"hostname": request.hostname,
"start_date": start_date,
"elapsed_time": elapsed_time,
"eta_time": eta_time,
"media_in_size": src_size,
"media_out_size": dst_size,
"percent": int(100 * ratio),
},
)
def transform_callback(status):
data_json = object2json({"task_id": request.id, "status": status}, include_properties=False)
if callback is None:
print("{0} [ERROR] Unable to callback orchestrator: {1}".format(request.id, data_json))
else:
r = callback.post(data_json)
print("{0} Code {1} {2} : {3}".format(request.id, r.status_code, r.reason, r._content))
# ------------------------------------------------------------------------------------------------------------------
RATIO_DELTA, TIME_DELTA = 0.01, 1 # Update status if at least 1% of progress and 1 second elapsed.
MAX_TIME_DELTA = 5 # Also ensure status update every 5 seconds.
DASHCAST_TIMEOUT_TIME = 10
try:
# Avoid 'referenced before assignment'
callback = dashcast_conf = None
encoder_out, request = "", current_task.request
# Let's the task begin !
print("{0} Transformation task started".format(request.id))
# Read current configuration to translate files uri to local paths
local_config = TransformLocalConfig.read(LOCAL_CONFIG_FILENAME, inspect_constructor=False)
print(object2json(local_config, include_properties=True))
# Load and check task parameters
callback = Callback.from_json(callback_json, inspect_constructor=True)
callback.is_valid(True)
# Update callback socket according to configuration
if local_config.api_nat_socket and len(local_config.api_nat_socket) > 0:
callback.replace_netloc(local_config.api_nat_socket)
media_in = Media.from_json(media_in_json, inspect_constructor=True)
media_out = Media.from_json(media_out_json, inspect_constructor=True)
profile = TransformProfile.from_json(profile_json, inspect_constructor=True)
media_in.is_valid(True)
media_out.is_valid(True)
profile.is_valid(True)
# Verify that media file can be accessed and create output path
media_in_path = local_config.storage_medias_path(media_in, generate=False)
if not media_in_path:
raise NotImplementedError(
to_bytes("Input media asset will not be readed from shared storage : {0}".format(media_in.uri))
)
media_out_path = local_config.storage_medias_path(media_out, generate=True)
if not media_out_path:
raise NotImplementedError(
to_bytes("Output media asset will not be written to shared storage : {0}".format(media_out.uri))
)
media_in_root = dirname(media_in_path)
media_out_root = dirname(media_out_path)
try_makedirs(media_out_root)
# Get input media duration and frames to be able to estimate ETA
media_in_duration = get_media_duration(media_in_path)
# NOT A REAL TRANSFORM : FILE COPY -----------------------------------------------------------------------------
if profile.encoder_name == "copy":
infos = recursive_copy(media_in_root, media_out_root, copy_callback, RATIO_DELTA, TIME_DELTA)
media_out_tmp = media_in_path.replace(media_in_root, media_out_root)
os.rename(media_out_tmp, media_out_path)
start_date = infos["start_date"]
elapsed_time = infos["elapsed_time"]
media_in_size = infos["src_size"]
# A REAL TRANSFORM : TRANSCODE WITH FFMPEG ---------------------------------------------------------------------
elif profile.encoder_name == "ffmpeg":
start_date, start_time = datetime_now(), time.time()
prev_ratio = prev_time = 0
# Get input media size to be able to estimate ETA
media_in_size = get_size(media_in_root)
# Create FFmpeg subprocess
cmd = 'ffmpeg -y -i "{0}" {1} "{2}"'.format(media_in_path, profile.encoder_string, media_out_path)
print(cmd)
ffmpeg = Popen(shlex.split(cmd), stderr=PIPE, close_fds=True)
make_async(ffmpeg.stderr)
while True:
# Wait for data to become available
select.select([ffmpeg.stderr], [], [])
chunk = ffmpeg.stderr.read()
encoder_out += chunk
elapsed_time = time.time() - start_time
match = FFMPEG_REGEX.match(chunk)
if match:
stats = match.groupdict()
media_out_duration = stats["time"]
try:
ratio = total_seconds(media_out_duration) / total_seconds(media_in_duration)
ratio = 0.0 if ratio < 0.0 else 1.0 if ratio > 1.0 else ratio
except ZeroDivisionError:
ratio = 1.0
delta_time = elapsed_time - prev_time
if (ratio - prev_ratio > RATIO_DELTA and delta_time > TIME_DELTA) or delta_time > MAX_TIME_DELTA:
prev_ratio, prev_time = ratio, elapsed_time
eta_time = int(elapsed_time * (1.0 - ratio) / ratio) if ratio > 0 else 0
transform_task.update_state(
state=TransformTask.PROGRESS,
meta={
"hostname": request.hostname,
"start_date": start_date,
"elapsed_time": elapsed_time,
"eta_time": eta_time,
"media_in_size": media_in_size,
"media_in_duration": media_in_duration,
"media_out_size": get_size(media_out_root),
"media_out_duration": media_out_duration,
"percent": int(100 * ratio),
"encoding_frame": stats["frame"],
"encoding_fps": stats["fps"],
"encoding_bitrate": stats["bitrate"],
"encoding_quality": stats["q"],
},
)
returncode = ffmpeg.poll()
if returncode is not None:
break
# FFmpeg output sanity check
if returncode != 0:
raise OSError(to_bytes("FFmpeg return code is {0}, encoding probably failed.".format(returncode)))
# Output media file sanity check
# media_out_duration = get_media_duration(media_out_path)
# if total_seconds(media_out_duration) / total_seconds(media_in_duration) > 1.5 or < 0.8:
# salut
# A REAL TRANSFORM : TRANSCODE WITH DASHCAST -------------------------------------------------------------------
elif profile.encoder_name == "dashcast":
start_date, start_time = datetime_now(), time.time()
prev_ratio = prev_time = 0
# Get input media size and frames to be able to estimate ETA
media_in_size = get_size(media_in_root)
try:
media_in_frames = int(get_media_tracks(media_in_path)["video"]["0:0"]["estimated_frames"])
media_out_frames = 0
except:
raise ValueError(to_bytes("Unable to estimate # frames of input media asset"))
# Create DashCast configuration file and subprocess
dashcast_conf = "dashcast_{0}.conf".format(uuid.uuid4())
with open(dashcast_conf, "w", "utf-8") as f:
f.write(profile.dash_config)
cmd = 'DashCast -conf {0} -av "{1}" {2} -out "{3}" -mpd "{4}"'.format(
dashcast_conf, media_in_path, profile.dash_options, media_out_root, media_out.filename
)
print(cmd)
dashcast = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE, close_fds=True)
make_async(dashcast.stdout.fileno())
make_async(dashcast.stderr.fileno())
while True:
# Wait for data to become available
select.select([dashcast.stdout.fileno()], [], [])
stdout, stderr = read_async(dashcast.stdout), read_async(dashcast.stderr)
elapsed_time = time.time() - start_time
match = DASHCAST_REGEX.match(stdout)
if match:
stats = match.groupdict()
media_out_frames = int(stats["frame"])
try:
ratio = float(media_out_frames) / media_in_frames
ratio = 0.0 if ratio < 0.0 else 1.0 if ratio > 1.0 else ratio
except ZeroDivisionError:
ratio = 1.0
delta_time = elapsed_time - prev_time
if (ratio - prev_ratio > RATIO_DELTA and delta_time > TIME_DELTA) or delta_time > MAX_TIME_DELTA:
prev_ratio, prev_time = ratio, elapsed_time
eta_time = int(elapsed_time * (1.0 - ratio) / ratio) if ratio > 0 else 0
transform_task.update_state(
state=TransformTask.PROGRESS,
meta={
"hostname": request.hostname,
"start_date": start_date,
"elapsed_time": elapsed_time,
"eta_time": eta_time,
"media_in_size": media_in_size,
"media_in_duration": media_in_duration,
"media_out_size": get_size(media_out_root),
"percent": int(100 * ratio),
"encoding_frame": media_out_frames,
},
)
match = DASHCAST_SUCCESS_REGEX.match(stdout)
returncode = dashcast.poll()
if returncode is not None or match:
encoder_out = "stdout: {0}\nstderr: {1}".format(stdout, stderr)
break
if media_out_frames == 0 and elapsed_time > DASHCAST_TIMEOUT_TIME:
encoder_out = "stdout: {0}\nstderr: {1}".format(stdout, stderr)
raise OSError(to_bytes("DashCast does not output frame number, encoding probably failed."))
# DashCast output sanity check
if not exists(media_out_path):
raise OSError(to_bytes("Output media asset not found, DashCast encoding probably failed."))
if returncode != 0:
raise OSError(to_bytes("DashCast return code is {0}, encoding probably failed.".format(returncode)))
# FIXME check duration too !
# Here all seem okay -------------------------------------------------------------------------------------------
media_out_size = get_size(media_out_root)
media_out_duration = get_media_duration(media_out_path)
print("{0} Transformation task successful, output media asset {1}".format(request.id, media_out.filename))
transform_callback(TransformTask.SUCCESS)
return {
"hostname": request.hostname,
"start_date": start_date,
"elapsed_time": elapsed_time,
"eta_time": 0,
"media_in_size": media_in_size,
"media_in_duration": media_in_duration,
"media_out_size": media_out_size,
"media_out_duration": media_out_duration,
"percent": 100,
}
except Exception as error:
# Here something went wrong
print("{0} Transformation task failed ".format(request.id))
transform_callback("ERROR\n{0}\n\nOUTPUT\n{1}".format(unicode(error), encoder_out))
raise
finally:
if dashcast_conf:
try_remove(dashcast_conf)
