def run(self):
while True:
# Get current time to retrieve state
env, now, now_string = self.environment, datetime_now(format=None), datetime_now()
try:
env.auto = True # Really better like that ;-)
index, event = env.events.get(now, default_value={})
print(u'[{0}] Update charts at index {1}.'.format(self.name, index))
for service, stats in env.statistics.iteritems():
label = SERVICE_TO_LABEL.get(service, service)
units_api, tasks_api = SERVICE_TO_UNITS_API[service], SERVICE_TO_TASKS_API[service]
planned = event.get(service, None)
api_client = env.api_client
api_client.auth = env.daemons_auth
if env.enable_units_status:
if env.enable_units_api:
units = getattr(api_client, units_api).list()
else:
units = env.get_units(service)
else:
units = {k: {u'agent-state': py_juju.STARTED} for k in range(planned)}
tasks = getattr(api_client, tasks_api).list(head=True) if env.enable_tasks_status else None
stats.update(now_string, planned, units, tasks)
stats.generate_units_pie_chart_by_status(env.charts_path)
stats.generate_units_line_chart(env.charts_path)
stats.generate_tasks_line_chart(env.charts_path)
stats.write()
except (ConnectionError, Timeout) as e:
# FIXME do something here ...
print(u'[{0}] WARNING! Communication error, details: {1}.'.format(self.name, e))
self.sleep()
def run(self):
while True:
# Get current time to retrieve state
env, now, now_string = self.environment, datetime_now(format=None), datetime_now()
try:
env.auto = True # Really better like that ;-)
index, event = env.events.get(now, default_value={})
print(u'[{0}] Handle scaling at index {1}.'.format(self.name, index))
for service, stats in env.statistics.iteritems():
label = SERVICE_TO_LABEL.get(service, service)
units_api = SERVICE_TO_UNITS_API[service]
planned = event.get(service, None)
if env.enable_units_api:
api_client = env.api_client
api_client.auth = env.daemons_auth
units = getattr(api_client, units_api).list()
else:
units = env.get_units(service)
if len(units) != planned:
print(u'[{0}] Ensure {1} instances of service {2}'.format(self.name, planned, label))
env.ensure_num_units(service, service, num_units=planned)
env.cleanup_machines() # Safer way to terminate machines !
else:
print(u'[{0}] Nothing to do !'.format(self.name))
# Recover faulty units
# FIXME only once and then destroy and warn admin by mail ...
for number, unit_dict in units.iteritems():
if unit_dict.get(u'agent-state') in ERROR_STATES:
unit = u'{0}/{1}'.format(service, number)
juju_do(u'resolved', environment=env.name, options=[u'--retry', unit], fail=False)
except (ConnectionError, Timeout) as e:
# FIXME do something here ...
print(u'[{0}] WARNING! Communication error, details: {1}.'.format(self.name, e))
self.sleep()
def main():
print(__doc__)
# Initialize statistics of all services of all environments
for environment, services_statistics in STATS.items():
for service, statistics in services_statistics.items():
STATS[environment][service] = ServiceStatistics(
environment=environment, service=service, maxlen=STATISTICS_MAXLEN, simulate=True,
simulated_units_start_latency_range=(10, 13), simulated_units_stop_latency_range=(1, 2),
simulated_tasks_per_unit_range=(2, 3), simulated_tasks_divider=3)
# Generate statistics of all services of all environments
for hour in xrange(STATISTICS_MAXLEN):
hour = hour + HOUR_SHIFT
for environment, events in EVENTS.iteritems():
for service, planned in events.events[hour % 24].iteritems():
for check in xrange(DAEMONS_CHECKS_PER_HOUR):
now_string = datetime_now(offset=timedelta(hours=hour, minutes=(60*check)/DAEMONS_CHECKS_PER_HOUR),
append_utc=False)
STATS[environment][service].update(now_string=now_string, planned=planned)
# Generate charts of all services of all environments
for services_statistics in STATS.itervalues():
for statistics in services_statistics.values():
statistics.generate_units_pie_chart_by_status(u'charts_simulated')
statistics.generate_units_line_chart(u'charts_simulated')
statistics.generate_tasks_line_chart(u'charts_simulated')
# Generate sum charts of all services
for service in SERVICES:
statistics_list = [s[service] for s in (STATS[MAAS], STATS[AMAZON])]
ServiceStatistics.generate_units_stacked_chart(statistics_list, u'charts_simulated', enable_current=False)
def launch_publisher_task(self, user_id, media_id, send_email, queue, callback_url):
if self.config.is_standalone:
user = self.get_user({u'_id': user_id}, {u'secret': 0})
if not user:
raise IndexError(to_bytes(u'No user with id {0}.'.format(user_id)))
media = self.get_media({u'_id': media_id})
if not media: # FIXME maybe a media access control here
raise IndexError(to_bytes(u'No media asset with id {0}.'.format(media_id)))
if not queue in self.config.publisher_queues:
raise IndexError(to_bytes(u'No publication queue with name {0}.'.format(queue)))
if media.status != Media.READY:
raise NotImplementedError(to_bytes(u"Cannot launch the task, input media asset's status is {0}.".format(
media.status)))
if len(media.public_uris) > 0:
raise NotImplementedError(to_bytes(u'Cannot launch the task, input media asset is already published.'))
other = self.get_publisher_task({u'media_id': media._id})
if other and other.status not in PublisherTask.FINAL_STATUS and other.status != PublisherTask.REVOKED:
raise NotImplementedError(to_bytes(u'Cannot launch the task, input media asset will be published by another'
' task with id {0}.'.format(other._id)))
# FIXME create a one-time password to avoid fixed secret authentication ...
callback = Callback(self.config.api_url + callback_url, u'node', self.config.node_secret)
if self.config.is_mock:
result_id = unicode(uuid.uuid4())
else:
result = PublisherWorker.publisher_task.apply_async(
args=(object2json(media, False), object2json(callback, False)), queue=queue)
result_id = result.id
if not result_id:
raise ValueError(to_bytes(u'Unable to transmit task to workers of queue {0}.'.format(queue)))
logging.info(u'New publication task {0} -> queue {1}.'.format(result_id, queue))
task = PublisherTask(user_id=user_id, media_id=media._id, send_email=send_email, _id=result_id)
task.statistic[u'add_date'] = datetime_now()
self._db.publisher_tasks.save(task.__dict__, safe=True)
return task
def revoke_publisher_task(publish_uri, callback_json):
def revoke_publish_callback(status, publish_uri):
data = {u'task_id': request.id, u'status': status}
if publish_uri:
data[u'publish_uri'] = publish_uri
data_json = object2json(data, False)
if callback is None:
print(u'{0} [ERROR] Unable to callback orchestrator: {1}'.format(request.id, data_json))
else:
r = callback.post(data_json)
print(u'{0} Code {1} {2} : {3}'.format(request.id, r.status_code, r.reason, r._content))
# ------------------------------------------------------------------------------------------------------------------
# Avoid 'referenced before assignment'
callback = None
request = current_task.request
try:
# Let's the task begin !
print(u'{0} Revoke publication task started'.format(request.id))
# Read current configuration to translate files URIs to local paths
local_config = PublisherLocalConfig.read(LOCAL_CONFIG_FILENAME, inspect_constructor=False)
print(object2json(local_config, True))
# Load and check task parameters
callback = Callback.from_json(callback_json, inspect_constructor=True)
callback.is_valid(True)
# Update callback socket according to configuration
if local_config.api_nat_socket and len(local_config.api_nat_socket) > 0:
callback.replace_netloc(local_config.api_nat_socket)
publish_root = dirname(local_config.publish_uri_to_path(publish_uri))
if not publish_root:
raise ValueError(to_bytes(u'Media asset is not hosted on this publication point.'))
# Remove publication directory
start_date, start_time = datetime_now(), time.time()
shutil.rmtree(publish_root, ignore_errors=True)
if valid_uri(publish_uri, check_404=True):
raise IOError(to_bytes(u'Media asset is reachable from publication URI {0}'.format(publish_uri)))
elapsed_time = time.time() - start_time
# Here all seem okay
print(u'{0} Revoke publication task successful, media asset unpublished from {1}'.format(
request.id, publish_uri))
revoke_publish_callback(PublisherTask.SUCCESS, publish_uri)
return {u'hostname': request.hostname, u'start_date': start_date, u'elapsed_time': elapsed_time, u'eta_time': 0,
u'percent': 100}
except Exception as error:
# Here something went wrong
print(u'{0} Revoke publication task failed'.format(request.id))
revoke_publish_callback(unicode(error), None)
raise
def run(self):
self.output_counter = 0
self.progress_tasks = (time.time(), [])
while True:
# Get current time to retrieve state
now, now_string = datetime_now(format=None), datetime_now()
try:
self.environment.auto = True # Really better like that ;-)
api_client = self.environment.api_client
api_client.auth = self.environment.daemons_auth
self.transform(api_client)
self.cleanup_transform_tasks(api_client, auto=True, cleanup_progress_time=60)
self.cleanup_media_assets(api_client)
except (ConnectionError, Timeout) as e:
# FIXME do something here ...
print(u'[{0}] WARNING! Communication error, details: {1}.'.format(self.name, e))
self.sleep()
def launch_transform_task(
self, user_id, media_in_id, profile_id, filename, metadata, send_email, queue, callback_url
):
if self.is_standalone:
user = self.get_user({"_id": user_id}, {"secret": 0})
if not user:
raise IndexError(to_bytes("No user with id {0}.".format(user_id)))
media_in = self.get_media({"_id": media_in_id})
if not media_in: # FIXME maybe a media access control here
raise IndexError(to_bytes("No media asset with id {0}.".format(media_in_id)))
profile = self.get_transform_profile({"_id": profile_id})
if not profile: # FIXME maybe a profile access control here
raise IndexError(to_bytes("No transformation profile with id {0}.".format(profile_id)))
if not queue in self.config.transform_queues:
raise IndexError(to_bytes("No transformation queue with name {0}.".format(queue)))
media_out = Media(
user_id=user_id, parent_id=media_in_id, filename=filename, metadata=metadata, status=Media.PENDING
)
media_out.uri = self.config.storage_medias_uri(media_out)
TransformTask.validate_task(media_in, profile, media_out)
self.save_media(media_out) # Save pending output media
# FIXME create a one-time password to avoid fixed secret authentication ...
callback = Callback(self.config.api_url + callback_url, "node", self.config.node_secret)
if self.is_mock:
result_id = unicode(uuid.uuid4())
else:
result = TransformWorker.transform_task.apply_async(
args=(
object2json(media_in, False),
object2json(media_out, False),
object2json(profile, False),
object2json(callback, False),
),
queue=queue,
)
result_id = result.id
if not result_id:
raise ValueError(to_bytes("Unable to transmit task to workers of queue {0}.".format(queue)))
logging.info("New transformation task {0} -> queue {1}.".format(result_id, queue))
task = TransformTask(
user_id=user_id,
media_in_id=media_in._id,
media_out_id=media_out._id,
profile_id=profile._id,
send_email=send_email,
_id=result_id,
)
task.statistic["add_date"] = datetime_now()
self._db.transform_tasks.save(task.__dict__, safe=True)
return task
def save_media(self, media):
media.is_valid(True)
if not media.get_metadata("title"):
raise ValueError(to_bytes("Title key is required in media asset's metadata."))
if media.status != Media.DELETED:
if self.is_mock:
size = randint(10 * 1024 * 1024, 10 * 1024 * 1024 * 1024)
duration = "%02d:%02d:%02d" % (randint(0, 2), randint(0, 59), randint(0, 59))
else:
size, duration = Storage.add_media(self.config, media)
else:
size, duration = (0, 0)
media.add_metadata("size", size, True)
if duration:
media.add_metadata("duration", duration, True)
media.add_metadata("add_date", datetime_now(), True)
try:
self._db.medias.save(media.__dict__, safe=True)
except DuplicateKeyError:
raise ValueError(to_bytes("The media URI {0} is already used by another media asset.".format(media.uri)))
def sleep(self):
now = datetime_now(format=None)
sleep_time = self.environment.events.sleep_time(now)
print(u'[{0}] Sleep {1} seconds ...'.format(self.name, sleep_time))
time.sleep(sleep_time)
def get_now(self):
return datetime_now()
def get_now():
return datetime_now()
def transform_task(media_in_json, media_out_json, profile_json, callback_json):
def copy_callback(start_date, elapsed_time, eta_time, src_size, dst_size, ratio):
transform_task.update_state(state=TransformTask.PROGRESS, meta={
u'hostname': request.hostname, 'start_date': start_date, u'elapsed_time': elapsed_time,
u'eta_time': eta_time, u'media_in_size': src_size, u'media_out_size': dst_size,
u'percent': int(100 * ratio)})
def transform_callback(status, measures):
data_json = object2json({u'task_id': request.id, u'status': status, u'measures': measures}, include_properties=False)
if callback is None:
print(u'{0} [ERROR] Unable to callback orchestrator: {1}'.format(request.id, data_json))
else:
r = callback.post(data_json)
print(u'{0} Code {1} {2} : {3}'.format(request.id, r.status_code, r.reason, r._content))
# ------------------------------------------------------------------------------------------------------------------
RATIO_DELTA, TIME_DELTA = 0.01, 1 # Update status if at least 1% of progress and 1 second elapsed.
MAX_TIME_DELTA = 5 # Also ensure status update every 5 seconds.
DASHCAST_TIMEOUT_TIME = 10
try:
# Avoid 'referenced before assignment'
callback = dashcast_conf = None
encoder_out, request = u'', current_task.request
# Let's the task begin !
print(u'{0} Transformation task started'.format(request.id))
# Read current configuration to translate files uri to local paths
local_config = TransformLocalConfig.read(LOCAL_CONFIG_FILENAME, inspect_constructor=False)
print(object2json(local_config, include_properties=True))
# Load and check task parameters
callback = Callback.from_json(callback_json, inspect_constructor=True)
callback.is_valid(True)
# Update callback socket according to configuration
if local_config.api_nat_socket and len(local_config.api_nat_socket) > 0:
callback.replace_netloc(local_config.api_nat_socket)
media_in = Media.from_json(media_in_json, inspect_constructor=True)
media_out = Media.from_json(media_out_json, inspect_constructor=True)
profile = TransformProfile.from_json(profile_json, inspect_constructor=True)
media_in.is_valid(True)
media_out.is_valid(True)
profile.is_valid(True)
# Verify that media file can be accessed and create output path
media_in_path = local_config.storage_medias_path(media_in, generate=False)
if not media_in_path:
raise NotImplementedError(to_bytes(u'Input media asset will not be readed from shared storage : {0}'.format(
media_in.uri)))
media_out_path = local_config.storage_medias_path(media_out, generate=True)
if not media_out_path:
raise NotImplementedError(to_bytes(u'Output media asset will not be written to shared storage : {0}'.format(
media_out.uri)))
media_in_root = dirname(media_in_path)
media_out_root = dirname(media_out_path)
try_makedirs(media_out_root)
# Get input media duration and frames to be able to estimate ETA
media_in_duration = get_media_duration(media_in_path)
# Keep potential PSNR status
measures = {}
# NOT A REAL TRANSFORM : FILE COPY -----------------------------------------------------------------------------
if profile.encoder_name == u'copy':
infos = recursive_copy(media_in_root, media_out_root, copy_callback, RATIO_DELTA, TIME_DELTA)
media_out_tmp = media_in_path.replace(media_in_root, media_out_root)
os.rename(media_out_tmp, media_out_path)
start_date = infos[u'start_date']
elapsed_time = infos[u'elapsed_time']
media_in_size = infos[u'src_size']
# A REAL TRANSFORM : TRANSCODE WITH FFMPEG ---------------------------------------------------------------------
elif profile.encoder_name == u'ffmpeg':
start_date, start_time = datetime_now(), time.time()
prev_ratio = prev_time = 0
# Get input media size to be able to estimate ETA
media_in_size = get_size(media_in_root)
# Create FFmpeg subprocess
cmd = u'ffmpeg -y -i "{0}" {1} "{2}"'.format(media_in_path, profile.encoder_string, media_out_path)
print(cmd)
ffmpeg = Popen(shlex.split(cmd), stderr=PIPE, close_fds=True)
make_async(ffmpeg.stderr)
while True:
# Wait for data to become available
select.select([ffmpeg.stderr], [], [])
chunk = ffmpeg.stderr.read()
encoder_out += chunk
elapsed_time = time.time() - start_time
match = FFMPEG_REGEX.match(chunk)
if match:
stats = match.groupdict()
media_out_duration = stats[u'time']
try:
ratio = total_seconds(media_out_duration) / total_seconds(media_in_duration)
ratio = 0.0 if ratio < 0.0 else 1.0 if ratio > 1.0 else ratio
except ZeroDivisionError:
ratio = 1.0
delta_time = elapsed_time - prev_time
if (ratio - prev_ratio > RATIO_DELTA and delta_time > TIME_DELTA) or delta_time > MAX_TIME_DELTA:
prev_ratio, prev_time = ratio, elapsed_time
eta_time = int(elapsed_time * (1.0 - ratio) / ratio) if ratio > 0 else 0
transform_task.update_state(
state=TransformTask.PROGRESS,
meta={u'hostname': request.hostname,
u'start_date': start_date,
u'elapsed_time': elapsed_time,
u'eta_time': eta_time,
u'media_in_size': media_in_size,
u'media_in_duration': media_in_duration,
u'media_out_size': get_size(media_out_root),
u'media_out_duration': media_out_duration,
u'percent': int(100 * ratio),
u'encoding_frame': stats[u'frame'],
u'encoding_fps': stats[u'fps'],
u'encoding_bitrate': stats[u'bitrate'],
u'encoding_quality': stats[u'q']})
returncode = ffmpeg.poll()
if returncode is not None:
break
# FFmpeg output sanity check
if returncode != 0:
raise OSError(to_bytes(u'FFmpeg return code is {0}, encoding probably failed.'.format(returncode)))
# compute stats about the video
measures['psnr'] = get_media_psnr(media_in_path, media_out_path)
measures['ssim'] = get_media_ssim(media_in_path, media_out_path)
# measures of the data and its metadata
measures['bitrate'] = get_media_bitrate(media_out_path)
# FIXME: fake git url, commit
measures['git_url'] = 'https://github.com/videolan/x265'
measures['git_commit'] = 'd2051f9544434612a105d2f5267db23018cb3454'
# Output media file sanity check
# media_out_duration = get_media_duration(media_out_path)
# if total_seconds(media_out_duration) / total_seconds(media_in_duration) > 1.5 or < 0.8:
# salut
elif profile.encoder_name == u'from_git':
start_date, start_time = datetime_now(), time.time()
prev_ratio = prev_time = 0
# Get input media size to be able to estimate ETA
media_in_size = get_size(media_in_root)
metadata = media_out.metadata
dirpath = tempfile.mkdtemp()
prepare_cmd = u'git clone --depth=1 "{0}" "{1}" && cd "{1}" && git checkout "{2}" && {3}'.format(metadata['git_url'], dirpath, metadata['git_commit'], metadata['build_cmds'])
check_call(prepare_cmd, shell=True)
# Templated parameter
encoder_string = profile.encoder_string.replace(u"BITRATE", str(metadata['input_bitrate']))
cmd = u'cd "{0}" && ffmpeg -y -i "{1}" -f yuv4mpegpipe - | {2} "{3}"'.format(dirpath, media_in_path, encoder_string, media_out_path)
returncode = call(cmd, shell=True)
if returncode != 0:
raise OSError(to_bytes(u'Encoding return code is {0}, encoding probably failed.'.format(returncode)))
# compute stats about the video
measures['psnr'] = get_media_psnr(media_in_path, media_out_path)
measures['ssim'] = get_media_ssim(media_in_path, media_out_path)
# measures of the data and its metadata
measures['bitrate'] = get_media_bitrate(media_out_path)
# FIXME: don't put this in measures
measures['git_url'] = metadata['git_url']
measures['git_commit'] = metadata['git_commit']
# A REAL TRANSFORM : TRANSCODE WITH DASHCAST -------------------------------------------------------------------
elif profile.encoder_name == u'dashcast':
start_date, start_time = datetime_now(), time.time()
prev_ratio = prev_time = 0
# Get input media size and frames to be able to estimate ETA
media_in_size = get_size(media_in_root)
try:
media_in_frames = int(get_media_tracks(media_in_path)[u'video'][u'0:0'][u'estimated_frames'])
media_out_frames = 0
except:
raise ValueError(to_bytes(u'Unable to estimate # frames of input media asset'))
# Create DashCast configuration file and subprocess
dashcast_conf = u'dashcast_{0}.conf'.format(uuid.uuid4())
with open(dashcast_conf, u'w', u'utf-8') as f:
f.write(profile.dash_config)
cmd = u'DashCast -conf {0} -av "{1}" {2} -out "{3}" -mpd "{4}"'.format(
dashcast_conf, media_in_path, profile.dash_options, media_out_root, media_out.filename)
print(cmd)
dashcast = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE, close_fds=True)
make_async(dashcast.stdout.fileno())
make_async(dashcast.stderr.fileno())
while True:
# Wait for data to become available
select.select([dashcast.stdout.fileno()], [], [])
stdout, stderr = read_async(dashcast.stdout), read_async(dashcast.stderr)
elapsed_time = time.time() - start_time
match = DASHCAST_REGEX.match(stdout)
if match:
stats = match.groupdict()
media_out_frames = int(stats[u'frame'])
try:
ratio = float(media_out_frames) / media_in_frames
ratio = 0.0 if ratio < 0.0 else 1.0 if ratio > 1.0 else ratio
except ZeroDivisionError:
ratio = 1.0
delta_time = elapsed_time - prev_time
if (ratio - prev_ratio > RATIO_DELTA and delta_time > TIME_DELTA) or delta_time > MAX_TIME_DELTA:
prev_ratio, prev_time = ratio, elapsed_time
eta_time = int(elapsed_time * (1.0 - ratio) / ratio) if ratio > 0 else 0
transform_task.update_state(
state=TransformTask.PROGRESS,
meta={u'hostname': request.hostname,
u'start_date': start_date,
u'elapsed_time': elapsed_time,
u'eta_time': eta_time,
u'media_in_size': media_in_size,
u'media_in_duration': media_in_duration,
u'media_out_size': get_size(media_out_root),
u'percent': int(100 * ratio),
u'encoding_frame': media_out_frames})
match = DASHCAST_SUCCESS_REGEX.match(stdout)
returncode = dashcast.poll()
if returncode is not None or match:
encoder_out = u'stdout: {0}\nstderr: {1}'.format(stdout, stderr)
break
if media_out_frames == 0 and elapsed_time > DASHCAST_TIMEOUT_TIME:
encoder_out = u'stdout: {0}\nstderr: {1}'.format(stdout, stderr)
raise OSError(to_bytes(u'DashCast does not output frame number, encoding probably failed.'))
# DashCast output sanity check
if not exists(media_out_path):
raise OSError(to_bytes(u'Output media asset not found, DashCast encoding probably failed.'))
if returncode != 0:
raise OSError(to_bytes(u'DashCast return code is {0}, encoding probably failed.'.format(returncode)))
# FIXME check duration too !
# Here all seem okay -------------------------------------------------------------------------------------------
elapsed_time = time.time() - start_time
media_out_size = get_size(media_out_root)
media_out_duration = get_media_duration(media_out_path)
print(u'{0} Transformation task successful, output media asset {1}'.format(request.id, media_out.filename))
transform_callback(TransformTask.SUCCESS, measures)
return {u'hostname': request.hostname, u'start_date': start_date, u'elapsed_time': elapsed_time,
u'eta_time': 0, u'media_in_size': media_in_size, u'media_in_duration': media_in_duration,
u'media_out_size': media_out_size, u'media_out_duration': media_out_duration, u'percent': 100 }
except Exception as error:
# Here something went wrong
print(u'{0} Transformation task failed '.format(request.id))
transform_callback(u'ERROR\n{0}\n\nOUTPUT\n{1}'.format(unicode(error), encoder_out), {})
raise
finally:
if dashcast_conf:
try_remove(dashcast_conf)
def transform_task(media_in_json, media_out_json, profile_json, callback_json):
def copy_callback(start_date, elapsed_time, eta_time, src_size, dst_size, ratio):
transform_task.update_state(
state=TransformTask.PROGRESS,
meta={
"hostname": request.hostname,
"start_date": start_date,
"elapsed_time": elapsed_time,
"eta_time": eta_time,
"media_in_size": src_size,
"media_out_size": dst_size,
"percent": int(100 * ratio),
},
)
def transform_callback(status):
data_json = object2json({"task_id": request.id, "status": status}, include_properties=False)
if callback is None:
print("{0} [ERROR] Unable to callback orchestrator: {1}".format(request.id, data_json))
else:
r = callback.post(data_json)
print("{0} Code {1} {2} : {3}".format(request.id, r.status_code, r.reason, r._content))
# ------------------------------------------------------------------------------------------------------------------
RATIO_DELTA, TIME_DELTA = 0.01, 1 # Update status if at least 1% of progress and 1 second elapsed.
MAX_TIME_DELTA = 5 # Also ensure status update every 5 seconds.
DASHCAST_TIMEOUT_TIME = 10
try:
# Avoid 'referenced before assignment'
callback = dashcast_conf = None
encoder_out, request = "", current_task.request
# Let's the task begin !
print("{0} Transformation task started".format(request.id))
# Read current configuration to translate files uri to local paths
local_config = TransformLocalConfig.read(LOCAL_CONFIG_FILENAME, inspect_constructor=False)
print(object2json(local_config, include_properties=True))
# Load and check task parameters
callback = Callback.from_json(callback_json, inspect_constructor=True)
callback.is_valid(True)
# Update callback socket according to configuration
if local_config.api_nat_socket and len(local_config.api_nat_socket) > 0:
callback.replace_netloc(local_config.api_nat_socket)
media_in = Media.from_json(media_in_json, inspect_constructor=True)
media_out = Media.from_json(media_out_json, inspect_constructor=True)
profile = TransformProfile.from_json(profile_json, inspect_constructor=True)
media_in.is_valid(True)
media_out.is_valid(True)
profile.is_valid(True)
# Verify that media file can be accessed and create output path
media_in_path = local_config.storage_medias_path(media_in, generate=False)
if not media_in_path:
raise NotImplementedError(
to_bytes("Input media asset will not be readed from shared storage : {0}".format(media_in.uri))
)
media_out_path = local_config.storage_medias_path(media_out, generate=True)
if not media_out_path:
raise NotImplementedError(
to_bytes("Output media asset will not be written to shared storage : {0}".format(media_out.uri))
)
media_in_root = dirname(media_in_path)
media_out_root = dirname(media_out_path)
try_makedirs(media_out_root)
# Get input media duration and frames to be able to estimate ETA
media_in_duration = get_media_duration(media_in_path)
# NOT A REAL TRANSFORM : FILE COPY -----------------------------------------------------------------------------
if profile.encoder_name == "copy":
infos = recursive_copy(media_in_root, media_out_root, copy_callback, RATIO_DELTA, TIME_DELTA)
media_out_tmp = media_in_path.replace(media_in_root, media_out_root)
os.rename(media_out_tmp, media_out_path)
start_date = infos["start_date"]
elapsed_time = infos["elapsed_time"]
media_in_size = infos["src_size"]
# A REAL TRANSFORM : TRANSCODE WITH FFMPEG ---------------------------------------------------------------------
elif profile.encoder_name == "ffmpeg":
start_date, start_time = datetime_now(), time.time()
prev_ratio = prev_time = 0
# Get input media size to be able to estimate ETA
media_in_size = get_size(media_in_root)
# Create FFmpeg subprocess
cmd = 'ffmpeg -y -i "{0}" {1} "{2}"'.format(media_in_path, profile.encoder_string, media_out_path)
print(cmd)
ffmpeg = Popen(shlex.split(cmd), stderr=PIPE, close_fds=True)
make_async(ffmpeg.stderr)
while True:
# Wait for data to become available
select.select([ffmpeg.stderr], [], [])
chunk = ffmpeg.stderr.read()
encoder_out += chunk
elapsed_time = time.time() - start_time
match = FFMPEG_REGEX.match(chunk)
if match:
stats = match.groupdict()
media_out_duration = stats["time"]
try:
ratio = total_seconds(media_out_duration) / total_seconds(media_in_duration)
ratio = 0.0 if ratio < 0.0 else 1.0 if ratio > 1.0 else ratio
except ZeroDivisionError:
ratio = 1.0
delta_time = elapsed_time - prev_time
if (ratio - prev_ratio > RATIO_DELTA and delta_time > TIME_DELTA) or delta_time > MAX_TIME_DELTA:
prev_ratio, prev_time = ratio, elapsed_time
eta_time = int(elapsed_time * (1.0 - ratio) / ratio) if ratio > 0 else 0
transform_task.update_state(
state=TransformTask.PROGRESS,
meta={
"hostname": request.hostname,
"start_date": start_date,
"elapsed_time": elapsed_time,
"eta_time": eta_time,
"media_in_size": media_in_size,
"media_in_duration": media_in_duration,
"media_out_size": get_size(media_out_root),
"media_out_duration": media_out_duration,
"percent": int(100 * ratio),
"encoding_frame": stats["frame"],
"encoding_fps": stats["fps"],
"encoding_bitrate": stats["bitrate"],
"encoding_quality": stats["q"],
},
)
returncode = ffmpeg.poll()
if returncode is not None:
break
# FFmpeg output sanity check
if returncode != 0:
raise OSError(to_bytes("FFmpeg return code is {0}, encoding probably failed.".format(returncode)))
# Output media file sanity check
# media_out_duration = get_media_duration(media_out_path)
# if total_seconds(media_out_duration) / total_seconds(media_in_duration) > 1.5 or < 0.8:
# salut
# A REAL TRANSFORM : TRANSCODE WITH DASHCAST -------------------------------------------------------------------
elif profile.encoder_name == "dashcast":
start_date, start_time = datetime_now(), time.time()
prev_ratio = prev_time = 0
# Get input media size and frames to be able to estimate ETA
media_in_size = get_size(media_in_root)
try:
media_in_frames = int(get_media_tracks(media_in_path)["video"]["0:0"]["estimated_frames"])
media_out_frames = 0
except:
raise ValueError(to_bytes("Unable to estimate # frames of input media asset"))
# Create DashCast configuration file and subprocess
dashcast_conf = "dashcast_{0}.conf".format(uuid.uuid4())
with open(dashcast_conf, "w", "utf-8") as f:
f.write(profile.dash_config)
cmd = 'DashCast -conf {0} -av "{1}" {2} -out "{3}" -mpd "{4}"'.format(
dashcast_conf, media_in_path, profile.dash_options, media_out_root, media_out.filename
)
print(cmd)
dashcast = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE, close_fds=True)
make_async(dashcast.stdout.fileno())
make_async(dashcast.stderr.fileno())
while True:
# Wait for data to become available
select.select([dashcast.stdout.fileno()], [], [])
stdout, stderr = read_async(dashcast.stdout), read_async(dashcast.stderr)
elapsed_time = time.time() - start_time
match = DASHCAST_REGEX.match(stdout)
if match:
stats = match.groupdict()
media_out_frames = int(stats["frame"])
try:
ratio = float(media_out_frames) / media_in_frames
ratio = 0.0 if ratio < 0.0 else 1.0 if ratio > 1.0 else ratio
except ZeroDivisionError:
ratio = 1.0
delta_time = elapsed_time - prev_time
if (ratio - prev_ratio > RATIO_DELTA and delta_time > TIME_DELTA) or delta_time > MAX_TIME_DELTA:
prev_ratio, prev_time = ratio, elapsed_time
eta_time = int(elapsed_time * (1.0 - ratio) / ratio) if ratio > 0 else 0
transform_task.update_state(
state=TransformTask.PROGRESS,
meta={
"hostname": request.hostname,
"start_date": start_date,
"elapsed_time": elapsed_time,
"eta_time": eta_time,
"media_in_size": media_in_size,
"media_in_duration": media_in_duration,
"media_out_size": get_size(media_out_root),
"percent": int(100 * ratio),
"encoding_frame": media_out_frames,
},
)
match = DASHCAST_SUCCESS_REGEX.match(stdout)
returncode = dashcast.poll()
if returncode is not None or match:
encoder_out = "stdout: {0}\nstderr: {1}".format(stdout, stderr)
break
if media_out_frames == 0 and elapsed_time > DASHCAST_TIMEOUT_TIME:
encoder_out = "stdout: {0}\nstderr: {1}".format(stdout, stderr)
raise OSError(to_bytes("DashCast does not output frame number, encoding probably failed."))
# DashCast output sanity check
if not exists(media_out_path):
raise OSError(to_bytes("Output media asset not found, DashCast encoding probably failed."))
if returncode != 0:
raise OSError(to_bytes("DashCast return code is {0}, encoding probably failed.".format(returncode)))
# FIXME check duration too !
# Here all seem okay -------------------------------------------------------------------------------------------
media_out_size = get_size(media_out_root)
media_out_duration = get_media_duration(media_out_path)
print("{0} Transformation task successful, output media asset {1}".format(request.id, media_out.filename))
transform_callback(TransformTask.SUCCESS)
return {
"hostname": request.hostname,
"start_date": start_date,
"elapsed_time": elapsed_time,
"eta_time": 0,
"media_in_size": media_in_size,
"media_in_duration": media_in_duration,
"media_out_size": media_out_size,
"media_out_duration": media_out_duration,
"percent": 100,
}
except Exception as error:
# Here something went wrong
print("{0} Transformation task failed ".format(request.id))
transform_callback("ERROR\n{0}\n\nOUTPUT\n{1}".format(unicode(error), encoder_out))
raise
finally:
if dashcast_conf:
try_remove(dashcast_conf)
def get_now(self):
return datetime_now()
