def launch_publisher_task(self, user_id, media_id, send_email, queue, callback_url): if self.config.is_standalone: user = self.get_user({u'_id': user_id}, {u'secret': 0}) if not user: raise IndexError(to_bytes(u'No user with id {0}.'.format(user_id))) media = self.get_media({u'_id': media_id}) if not media: # FIXME maybe a media access control here raise IndexError(to_bytes(u'No media asset with id {0}.'.format(media_id))) if not queue in self.config.publisher_queues: raise IndexError(to_bytes(u'No publication queue with name {0}.'.format(queue))) if media.status != Media.READY: raise NotImplementedError(to_bytes(u"Cannot launch the task, input media asset's status is {0}.".format( media.status))) if len(media.public_uris) > 0: raise NotImplementedError(to_bytes(u'Cannot launch the task, input media asset is already published.')) other = self.get_publisher_task({u'media_id': media._id}) if other and other.status not in PublisherTask.FINAL_STATUS and other.status != PublisherTask.REVOKED: raise NotImplementedError(to_bytes(u'Cannot launch the task, input media asset will be published by another' ' task with id {0}.'.format(other._id))) # FIXME create a one-time password to avoid fixed secret authentication ... callback = Callback(self.config.api_url + callback_url, u'node', self.config.node_secret) if self.config.is_mock: result_id = unicode(uuid.uuid4()) else: result = PublisherWorker.publisher_task.apply_async( args=(object2json(media, False), object2json(callback, False)), queue=queue) result_id = result.id if not result_id: raise ValueError(to_bytes(u'Unable to transmit task to workers of queue {0}.'.format(queue))) logging.info(u'New publication task {0} -> queue {1}.'.format(result_id, queue)) task = PublisherTask(user_id=user_id, media_id=media._id, send_email=send_email, _id=result_id) task.statistic[u'add_date'] = int(time()) self._db.publisher_tasks.save(task.__dict__, safe=True) return task
def launch_transform_task( self, user_id, media_in_id, profile_id, filename, metadata, send_email, queue, callback_url ): if self.is_standalone: user = self.get_user({"_id": user_id}, {"secret": 0}) if not user: raise IndexError(to_bytes("No user with id {0}.".format(user_id))) media_in = self.get_media({"_id": media_in_id}) if not media_in: # FIXME maybe a media access control here raise IndexError(to_bytes("No media asset with id {0}.".format(media_in_id))) profile = self.get_transform_profile({"_id": profile_id}) if not profile: # FIXME maybe a profile access control here raise IndexError(to_bytes("No transformation profile with id {0}.".format(profile_id))) if not queue in self.config.transform_queues: raise IndexError(to_bytes("No transformation queue with name {0}.".format(queue))) media_out = Media( user_id=user_id, parent_id=media_in_id, filename=filename, metadata=metadata, status=Media.PENDING ) media_out.uri = self.config.storage_medias_uri(media_out) TransformTask.validate_task(media_in, profile, media_out) self.save_media(media_out) # Save pending output media # FIXME create a one-time password to avoid fixed secret authentication ... callback = Callback(self.config.api_url + callback_url, "node", self.config.node_secret) if self.is_mock: result_id = unicode(uuid.uuid4()) else: result = TransformWorker.transform_task.apply_async( args=( object2json(media_in, False), object2json(media_out, False), object2json(profile, False), object2json(callback, False), ), queue=queue, ) result_id = result.id if not result_id: raise ValueError(to_bytes("Unable to transmit task to workers of queue {0}.".format(queue))) logging.info("New transformation task {0} -> queue {1}.".format(result_id, queue)) task = TransformTask( user_id=user_id, media_in_id=media_in._id, media_out_id=media_out._id, profile_id=profile._id, send_email=send_email, _id=result_id, ) task.statistic["add_date"] = datetime_now() self._db.transform_tasks.save(task.__dict__, safe=True) return task
def revoke_publisher_task(self, task, callback_url, terminate=False, remove=False): u""" This do not delete tasks from tasks database (if remove=False) but set revoked attribute in tasks database and broadcast revoke request to publication units with celery. If the task is actually running it will be cancelled if terminated = True. In any case, the output media asset will be deleted (task running or successfully finished). """ if valid_uuid(task, none_allowed=False): task = self.get_publisher_task({u'_id': task}) task.is_valid(True) if task.status in PublisherTask.CANCELED_STATUS: raise ValueError(to_bytes(u'Cannot revoke a publication task with status {0}.'.format(task.status))) if not self.config.is_mock: revoke(task._id, terminate=terminate) if task.status == PublisherTask.SUCCESS and not self.config.is_mock: # Send revoke task to the worker that published the media callback = Callback(self.config.api_url + callback_url, u'node', self.config.node_secret) queue = task.get_hostname() result = PublisherWorker.revoke_publisher_task.apply_async( args=(task.publish_uri, object2json(callback, False)), queue=queue) if not result.id: raise ValueError(to_bytes(u'Unable to transmit task to queue {0}.'.format(queue))) logging.info(u'New revoke publication task {0} -> queue {1}.'.format(result.id, queue)) self.update_publisher_task_and_media(task, revoke_task_id=result.id, status=PublisherTask.REVOKING) else: self.update_publisher_task_and_media(task, status=PublisherTask.REVOKED) if remove: self._db.publisher_tasks.remove({u'_id': task._id})
def transform_callback(status, measures): data_json = object2json({u'task_id': request.id, u'status': status, u'measures': measures}, include_properties=False) if callback is None: print(u'{0} [ERROR] Unable to callback orchestrator: {1}'.format(request.id, data_json)) else: r = callback.post(data_json) print(u'{0} Code {1} {2} : {3}'.format(request.id, r.status_code, r.reason, r._content))
def dispatch_request(self, *args, **kwargs): if not check_ip(request): return objResponse = {} # Template information objResponse['template_tag'] = ("" if self.action.pi_api_template == "" else md5Checksum('templates/' + self.action.pi_api_template)) for attribute in (u'only_logged_user', u'only_member_user', u'only_admin_user', u'only_orga_member_user', u'only_orga_admin_user', # User restrictions u'cache_time', u'cache_by_user', # Cache information u'user_info', u'json_only', 'no_template'): # Requested user infos + JSON-only if hasattr(self.action, u'pi_api_' + attribute): objResponse[attribute] = getattr(self.action, u'pi_api_' + attribute) # Add the cache headers response = make_response(object2json(objResponse, include_properties=False)) expires = datetime.utcnow() + timedelta(seconds=PI_META_CACHE) expires = expires.strftime("%a, %d %b %Y %H:%M:%S GMT") response.headers['Expire'] = expires response.headers['Cache-Control'] = 'public, max-age=' + str(PI_META_CACHE) # Return the final response return response
def transform_callback(status): data_json = object2json({"task_id": request.id, "status": status}, include_properties=False) if callback is None: print("{0} [ERROR] Unable to callback orchestrator: {1}".format(request.id, data_json)) else: r = callback.post(data_json) print("{0} Code {1} {2} : {3}".format(request.id, r.status_code, r.reason, r._content))
def revoke_publisher_task(publish_uri, callback_json): def revoke_publish_callback(status, publish_uri): data = {u'task_id': request.id, u'status': status} if publish_uri: data[u'publish_uri'] = publish_uri data_json = object2json(data, False) if callback is None: print(u'{0} [ERROR] Unable to callback orchestrator: {1}'.format(request.id, data_json)) else: r = callback.post(data_json) print(u'{0} Code {1} {2} : {3}'.format(request.id, r.status_code, r.reason, r._content)) # ------------------------------------------------------------------------------------------------------------------ # Avoid 'referenced before assignment' callback = None request = current_task.request try: # Let's the task begin ! print(u'{0} Revoke publication task started'.format(request.id)) # Read current configuration to translate files URIs to local paths local_config = PublisherLocalConfig.read(LOCAL_CONFIG_FILENAME, inspect_constructor=False) print(object2json(local_config, True)) # Load and check task parameters callback = Callback.from_json(callback_json, inspect_constructor=True) callback.is_valid(True) # Update callback socket according to configuration if local_config.api_nat_socket and len(local_config.api_nat_socket) > 0: callback.replace_netloc(local_config.api_nat_socket) publish_root = dirname(local_config.publish_uri_to_path(publish_uri)) if not publish_root: raise ValueError(to_bytes(u'Media asset is not hosted on this publication point.')) # Remove publication directory start_date, start_time = datetime_now(), time.time() shutil.rmtree(publish_root, ignore_errors=True) if valid_uri(publish_uri, check_404=True): raise IOError(to_bytes(u'Media asset is reachable from publication URI {0}'.format(publish_uri))) elapsed_time = time.time() - start_time # Here all seem okay print(u'{0} Revoke publication task successful, media asset unpublished from {1}'.format( request.id, publish_uri)) revoke_publish_callback(PublisherTask.SUCCESS, publish_uri) return {u'hostname': request.hostname, u'start_date': start_date, u'elapsed_time': elapsed_time, u'eta_time': 0, u'percent': 100} except Exception as error: # Here something went wrong print(u'{0} Revoke publication task failed'.format(request.id)) revoke_publish_callback(unicode(error), None) raise
def list(self, head=False, **data): values = [] response_dict = self.api_client.do_request(get, self.get_url(extra=(u'HEAD' if head else None)), data=object2json(data, include_properties=False)) if self.cls is None: return response_dict for value_dict in response_dict: values.append(dict2object(self.cls, value_dict, inspect_constructor=True)) return values
def publish_callback(status, publish_uri): data = {u'task_id': request.id, u'status': status} if publish_uri: data[u'publish_uri'] = publish_uri data_json = object2json(data, include_properties=False) if callback is None: print(u'{0} [ERROR] Unable to callback orchestrator: {1}'.format(request.id, data_json)) else: r = callback.post(data_json) print(u'{0} Code {1} {2} : {3}'.format(request.id, r.status_code, r.reason, r._content))
def add(self, *args, **kwargs): if not(bool(args) ^ bool(kwargs)): raise ValueError(to_bytes(u'You must set args OR kwargs.')) if args and len(args) != 1: raise ValueError(to_bytes(u'args should contain only 1 value.')) value = args[0] if args else kwargs response = self.api_client.do_request(post, self.get_url(), data=object2json(value, include_properties=False)) instance = dict2object(self.cls, response, inspect_constructor=True) if self.cls else response # Recover user's secret if isinstance(instance, User): instance.secret = value.secret if args else kwargs[u'secret'] return instance
def dispatch_request(self, *args, **kwargs): if not check_ip(request): return # Call the action result = self.action(request, *args, **kwargs) # Is it a redirect ? if isinstance(result, PlugItRedirect): response = make_response("") response.headers['EbuIo-PlugIt-Redirect'] = result.url if result.no_prefix: response.headers['EbuIo-PlugIt-Redirect-NoPrefix'] = 'True' return response elif isinstance(result, PlugItSendFile): response = send_file(result.filename, mimetype=result.mimetype, as_attachment=result.as_attachment, attachment_filename=result.attachment_filename) response.headers['EbuIo-PlugIt-ItAFile'] = 'True' return response return object2json(result, include_properties=False)
def publisher_task(media_json, callback_json): def copy_callback(start_date, elapsed_time, eta_time, src_size, dst_size, ratio): publisher_task.update_state(state=PublisherTask.PROGRESS, meta={ u'hostname': request.hostname, u'start_date': start_date, u'elapsed_time': elapsed_time, u'eta_time': eta_time, u'media_size': src_size, u'publish_size': dst_size, u'percent': int(100 * ratio)}) def publish_callback(status, publish_uri): data = {u'task_id': request.id, u'status': status} if publish_uri: data[u'publish_uri'] = publish_uri data_json = object2json(data, include_properties=False) if callback is None: print(u'{0} [ERROR] Unable to callback orchestrator: {1}'.format(request.id, data_json)) else: r = callback.post(data_json) print(u'{0} Code {1} {2} : {3}'.format(request.id, r.status_code, r.reason, r._content)) # ------------------------------------------------------------------------------------------------------------------ RATIO_DELTA = 0.01 # Update status if at least 1% of progress TIME_DELTA = 1 # Update status if at least 1 second(s) elapsed # Avoid 'referenced before assignment' callback = publish_root = None request = current_task.request try: # Let's the task begin ! print(u'{0} Publication task started'.format(request.id)) # Read current configuration to translate files URIs to local paths local_config = PublisherLocalConfig.read(LOCAL_CONFIG_FILENAME, inspect_constructor=False) print(object2json(local_config, include_properties=True)) # Load and check task parameters callback = Callback.from_json(callback_json, inspect_constructor=True) callback.is_valid(True) # Update callback socket according to configuration if local_config.api_nat_socket and len(local_config.api_nat_socket) > 0: callback.replace_netloc(local_config.api_nat_socket) media = Media.from_json(media_json, inspect_constructor=True) media.is_valid(True) # Verify that media file can be accessed media_path = local_config.storage_medias_path(media, generate=False) if not media_path: raise NotImplementedError(to_bytes(u'Media asset will not be readed from shared storage : {0}'.format( media.uri))) publish_path, publish_uri = local_config.publish_point(media) media_root, publish_root = dirname(media_path), dirname(publish_path) infos = recursive_copy(media_root, publish_root, copy_callback, RATIO_DELTA, TIME_DELTA) if not valid_uri(publish_uri, check_404=True): raise IOError(to_bytes(u'Media asset is unreachable from publication URI {0}'.format(publish_uri))) # Here all seem okay print(u'{0} Publication task successful, media asset published as {1}'.format(request.id, publish_uri)) publish_callback(PublisherTask.SUCCESS, publish_uri) return {u'hostname': request.hostname, u'start_date': infos[u'start_date'], u'elapsed_time': infos[u'elapsed_time'], u'eta_time': 0, u'media_size': infos[u'src_size'], u'publish_size': infos[u'src_size'], u'percent': 100} except Exception as error: # Here something went wrong print(u'{0} Publication task failed'.format(request.id)) if publish_root: shutil.rmtree(publish_root, ignore_errors=True) publish_callback(unicode(error), None) raise
def count(self, **data): return self.api_client.do_request(get, self.get_url(extra=u'count'), data=object2json(data, include_properties=False))
def __setitem__(self, index, value): return self.api_client.do_request(patch, self.get_url(index), data=object2json(value, include_properties=True))
def transform_task(media_in_json, media_out_json, profile_json, callback_json): def copy_callback(start_date, elapsed_time, eta_time, src_size, dst_size, ratio): transform_task.update_state(state=TransformTask.PROGRESS, meta={ u'hostname': request.hostname, 'start_date': start_date, u'elapsed_time': elapsed_time, u'eta_time': eta_time, u'media_in_size': src_size, u'media_out_size': dst_size, u'percent': int(100 * ratio)}) def transform_callback(status, measures): data_json = object2json({u'task_id': request.id, u'status': status, u'measures': measures}, include_properties=False) if callback is None: print(u'{0} [ERROR] Unable to callback orchestrator: {1}'.format(request.id, data_json)) else: r = callback.post(data_json) print(u'{0} Code {1} {2} : {3}'.format(request.id, r.status_code, r.reason, r._content)) # ------------------------------------------------------------------------------------------------------------------ RATIO_DELTA, TIME_DELTA = 0.01, 1 # Update status if at least 1% of progress and 1 second elapsed. MAX_TIME_DELTA = 5 # Also ensure status update every 5 seconds. DASHCAST_TIMEOUT_TIME = 10 try: # Avoid 'referenced before assignment' callback = dashcast_conf = None encoder_out, request = u'', current_task.request # Let's the task begin ! print(u'{0} Transformation task started'.format(request.id)) # Read current configuration to translate files uri to local paths local_config = TransformLocalConfig.read(LOCAL_CONFIG_FILENAME, inspect_constructor=False) print(object2json(local_config, include_properties=True)) # Load and check task parameters callback = Callback.from_json(callback_json, inspect_constructor=True) callback.is_valid(True) # Update callback socket according to configuration if local_config.api_nat_socket and len(local_config.api_nat_socket) > 0: callback.replace_netloc(local_config.api_nat_socket) media_in = Media.from_json(media_in_json, inspect_constructor=True) media_out = Media.from_json(media_out_json, inspect_constructor=True) profile = TransformProfile.from_json(profile_json, inspect_constructor=True) media_in.is_valid(True) media_out.is_valid(True) profile.is_valid(True) # Verify that media file can be accessed and create output path media_in_path = local_config.storage_medias_path(media_in, generate=False) if not media_in_path: raise NotImplementedError(to_bytes(u'Input media asset will not be readed from shared storage : {0}'.format( media_in.uri))) media_out_path = local_config.storage_medias_path(media_out, generate=True) if not media_out_path: raise NotImplementedError(to_bytes(u'Output media asset will not be written to shared storage : {0}'.format( media_out.uri))) media_in_root = dirname(media_in_path) media_out_root = dirname(media_out_path) try_makedirs(media_out_root) # Get input media duration and frames to be able to estimate ETA media_in_duration = get_media_duration(media_in_path) # Keep potential PSNR status measures = {} # NOT A REAL TRANSFORM : FILE COPY ----------------------------------------------------------------------------- if profile.encoder_name == u'copy': infos = recursive_copy(media_in_root, media_out_root, copy_callback, RATIO_DELTA, TIME_DELTA) media_out_tmp = media_in_path.replace(media_in_root, media_out_root) os.rename(media_out_tmp, media_out_path) start_date = infos[u'start_date'] elapsed_time = infos[u'elapsed_time'] media_in_size = infos[u'src_size'] # A REAL TRANSFORM : TRANSCODE WITH FFMPEG --------------------------------------------------------------------- elif profile.encoder_name == u'ffmpeg': start_date, start_time = datetime_now(), time.time() prev_ratio = prev_time = 0 # Get input media size to be able to estimate ETA media_in_size = get_size(media_in_root) # Create FFmpeg subprocess cmd = u'ffmpeg -y -i "{0}" {1} "{2}"'.format(media_in_path, profile.encoder_string, media_out_path) print(cmd) ffmpeg = Popen(shlex.split(cmd), stderr=PIPE, close_fds=True) make_async(ffmpeg.stderr) while True: # Wait for data to become available select.select([ffmpeg.stderr], [], []) chunk = ffmpeg.stderr.read() encoder_out += chunk elapsed_time = time.time() - start_time match = FFMPEG_REGEX.match(chunk) if match: stats = match.groupdict() media_out_duration = stats[u'time'] try: ratio = total_seconds(media_out_duration) / total_seconds(media_in_duration) ratio = 0.0 if ratio < 0.0 else 1.0 if ratio > 1.0 else ratio except ZeroDivisionError: ratio = 1.0 delta_time = elapsed_time - prev_time if (ratio - prev_ratio > RATIO_DELTA and delta_time > TIME_DELTA) or delta_time > MAX_TIME_DELTA: prev_ratio, prev_time = ratio, elapsed_time eta_time = int(elapsed_time * (1.0 - ratio) / ratio) if ratio > 0 else 0 transform_task.update_state( state=TransformTask.PROGRESS, meta={u'hostname': request.hostname, u'start_date': start_date, u'elapsed_time': elapsed_time, u'eta_time': eta_time, u'media_in_size': media_in_size, u'media_in_duration': media_in_duration, u'media_out_size': get_size(media_out_root), u'media_out_duration': media_out_duration, u'percent': int(100 * ratio), u'encoding_frame': stats[u'frame'], u'encoding_fps': stats[u'fps'], u'encoding_bitrate': stats[u'bitrate'], u'encoding_quality': stats[u'q']}) returncode = ffmpeg.poll() if returncode is not None: break # FFmpeg output sanity check if returncode != 0: raise OSError(to_bytes(u'FFmpeg return code is {0}, encoding probably failed.'.format(returncode))) # compute stats about the video measures['psnr'] = get_media_psnr(media_in_path, media_out_path) measures['ssim'] = get_media_ssim(media_in_path, media_out_path) # measures of the data and its metadata measures['bitrate'] = get_media_bitrate(media_out_path) # FIXME: fake git url, commit measures['git_url'] = 'https://github.com/videolan/x265' measures['git_commit'] = 'd2051f9544434612a105d2f5267db23018cb3454' # Output media file sanity check # media_out_duration = get_media_duration(media_out_path) # if total_seconds(media_out_duration) / total_seconds(media_in_duration) > 1.5 or < 0.8: # salut elif profile.encoder_name == u'from_git': start_date, start_time = datetime_now(), time.time() prev_ratio = prev_time = 0 # Get input media size to be able to estimate ETA media_in_size = get_size(media_in_root) metadata = media_out.metadata dirpath = tempfile.mkdtemp() prepare_cmd = u'git clone --depth=1 "{0}" "{1}" && cd "{1}" && git checkout "{2}" && {3}'.format(metadata['git_url'], dirpath, metadata['git_commit'], metadata['build_cmds']) check_call(prepare_cmd, shell=True) # Templated parameter encoder_string = profile.encoder_string.replace(u"BITRATE", str(metadata['input_bitrate'])) cmd = u'cd "{0}" && ffmpeg -y -i "{1}" -f yuv4mpegpipe - | {2} "{3}"'.format(dirpath, media_in_path, encoder_string, media_out_path) returncode = call(cmd, shell=True) if returncode != 0: raise OSError(to_bytes(u'Encoding return code is {0}, encoding probably failed.'.format(returncode))) # compute stats about the video measures['psnr'] = get_media_psnr(media_in_path, media_out_path) measures['ssim'] = get_media_ssim(media_in_path, media_out_path) # measures of the data and its metadata measures['bitrate'] = get_media_bitrate(media_out_path) # FIXME: don't put this in measures measures['git_url'] = metadata['git_url'] measures['git_commit'] = metadata['git_commit'] # A REAL TRANSFORM : TRANSCODE WITH DASHCAST ------------------------------------------------------------------- elif profile.encoder_name == u'dashcast': start_date, start_time = datetime_now(), time.time() prev_ratio = prev_time = 0 # Get input media size and frames to be able to estimate ETA media_in_size = get_size(media_in_root) try: media_in_frames = int(get_media_tracks(media_in_path)[u'video'][u'0:0'][u'estimated_frames']) media_out_frames = 0 except: raise ValueError(to_bytes(u'Unable to estimate # frames of input media asset')) # Create DashCast configuration file and subprocess dashcast_conf = u'dashcast_{0}.conf'.format(uuid.uuid4()) with open(dashcast_conf, u'w', u'utf-8') as f: f.write(profile.dash_config) cmd = u'DashCast -conf {0} -av "{1}" {2} -out "{3}" -mpd "{4}"'.format( dashcast_conf, media_in_path, profile.dash_options, media_out_root, media_out.filename) print(cmd) dashcast = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE, close_fds=True) make_async(dashcast.stdout.fileno()) make_async(dashcast.stderr.fileno()) while True: # Wait for data to become available select.select([dashcast.stdout.fileno()], [], []) stdout, stderr = read_async(dashcast.stdout), read_async(dashcast.stderr) elapsed_time = time.time() - start_time match = DASHCAST_REGEX.match(stdout) if match: stats = match.groupdict() media_out_frames = int(stats[u'frame']) try: ratio = float(media_out_frames) / media_in_frames ratio = 0.0 if ratio < 0.0 else 1.0 if ratio > 1.0 else ratio except ZeroDivisionError: ratio = 1.0 delta_time = elapsed_time - prev_time if (ratio - prev_ratio > RATIO_DELTA and delta_time > TIME_DELTA) or delta_time > MAX_TIME_DELTA: prev_ratio, prev_time = ratio, elapsed_time eta_time = int(elapsed_time * (1.0 - ratio) / ratio) if ratio > 0 else 0 transform_task.update_state( state=TransformTask.PROGRESS, meta={u'hostname': request.hostname, u'start_date': start_date, u'elapsed_time': elapsed_time, u'eta_time': eta_time, u'media_in_size': media_in_size, u'media_in_duration': media_in_duration, u'media_out_size': get_size(media_out_root), u'percent': int(100 * ratio), u'encoding_frame': media_out_frames}) match = DASHCAST_SUCCESS_REGEX.match(stdout) returncode = dashcast.poll() if returncode is not None or match: encoder_out = u'stdout: {0}\nstderr: {1}'.format(stdout, stderr) break if media_out_frames == 0 and elapsed_time > DASHCAST_TIMEOUT_TIME: encoder_out = u'stdout: {0}\nstderr: {1}'.format(stdout, stderr) raise OSError(to_bytes(u'DashCast does not output frame number, encoding probably failed.')) # DashCast output sanity check if not exists(media_out_path): raise OSError(to_bytes(u'Output media asset not found, DashCast encoding probably failed.')) if returncode != 0: raise OSError(to_bytes(u'DashCast return code is {0}, encoding probably failed.'.format(returncode))) # FIXME check duration too ! # Here all seem okay ------------------------------------------------------------------------------------------- elapsed_time = time.time() - start_time media_out_size = get_size(media_out_root) media_out_duration = get_media_duration(media_out_path) print(u'{0} Transformation task successful, output media asset {1}'.format(request.id, media_out.filename)) transform_callback(TransformTask.SUCCESS, measures) return {u'hostname': request.hostname, u'start_date': start_date, u'elapsed_time': elapsed_time, u'eta_time': 0, u'media_in_size': media_in_size, u'media_in_duration': media_in_duration, u'media_out_size': media_out_size, u'media_out_duration': media_out_duration, u'percent': 100 } except Exception as error: # Here something went wrong print(u'{0} Transformation task failed '.format(request.id)) transform_callback(u'ERROR\n{0}\n\nOUTPUT\n{1}'.format(unicode(error), encoder_out), {}) raise finally: if dashcast_conf: try_remove(dashcast_conf)
def transform_task(media_in_json, media_out_json, profile_json, callback_json): def copy_callback(start_date, elapsed_time, eta_time, src_size, dst_size, ratio): transform_task.update_state( state=TransformTask.PROGRESS, meta={ "hostname": request.hostname, "start_date": start_date, "elapsed_time": elapsed_time, "eta_time": eta_time, "media_in_size": src_size, "media_out_size": dst_size, "percent": int(100 * ratio), }, ) def transform_callback(status): data_json = object2json({"task_id": request.id, "status": status}, include_properties=False) if callback is None: print("{0} [ERROR] Unable to callback orchestrator: {1}".format(request.id, data_json)) else: r = callback.post(data_json) print("{0} Code {1} {2} : {3}".format(request.id, r.status_code, r.reason, r._content)) # ------------------------------------------------------------------------------------------------------------------ RATIO_DELTA, TIME_DELTA = 0.01, 1 # Update status if at least 1% of progress and 1 second elapsed. MAX_TIME_DELTA = 5 # Also ensure status update every 5 seconds. DASHCAST_TIMEOUT_TIME = 10 try: # Avoid 'referenced before assignment' callback = dashcast_conf = None encoder_out, request = "", current_task.request # Let's the task begin ! print("{0} Transformation task started".format(request.id)) # Read current configuration to translate files uri to local paths local_config = TransformLocalConfig.read(LOCAL_CONFIG_FILENAME, inspect_constructor=False) print(object2json(local_config, include_properties=True)) # Load and check task parameters callback = Callback.from_json(callback_json, inspect_constructor=True) callback.is_valid(True) # Update callback socket according to configuration if local_config.api_nat_socket and len(local_config.api_nat_socket) > 0: callback.replace_netloc(local_config.api_nat_socket) media_in = Media.from_json(media_in_json, inspect_constructor=True) media_out = Media.from_json(media_out_json, inspect_constructor=True) profile = TransformProfile.from_json(profile_json, inspect_constructor=True) media_in.is_valid(True) media_out.is_valid(True) profile.is_valid(True) # Verify that media file can be accessed and create output path media_in_path = local_config.storage_medias_path(media_in, generate=False) if not media_in_path: raise NotImplementedError( to_bytes("Input media asset will not be readed from shared storage : {0}".format(media_in.uri)) ) media_out_path = local_config.storage_medias_path(media_out, generate=True) if not media_out_path: raise NotImplementedError( to_bytes("Output media asset will not be written to shared storage : {0}".format(media_out.uri)) ) media_in_root = dirname(media_in_path) media_out_root = dirname(media_out_path) try_makedirs(media_out_root) # Get input media duration and frames to be able to estimate ETA media_in_duration = get_media_duration(media_in_path) # NOT A REAL TRANSFORM : FILE COPY ----------------------------------------------------------------------------- if profile.encoder_name == "copy": infos = recursive_copy(media_in_root, media_out_root, copy_callback, RATIO_DELTA, TIME_DELTA) media_out_tmp = media_in_path.replace(media_in_root, media_out_root) os.rename(media_out_tmp, media_out_path) start_date = infos["start_date"] elapsed_time = infos["elapsed_time"] media_in_size = infos["src_size"] # A REAL TRANSFORM : TRANSCODE WITH FFMPEG --------------------------------------------------------------------- elif profile.encoder_name == "ffmpeg": start_date, start_time = datetime_now(), time.time() prev_ratio = prev_time = 0 # Get input media size to be able to estimate ETA media_in_size = get_size(media_in_root) # Create FFmpeg subprocess cmd = 'ffmpeg -y -i "{0}" {1} "{2}"'.format(media_in_path, profile.encoder_string, media_out_path) print(cmd) ffmpeg = Popen(shlex.split(cmd), stderr=PIPE, close_fds=True) make_async(ffmpeg.stderr) while True: # Wait for data to become available select.select([ffmpeg.stderr], [], []) chunk = ffmpeg.stderr.read() encoder_out += chunk elapsed_time = time.time() - start_time match = FFMPEG_REGEX.match(chunk) if match: stats = match.groupdict() media_out_duration = stats["time"] try: ratio = total_seconds(media_out_duration) / total_seconds(media_in_duration) ratio = 0.0 if ratio < 0.0 else 1.0 if ratio > 1.0 else ratio except ZeroDivisionError: ratio = 1.0 delta_time = elapsed_time - prev_time if (ratio - prev_ratio > RATIO_DELTA and delta_time > TIME_DELTA) or delta_time > MAX_TIME_DELTA: prev_ratio, prev_time = ratio, elapsed_time eta_time = int(elapsed_time * (1.0 - ratio) / ratio) if ratio > 0 else 0 transform_task.update_state( state=TransformTask.PROGRESS, meta={ "hostname": request.hostname, "start_date": start_date, "elapsed_time": elapsed_time, "eta_time": eta_time, "media_in_size": media_in_size, "media_in_duration": media_in_duration, "media_out_size": get_size(media_out_root), "media_out_duration": media_out_duration, "percent": int(100 * ratio), "encoding_frame": stats["frame"], "encoding_fps": stats["fps"], "encoding_bitrate": stats["bitrate"], "encoding_quality": stats["q"], }, ) returncode = ffmpeg.poll() if returncode is not None: break # FFmpeg output sanity check if returncode != 0: raise OSError(to_bytes("FFmpeg return code is {0}, encoding probably failed.".format(returncode))) # Output media file sanity check # media_out_duration = get_media_duration(media_out_path) # if total_seconds(media_out_duration) / total_seconds(media_in_duration) > 1.5 or < 0.8: # salut # A REAL TRANSFORM : TRANSCODE WITH DASHCAST ------------------------------------------------------------------- elif profile.encoder_name == "dashcast": start_date, start_time = datetime_now(), time.time() prev_ratio = prev_time = 0 # Get input media size and frames to be able to estimate ETA media_in_size = get_size(media_in_root) try: media_in_frames = int(get_media_tracks(media_in_path)["video"]["0:0"]["estimated_frames"]) media_out_frames = 0 except: raise ValueError(to_bytes("Unable to estimate # frames of input media asset")) # Create DashCast configuration file and subprocess dashcast_conf = "dashcast_{0}.conf".format(uuid.uuid4()) with open(dashcast_conf, "w", "utf-8") as f: f.write(profile.dash_config) cmd = 'DashCast -conf {0} -av "{1}" {2} -out "{3}" -mpd "{4}"'.format( dashcast_conf, media_in_path, profile.dash_options, media_out_root, media_out.filename ) print(cmd) dashcast = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE, close_fds=True) make_async(dashcast.stdout.fileno()) make_async(dashcast.stderr.fileno()) while True: # Wait for data to become available select.select([dashcast.stdout.fileno()], [], []) stdout, stderr = read_async(dashcast.stdout), read_async(dashcast.stderr) elapsed_time = time.time() - start_time match = DASHCAST_REGEX.match(stdout) if match: stats = match.groupdict() media_out_frames = int(stats["frame"]) try: ratio = float(media_out_frames) / media_in_frames ratio = 0.0 if ratio < 0.0 else 1.0 if ratio > 1.0 else ratio except ZeroDivisionError: ratio = 1.0 delta_time = elapsed_time - prev_time if (ratio - prev_ratio > RATIO_DELTA and delta_time > TIME_DELTA) or delta_time > MAX_TIME_DELTA: prev_ratio, prev_time = ratio, elapsed_time eta_time = int(elapsed_time * (1.0 - ratio) / ratio) if ratio > 0 else 0 transform_task.update_state( state=TransformTask.PROGRESS, meta={ "hostname": request.hostname, "start_date": start_date, "elapsed_time": elapsed_time, "eta_time": eta_time, "media_in_size": media_in_size, "media_in_duration": media_in_duration, "media_out_size": get_size(media_out_root), "percent": int(100 * ratio), "encoding_frame": media_out_frames, }, ) match = DASHCAST_SUCCESS_REGEX.match(stdout) returncode = dashcast.poll() if returncode is not None or match: encoder_out = "stdout: {0}\nstderr: {1}".format(stdout, stderr) break if media_out_frames == 0 and elapsed_time > DASHCAST_TIMEOUT_TIME: encoder_out = "stdout: {0}\nstderr: {1}".format(stdout, stderr) raise OSError(to_bytes("DashCast does not output frame number, encoding probably failed.")) # DashCast output sanity check if not exists(media_out_path): raise OSError(to_bytes("Output media asset not found, DashCast encoding probably failed.")) if returncode != 0: raise OSError(to_bytes("DashCast return code is {0}, encoding probably failed.".format(returncode))) # FIXME check duration too ! # Here all seem okay ------------------------------------------------------------------------------------------- media_out_size = get_size(media_out_root) media_out_duration = get_media_duration(media_out_path) print("{0} Transformation task successful, output media asset {1}".format(request.id, media_out.filename)) transform_callback(TransformTask.SUCCESS) return { "hostname": request.hostname, "start_date": start_date, "elapsed_time": elapsed_time, "eta_time": 0, "media_in_size": media_in_size, "media_in_duration": media_in_duration, "media_out_size": media_out_size, "media_out_duration": media_out_duration, "percent": 100, } except Exception as error: # Here something went wrong print("{0} Transformation task failed ".format(request.id)) transform_callback("ERROR\n{0}\n\nOUTPUT\n{1}".format(unicode(error), encoder_out)) raise finally: if dashcast_conf: try_remove(dashcast_conf)