def run_checks_prepare(self, checks, cpl, asset):
_, asset_node = asset
path = os.path.join(self.dcp.path, asset_node['Path'])
can_unwrap = path.endswith('.mxf') and os.path.isfile(path)
asset_stack = [cpl['FileName'], asset[1].get('Path', asset[1]['Id'])]
if self.dcp.schema == 'SMPTE' and can_unwrap:
unwrap_args = []
try:
if asset_node['Encrypted']:
k = get_contentkey_for_asset(self.dcp, asset_node)
unwrap_args = ['-k', k]
except Exception as e:
get_log().info('Subtitle inspection skipped : {}'.format(
str(e)))
return
with unwrap_mxf(path, args=unwrap_args) as folder:
[
self.run_check(check,
cpl,
asset,
folder,
stack=asset_stack) for check in checks
]
elif self.dcp.schema == 'Interop':
folder = os.path.dirname(path)
[
self.run_check(check, cpl, asset, folder, stack=asset_stack)
for check in checks
]
def execute_command(cmd_args):
""" Execute command and returns the result.
Args:
cmd_args (list): Command argument list.
Returns:
Tuple (stdout, stderr).
Raises:
ValueError: If ``cmd_args`` is empty.
ValueError: In case of non-zero return code.
"""
if not cmd_args:
return ValueError("Invalid arguments")
p = subprocess.Popen(
cmd_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if p.returncode:
raise ValueError("Error calling process : {}".format(cmd_args[0]))
stdout, stderr = p.communicate()
get_log().debug("Executed command with return code ({})\n{}".format(
p.returncode, " ".join(cmd_args)))
if p.returncode != 0:
get_log().warning(stderr)
return stdout, stderr
def check_xml(xml_path, xml_ns, schema_type, schema_dcp):
# Correct file type (magic number)
xml_magic = magic.from_file(xml_path)
if "XML" not in xml_magic:
raise CheckException(
"File type unknown, expected XML Document but got {}".format(
xml_magic))
# Correct namespace
schema_id = get_schema(xml_ns)
if not schema_id:
raise CheckException("Namespace unknown : {}".format(xml_ns))
# Coherence with package schema
if schema_type != schema_dcp:
raise CheckException(
"Schema is not valid got {} but was expecting {}".format(
schema_type, schema_dcp))
# XSD schema validation
try:
validate_xml(xml_path, schema_id)
except LookupError as e:
get_log().info("Schema validation skipped : {}".format(xml_path))
except Exception as e:
raise CheckException("Schema validation error : {}".format(str(e)))
def generic_parse(
path,
root_name,
force_list=(),
namespaces=DCP_SETTINGS['xmlns']
):
""" Parse an XML and returns a Python Dictionary """
try:
res_dict = parse_xml(
path,
namespaces=namespaces,
force_list=force_list)
if res_dict and root_name in res_dict:
node = res_dict[root_name]
discover_schema(node)
return {
'FileName': os.path.basename(path),
'FilePath': path,
'Info': {
root_name: node
}
}
except Exception as e:
get_log().info("Error parsing XML {} : {}".format(path, str(e)))
def parse_xml(xml_path, namespaces={}, force_list=(), xml_attribs=True):
""" Parse a XML document and returns a dict with proper formating.
Args:
xml_path (str): XML file absolute path.
namespaces (dict): Namespace mapping dict, prefix: namespace. All
matching namespaces found in the XML document will be processed
and replaced by prefix.
force_list (tuple): Tuple containing XML element name that needs
to appear as list node in the generated dict. This force a list
even if only one such element is found in a particular XML
file.
xml_attribs (boolean): If True, completly ignore all attributes
found in the XML file.
Returns:
A dict representation of the input XML file.
Raises:
ValueError: If ``xml_path`` is not a valid file.
"""
if not os.path.isfile(xml_path):
raise ValueError("{} is not a file".format(xml_path))
try:
with open(xml_path, encoding="utf-8-sig") as file:
readed_file = file.read()
# Collapse these namespace
namespaces = {v: k for k, v in six.iteritems(namespaces)}
xml_dict = xmltodict.parse(readed_file,
process_namespaces=True,
namespaces=namespaces,
force_list=force_list,
xml_attribs=xml_attribs,
postprocessor=post_parse_node,
dict_constructor=dict,
namespace_separator=_DEFAULT_NS_SEP)
if xml_attribs:
xml_dict = post_parse_attr(xml_dict)
return xml_dict
except (Exception, ExpatError) as e:
get_log().error("Error parsing XML {} : {}".format(xml_path, str(e)))
def __init__(self,
dcp,
ov_path=None,
hash_callback=None,
bypass_list=None):
""" CheckerBase constructor.
Args:
dcp (clairmeta.DCP): DCP object.
ov_path (str, optional): Absolute path of OriginalVersion DCP.
hash_callback (function, optional): Callback function to report
file hash progression.
bypass_list (list, optional): List of checks to bypass.
"""
self.dcp = dcp
self.log = get_log()
self.checks = []
self.errors = []
self.report = None
self.bypass_list = bypass_list if bypass_list else []
self.check_modules = {}
self.ov_path = ov_path
self.ov_dcp = None
self.hash_callback = hash_callback
if not self.hash_callback:
pass
elif isinstance(self.hash_callback, ConsoleProgress):
self.hash_callback._total_size = self.dcp.size
elif inspect.isclass(self.hash_callback):
raise CheckException(
"Invalid callback, please provide a function"
" or instance of ConsoleProgress (or derivate).")
def __init__(self, path, kdm=None, pkey=None):
""" DCP constructor.
Args:
path (str): Absolute path to directory.
kdm (str): Absolute path to KDM file.
pkey (str): Absolute path to private key, this should be the
KDM recipient private key.
Raises:
ClairMetaException: ``path`` directory not found.
"""
if not os.path.isdir(path):
raise ClairMetaException("{} is not a valid folder".format(path))
self.path = os.path.normpath(path)
self.kdm = os.path.normpath(kdm) if kdm else None
self.pkey = os.path.normpath(pkey) if pkey else None
self.schema = 'Unknown'
self.package_type = 'Unknown'
self.foreign_files = []
self.size = folder_size(path)
self.log = get_log()
self._probeb = False
self._parsed = False
def check_xml_constraints(checker, xml_path):
""" Check D-Cinema XML Contraints
References:
TI Subtitle Operational Recommendation for DLP Cinema Projectors (Draft A)
https://web.archive.org/web/20140924153620/http://dlp.com/downloads/pdf_dlp_cinema_subtitle_operational_recommendation_rev_a.pdf
SMPTE ST 429-17:2017
W3C Extensible Markup Language v (1.0)
"""
# Follow the XML spec precicely for the definition of XMLDecl, except for:
# VersionNum := '1.0'
# EncName := 'UTF-8'
# EncodingDecl not optional
# SDDecl must have 'no'
RE_XML_S = r'([\x20\x09\x0D\x0A])'
RE_XML_Eq = '(' + RE_XML_S + '?=' + RE_XML_S + '?)'
RE_XML_SDDecl = '(' + RE_XML_S + 'standalone' + RE_XML_Eq + r'(\'no\'|"no"))'
RE_XML_EncName = r'(UTF\-8)'
RE_XML_EncodingDecl = '(' + RE_XML_S + 'encoding' + RE_XML_Eq + '("' + RE_XML_EncName + r'"|\'' + RE_XML_EncName + r'\'))'
RE_XML_VersionNum = r'(1\.0)'
RE_XML_VersionInfo = '(' + RE_XML_S + 'version' + RE_XML_Eq + r'(\'' + RE_XML_VersionNum + r'\'|"' + RE_XML_VersionNum + '"))'
RE_XML_XMLDecl = r'<\?xml' + RE_XML_VersionInfo + RE_XML_EncodingDecl + RE_XML_SDDecl + '?' + RE_XML_S + '?' + r'\?>'
try:
with open(xml_path) as file:
xml_file = file.read()
newlines = file.newlines
except IOError as e:
get_log().error("Error opening XML file {} : {}".format(
xml_path, str(e)))
return
if re.match('\ufeff', xml_file):
checker.error("BOM not allowed in XML file", "constraints_bom")
if not (re.match(RE_XML_XMLDecl, xml_file)
or re.match('\ufeff' + RE_XML_XMLDecl, xml_file)):
checker.error("Invalid XML Declaration", "constraints_declaration")
# Some files might not have newlines at all (single line)
if not newlines in ['\n', '\r\n', None]:
checker.error(
"XML file has invalid ending: {}".format(repr(file.newlines)),
"constraints_line_ending")
def check_xml(xml_path, xml_ns, schema_type, schema_dcp):
# Correct namespace
schema_id = get_schema(xml_ns)
if not schema_id:
raise CheckException("Namespace unknown : {}".format(xml_ns))
# Coherence with package schema
if schema_type != schema_dcp:
raise CheckException(
"Schema is not valid got {} but was expecting {}".format(
schema_type, schema_dcp))
# XSD schema validation
try:
validate_xml(xml_path, schema_id)
except LookupError as e:
get_log().info("Schema validation skipped : {}".format(xml_path))
except Exception as e:
raise CheckException("Schema validation error : {}".format(str(e)))
def __init__(self, dcp, profile):
""" CheckerBase constructor.
Args:
dcp (clairmeta.DCP): DCP object.
profile (dict): Checker profile.
"""
self.dcp = dcp
self.check_profile = profile
self.check_log = get_log()
self.check_executions = []
self.check_report = {}
self.hash_callback = None
def check_xml(checker, xml_path, xml_ns, schema_type, schema_dcp):
# XML constraints
check_xml_constraints(checker, xml_path)
# Correct namespace
schema_id = get_schema(xml_ns)
if not schema_id:
checker.error("Namespace unknown : {}".format(xml_ns), "namespace")
# Coherence with package schema
if schema_type != schema_dcp:
message = "Schema is not valid got {} but was expecting {}".format(
schema_type, schema_dcp)
checker.error(message, "schema_coherence")
# XSD schema validation
try:
validate_xml(xml_path, schema_id)
except LookupError as e:
get_log().info("Schema validation skipped : {}".format(xml_path))
except Exception as e:
message = ("Schema validation error : {}\n"
"Using schema : {}".format(str(e), schema_id))
checker.error(message, "schema_validation")
def cli_copy(args):
dcp = DCP(args.source)
dcp_size = dcp.size
try:
log = get_log()
log.info("Copy {} to {}".format(args.source, args.dest))
start = time.time()
progress = ConsoleProgress()
progress._total_size = dcp_size
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
future = executor.submit(
shutil.copytree,
args.source,
args.dest,
dirs_exist_ok=args.overwrite
)
while future.running():
copy_size = folder_size(args.dest)
elapsed = time.time() - start
progress(args.source, copy_size, dcp_size, elapsed)
time.sleep(1.0)
future.result()
progress(args.source, dcp_size, dcp_size, elapsed)
log.info("Total time : {:.2f} sec".format(time.time() - start))
dcp_dst = DCP(args.dest)
status, report = dcp.check(hash_callback=ConsoleProgress())
return status
except Exception as e:
print(str(e))
return False
def __init__(self, path):
""" DCP constructor.
Args:
path (str): Absolute path to directory.
Raises:
ValueError: ``path`` directory not found.
"""
if not os.path.isdir(path):
raise ValueError("{} is not a valid folder".format(path))
self.path = os.path.normpath(path)
self.schema = 'Unknown'
self.package_type = 'Unknown'
self.foreign_files = []
self.size = folder_size(path)
self.log = get_log()
self._probeb = False
self._parsed = False
# Clairmeta - (C) YMAGIS S.A.
# See LICENSE for more information
from clairmeta.info import __license__, __author__, __version__
from clairmeta.dcp import DCP
from clairmeta.sequence import Sequence
from clairmeta.logger import get_log
from clairmeta.utils.probe import check_command, PROBE_DEPS
from clairmeta.dcp_parse import (volindex_parse, assetmap_parse, pkl_parse,
cpl_parse, kdm_parse)
from clairmeta.exception import ClairMetaException
__all__ = ['DCP', 'Sequence']
__license__ = __license__
__author__ = __author__
__version__ = __version__
# External dependencies check
for d in PROBE_DEPS:
if not check_command(d):
get_log().warning("Missing dependency : {}".format(d))
