def init():
"""Initiates the logging subsystem."""
from jomiel.cache import logger_paths, opts
from jomiel_kore.log import log_init
(logger_file, logger_idents) = log_init(logger_paths)
from jomiel.log import lg
lg().debug(
"subsys/log: configuration file loaded from '%s'",
logger_file,
)
if opts.logger_idents:
from jomiel_kore.app import dump_logger_identities
dump_logger_identities(
logger_idents,
opts.logger_idents_verbose,
)
if opts.plugin_list:
# Prevent INFO lines from being printed to the output with
# --plugin-list.
from logging import WARNING
lg().level = WARNING
lg().info("log subsystem initiated")
def init():
"""Initiates the application subsystems."""
from jomiel.subsys import log, hypertext, plugin, broker
log.init()
plugin.init()
hypertext.init()
broker.init()
from jomiel.log import lg
lg().info("exit normally")
def init():
"""Initiates the HTTP subsystem."""
from jomiel.cache import opts
from jomiel.log import lg
if opts.http_debug:
from jomiel.hypertext import be_verbose
be_verbose()
lg().debug("enable http logging")
lg().info("http subsystem initiated")
def http_post(uri, payload, params=None, **kwargs):
"""Make a new HTTP/POST request.
Args:
uri (string): URI to send the payload to
payload (dict): JSON payload to send to the HTTP server
params (dict): URI query parameters
Returns:
obj: requests.Response
"""
headers = http_headers(**kwargs)
lg().debug("http<post>: '%s'", log_sanitize_string(uri))
lg().debug("http<post/params>: '%s'", log_sanitize_string(params))
lg().debug("http<post/headers>: '%s'", log_sanitize_string(headers))
lg().debug("http<post/payload>: '%s'", log_sanitize_string(payload))
result = post(
uri,
allow_redirects=opts.http_allow_redirects,
timeout=opts.http_timeout,
headers=headers,
params=params,
json=payload,
)
result.raise_for_status()
return result
def log(self, text, msgtype="debug"):
"""Write a new (debug) worker entry to the logger."""
logger = getattr(lg(), msgtype)
logger(
"subsystem/broker<worker#%03d>: %s",
self.worker_id,
text,
)
def script_dispatcher(input_uri):
"""Match input URI to a handling (media) script.
Args:
input_uri (string): the input URI to match
Returns:
obj: The parsed media meta data in a PluginMediaParser subclass
Raises:
NoParserError if no matching handler could not be found
"""
lg().debug(
"dispatcher<%s>: match '%s'",
NS_NAME,
log_sanitize_string(input_uri),
)
(uri_handlers, uri_components) = (
plugin_handlers[NS_NAME],
urlparse(input_uri),
)
for handler in uri_handlers:
try:
# Either return a new subclassed PluginMediaParser object, or
# raise the CannotParseError exception.
#
return handler.inquire(uri_components)
except CannotParseError:
# Rinse and repeat until we run out of handlers.
#
pass
except:
# Fail at all other exceptions by passing the raised
# exception.
#
raise
# When we run out of handlers, inform the caller that we could
# not find a matching parser for the given input URI.
#
raise NoParserError(
"Unable to find a matching parser for URI <%s>" % input_uri,
)
def init():
"""Initiates the plugin subsystem."""
from jomiel.plugin import load
import jomiel.cache as cache
from jomiel.log import lg
def log(text):
"""Write a new (debug) entry to the logger."""
lg().debug("subsystem/plugin: %s", text)
cache.plugin_packages = {}
cache.plugin_handlers = {}
from importlib import import_module
namespace_packages = [import_module("jomiel.plugin.media")]
for ns_pkg in namespace_packages:
ns_name = ns_pkg.__name__
cache.plugin_packages[ns_pkg] = load(ns_pkg)
cache.plugin_handlers[ns_name] = []
for pkg_name in cache.plugin_packages[ns_pkg]:
module = cache.plugin_packages[ns_pkg][pkg_name]
handler = module.Handler()
cache.plugin_handlers[ns_name].append(handler)
log(f"<{pkg_name}> loaded {ns_pkg}")
num_handlers = len(cache.plugin_handlers[ns_name])
log("<%s> cached %d handler(s)" % (ns_name, num_handlers))
no_packages = len(cache.plugin_packages)
log("cached %d package(s)" % no_packages)
lg().info("plugin subsystem initiated")
from jomiel.cache import opts, dump_plugins
if opts.plugin_list:
dump_plugins()
def message_log_serialized(self, prefix, message):
"""Logs the given serialized message in hex format.
Args:
message (obj): Message to be logged
"""
if lg().level <= DEBUG:
_len = len(message)
_hex = hexlify(bytearray(message))
self.log(
"<%s:serialized> [%s] %s" %
(prefix, _len, log_sanitize_string(_hex)), )
def message_dump(self, logtext, message):
"""Dump the message details in JSON to the logger
Ignored unless application uses the debug level.
Args:
logtext (string): log entry text to write
message (obj): the message to log
"""
if lg().level <= DEBUG:
json = to_json(message, minified=opts.debug_minify_json)
self.log(logtext % log_sanitize_string(json))
def http_get(uri, **kwargs):
"""Make a new HTTP/GET request.
Args:
uri (string): URI to retrieve
Returns:
obj: requests.Response
"""
headers = http_headers(**kwargs)
lg().debug("http<get>: '%s'", log_sanitize_string(uri))
lg().debug("http<get/headers>: '%s'", log_sanitize_string(headers))
result = get(
uri,
allow_redirects=opts.http_allow_redirects,
timeout=opts.http_timeout,
headers=headers,
)
result.raise_for_status()
return result
def auth_init():
"""Start an authenticator for this context."""
from zmq.auth.thread import ThreadAuthenticator
from jomiel.log import lg
auth = ThreadAuthenticator(ctx, log=lg())
auth.start()
auth.allow(opts.curve_allow)
# Tell the authenticator to use the client certificates in the
# specified directory.
#
from os.path import abspath
pubdir = abspath(opts.curve_public_key_dir)
auth.configure_curve(domain=opts.curve_domain, location=pubdir)
return auth
def log(text):
"""Write a new (debug) entry to the logger."""
lg().debug("subsystem/plugin: %s", text)
def log(text, msgtype="debug"):
"""Write a new (debug) entry to the logger."""
logger = getattr(lg(), msgtype)
logger("subsystem/broker: %s", text)
def parse(self, uri_components):
"""Parses the relevant metadata for the media.
Args:
uri_components (dict): The input URI components
Raises:
jomiel.error.ParseError if a parsing error occurred
"""
def _parse_metadata(video_info):
"""Parse meta data from the video info."""
def _value_from(d, key_name):
"""Return value from a dictionary, or raise an error."""
if key_name in d:
return d.get(key_name)
raise ParseError(f"'{key_name}' not found")
def _check_playability_status():
"""Check the 'playability status' of the video."""
playability_status = _value_from(
video_info,
"playabilityStatus",
)
if playability_status["status"] == "ERROR":
raise ParseError(playability_status["reason"])
def _parse_video_details():
"""Return video details."""
def _int(key_name):
"""Return int from 'vd' or 0."""
value = vd.get(key_name, 0)
return int(value)
def _float(key_name):
"""Return float from 'vd' or 0."""
value = vd.get(key_name, 0)
return float(value)
def _str(key_name):
"""Return str from 'vd' or ''."""
return vd.get(key_name, "")
vd = _value_from(video_info, "videoDetails")
self.media.statistics.average_rating = _float(
"averageRating", )
self.media.statistics.view_count = _int("viewCount")
self.media.length_seconds = _int("lengthSeconds")
self.media.description = _str("shortDescription")
self.media.author.channel_id = _str("channelId")
self.media.author.name = _str("author")
self.media.title = _str("title")
thumbnail = vd.get("thumbnail")
if thumbnail:
thumbnails = thumbnail.get("thumbnails", [])
# Re-use 'vd' so that _int() works out of the box.
for vd in thumbnails:
thumb = self.media.thumbnail.add()
thumb.width = _int("width")
thumb.height = _int("height")
thumb.uri = vd["url"]
def _parse_streaming_data():
"""Parse 'streaming data'."""
def _parse(key_name):
"""Parse an element of the 'streaming data'."""
def _parse_format():
"""Parse 'format' of streaming data."""
def _profile():
"""Generate the stream profile string."""
profile = _fmt.get(
"qualityLabel",
_fmt.get("quality", "undefined"),
)
return f"{profile} (itag={_fmt['itag']})"
def _int(key_name):
"""Return int from '_fmt' dict or 0."""
value = _fmt.get(key_name, 0)
return int(value)
stream = self.media.stream.add()
stream.content_length = _int("contentLength")
stream.quality.bitrate = _int("bitrate")
stream.quality.height = _int("height")
stream.quality.width = _int("width")
stream.quality.profile = _profile()
stream.mime_type = _fmt["mimeType"]
stream.uri = _fmt["url"]
for _fmt in streaming_data[key_name]:
_parse_format()
streaming_data = _value_from(
video_info,
"streamingData",
)
_parse("adaptiveFormats")
_parse("formats")
# json_pprint(video_info)
_check_playability_status()
_parse_video_details()
_parse_streaming_data()
def _parse_player_response():
"""Return 'player_response'."""
if "player_response" in video_info:
pr = video_info["player_response"]
if isinstance(pr, list):
if len(pr) > 0:
return pr[0]
raise ParseError("'player_response' is empty")
raise ParseError("'player_response' is not 'list'")
raise ParseError("'player_response' not found")
def _parse_video_id():
"""Return video identifier."""
result = re_match(r"v=([\w\-_]{11})", uri_components.query)
if not result:
raise ParseError("unable to match video ID")
self.media.identifier = result.group(1)
def _video_info_request():
"""Make a GET request to the /get_video_info endpoint."""
v_id = self.media.identifier
data = urlencode(
{
"video_id": v_id,
"eurl": f"https://youtube.googleapis.com/v/{v_id}",
"html5": 1,
}, )
uri = f"https://www.youtube.com/get_video_info?{data}"
return http_get(uri).text
def _youtubei_player_request():
"""Make a POST request to the /youtubei/player endpoint."""
uri = "https://www.youtube.com/youtubei/v1/player"
params = {"key": "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"}
payload = {
"context": {
"client": {
"clientName": "WEB",
"clientVersion": "2.20201021.03.00",
},
},
}
payload.update({"videoId": self.media.identifier})
return http_post(uri, payload, params=params).text
_parse_video_id()
try:
video_info = _video_info_request()
video_info = parse_qs(video_info)
video_info = _parse_player_response()
except HTTPError:
# /get_video_info endpoint failed. Try /youtubei/player.
lg().debug("http<get>: /get_video_info failed")
video_info = _youtubei_player_request()
json = loads(video_info)
_parse_metadata(json)