def init(): """Initiates the logging subsystem.""" from jomiel.cache import logger_paths, opts from jomiel_kore.log import log_init (logger_file, logger_idents) = log_init(logger_paths) from jomiel.log import lg lg().debug( "subsys/log: configuration file loaded from '%s'", logger_file, ) if opts.logger_idents: from jomiel_kore.app import dump_logger_identities dump_logger_identities( logger_idents, opts.logger_idents_verbose, ) if opts.plugin_list: # Prevent INFO lines from being printed to the output with # --plugin-list. from logging import WARNING lg().level = WARNING lg().info("log subsystem initiated")
def init(): """Initiates the application subsystems.""" from jomiel.subsys import log, hypertext, plugin, broker log.init() plugin.init() hypertext.init() broker.init() from jomiel.log import lg lg().info("exit normally")
def init(): """Initiates the HTTP subsystem.""" from jomiel.cache import opts from jomiel.log import lg if opts.http_debug: from jomiel.hypertext import be_verbose be_verbose() lg().debug("enable http logging") lg().info("http subsystem initiated")
def http_post(uri, payload, params=None, **kwargs): """Make a new HTTP/POST request. Args: uri (string): URI to send the payload to payload (dict): JSON payload to send to the HTTP server params (dict): URI query parameters Returns: obj: requests.Response """ headers = http_headers(**kwargs) lg().debug("http<post>: '%s'", log_sanitize_string(uri)) lg().debug("http<post/params>: '%s'", log_sanitize_string(params)) lg().debug("http<post/headers>: '%s'", log_sanitize_string(headers)) lg().debug("http<post/payload>: '%s'", log_sanitize_string(payload)) result = post( uri, allow_redirects=opts.http_allow_redirects, timeout=opts.http_timeout, headers=headers, params=params, json=payload, ) result.raise_for_status() return result
def log(self, text, msgtype="debug"): """Write a new (debug) worker entry to the logger.""" logger = getattr(lg(), msgtype) logger( "subsystem/broker<worker#%03d>: %s", self.worker_id, text, )
def script_dispatcher(input_uri): """Match input URI to a handling (media) script. Args: input_uri (string): the input URI to match Returns: obj: The parsed media meta data in a PluginMediaParser subclass Raises: NoParserError if no matching handler could not be found """ lg().debug( "dispatcher<%s>: match '%s'", NS_NAME, log_sanitize_string(input_uri), ) (uri_handlers, uri_components) = ( plugin_handlers[NS_NAME], urlparse(input_uri), ) for handler in uri_handlers: try: # Either return a new subclassed PluginMediaParser object, or # raise the CannotParseError exception. # return handler.inquire(uri_components) except CannotParseError: # Rinse and repeat until we run out of handlers. # pass except: # Fail at all other exceptions by passing the raised # exception. # raise # When we run out of handlers, inform the caller that we could # not find a matching parser for the given input URI. # raise NoParserError( "Unable to find a matching parser for URI <%s>" % input_uri, )
def init(): """Initiates the plugin subsystem.""" from jomiel.plugin import load import jomiel.cache as cache from jomiel.log import lg def log(text): """Write a new (debug) entry to the logger.""" lg().debug("subsystem/plugin: %s", text) cache.plugin_packages = {} cache.plugin_handlers = {} from importlib import import_module namespace_packages = [import_module("jomiel.plugin.media")] for ns_pkg in namespace_packages: ns_name = ns_pkg.__name__ cache.plugin_packages[ns_pkg] = load(ns_pkg) cache.plugin_handlers[ns_name] = [] for pkg_name in cache.plugin_packages[ns_pkg]: module = cache.plugin_packages[ns_pkg][pkg_name] handler = module.Handler() cache.plugin_handlers[ns_name].append(handler) log(f"<{pkg_name}> loaded {ns_pkg}") num_handlers = len(cache.plugin_handlers[ns_name]) log("<%s> cached %d handler(s)" % (ns_name, num_handlers)) no_packages = len(cache.plugin_packages) log("cached %d package(s)" % no_packages) lg().info("plugin subsystem initiated") from jomiel.cache import opts, dump_plugins if opts.plugin_list: dump_plugins()
def message_log_serialized(self, prefix, message): """Logs the given serialized message in hex format. Args: message (obj): Message to be logged """ if lg().level <= DEBUG: _len = len(message) _hex = hexlify(bytearray(message)) self.log( "<%s:serialized> [%s] %s" % (prefix, _len, log_sanitize_string(_hex)), )
def message_dump(self, logtext, message): """Dump the message details in JSON to the logger Ignored unless application uses the debug level. Args: logtext (string): log entry text to write message (obj): the message to log """ if lg().level <= DEBUG: json = to_json(message, minified=opts.debug_minify_json) self.log(logtext % log_sanitize_string(json))
def http_get(uri, **kwargs): """Make a new HTTP/GET request. Args: uri (string): URI to retrieve Returns: obj: requests.Response """ headers = http_headers(**kwargs) lg().debug("http<get>: '%s'", log_sanitize_string(uri)) lg().debug("http<get/headers>: '%s'", log_sanitize_string(headers)) result = get( uri, allow_redirects=opts.http_allow_redirects, timeout=opts.http_timeout, headers=headers, ) result.raise_for_status() return result
def auth_init(): """Start an authenticator for this context.""" from zmq.auth.thread import ThreadAuthenticator from jomiel.log import lg auth = ThreadAuthenticator(ctx, log=lg()) auth.start() auth.allow(opts.curve_allow) # Tell the authenticator to use the client certificates in the # specified directory. # from os.path import abspath pubdir = abspath(opts.curve_public_key_dir) auth.configure_curve(domain=opts.curve_domain, location=pubdir) return auth
def log(text): """Write a new (debug) entry to the logger.""" lg().debug("subsystem/plugin: %s", text)
def log(text, msgtype="debug"): """Write a new (debug) entry to the logger.""" logger = getattr(lg(), msgtype) logger("subsystem/broker: %s", text)
def parse(self, uri_components): """Parses the relevant metadata for the media. Args: uri_components (dict): The input URI components Raises: jomiel.error.ParseError if a parsing error occurred """ def _parse_metadata(video_info): """Parse meta data from the video info.""" def _value_from(d, key_name): """Return value from a dictionary, or raise an error.""" if key_name in d: return d.get(key_name) raise ParseError(f"'{key_name}' not found") def _check_playability_status(): """Check the 'playability status' of the video.""" playability_status = _value_from( video_info, "playabilityStatus", ) if playability_status["status"] == "ERROR": raise ParseError(playability_status["reason"]) def _parse_video_details(): """Return video details.""" def _int(key_name): """Return int from 'vd' or 0.""" value = vd.get(key_name, 0) return int(value) def _float(key_name): """Return float from 'vd' or 0.""" value = vd.get(key_name, 0) return float(value) def _str(key_name): """Return str from 'vd' or ''.""" return vd.get(key_name, "") vd = _value_from(video_info, "videoDetails") self.media.statistics.average_rating = _float( "averageRating", ) self.media.statistics.view_count = _int("viewCount") self.media.length_seconds = _int("lengthSeconds") self.media.description = _str("shortDescription") self.media.author.channel_id = _str("channelId") self.media.author.name = _str("author") self.media.title = _str("title") thumbnail = vd.get("thumbnail") if thumbnail: thumbnails = thumbnail.get("thumbnails", []) # Re-use 'vd' so that _int() works out of the box. for vd in thumbnails: thumb = self.media.thumbnail.add() thumb.width = _int("width") thumb.height = _int("height") thumb.uri = vd["url"] def _parse_streaming_data(): """Parse 'streaming data'.""" def _parse(key_name): """Parse an element of the 'streaming data'.""" def _parse_format(): """Parse 'format' of streaming data.""" def _profile(): """Generate the stream profile string.""" profile = _fmt.get( "qualityLabel", _fmt.get("quality", "undefined"), ) return f"{profile} (itag={_fmt['itag']})" def _int(key_name): """Return int from '_fmt' dict or 0.""" value = _fmt.get(key_name, 0) return int(value) stream = self.media.stream.add() stream.content_length = _int("contentLength") stream.quality.bitrate = _int("bitrate") stream.quality.height = _int("height") stream.quality.width = _int("width") stream.quality.profile = _profile() stream.mime_type = _fmt["mimeType"] stream.uri = _fmt["url"] for _fmt in streaming_data[key_name]: _parse_format() streaming_data = _value_from( video_info, "streamingData", ) _parse("adaptiveFormats") _parse("formats") # json_pprint(video_info) _check_playability_status() _parse_video_details() _parse_streaming_data() def _parse_player_response(): """Return 'player_response'.""" if "player_response" in video_info: pr = video_info["player_response"] if isinstance(pr, list): if len(pr) > 0: return pr[0] raise ParseError("'player_response' is empty") raise ParseError("'player_response' is not 'list'") raise ParseError("'player_response' not found") def _parse_video_id(): """Return video identifier.""" result = re_match(r"v=([\w\-_]{11})", uri_components.query) if not result: raise ParseError("unable to match video ID") self.media.identifier = result.group(1) def _video_info_request(): """Make a GET request to the /get_video_info endpoint.""" v_id = self.media.identifier data = urlencode( { "video_id": v_id, "eurl": f"https://youtube.googleapis.com/v/{v_id}", "html5": 1, }, ) uri = f"https://www.youtube.com/get_video_info?{data}" return http_get(uri).text def _youtubei_player_request(): """Make a POST request to the /youtubei/player endpoint.""" uri = "https://www.youtube.com/youtubei/v1/player" params = {"key": "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"} payload = { "context": { "client": { "clientName": "WEB", "clientVersion": "2.20201021.03.00", }, }, } payload.update({"videoId": self.media.identifier}) return http_post(uri, payload, params=params).text _parse_video_id() try: video_info = _video_info_request() video_info = parse_qs(video_info) video_info = _parse_player_response() except HTTPError: # /get_video_info endpoint failed. Try /youtubei/player. lg().debug("http<get>: /get_video_info failed") video_info = _youtubei_player_request() json = loads(video_info) _parse_metadata(json)