class VLANS:
'''
Manage a bunch of VLANs, as a dictionary
'''
def __init__(self, vlans=None, delemiter=",", range_delemiter="-"):
super().__init__()
self._delemiter = delemiter
self._range_delemiter = range_delemiter
self._vlans = AttrDict()
if vlans:
self.__iadd__(vlans)
def __add__(self, other):
"""
Add two VLANS to each other
"""
if not isinstance(other, VLANS):
raise TypeError("Error: Can only handle object of VLANS()")
tmp = self.copy()
for vlan in other._vlans.values():
tmp._vlans[vlan.id] = vlan
return tmp
def __iadd__(self, other):
if isinstance(other, VLANS):
for vlan in other._vlans.values():
self._vlans[vlan.id] = vlan
elif isinstance(other, VLAN):
self._vlans[other.id] = other
else:
raise TypeError(
"Error: Can only handle object of VLANS() or VLAN() got %s" %
type(other))
return self
def __str__(self):
return dict_to_vlan_str(self._vlans,
delemiter=self._delemiter,
range_delemiter=self._range_delemiter)
def __repr__(self):
s = ""
for vlan in self._vlans.values():
s += "(%s)" % vlan.to_str()
return "VLANS(%s)" % s
def __iter__(self):
return iter(self.__dict__)
def items(self):
for item in self._vlans.items():
yield item
def keys(self):
for item in self._vlans.keys():
yield item
def values(self):
for item in self._vlans.values():
yield item
def __init__(self, *args, **kwargs):
self._view = None
self._session = None
self._active = False
self._filters = AttrDict(
{n: f(provider=self, name=n)
for n, f in self.FILTERS.items()})
rules = AttrDict(self.config.rules.label,
**config.settings.profile.rules.label)
labels = AttrDict(self.config.labels, **config.settings.profile.labels)
self.rule_map = AttrDict([(re.compile(k, re.IGNORECASE), v)
for k, v in [(r, rules[r])
for r in rules.keys()]])
self.highlight_map = AttrDict([(re.compile(k,
re.IGNORECASE), labels[v])
for k, v in rules.items()])
self.highlight_re = re.compile(
"(" + "|".join([k.pattern
for k in self.highlight_map.keys()]) + ")",
re.IGNORECASE)
def test_change_chart(self):
# Test case for all native charts charts.
with assert_raises(AttributeError):
pptgen.pptgen(
source=self.input,
only=25,
data={'data': {}},
change={'Invalid Input': {
'chart': {
'data': 'data'
}
}})
xaxis, opacity = 'city', 0.5
slidenumbers = AttrDict(Bar_Chart=2,
Column_Chart=10,
Line_Chart=11,
Area_Chart=12,
Scatter_Chart=13,
Bubble_Chart=14,
Bubble_Chart_3D=15,
Radar_Chart=16,
Donut_Chart=17,
Pie_Chart=18)
for chart_name, slidenumber in slidenumbers.items():
# Replacing `_` with a white space. Because chart names in input slides contains
# spaces not `_`.
chart_name = chart_name.replace('_', ' ')
if chart_name in ['Pie Chart', 'Donut Chart']:
series = ['sales']
chart_colors = {
'Singapore': '#D34817',
'Hyderabad': '#9B2D1F',
'Bangalore': '#A28E6A',
'Coimbatore': '#956251',
'Newport Beach': '#918485',
'South Plainfield': '#855D5D'
}
else:
series = ['growth', 'sales']
chart_colors = {
'sales': '#D34817',
'growth': '#9B2D1F',
}
data = self.data.groupby(xaxis, as_index=False)[series].sum()
rule = {
chart_name: {
'chart': {
'data': 'data["data"]',
'x': xaxis,
'color': chart_colors,
'opacity': opacity
}
}
}
target = self.draw_chart(slidenumber, data, rule)
shape = self.get_shape(target, chart_name)[0]
self.check_chart(shape, data, series, xaxis, chart_name,
chart_colors, opacity)
class VLANS:
'''
Manage a bunch of VLANs, as a dictionary
'''
def __init__(self, vlans=None, delemiter=",", range_delemiter="-"):
super().__init__()
self._delemiter = delemiter
self._range_delemiter = range_delemiter
self._vlans = AttrDict()
if vlans:
self.__iadd__(vlans)
def __add__(self, other):
"""
Add two VLANS to each other
"""
if not isinstance(other, VLANS):
raise TypeError("Error: Can only handle object of VLANS()")
tmp = self.copy()
for vlan in other._vlans.values():
tmp._vlans[vlan.id] = vlan
return tmp
def __iadd__(self, other):
if isinstance(other, VLANS):
for vlan in other._vlans.values():
self._vlans[vlan.id] = vlan
elif isinstance(other, VLAN):
self._vlans[other.id] = other
else:
raise TypeError("Error: Can only handle object of VLANS() or VLAN() got %s" % type(other))
return self
def __str__(self):
return dict_to_vlan_str(self._vlans,
delemiter=self._delemiter,
range_delemiter=self._range_delemiter)
def __repr__(self):
s = ""
for vlan in self._vlans.values():
s += "(%s)" % vlan.to_str()
return "VLANS(%s)" % s
def __iter__(self):
return iter(self.__dict__)
def items(self):
for item in self._vlans.items():
yield item
def keys(self):
for item in self._vlans.keys():
yield item
def values(self):
for item in self._vlans.values():
yield item
def get(self):
limit = int(self.getq('limit', [100])[0])
if isinstance(self.query, dict):
# Bind all queries and run them in parallel
args = {
key: self.getq(key, [''])[0]
for name, query in self.query.items()
for key, _bindparams in query._bindparams.items()
}
stmts = AttrDict([
(key,
q.bindparams(
**{k: v
for k, v in args.items() if k in q._bindparams}))
for key, q in self.query.items()
])
if self.thread:
futures = AttrDict([
(key,
gramex.service.threadpool.submit(self.run, stmt, limit))
for key, stmt in stmts.items()
])
self.result = AttrDict()
for key, future in futures.items():
self.result[key] = yield future
else:
self.result = AttrDict([(key, self.run(stmt, limit))
for key, stmt in stmts.items()])
self.renderdatas()
else:
# Bind query and run it
args = {
key: self.getq(key, [''])[0]
for key in self.query._bindparams
}
stmt = self.query.bindparams(
**
{k: v
for k, v in args.items() if k in self.query._bindparams})
if self.thread:
self.result = yield gramex.service.threadpool.submit(
self.run, stmt, limit)
else:
self.result = self.run(stmt, limit)
self.renderdata()
def prune_keys(conf, keys={}):
'''
Returns a deep copy of a configuration removing specified keys.
``prune_keys(conf, {'comment'})`` drops the "comment" key from any dict or sub-dict.
'''
if isinstance(conf, dict):
conf = AttrDict(
{k: prune_keys(v, keys)
for k, v in conf.items() if k not in keys})
elif isinstance(conf, (list, tuple)):
conf = [prune_keys(v, keys) for v in conf]
return conf
class HighlightRuleConfig(object):
# def __init__(self, config):
def __init__(self, config_file):
self._config_file = config_file
self.config = config.Config(
self._config_file
)
self.flags = 0 if self.config.match.case_sensitive else re.IGNORECASE
# import ipdb; ipdb.set_trace()
self.rules = AttrDict([
(label,
HighlightRuleList(
self.highlight_config.get(label),
[ dict(subject=k, **(v or {})) for k, v in rule_dict.items() ],
config=self.config
)
)
for label, rule_dict in self.label_config.items()
])
self.RE_MAP = dict()
@property
def highlight_config(self):
return self.config.get("highlight", {})
@property
def label_config(self):
return self.config.get("label", {})
@property
def labels(self):
return self.label_config.keys()
def __getitem__(self, key):
return self.rules[key]
def add_rule(self, label, subject, group=None, patterns=None):
targets = [subject] + (patterns if patterns else [])
self.remove_rule(targets)
rule = HighlightRule(subject, group=group, patterns=patterns)
self.rules[label].append(rule)
self.save()
def remove_rule(self, targets):
if not isinstance(targets, list):
targets = [targets]
self.rules = AttrDict([
(label, HighlightRuleList(
self.highlight_config.get(label),
[
r for r in self.rules[label]
if r.subject not in targets
and not any(pattern in targets for pattern in r.patterns)
],
config=self.config
))
for label, rule_list in self.rules.items()
])
self.save()
def save(self):
self.config.label = {
label: {
d.subject: {
k: v for k, v in d.items()
if k != "subject"
} or None
for d in [
rule.to_dict()
for rule in sorted(rule_list)
]
}
for label, rule_list in self.rules.items()
}
self.config.save()
# temp_config = config.Config(
# self._config_file + ".new.yaml"
# )
# temp_config.update(self.config.tree)
# temp_config.label = {
# label: {
# d.subject: {
# k: v for k, v in d.items()
# if k != "subject"
# } or None
# for d in [
# rule.to_dict()
# for rule in sorted(rule_list)
# ]
# }
# for label, rule_list in self.rules.items()
# }
# temp_config.save()
def get_regex(self, rules):
rules_set = frozenset(rules.items())
if rules_set not in self.RE_MAP:
pattern = '|'.join(
f"{rules_list.pattern}"
for label, rules_list in rules.items()
if len(rules_list)
)
pattern_grouped = '|'.join(
f"(?P<{label}>{rules_list.pattern})"
for label, rules_list in rules.items()
if len(rules_list)
)
# pattern_tokens = f"{pattern_grouped}|(?P<none>(?!{pattern})(.+?))"
pattern_tokens = "|".join([
p for p in [pattern_grouped, "(?P<none>(?!{pattern})(.+?))"]
if len(p)
])
self.RE_MAP[rules_set] = tuple(
re.compile(p, self.flags)
for p in [pattern, pattern_grouped, pattern_tokens]
)
return self.RE_MAP[rules_set]
@property
def pattern_rules(self):
return self.get_regex(self.rules)[0]
@property
def pattern_rules_grouped(self):
return self.get_regex(self.rules)[1]
@property
def pattern_rules_tokens(self):
return self.get_regex(self.rules)[2]
def search(self, text):
return next(
(
match for match in (
rules.search(text)
for rules in self.rules.values()
)
if match
),
None
)
def tokenize(self, text, candidates=[], aliases={}):
for subject, alias_list in aliases.items():
text = re.sub("|".join(
(
alias
for alias in alias_list
)
), subject, text)
if candidates:
rules = AttrDict([
(label, HighlightRuleList(
self.highlight_config[label],
[
r for r in rule_list
if r.subject in candidates
],
config=self.config
))
for label, rule_list in self.rules.items()
])
else:
rules = self.rules
(pattern, pattern_grouped, pattern_tokens) = self.get_regex(rules)
tokens = (
(match.lastgroup, match.group())
for match in re.finditer(pattern_tokens, text)
)
out = []
for k, g in groupby(tokens, lambda x: x[0] == "none"):
if k:
out.append(((None, None), "".join(item[1] for item in g)))
else:
(attr, text) = next(g)
label, rule = self.rule_for_token(text)
attr = rule.attr if (rule and rule.attr) else attr
out.append(((label, attr), text))
return out
def apply(self, text, candidates=[], aliases={}):
return [
(attr, token) if attr else token
for attr, token in [
(self.highlight_config.get(attr, attr), token)
for ( (label, attr), token) in self.tokenize(text, candidates=candidates, aliases=aliases)
]
]
def get_tokens(self, text, candidates=[], aliases={}):
return [
token
for (attr, token) in self.tokenize(text, candidates=candidates, aliases=aliases)
if attr
]
def rule_for_token(self, token):
return next(
(
(label, rule) for label, rule in (
(label, rules.rule_for_token(token))
for label, rules in self.rules.items()
)
if rule
),
(None, None)
)
class BaseProvider(abc.ABC):
"""
Abstract base class from which providers should inherit from
"""
SESSION_CLASS = StreamSession
ITEM_CLASS = model.MediaItem
# VIEW_CLASS = SimpleProviderView
FILTERS = AttrDict()
ATTRIBUTES = AttrDict(title={"width": ("weight", 1)})
MEDIA_TYPES = None
def __init__(self, *args, **kwargs):
self._view = None
self._session = None
self._active = False
self._filters = AttrDict(
{n: f(provider=self, name=n)
for n, f in self.FILTERS.items()})
rules = AttrDict(self.config.rules.label,
**config.settings.profile.rules.label)
labels = AttrDict(self.config.labels, **config.settings.profile.labels)
self.rule_map = AttrDict([(re.compile(k, re.IGNORECASE), v)
for k, v in [(r, rules[r])
for r in rules.keys()]])
self.highlight_map = AttrDict([(re.compile(k,
re.IGNORECASE), labels[v])
for k, v in rules.items()])
self.highlight_re = re.compile(
"(" + "|".join([k.pattern
for k in self.highlight_map.keys()]) + ")",
re.IGNORECASE)
def init_config(self):
pass
@property
def LISTING_CLASS(self):
for cls in [self.__class__] + list(self.__class__.__bases__):
pkg = sys.modules.get(cls.__module__)
pkgname = pkg.__name__.split(".")[-1]
try:
return next(
v for k, v in pkg.__dict__.items()
if pkgname in k.lower() and k.endswith("MediaListing"))
except StopIteration:
continue
return model.MediaListing
@property
def MEDIA_SOURCE_CLASS(self):
for cls in [self.__class__] + list(self.__class__.mro()):
pkg = sys.modules.get(cls.__module__)
pkgname = pkg.__name__.split(".")[-1]
try:
return next(
v for k, v in pkg.__dict__.items()
if pkgname in k.lower() and k.endswith("MediaSource"))
except (StopIteration, AttributeError):
continue
return model.MediaSource
@property
def session_params(self):
return {"proxies": config.settings.profile.get("proxies")}
@property
def session(self):
if self._session is None:
session_params = self.session_params
self._session = self.SESSION_CLASS.new(self.IDENTIFIER,
**session_params)
return self._session
@property
def gui(self):
return self._view is not None
@property
def filters(self):
return self._filters
@property
def view(self):
if not self._view:
self._view = self.make_view()
self._view.update()
return self._view
@property
def is_active(self):
return self._active
def activate(self):
if self.is_active:
return
self._active = True
self.on_activate()
def deactivate(self):
if not self.is_active:
return
self.on_deactivate()
self._active = False
def on_activate(self):
pass
def on_deactivate(self):
pass
@abc.abstractmethod
def make_view(self):
pass
@classproperty
def IDENTIFIER(cls):
return next(c.__module__ for c in cls.__mro__
if __package__ in c.__module__).split(".")[-1]
@classproperty
@abc.abstractmethod
def NAME(cls):
return cls.__name__.replace("Provider", "")
@property
def FILTERS_BROWSE(self):
return AttrDict()
@property
def FILTERS_OPTIONS(self):
return AttrDict()
@property
def FILTERS(self):
d = getattr(self, "FILTERS_BROWSE", AttrDict())
d.update(getattr(self, "FILTERS_OPTIONS", {}))
return d
def parse_identifier(self, identifier):
return
def new_media_source(self, *args, **kwargs):
return self.MEDIA_SOURCE_CLASS(self.IDENTIFIER, *args, **kwargs)
def new_listing(self, **kwargs):
return self.LISTING_CLASS(self.IDENTIFIER, **kwargs)
@abc.abstractmethod
def listings(self, filters=None):
pass
def should_download(self, listing):
return listing.label in (list(self.config.rules) +
list(config.settings.profile.rules.download))
def on_new_listing(self, listing):
try:
label = next(l for r, l in self.rule_map.items()
if r.search(listing.title))
listing.label = label
if self.should_download(listing):
self.download(listing)
except StopIteration:
pass
@property
def config(self):
return config.ConfigTree(
config.settings.profile.providers.get(self.IDENTIFIER, {}))
@property
def config_is_valid(self):
def check_config(required, cfg):
if isinstance(required, dict):
for k, v in required.items():
if not k in cfg:
return False
if not check_config(required[k], cfg[k]):
return False
else:
for k in required:
if not k in cfg:
return False
return True
# return all([ self.config.get(x, None) is not None
# for x in getattr(self, "REQUIRED_CONFIG", [])
return check_config(getattr(self, "REQUIRED_CONFIG", []), self.config)
def parse_options(self, options):
if not options:
return AttrDict()
return AttrDict(
[(list(self.FILTERS_OPTIONS.keys())[n], v) for n, v in enumerate(
[o for o in options.split(",") if "=" not in o])],
**dict(o.split("=") for o in options.split(",") if "=" in o))
def get_source(self, selection, **kwargs):
source = selection.content
if not isinstance(source, list):
source = [source]
return source
def play_args(self, selection, **kwargs):
source = self.get_source(selection, **kwargs)
kwargs = {
k: v
for k, v in list(kwargs.items()) +
[(f, self.filters[f].value)
for f in self.filters if f not in kwargs]
}
return (source, kwargs)
def play(self, selection, no_task_manager=False, **kwargs):
try:
sources, kwargs = self.play_args(selection, **kwargs)
except SGStreamNotFound as e:
logger.error(f"stream not found: {e}")
return
# media_type = kwargs.pop("media_type", None)
# FIXME: For now, we just throw playlists of media items at the default
# player program and hope it can handle all of them.
player_spec = None
helper_spec = None
if not isinstance(sources, list):
sources = [sources]
for s in sources:
if not s.media_type:
# Try to set the content types of the source(s) with a HTTP HEAD
# request if the provider didn't specify one.
s.media_type = self.session.head(
s.locator).headers.get("Content-Type").split("/")[0]
media_types = set([s.media_type for s in sources if s.media_type])
player_spec = {"media_types": media_types}
if media_types == {"image"}:
helper_spec = {None: None}
else:
helper_spec = getattr(self.config, "helpers",
None) or sources[0].helper
task = model.PlayMediaTask(provider=self.NAME,
title=selection.title,
sources=sources)
if not (no_task_manager or state.options.debug_console):
return state.task_manager.play(task, player_spec, helper_spec,
**kwargs)
else:
return state.asyncio_loop.create_task(
Player.play(task, player_spec, helper_spec, **kwargs))
def download(self, selection, no_task_manager=False, **kwargs):
sources, kwargs = self.play_args(selection, **kwargs)
# filename = selection.download_filename
if not isinstance(sources, list):
sources = [sources]
for i, s in enumerate(sources):
# filename = s.download_filename
# kwargs = {"ext": getattr(s, "ext", None)}
if len(sources):
kwargs["index"] = i
try:
filename = selection.download_filename(**kwargs)
except SGInvalidFilenameTemplate as e:
logger.warn(
f"filename template for provider {self.IDENTIFIER} is invalid: {e}"
)
helper_spec = getattr(self.config, "helpers") or s.download_helper
# logger.info(f"helper: {helper_spec}")
task = model.DownloadMediaTask(provider=self.NAME,
title=selection.title,
sources=[s],
dest=filename)
# s = AttrDict(dataclasses.asdict(s))
# s.provider = self.NAME
# s.title = selection.title
# s.dest = filename
if not (no_task_manager or state.options.debug_console):
return state.task_manager.download(task, filename, helper_spec,
**kwargs)
else:
return state.asyncio_loop.create_task(
Downloader.download(task, filename, helper_spec, **kwargs))
def on_select(self, widget, selection):
self.play(selection)
@property
def limit(self):
return None
def refresh(self):
self.view.refresh()
def reset(self):
self.view.reset()
def __str__(self):
return self.NAME
def __repr__(self):
return f"<{type(self)}: {self.NAME}>"
def _call(self, inp, inp_features, is_training, is_posterior=True, prop_state=None):
print("\n" + "-" * 10 + " ConvGridObjectLayer(is_posterior={}) ".format(is_posterior) + "-" * 10)
# --- set up sub networks and attributes ---
self.maybe_build_subnet("box_network", builder=cfg.build_conv_lateral, key="box")
self.maybe_build_subnet("attr_network", builder=cfg.build_conv_lateral, key="attr")
self.maybe_build_subnet("z_network", builder=cfg.build_conv_lateral, key="z")
self.maybe_build_subnet("obj_network", builder=cfg.build_conv_lateral, key="obj")
self.maybe_build_subnet("object_encoder")
_, H, W, _, n_channels = tf_shape(inp_features)
if self.B != 1:
raise Exception("NotImplemented")
if not self.initialized:
# Note this limits the re-usability of this module to images
# with a fixed shape (the shape of the first image it is used on)
self.batch_size, self.image_height, self.image_width, self.image_depth = tf_shape(inp)
self.H = H
self.W = W
self.HWB = H*W
self.batch_size = tf.shape(inp)[0]
self.is_training = is_training
self.float_is_training = tf.to_float(is_training)
inp_features = tf.reshape(inp_features, (self.batch_size, H, W, n_channels))
is_posterior_tf = tf.ones_like(inp_features[..., :2])
if is_posterior:
is_posterior_tf = is_posterior_tf * [1, 0]
else:
is_posterior_tf = is_posterior_tf * [0, 1]
objects = AttrDict()
base_features = tf.concat([inp_features, is_posterior_tf], axis=-1)
# --- box ---
layer_inp = base_features
n_features = self.n_passthrough_features
output_size = 8
network_output = self.box_network(layer_inp, output_size + n_features, self.is_training)
rep_input, features = tf.split(network_output, (output_size, n_features), axis=-1)
_objects = self._build_box(rep_input, self.is_training)
objects.update(_objects)
# --- attr ---
if is_posterior:
# --- Get object attributes using object encoder ---
yt, xt, ys, xs = tf.split(objects['normalized_box'], 4, axis=-1)
yt, xt, ys, xs = coords_to_image_space(
yt, xt, ys, xs, (self.image_height, self.image_width), self.anchor_box, top_left=False)
transform_constraints = snt.AffineWarpConstraints.no_shear_2d()
warper = snt.AffineGridWarper(
(self.image_height, self.image_width), self.object_shape, transform_constraints)
_boxes = tf.concat([xs, 2*xt - 1, ys, 2*yt - 1], axis=-1)
_boxes = tf.reshape(_boxes, (self.batch_size*H*W, 4))
grid_coords = warper(_boxes)
grid_coords = tf.reshape(grid_coords, (self.batch_size, H, W, *self.object_shape, 2,))
if self.edge_resampler:
glimpse = resampler_edge.resampler_edge(inp, grid_coords)
else:
glimpse = tf.contrib.resampler.resampler(inp, grid_coords)
else:
glimpse = tf.zeros((self.batch_size, H, W, *self.object_shape, self.image_depth))
# Create the object encoder network regardless of is_posterior, otherwise messes with ScopedFunction
encoded_glimpse = apply_object_wise(
self.object_encoder, glimpse, n_trailing_dims=3, output_size=self.A, is_training=self.is_training)
if not is_posterior:
encoded_glimpse = tf.zeros_like(encoded_glimpse)
layer_inp = tf.concat([base_features, features, encoded_glimpse, objects['local_box']], axis=-1)
network_output = self.attr_network(layer_inp, 2 * self.A + n_features, self.is_training)
attr_mean, attr_log_std, features = tf.split(network_output, (self.A, self.A, n_features), axis=-1)
attr_std = self.std_nonlinearity(attr_log_std)
attr = Normal(loc=attr_mean, scale=attr_std).sample()
objects.update(attr_mean=attr_mean, attr_std=attr_std, attr=attr, glimpse=glimpse)
# --- z ---
layer_inp = tf.concat([base_features, features, objects['local_box'], objects['attr']], axis=-1)
n_features = self.n_passthrough_features
network_output = self.z_network(layer_inp, 2 + n_features, self.is_training)
z_mean, z_log_std, features = tf.split(network_output, (1, 1, n_features), axis=-1)
z_std = self.std_nonlinearity(z_log_std)
z_mean = self.training_wheels * tf.stop_gradient(z_mean) + (1-self.training_wheels) * z_mean
z_std = self.training_wheels * tf.stop_gradient(z_std) + (1-self.training_wheels) * z_std
z_logit = Normal(loc=z_mean, scale=z_std).sample()
z = self.z_nonlinearity(z_logit)
objects.update(z_logit_mean=z_mean, z_logit_std=z_std, z_logit=z_logit, z=z)
# --- obj ---
layer_inp = tf.concat([base_features, features, objects['local_box'], objects['attr'], objects['z']], axis=-1)
rep_input = self.obj_network(layer_inp, 1, self.is_training)
_objects = self._build_obj(rep_input, self.is_training)
objects.update(_objects)
# --- final ---
_objects = AttrDict()
for k, v in objects.items():
_, _, _, *trailing_dims = tf_shape(v)
_objects[k] = tf.reshape(v, (self.batch_size, self.HWB, *trailing_dims))
objects = _objects
if prop_state is not None:
objects.prop_state = tf.tile(prop_state[0:1, None], (self.batch_size, self.HWB, 1))
objects.prior_prop_state = tf.tile(prop_state[0:1, None], (self.batch_size, self.HWB, 1))
# --- misc ---
objects.n_objects = tf.fill((self.batch_size,), self.HWB)
objects.pred_n_objects = tf.reduce_sum(objects.obj, axis=(1, 2))
objects.pred_n_objects_hard = tf.reduce_sum(tf.round(objects.obj), axis=(1, 2))
return objects
class BaseProvider(abc.ABC, Observable):
"""
Abstract base class from which providers should inherit from
"""
SESSION_CLASS = StreamSession
LISTING_CLASS = model.TitledMediaListing
# VIEW_CLASS = SimpleProviderView
# FILTERS = AttrDict()
ATTRIBUTES = AttrDict(title={"width": ("weight", 1)})
MEDIA_TYPES = None
RPC_METHODS = []
def __init__(self, *args, **kwargs):
self._view = None
self._session = None
self._active = False
self._filters = AttrDict(
{n: f(provider=self, name=n)
for n, f in self.FILTERS.items()})
rules = AttrDict(self.config.rules.label or {},
**config.settings.profile.rules.label or {})
labels = AttrDict(self.config.labels, **config.settings.profile.labels)
self.rule_map = AttrDict([(re.compile(k, re.IGNORECASE), v)
for k, v in [(r, rules[r])
for r in rules.keys()]])
self.highlight_map = AttrDict([(re.compile(k,
re.IGNORECASE), labels[v])
for k, v in rules.items()])
self.highlight_re = re.compile(
"(" + "|".join([k.pattern
for k in self.highlight_map.keys()]) + ")",
re.IGNORECASE)
# print(self.filters)
self.filters["search"].connect("changed", self.on_search_change)
def init_config(self):
with db_session:
try:
self.provider_data = model.ProviderData.get(
name=self.IDENTIFIER).settings
except AttributeError:
self.provider_data = model.ProviderData(
name=self.IDENTIFIER).settings
for name, f in self.filters.items():
value = self.default_filter_values.get(name, None)
if value:
try:
f.value = value
except (ValueError, ):
# import ipdb; ipdb.set_trace()
pass
@property
def default_filter_values(self):
return AttrDict()
@db_session
def save_provider_data(self):
model.ProviderData.get(
name=self.IDENTIFIER).settings = self.provider_data
commit()
@property
def LISTING_CLASS(self):
for cls in [self.__class__] + list(self.__class__.__bases__):
pkg = sys.modules.get(cls.__module__)
pkgname = pkg.__name__.split(".")[-1]
try:
return next(
v for k, v in pkg.__dict__.items()
if pkgname in k.lower() and k.endswith("MediaListing"))
except StopIteration:
continue
return model.TitledMediaListing
@property
def MEDIA_SOURCE_CLASS(self):
for cls in [self.__class__] + list(self.__class__.mro()):
pkg = sys.modules.get(cls.__module__)
pkgname = pkg.__name__.split(".")[-1]
try:
return next(
v for k, v in pkg.__dict__.items()
if pkgname in k.lower() and k.endswith("MediaSource"))
except (StopIteration, AttributeError):
continue
return model.MediaSource
@property
def helper(self):
return None
@property
def session_params(self):
return {"proxies": config.settings.profile.get("proxies")}
@property
def PREVIEW_TYPES(self):
return ["default"]
@property
def session(self):
if self._session is None:
session_params = self.session_params
self._session = self.SESSION_CLASS.new(self.IDENTIFIER,
**session_params)
return self._session
@property
def gui(self):
return self._view is not None
@property
def filters(self):
return self._filters
@property
def view(self):
if not self._view:
self._view = self.make_view()
self._view.update()
return self._view
@property
def toolbar(self):
return self.view.toolbar
@property
def is_active(self):
return self._active
def activate(self):
if self.is_active:
return
self._active = True
self.on_activate()
def deactivate(self):
if not self.is_active:
return
self.on_deactivate()
self._active = False
def on_activate(self):
self.reset()
def on_deactivate(self):
self.view.on_deactivate()
@property
def VIEW(self):
return SimpleProviderView(self, ProviderDataTable(self))
# @abc.abstractmethod
def make_view(self):
if not self.config_is_valid:
return InvalidConfigView(self.NAME, self.REQUIRED_CONFIG)
return self.VIEW
@classproperty
def IDENTIFIER(cls):
return next(c.__module__ for c in cls.__mro__
if __package__ in c.__module__).split(".")[-1]
@classproperty
@abc.abstractmethod
def NAME(cls):
return cls.__name__.replace("Provider", "")
@property
def FILTERS_BROWSE(self):
return AttrDict()
@property
def FILTERS_OPTIONS(self):
return AttrDict([("search", TextFilter)])
@property
def FILTERS(self):
d = getattr(self, "FILTERS_BROWSE", AttrDict())
d.update(getattr(self, "FILTERS_OPTIONS", {}))
return d
def on_search_change(self, value, *args):
if getattr(self, "search_task", False):
self.search_task.cancel()
async def apply_search_async():
await asyncio.sleep(1)
await self.apply_search_query(value)
self.search_task = state.event_loop.create_task(apply_search_async())
async def apply_search_query(self, query):
self.view.apply_search_query(query)
def parse_spec(self, spec):
(identifier, options) = MEDIA_SPEC_RE.search(spec).groups()
try:
selection, filters, identifier_options = self.parse_identifier(
identifier)
self.apply_identifier(selection, filters, identifier_options)
except SGIncompleteIdentifier:
selection, identifier_options = None, {}
options = AttrDict(identifier_options, **self.parse_options(options))
self.apply_options(options)
return (selection, options)
def parse_identifier(self, identifier):
return (None, identifier, {})
def apply_identifier(self, selection, filters, options):
if filters:
selected_filters = zip(self.filters.keys(), filters)
for f, value in selected_filters:
if value is None or value in [
getattr(self.filters[f], "selected_label", None),
self.filters[f].value
]:
continue
try:
self.filters[f].selected_label = value
except ValueError:
self.filters[f].value = value
def parse_options(self, options):
if not options:
options = ""
d = AttrDict(
[(list(self.FILTERS_OPTIONS.keys())[n], v) for n, v in enumerate(
[o for o in options.split(",") if "=" not in o]) if v],
**dict(o.split("=") for o in options.split(",") if "=" in o))
return d
def apply_options(self, options):
for k, v in options.items():
if v is None:
continue
if k in self.filters:
logger.debug(f"option: {k}={v}")
try:
if self.filters[k].value != v:
self.filters[k].value = v
except StopIteration:
raise SGException("invalid value for %s: %s" % (k, v))
def new_media_source(self, *args, **kwargs):
return self.MEDIA_SOURCE_CLASS.attr_class(provider_id=self.IDENTIFIER,
*args,
**kwargs)
def new_listing(self, **kwargs):
return self.LISTING_CLASS.attr_class(provider_id=self.IDENTIFIER,
**kwargs)
async def play(self, listing, **kwargs):
# sources, kwargs = self.extract_sources(listing, **kwargs)
task = self.create_play_task(listing, **kwargs)
yield state.task_manager.play(task)
async def download(self,
listing,
index=None,
no_task_manager=False,
**kwargs):
for task in self.create_download_tasks(listing, index=index, **kwargs):
yield state.task_manager.download(task)
def translate_template(self, template):
return template
# def new_listing_attr(self, **kwargs):
# return self.LISTING_CLASS.attr_class(
# provider_id = self.IDENTIFIER,
# **kwargs
# )
def sort(self, field, reverse=False):
self.view.sort(field, reverse=reverse)
@abc.abstractmethod
def listings(self, filters=None):
pass
def should_download(self, listing):
return listing.label in (list(self.config.rules) +
list(config.settings.profile.rules.download))
def on_new_listing(self, listing):
try:
label = next(l for r, l in self.rule_map.items()
if r.search(listing.title))
listing.label = label
if self.should_download(listing):
self.download(listing)
except StopIteration:
pass
@property
def config(self):
return config.ConfigTree(
config.settings.profile.providers.get(self.IDENTIFIER, {}))
@property
def config_is_valid(self):
def check_config(required, cfg):
if isinstance(required, dict):
for k, v in required.items():
if not k in cfg:
return False
if not check_config(required[k], cfg[k]):
return False
else:
for k in required:
if not k in cfg:
return False
return True
# return all([ self.config.get(x, None) is not None
# for x in getattr(self, "REQUIRED_CONFIG", [])
return check_config(getattr(self, "REQUIRED_CONFIG", []), self.config)
def get_source(self, selection, **kwargs):
sources = sorted(selection.sources,
key=lambda s: getattr(s, "rank", 0))
if not isinstance(sources, list):
sources = [sources]
return sources
def play_args(self, selection, **kwargs):
source = self.get_source(selection, **kwargs)
return (source, kwargs)
def filter_args(self):
return {f: self.filters[f].value for f in self.filters}
def extract_sources(self, listing, **kwargs):
try:
sources, kwargs = self.play_args(listing, **kwargs)
kwargs.update({
k: v
for k, v in self.filter_args().items() if k not in kwargs
})
except SGStreamNotFound as e:
logger.error(f"stream not found: {e}")
return
# FIXME: For now, we just throw playlists of media items at the default
# player program and hope it can handle all of them.
player_spec = None
downloader_spec = None
if not isinstance(sources, list):
sources = [sources]
for s in sources:
if not s.media_type:
# Try to set the content types of the source(s) with a HTTP HEAD
# request if the provider didn't specify one.
s.media_type = self.session.head(
s.locator).headers.get("Content-Type").split("/")[0]
return sources, kwargs
def create_play_task(self, listing, **kwargs):
sources, kwargs = self.extract_sources(listing, **kwargs)
media_types = set([s.media_type for s in sources if s.media_type])
player_spec = {"media_types": media_types}
if media_types == {"image"}:
downloader_spec = {None: None}
else:
downloader_spec = (getattr(self.config, "helpers", None)
or getattr(sources[0], "helper", None)
or self.helper)
return ListingsPlayMediaTask.attr_class(provider=self.NAME,
title=listing.title,
listing=listing,
sources=sources,
args=(player_spec,
downloader_spec),
kwargs=kwargs)
@property
def limit(self):
return None
def refresh(self):
self.view.refresh()
def reset(self):
self.view.reset()
def __str__(self):
return self.NAME
def __repr__(self):
return f"<{type(self)}: {self.NAME}>"
@property
def playlist_title(self):
return f"[{self.IDENTIFIER}"
@property
def auto_preview_enabled(self):
return not self.config.auto_preview.disabled
@property
def auto_preview_default(self):
return self.config.auto_preview.default if self.auto_preview_enabled else "default"
@property
def strip_emoji(self):
return (self.config.get("strip_emoji")
or config.settings.profile.tables.get("strip_emoji") or False)
@property
def translate(self):
return (self.config.get("translate")
or config.settings.profile.tables.get("translate") or False)
@property
def translate_src(self):
return "auto"
@property
def translate_dest(self):
return (self.config.get("translate_dest")
or config.settings.profile.tables.get("translate_dest")
or "en")
@property
def output_path(self):
return (self.config.get_path("output.path")
or config.settings.profile.get_path("output.path"))
class CachedFeedProvider(BackgroundTasksMixin, TabularProviderMixin,
FeedProvider):
UPDATE_INTERVAL = (60 * 60 * 4)
RATE_LIMIT = 5
BURST_LIMIT = 5
DEFAULT_FETCH_LIMIT = 50
TASKS = [
# ("update", UPDATE_INTERVAL, [], {"force": True})
("update", UPDATE_INTERVAL)
]
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.search_filter = None
self.items_query = None
self.custom_filters = AttrDict()
urwid.connect_signal(self.view, "feed_change", self.on_feed_change)
urwid.connect_signal(self.view, "feed_select", self.on_feed_select)
self.filters["status"].connect("changed", self.on_status_change)
self.filters["filters"].connect("changed", self.on_custom_change)
self.pagination_cursor = None
self.limiter = get_limiter(rate=self.RATE_LIMIT,
capacity=self.BURST_LIMIT)
self.listing_lock = asyncio.Lock()
@property
def VIEW(self):
return FeedProviderView(
self,
CachedFeedProviderBodyView(self,
CachedFeedProviderDataTable(self)))
def init_config(self):
super().init_config()
if config.settings.profile.cache.max_age > 0:
with db_session(optimistic=False):
FeedMediaChannel.purge_all(
min_items=config.settings.profile.cache.min_items,
max_items=config.settings.profile.cache.max_items,
max_age=config.settings.profile.cache.max_age)
def format_feed(feed):
return feed.name if hasattr(feed, "name") else ""
ATTRIBUTES = AttrDict(
media_listing_id={"hide": True},
feed={
"width": 30,
"format_fn": format_feed
},
created={"width": 19},
title={
"width": ("weight", 1),
"truncate": True
},
)
# @property
# def ATTRIBUTES(self):
# def format_feed(feed):
# return feed.name if hasattr(feed, "name") else ""
# return AttrDict(
# media_listing_id = {"hide": True},
# feed = {"width": 32, "format_fn": format_feed },
# created = {"width": 19},
# title = {"width": ("weight", 1), "truncate": False},
# )
# @property
# def RPC_METHODS(self):
# return [
# ("mark_items_read", self.mark_items_read)
# ]
@property
def status(self):
return self.filters["status"].value
# @property
# def search_string(self):
# return self.filters["search"].value
@property
def feeds(self):
locators = [c.locator for c in self.view.selected_channels]
with db_session:
return [
self.FEED_CLASS.get(locator=locator) for locator in locators
]
@property
def fetch_limit(self):
return self.config.fetch_limit or self.DEFAULT_FETCH_LIMIT
@property
def translate(self):
return (self.translate_src and super().translate)
def create_feeds(self):
all_channels = list(self.view.all_channels)
with db_session:
for channel in all_channels:
feed = self.FEED_CLASS.get(provider_id=self.IDENTIFIER,
locator=channel.locator)
if not feed:
feed = self.FEED_CLASS(provider_id=self.IDENTIFIER,
locator=channel.locator)
feed.name = channel.name
feed.attrs.update(channel.attrs)
for channel in self.FEED_CLASS.select():
if channel.locator not in [c.get_key() for c in all_channels]:
self.FEED_CLASS[channel.channel_id].delete()
def feed_attrs(self, feed_name):
return {}
@property
def feed_filters(self):
return None
def on_feed_change(self, source, selection):
self.provider_data["selected_feed"] = [n.identifier for n in selection]
self.save_provider_data()
def on_feed_select(self, source, selection):
if not self.is_active:
return
self.update_count = True
self.reset()
def on_status_change(self, status, *args):
with db_session:
self.provider_data["selected_status"] = status
self.save_provider_data()
if not self.is_active:
return
self.reset()
def on_custom_change(self, custom, *args):
logger.info(f"{custom}, {args}")
self.custom_filters = custom
self.reset()
def open_popup(self, text):
class UpdateMessage(BasePopUp):
def __init__(self):
self.text = urwid.Text(text, align="center")
self.filler = urwid.Filler(self.text)
super().__init__(self.filler)
def selectable(self):
return False
self.message = UpdateMessage()
self.view.open_popup(self.message, width=24, height=5)
def close_popup(self):
self.view.close_popup()
async def update(self, force=False, resume=False, replace=False):
logger.info(f"update: force={force} resume={resume}")
state.loop.draw_screen()
# self.open_popup("Updating feeds...")
# asyncio.create_task(
await self.update_feeds(force=force, resume=resume, replace=replace)
# )
# self.close_popup()
self.reset()
# update_task = state.event_loop.run_in_executor(None, update_feeds)
async def update_feeds(self, force=False, resume=False, replace=False):
logger.info(f"update_feeds: {force} {resume} {replace}")
with db_session:
# feeds = select(f for f in self.FEED_CLASS if f in self.selected_channels)
# if not self.feed_entity:
# feeds = self.FEED_CLASS.select()
# else:
# feeds = [self.feed_entity]
for feed in self.selected_channels:
if (force or feed.updated is None
or datetime.now() - feed.updated >
timedelta(seconds=feed.update_interval)):
logger.info(f"updating {feed.locator}")
with limit(self.limiter):
await feed.update(resume=resume, replace=replace)
# f.updated = datetime.now()
# commit()
def refresh(self):
logger.info("+feed provider refresh")
# self.update_query()
self.view.refresh()
# state.loop.draw_screen()
logger.info("-feed provider refresh")
def reset(self):
logger.info("provider reset")
self.pagination_cursor = None
self.update_query()
super().reset()
def on_activate(self):
super().on_activate()
self.create_feeds()
# self.reset()
# def on_deactivate(self):
# if self.view.player:
# self.view.quit_player()
# super().on_deactivate()
@property
def total_item_count(self):
with db_session:
return self.all_items_query.count()
@property
def feed_item_count(self):
with db_session:
return self.feed_items_query.count()
@db_session
def update_query(self, sort=None, cursor=None):
logger.info(f"update_query: {cursor}")
status_filters = {
"all": lambda: True,
"unread": lambda i: i.read is None,
"not_downloaded": lambda i: i.downloaded is None
}
self.all_items_query = (self.LISTING_CLASS.select())
if self.selected_channels:
self.feed_items_query = self.all_items_query.filter(
lambda i: i.channel in self.selected_channels)
else:
self.feed_items_query = self.all_items_query
self.items_query = self.feed_items_query
if self.feed_filters:
for f in self.feed_filters:
self.items_query = self.items_query.filter(f)
self.items_query = self.items_query.filter(
status_filters[self.filters.status.value])
if self.search_filter:
(field, query) = re.search("(?:(\w+):)?(.*)",
self.search_filter).groups()
if field and field in [a.name for a in self.LISTING_CLASS._attrs_]:
self.items_query = self.items_query.filter(
lambda i: getattr(i, field) == query)
else:
# raise Exception(query, self.pagination_cursor)
self.items_query = self.items_query.filter(
lambda i: query.lower() in i.title.lower())
if self.custom_filters:
for k, v in self.custom_filters.items():
self.items_query = self.items_query.filter(
lambda i: v in getattr(i, k))
(sort_field, sort_desc) = sort if sort else self.view.sort_by
if cursor:
op = "<" if sort_desc else ">"
self.items_query = self.items_query.filter(
raw_sql(f"{sort_field} {op} '{cursor}'"))
if sort_field:
if sort_desc:
sort_fn = lambda i: desc(getattr(i, sort_field))
else:
sort_fn = lambda i: getattr(i, sort_field)
# break ties with primary key to ensure pagination cursor is correct
pk_sort_attr = self.LISTING_CLASS._pk_
if sort_desc:
pk_sort_attr = desc(pk_sort_attr)
# self.items_query = self.items_query.order_by(pk_sort_attr)
self.items_query = self.items_query.order_by(sort_fn)
# logger.info(self.items_query.get_sql())
self.view.update_count = True
async def apply_search_query(self, query):
self.pagination_cursor = None
self.search_filter = query
self.update_query()
self.reset()
def update_fetch_indicator(self, num):
self.view.update_fetch_indicator(num, self.fetch_limit)
def show_message(self, message):
self.view.show_message(message)
def listings(self,
sort=None,
cursor=None,
offset=None,
limit=None,
*args,
**kwargs):
count = 0
# cursor = None
if not offset:
offset = 0
if not limit:
limit = self.limit
with db_session(optimistic=False):
self.update_query(sort=sort, cursor=cursor)
for listing in self.items_query[:limit]:
sources = [
source.detach() for source in
listing.sources.select().order_by(lambda s: s.rank)
]
listing = listing.detach()
listing.channel = listing.channel.detach()
listing.channel.listings = None
listing.sources = sources
# get last item's sort key and store it as our pagination cursor
# cursor = getattr(listing, self.view.sort_by[0])
# if not listing.check():
# logger.debug("listing broken, fixing...")
# listing.refresh()
# # have to force a reload here since sources may have changed
# listing = listing.attach().detach()
yield listing
self.update_query(sort=sort, cursor=cursor)
self.pagination_cursor = cursor
# self.update_query(cursor, sort=sort)
@db_session
async def mark_items_read(self, request):
media_listing_ids = list(set(request.params))
logger.info(f"mark_items_read: {media_listing_ids}")
with db_session:
try:
for item in self.LISTING_CLASS.select(
lambda i: i.media_listing_id in media_listing_ids):
item.read = datetime.now()
commit()
self.reset()
except pony.orm.core.ObjectNotFound:
logger.info(
f("mark_item_read: item {media_listing_id} not found"))
@property
def playlist_title(self):
return f"[{self.IDENTIFIER}]"
