def stepFrame(self, framerate, frames_offset):
"""
Seek backwards or forwards a certain amount of frames (frames_offset).
This clamps the playhead to the project frames.
"""
cur_frame = int(round(self.getPosition() * framerate.num / float(Gst.SECOND * framerate.denom), 2))
new_frame = cur_frame + frames_offset
new_pos = long(new_frame * Gst.SECOND * framerate.denom / framerate.num)
Loggable.info(self, "From frame %d to %d at %f fps, seek to %s s" % (cur_frame,
new_frame, framerate.num / framerate.denom,
new_pos / float(Gst.SECOND)))
self.simple_seek(new_pos)
def test_basic_progress_bar(self):
log = Loggable("test")
for x in tqdm(range(10)):
log.info(x)
time.sleep(0.01)
log.info("This is more information")
class EdgeWeightContainer:
def __init__(self, browser, edge_hash, node_hash, plans, plan_ids=None):
"""EdgeCalculator initializer.
:param browser: the Browser object
:type browser: Browser
:param edge_hash: The edge hashing function. Should take exactly 2 arguments.
:type edge_hash: function
:param node_hash: The node hashing function. Should take exactly 1 argument.
:type node_hash: function
:param plans: optional list of plans
:type plans: list
"""
self.browser = browser
self.log = Loggable(
self, "EdgeWeightContainer({})".format(self.browser.session)
)
# filter only those plans that have operations
self._plans = []
if plan_ids is not None:
self._plan_ids = plan_ids
else:
self._plan_ids = []
if plans is not None:
self.plans = plans
def new_edge_hash(pair):
h = edge_hash(pair)
return "{}_{}_{}".format(
pair[0].field_type.parent_id, h, pair[1].field_type.parent_id
)
self.edges = []
self._edge_counter = HashCounter(new_edge_hash)
self._node_counter = HashCounter(node_hash)
self._weights = {}
self.created_at = str(arrow.now())
self.updated_at = None
self.is_cached = False
@property
def plans(self):
return self._plans
@plans.setter
def plans(self, new_plans):
self._plans = new_plans
self._plan_ids = [p.id for p in new_plans]
def _was_updated(self):
self.updated_at = str(arrow.now())
def reset(self):
self.is_cached = False
self._edge_counter.clear()
self._node_counter.clear()
def recompute(self):
"""Reset the counters and recompute weights."""
self._edge_counter.clear()
self._node_counter.clear()
return self.compute()
def update(self, plans, only_unique=False):
if only_unique:
existing_plan_ids = [p.id for p in self.plans]
plan_ids = [p.id for p in plans]
unique_plans = set(plan_ids).difference(existing_plan_ids)
num_ignored = len(plans) - len(unique_plans)
plans = list(unique_plans)
self.log.info("Ignoring {} existing plans".format(num_ignored))
self.log.info("Updating edge counter with {} new plans".format(len(plans)))
self.cache_plans()
wires = self.collect_wires()
self.log.info(" {} wires loaded".format(len(wires)))
operations = self.collect_operations()
self.log.info(" {} operations loaded".format(len(operations)))
edges = self.to_edges(wires, operations)
self.update_tally(edges)
self.plans += plans
self.edges += edges
self.save_weights(self.edges)
def compute(self):
"""Compute the weights.
If previously computed, this function will avoid re-caching
plans and wires.
"""
self.log.info("Computing weights for {} plans".format(len(self.plans)))
self._was_updated()
if not self.is_cached:
self.cache_plans()
else:
self.log.info(" Plans already cached. Skipping...")
wires = self.collect_wires()
self.log.info(" {} wires loaded".format(len(wires)))
operations = self.collect_operations()
self.log.info(" {} operations loaded".format(len(operations)))
self.is_cached = True
self.edges = self.to_edges(wires, operations)
self.update_tally(self.edges)
self.save_weights(self.edges)
def cache_plans(self):
self.log.info(" Caching plans...")
self.browser.get("Plan", {"operations": {"field_values"}})
self.browser.get("Wire", {"source", "destination"})
self.browser.get("FieldValue", {"allowable_field_type": "field_type"})
self.browser.get("Operation", "operation_type")
def collect_wires(self):
return self.browser.get("Wire")
def collect_operations(self):
return self.browser.get("Operation")
@staticmethod
def to_edges(wires, operations):
"""Wires and operations to a list of edges."""
edges = []
for wire in wires: # external wires
if wire.source and wire.destination:
edges.append(
(
wire.source.allowable_field_type,
wire.destination.allowable_field_type,
)
)
for op in operations: # internal wires
for i in op.inputs:
for o in op.outputs:
edges.append((i.allowable_field_type, o.allowable_field_type))
# due to DB inconsitencies, some wires and operations have not AFTS
edges = [(n1, n2) for n1, n2 in edges if n1 is not None and n2 is not None]
return edges
def update_tally(self, edges):
self.log.info("Hashing and counting edges...")
for n1, n2 in tqdm(edges, desc="counting edges"):
if n1 and n2:
self._edge_counter[(n1, n2)] += 1
self._node_counter[n2] += 1
def save_weights(self, edges):
for n1, n2 in edges:
if n1 and n2:
self._weights[self._edge_counter.hash_function((n1, n2))] = self.cost(
n1, n2
)
def cost(self, n1, n2):
n = self._edge_counter[(n1, n2)] * 1.0
t = self._node_counter[n1] * 1.0
return self.cost_function(n, t)
def cost_function(self, n, t):
n = max(n, 0)
t = max(t, 0)
p = 10e-6
if t > 0:
p = n / t
w = (1 - p) / (1 + p)
return 10 / (1.000001 - w)
def get_weight(self, n1, n2):
if not self.is_cached:
raise AutoPlannerException("The tally and weights have not been computed")
ehash = self._edge_counter.hash_function((n1, n2))
return self._weights.get(ehash, self.cost(n1, n2))
def copy(self):
return self.__copy__()
def __copy__(self):
new = self.__class__(self.browser, None, None, self.plans)
new._edge_counter = self._edge_counter.copy()
new._node_counter = self._node_counter.copy()
new.is_cached = self.is_cached
return new
def __add__(self, other):
new = self.copy()
new._edge_counter = self._edge_counter + other._edge_counter
new._node_counter = self._node_counter + other._node_counter
new._was_updated()
return new
def __sub__(self, other):
new = self.copy()
new._edge_counter = self._edge_counter - other._edge_counter
new._node_counter = self._node_counter - other._node_counter
# make sure there are no negative values
for k, v in new._edge_counter.counter:
new._edge_counter.counter[k] = max(v, 0)
for k, v in new._node_counter.counter:
new._node_counter.counter[k] = max(v, 0)
new._was_updated()
return new
def __mul__(self, num):
new = self.copy()
new._edge_counter = self._edge_counter * num
new._node_counter = self._node_counter * num
new._was_updated()
return new
def __getstate__(self):
return {
"_edge_counter": self._edge_counter,
"_node_counter": self._node_counter,
"plan_ids": self._plan_ids,
"is_cached": self.is_cached,
"_weights": self._weights,
"updated_at": self.updated_at,
"created_at": self.created_at,
"edges": self.edges,
}
def __setstate__(self, state):
self._edge_counter = state["_edge_counter"]
self._node_counter = state["_node_counter"]
self.edges = state["edges"]
self._plans = []
self._plan_ids = state["plan_ids"]
self.is_cached = state["is_cached"]
self._weights = state["_weights"]
self.updated_at = state["updated_at"]
self.created_at = state["created_at"]
# self.browser = state['browser']
self.log = Loggable(self)
class AutoPlannerModel:
"""Builds a model from historical plan data."""
EXCLUDE_FILTER = "exclude"
FILTER_MODEL_CLASS = "model_class"
FILTER_FUNCTION = "function"
VALID_FILTERS = [EXCLUDE_FILTER]
def __init__(self, browser, plans=None, name=None):
self.browser = browser
self.log = Loggable(
self, "AutoPlanner@{url}".format(url=self.browser.session.url)
)
if plans:
self.plans = plans
self.browser.update_cache(plans)
self.weight_container = EdgeWeightContainer(
self.browser, self._hash_afts, self._external_aft_hash, plans=plans
)
self._template_graph = None
self.model_filters = {}
if name is None:
name = "unnamed_{}".format(id(self))
self.name = name
self._version = __version__
self.created_at = str(arrow.now())
self.updated_at = str(arrow.now())
def info(self):
return {
"name": self.name,
"version": self.version,
"created_at": self.created_at,
"updated_at": self.updated_at,
"has_template_graph": self._template_graph is not None,
"num_plans": len(self.weight_container._plan_ids),
}
def print_info(self):
print(json.dumps(self.info(), indent=2))
# TODO: method for printing statistics and plots for the model
def plots(self):
raise NotImplementedError()
@property
def version(self):
return self._version
def set_plans(self, plans):
self.weight_container.plans = plans
self._template_graph = None
@staticmethod
def _external_aft_hash(aft):
"""A has function representing two 'external'.
:class:`pydent.models.AllowableFieldType` models (i.e. a wire)
"""
if not aft.field_type:
return str(uuid4())
if aft.field_type.part:
part = True
else:
part = False
return "{object_type}-{sample_type}-{part}".format(
object_type=aft.object_type_id, sample_type=aft.sample_type_id, part=part
)
@staticmethod
def _internal_aft_hash(aft):
"""A has function representing two 'internal'.
:class:`pydent.models.AllowableFieldType` models (i.e. an operation)
"""
return "{operation_type}".format(
operation_type=aft.field_type.parent_id,
routing=aft.field_type.routing,
sample_type=aft.sample_type_id,
)
@classmethod
def _hash_afts(cls, pair):
"""Make a unique hash for a :class:`pydent.models.AllowableFieldType`
pair."""
source_hash = cls._external_aft_hash(pair[0])
dest_hash = cls._external_aft_hash(pair[1])
return "{}->{}".format(source_hash, dest_hash)
def _cache_afts(self):
"""Cache :class:`AllowableFieldType`"""
ots = self.browser.where({"deployed": True}, "OperationType")
self.log.info(
"Caching all AllowableFieldTypes from {} deployed operation types".format(
len(ots)
)
)
results = self.browser.recursive_retrieve(
ots,
{
"field_types": {
"allowable_field_types": {
"object_type": [],
"sample_type": [],
"field_type": [],
}
}
},
strict=False,
)
fts = [ft for ft in results["field_types"] if ft.ftype == "sample"]
inputs = [ft for ft in fts if ft.role == "input"]
outputs = [ft for ft in fts if ft.role == "output"]
input_afts = []
for i in inputs:
for aft in i.allowable_field_types:
if aft not in input_afts:
input_afts.append(aft)
output_afts = []
for o in outputs:
for aft in o.allowable_field_types:
if aft not in output_afts:
output_afts.append(aft)
return input_afts, output_afts
@classmethod
def _match_internal_afts(
cls, input_afts: List[AllowableFieldType], output_afts: List[AllowableFieldType]
) -> Tuple[AllowableFieldType, AllowableFieldType]:
internal_groups = {}
for aft in output_afts:
internal_groups.setdefault(cls._internal_aft_hash(aft), []).append(aft)
edges = []
for iaft in input_afts:
hsh = cls._internal_aft_hash(iaft)
internals = internal_groups.get(hsh, [])
for aft in internals:
edges.append((iaft, aft))
return edges
@classmethod
def _match_external_afts(
cls, input_afts: List[AllowableFieldType], output_afts: List[AllowableFieldType]
) -> Tuple[AllowableFieldType, AllowableFieldType]:
external_groups = {}
for aft in input_afts:
external_groups.setdefault(cls._external_aft_hash(aft), []).append(aft)
edges = []
for oaft in output_afts:
hsh = cls._external_aft_hash(oaft)
externals = external_groups.get(hsh, [])
for aft in externals:
edges.append((oaft, aft))
return edges
@classmethod
def _match_afts(
cls, input_afts: List[AllowableFieldType], output_afts: List[AllowableFieldType]
) -> Tuple[AllowableFieldType, AllowableFieldType]:
return cls._match_internal_afts(
input_afts, output_afts
) + cls._match_external_afts(input_afts, output_afts)
def _get_aft_pairs(self):
"""Construct edges from all deployed allowable_field_types.
:return: list of tuples representing connections between AllowableFieldType
:rtype: list
"""
input_afts, output_afts = self._cache_afts()
return self._match_afts(input_afts, output_afts)
def _excluded_nodes(self):
graph = self._template_graph
excluded_nodes = set()
if self.EXCLUDE_FILTER in self.model_filters:
for f in self.model_filters[self.EXCLUDE_FILTER]:
excluded_nodes.update(
graph.select_nodes(
model_class=f[self.FILTER_MODEL_CLASS],
key=f[self.FILTER_FUNCTION],
)
)
return excluded_nodes
@property
def template_graph(self):
if self._template_graph is None:
self.construct_template_graph()
graph = self._template_graph
excluded = self._excluded_nodes()
return graph.difference(excluded)
def add_model_filter(self, model_class: str, filter_type: str, func: callable):
if filter_type not in self.VALID_FILTERS:
raise ValueError(
"Filter type '{}' not recognized. Select from {}".format(
filter_type, self.VALID_FILTERS
)
)
self.model_filters.setdefault(filter_type, []).append(
{self.FILTER_FUNCTION: func, self.FILTER_MODEL_CLASS: model_class}
)
def exclude_operation_types(self, operation_types: List[OperationType]):
def is_blacklisted(aft):
return aft.field_type.parent_id in [ot.id for ot in operation_types]
self.add_model_filter("AllowableFieldType", self.EXCLUDE_FILTER, is_blacklisted)
return self
def exclude_categories(self, categories: List[str]):
ots = self.browser.session.Operation.where({"category": categories})
return self.exclude_operation_type(ots)
def reset_model_filters(self):
self.model_filters = []
def update_weights(
self, graph: BrowserGraph, weight_container: EdgeWeightContainer
):
for aft1, aft2 in self._get_aft_pairs():
edge_type = None
roles = [aft.field_type.role for aft in [aft1, aft2]]
if roles == ("input", "output"):
edge_type = "internal"
elif roles == ("output", "input"):
edge_type = "external"
graph.add_edge_from_models(
aft1,
aft2,
edge_type=edge_type,
weight=weight_container.get_weight(aft1, aft2),
)
def build(self):
"""Construct a graph of all possible Operation connections."""
# computer weights
self.weight_container.compute()
self.log.info("Building Graph:")
G = BrowserGraph(self.browser)
self.update_weights(G, self.weight_container)
self.log.info(" {} edges".format(len(list(G.edges()))))
self.log.info(" {} nodes".format(len(G)))
self._template_graph = G
self._was_updated()
return self
def _collect_afts(
self, graph: BrowserGraph
) -> Tuple[List[AllowableFieldType], List[AllowableFieldType]]:
"""Collect :class:`pydent.models.AllowableFieldType` models from graph.
:param graph: a browser graph
:type graph: BrowserGraph
:return: list of tuples of input vs output allowable field types in the graph
:rtype: list
"""
afts = graph.models("AllowableFieldType")
input_afts = [aft for aft in afts if aft.field_type.role == "input"]
output_afts = [aft for aft in afts if aft.field_type.role == "output"]
return input_afts, output_afts
def print_path(self, path: List[str], graph: BrowserGraph):
ots = []
for n, ndata in graph.iter_model_data("AllowableFieldType", nbunch=path):
aft = ndata["model"]
ot = self.browser.find(aft.field_type.parent_id, "OperationType")
ots.append("{ot} in '{category}'".format(category=ot.category, ot=ot.name))
edge_weights = [
graph.get_edge(x, y)["weight"] for x, y in zip(path[:-1], path[1:])
]
print("PATH: {}".format(path))
print("WEIGHTS: {}".format(edge_weights))
print("NUM NODES: {}".format(len(path)))
print("OP TYPES:\n{}".format(ots))
def search_graph(
self,
goal_sample: Sample,
goal_object_type: ObjectType,
start_object_type: ObjectType,
):
graph = self.template_graph.copy()
# filter afts
obj1 = start_object_type
obj2 = goal_object_type
# Add terminal nodes
graph.add_special_node("START", "START")
graph.add_special_node("END", "END")
for n, ndata in graph.iter_model_data(
"AllowableFieldType",
object_type_id=obj1.id,
sample_type_id=obj1.sample_type_id,
):
graph.add_edge("START", n, weight=0)
for n, ndata in graph.iter_model_data(
"AllowableFieldType",
object_type_id=obj2.id,
sample_type_id=obj2.sample_type_id,
):
graph.add_edge(n, "END", weight=0)
# find and sort shortest paths
shortest_paths = []
for n1 in graph.graph.successors("START"):
n2 = "END"
try:
path = nx.dijkstra_path(graph.graph, n1, n2, weight="weight")
path_length = nx.dijkstra_path_length(
graph.graph, n1, n2, weight="weight"
)
shortest_paths.append((path, path_length))
except nx.exception.NetworkXNoPath:
pass
shortest_paths = sorted(shortest_paths, key=lambda x: x[1])
# print the results
print()
print("*" * 50)
print("{} >> {}".format(obj1.name, obj2.name))
print("*" * 50)
print()
for path, pathlen in shortest_paths[:10]:
print(pathlen)
self.print_path(path, graph)
@SetRecusion.set_recursion_limit(10000)
def dump(self, path: str):
with open(path, "wb") as f:
dill.dump(
{
"browser": self.browser,
"template_graph": self.template_graph,
"version": self._version,
"name": self.name,
"created_at": self.created_at,
"updated_at": self.updated_at,
"weight_container": self.weight_container,
},
f,
)
statinfo = stat(path)
self.log.info("{} bytes written to '{}'".format(statinfo.st_size, path))
def save(self, path: str):
return self.dump(path)
@classmethod
@SetRecusion.set_recursion_limit(10000)
def load(cls, path: str):
with open(path, "rb") as f:
try:
data = dill.load(f)
if data["version"] != __version__:
warnings.warn(
"Version number of saved model ('{}') does not match current "
"model version ('{}')".format(data["version"], __version__)
)
browser = data["browser"]
model = cls(browser.session)
statinfo = stat(path)
model.log.info(
"{} bytes loaded from '{}' to new AutoPlanner (id={})".format(
statinfo.st_size, path, id(model)
)
)
model.browser = browser
model.weight_container = data["weight_container"]
model._template_graph = data["template_graph"]
model._version = data["version"]
model.name = data.get("name", None)
model.updated_at = data.get("updated_at", None)
model.created_at = data.get("created_at", None)
return model
except Exception as e:
raise e
msg = "An error occurred while loading an {} model:\n{}".format(
cls.__name__, str(e)
)
if "version" in data and data["version"] != __version__:
msg = (
"Version notice: This may have occurred since saved model "
"version {} does not match current version {}".format(
data["version"], __version__
)
)
raise AutoPlannerLoadingError(msg) from e
def _was_updated(self):
self.updated_at = str(arrow.now())
def copy(self):
return self.__copy__()
def __copy__(self):
new = self.__class__(self.browser, None, self.name + "_copy")
new.weight_container = self.weight_container
if self._template_graph:
new._template_graph = self._template_graph.copy()
return new
def __add__(self, other):
new = self.copy()
new.weight_container = self.weight_container + other.weight_container
if self._template_graph:
new.update_weights(self._template_graph, new.weight_container)
return new
def __mul__(self, num):
new = self.copy()
new.weight_container = self.weight_container * num
if self._template_graph:
new.update_weights(self._template_graph, new.weight_container)
return new
