def get_links(names, html):
"""
Return a SortedSet of computer scientist names that are linked from this
html page. The return set is restricted to those people in the provided
set of names. The returned list should contain no duplicates.
Params:
names....A SortedSet of computer scientist names, one per filename.
html.....A string representing one html page.
Returns:
A SortedSet of names of linked computer scientists on this html page, restricted to
elements of the set of provided names.
>>> get_links({'Gerald_Jay_Sussman'},
... '''<a href="/wiki/Gerald_Jay_Sussman">xx</a> and <a href="/wiki/Not_Me">xx</a>''')
SortedSet(['Gerald_Jay_Sussman'], key=None, load=1000)
"""
pagenames = SortedSet()
for link in BeautifulSoup(html, "html.parser", parse_only=SoupStrainer('a')):
if link.has_attr('href'):
name = link['href'].split('/')[-1]
if name in names:
pagenames.add(name)
return pagenames
def test_copy():
temp = SortedSet(range(100))
temp._reset(7)
that = temp.copy()
that.add(1000)
assert len(temp) == 100
assert len(that) == 101
def test_delitem_slice():
vals = list(range(100))
temp = SortedSet(vals)
temp._reset(7)
del vals[20:40:2]
del temp[20:40:2]
assert temp == set(vals)
def get_links(names, html):
"""
Return a SortedSet of computer scientist names that are linked from this
html page. The return set is restricted to those people in the provided
set of names. The returned list should contain no duplicates.
Params:
names....A SortedSet of computer scientist names, one per filename.
html.....A string representing one html page.
Returns:
A SortedSet of names of linked computer scientists on this html page, restricted to
elements of the set of provided names.
>>> get_links({'Gerald_Jay_Sussman'},
... '''<a href="/wiki/Gerald_Jay_Sussman">xx</a> and <a href="/wiki/Not_Me">xx</a>''')
SortedSet(['Gerald_Jay_Sussman'], key=None, load=1000)
"""
soup = BeautifulSoup(html,"html.parser")
list = [l['href'] for l in soup.find_all('a') if l.get('href')]
res = SortedSet()
for l in list:
if l.startswith('/wiki/'):
tokens = l.split('/')
if tokens[2] in names:
res.add(tokens[2])
return res
def get_links(names, html):
"""
Return a SortedSet of computer scientist names that are linked from this
html page. The return set is restricted to those people in the provided
set of names. The returned list should contain no duplicates.
Params:
names....A SortedSet of computer scientist names, one per filename.
html.....A string representing one html page.
Returns:
A SortedSet of names of linked computer scientists on this html page, restricted to
elements of the set of provided names.
>>> get_links({'Gerald_Jay_Sussman'},
... '''<a href="/wiki/Gerald_Jay_Sussman">xx</a> and <a href="/wiki/Not_Me">xx</a>''')
SortedSet(['Gerald_Jay_Sussman'], key=None, load=1000)
"""
###TODO
found = SortedSet()
bs = BeautifulSoup(html, "html.parser")
for link in bs.find_all('a'):
if link.get('href'):
href = link.get('href')
nl = re.split('|'.join(['/', '=', '&', ':', '/d+_']), href)
cleared_nl = [n for n in nl if n != '']
found = found.union([name for name in names if name in cleared_nl])
return found
pass
def test_union():
temp = SortedSet(range(0, 50), load=7)
that = SortedSet(range(50, 100), load=9)
result = temp.union(that)
assert all(result[val] == val for val in range(100))
assert all(temp[val] == val for val in range(50))
assert all(that[val] == (val + 50) for val in range(50))
def test_ne():
alpha = SortedSet(range(100), load=7)
beta = SortedSet(range(99), load=17)
assert alpha != beta
beta.add(100)
assert alpha != beta
assert alpha != beta._set
def test_eq():
alpha = SortedSet(range(100), load=7)
beta = SortedSet(range(100), load=17)
assert alpha == beta
assert alpha == beta._set
beta.add(101)
assert not (alpha == beta)
def __init__(self, root_path):
self.code_graph = nx.MultiDiGraph()
self.metrics = {}
self.root_path = Path(root_path)
self.root_arch_ids = []
self.entity_kinds = SortedSet()
self.ref_kinds = SortedSet()
def test_pickle():
import pickle
alpha = SortedSet(range(10000), key=negate)
alpha._reset(500)
data = pickle.dumps(alpha)
beta = pickle.loads(data)
assert alpha == beta
assert alpha._key == beta._key
def test_symmetric_difference():
temp = SortedSet(range(0, 75), load=7)
that = SortedSet(range(25, 100), load=9)
result = temp.symmetric_difference(that)
assert all(result[val] == val for val in range(25))
assert all(result[val + 25] == (val + 75) for val in range(25))
assert all(temp[val] == val for val in range(75))
assert all(that[val] == (val + 25) for val in range(75))
def test_delitem_key():
temp = SortedSet(range(100), key=modulo)
temp._reset(7)
values = sorted(range(100), key=modulo)
for val in range(10):
del temp[val]
del values[val]
assert list(temp) == list(values)
def test_copy_copy():
import copy
temp = SortedSet(range(100))
temp._reset(7)
that = copy.copy(temp)
that.add(1000)
assert len(temp) == 100
assert len(that) == 101
class DictGraph(Graph):
"""Graph that supports nonconsecutive vertex ids."""
def __init__(self, nodes: Set[int]=None, r: int=1) -> None:
"""Make a new graph."""
if nodes is None:
self.nodes = SortedSet() # type: Set[int]
else:
self.nodes = nodes
self.radius = r
self.inarcs_by_weight = [defaultdict(SortedSet) for _ in range(self.radius)]
def __len__(self):
"""len() support."""
return len(self.nodes)
def __iter__(self):
"""Iteration support."""
return iter(self.nodes)
def __contains__(self, v):
"""Support for `if v in graph`."""
return v in self.nodes
def add_node(self, u: int):
"""Add a new node."""
self.nodes.add(u)
def arcs(self, weight: int=None):
"""
Return all the arcs in the graph.
restrict to a given weight when provided
"""
if weight:
return [(x, y) for x in self.nodes
for y in self.inarcs_by_weight[weight-1][x]]
else:
return [(x, y, w+1) for w, arc_list in
enumerate(self.inarcs_by_weight)
for x in self.nodes
for y in arc_list[x]]
def remove_isolates(self) -> List[int]:
"""
Remove all isolated vertices and return a list of vertices removed.
Precondition: the graph is bidirectional
"""
isolates = [v for v in self if self.in_degree(v, 1) == 0]
for v in isolates:
self.nodes.remove(v)
for N in self.inarcs_by_weight:
if v in N:
del N[v]
return isolates
def test_add():
temp = SortedSet(range(100))
temp._reset(7)
temp.add(100)
temp.add(90)
temp._check()
assert all(val == temp[val] for val in range(101))
def get_links(names, html):
"""
Return a SortedSet of computer scientist names that are linked from this
html page. The return set is restricted to those people in the provided
set of names. The returned list should contain no duplicates.
Params:
names....A SortedSet of computer scientist names, one per filename.
html.....A string representing one html page.
Returns:
A SortedSet of names of linked computer scientists on this html page, restricted to
elements of the set of provided names.
>>> get_links({'Gerald_Jay_Sussman'},
... '''<a href="/wiki/Gerald_Jay_Sussman">xx</a> and <a href="/wiki/Not_Me">xx</a>''')
SortedSet(['Gerald_Jay_Sussman'], key=None, load=1000)
"""
###TODO
#remove BeautifulSoap - later
listofHrefs = []
listofHrefTexts = []
FinalSortedSet = SortedSet()
splice_char = '/'
#getting all the tags using the BeautifulSoup
#soup = BeautifulSoup(html, "html.parser")
#fectching all the links in anchor tags
#for link in soup.find_all('a'):
#listofHrefs.append(link.get('href'))
for i in range(0,len(listofHrefs)):
value = listofHrefs[i][6:]
listofHrefTexts.append(value)
listofHrefTexts = re.findall(r'href="([^"]*)', html)
#print(listofHrefTexts)
for i in listofHrefTexts:
#print(i)
value = i[6:]
listofHrefs.append(value)
#print(listofHrefs)
listofHrefs = list(set(listofHrefs))
#print(len(listofHrefs))
for href in listofHrefs:
for name in names:
#windows OS handling
if(name == "Guy_L._Steele,_Jr"):
names.remove(name)
names.add("Guy_L._Steele,_Jr.")
if(href == name):
FinalSortedSet.add(name)
return FinalSortedSet
pass
def __init__(self, software_system, logical_name, size):
"""
Constructs a new feature specification, initially with no corresponding chunks in the system.
"""
self._logical_name = logical_name
self.software_system = software_system
self.size = size
self.chunks = SortedSet(key=lambda c: c.logical_name)
self.tests = SortedSet(key=lambda t: t.logical_name)
def __init__(self, logical_name, feature, local_content=None):
self.logical_name = logical_name
self.feature = feature
self.local_content = local_content
self.dependencies = SortedSet(key=lambda d: d.fully_qualified_name)
self.bugs = SortedSet(key=lambda b: b.logical_name)
self.bug_count = 0
def compute_domset(graph: Graph, radius: int):
"""
Compute a d-dominating set using Dvorak's approximation algorithm
for dtf-graphs (see `Structural Sparseness and Complex Networks').
Graph needs a distance-d dtf augmentation (see rdomset() for usage).
"""
domset = SortedSet()
infinity = float('inf')
# minimum distance to a dominating vertex, obviously infinite at start
domdistance = defaultdict(lambda: infinity) # type: Dict[int, float]
# counter that keeps track of how many neighbors have made it into the
# domset
domcounter = defaultdict(int) # type: Dict[int, int]
# cutoff for how many times a vertex needs to have its neighbors added to
# the domset before it does. We choose radius^2 as a convenient "large"
# number
c = (2*radius)**2
# Sort the vertices by indegree so we take fewer vertices
order = sorted([v for v in graph], key=lambda x: graph.in_degree(x),
reverse=True)
# vprops = [(v,graph.in_degree(v)) for v in nodes]
# vprops.sort(key=itemgetter(1),reverse=False)
# order = map(itemgetter(0),vprops)
for v in order:
# look at the in neighbors to update the distance
for r in range(1, radius + 1):
for u in graph.in_neighbors(v, r):
domdistance[v] = min(domdistance[v], r+domdistance[u])
# if v is already dominated at radius, no need to work
if domdistance[v] <= radius:
continue
# if v is not dominated at radius, put v in the dominating set
domset.add(v)
domdistance[v] = 0
# update distances of neighbors of v if v is closer if u has had too
# many of its neighbors taken into the domset, include it too.
for r in range(1, graph.radius + 1):
for u in graph.in_neighbors(v, r):
domcounter[u] += 1
domdistance[u] = min(domdistance[u], r)
if domcounter[u] > c and u not in domset:
# add u to domset
domset.add(u)
domdistance[u] = 0
for x, rx in graph.in_neighbors(u):
domdistance[x] = min(domdistance[x], rx)
# only need to update domdistance if u didn't get added
return domset
def test_discard():
temp = SortedSet(range(100), load=7)
temp.discard(0)
temp.discard(99)
temp.discard(50)
temp.discard(1000)
temp._check()
assert len(temp) == 97
def test_count():
temp = SortedSet(range(100), load=7)
assert all(temp.count(val) == 1 for val in range(100))
assert temp.count(100) == 0
assert temp.count(0) == 1
temp.add(0)
assert temp.count(0) == 1
temp._check()
class Server:
_ids = 0
def __init__(self):
self.queue = SortedSet(key = lambda job: job.arrivalTime)
self.numServers = 1
Server._ids +=1
self.busyServers = 0
self.serviceTimeDistribution = None
self.name = "Server {}".format(Server._ids)
self.In = None
self.Out = None
self.scheduler = None
def receive(self, m):
if m.event == "end": # end of service
self.send(m.job)
if len(self.queue) > 0:
assert self.busyServers == self.numServers
job = self.queue.pop(0)
self.startService(job)
else:
assert self.busyServers > 0
self.busyServers -= 1
#self.departureStats()
else: # receive new job
assert "job" in m.event
job = m.job
job.setArrivalTime(self.scheduler.now())
serviceTime = self.serviceTimeDistribution.rvs()
job.setServiceTime(serviceTime)
job.log(self.scheduler.now(), "a", self.busyServers + len(self.queue))
if self.busyServers < self.numServers:
self.busyServers += 1
self.startService(job)
else:
self.queue.add(job)
def startService(self, job):
job.log(self.scheduler.now(), "s", len(self.queue))
t = self.scheduler.now() + job.serviceTime
m = Event(self, self, t, job = job, event = "end")
self.scheduler.add(m)
def send(self, job): # job departure
job.log(self.scheduler.now(), "d", len(self.queue))
m = Event(self, self.Out, self.scheduler.now(), job = job, event = "job")
self.scheduler.add(m)
def setServiceTimeDistribution(self, distribution):
self.serviceTimeDistribution = distribution
def __init__(self, g, prefix, port_filter, property_filter, syncpoint_run_last=True):
"""
:param prefix: Prefix for the task properties used by the implementation
to keep track of the state. If several decorators of this type
are wrapped around the same graph, then different prefixes must be used.
:param port_filter: Function to filter the relevant input ports. The function
takes three arguments:
* The graph (`self`)
* The tick of the task.
* The name of the port.
If the function returns `False` Then the port is ignored.
:param property_filter: Function that filters tasks which are collected.
The function takes three
* The graph (`self`)
* The tick of the task.
* The properties of the task (`graph.get_task_properties(tick)`).
Only tasks for which this function returns `True` are returned. The function
is reevaluated if the properties of a task change.
:param syncpoint_run_last: If `True` then tasks with syncpoints only
run once all tasks with a lower tick have completed.
"""
AbstractGraphDecorator.__init__(self, g)
self._prefix = prefix
self._port_filter = port_filter
self._property_filter = property_filter
self._syncpoint_run_last = syncpoint_run_last
#: ticks of all tasks that have data for all inputs.
#: That is, if every output port connected to each input port
#: had data set with `set_output_data`.
self._queue = SortedSet()
#: ticks of all tasks with
#: * `properties["syncpoint"] == True`
#: * `set_output_data` not yet called.
self._pending_syncpoints = SortedSet()
#: ticks of all tasks with
#: * `set_output_data` not yet called.
self._pending_ticks = SortedSet()
self._collected_prop = self._prefix + "_collected"
self._count_prop = self._prefix + "_count"
self._ready_prop = self._prefix + "_ready"
self.g.set_task_property(graph.FINAL_TICK, self._count_prop, 0)
self.g.set_task_property(graph.FINAL_TICK, self._ready_prop, 0)
self.g.set_task_property(graph.FINAL_TICK, self._collected_prop, False)
self._consider(graph.FINAL_TICK)
self._pending_ticks.add(graph.FINAL_TICK)
def euler():
fracs = SortedSet()
limit = 10**6
for d in range(5, limit+1):
target = d*3/7
for n in range(int(floor(target)), int(ceil(target))+1):
if gcd(n, d) == 1:
fracs.add(Fraction(n, d))
result = 0
compare = Fraction(3, 7)
for i, f in enumerate(fracs):
if f == compare:
result = i-1
break
print(fracs[result])
def euler50():
limit = 100
primes = SortedSet(sieve.primerange(1, limit))
prime_sums = SortedSet()
prime_sums.add(0)
consecutive = 0
for p in primes:
prime_sums.add(prime_sums[-1] + p)
for i in range(consecutive, len(prime_sums)):
for j in range(i - consecutive - 1, -1, -1):
if prime_sums[i] - prime_sums[j] > limit:
break
if isprime(prime_sums[i] - prime_sums[j]):
consecutive = i - j
result = prime_sums[i] - prime_sums[j]
print(result)
def __init__(self):
state = ['Up', 'Failed', 'Maintenance', 'Blocked']
Machine.__init__(self, states = state, initial='Up')
self.add_transition('start', 'Up', 'Up', after = "startJob")
self.add_transition('fail', 'Up', 'Failed', after = 'startFail')
self.add_transition('repair', 'Failed', 'Up', after = 'rep')
self.add_transition('maint', 'Up', 'Maintenance', after='startmaint')
self.add_transition('maintcpl', 'Maintenance', 'Up')
self.add_transition('interrep', 'Failed', 'Maintenance', after='startmaint')
self.add_transition('block', 'Up', 'Blocked')
self.add_transition('unblock', 'Blocked', 'Up')
self.queue = SortedSet(key = lambda job: job.arrivalTime)
self.numServers = 1
self.busyServers = 0
self.prevState = None
Server._ids +=1
self.serviceTimeDistribution = None
self.name = "Server {}".format(Server._ids)
self.In = None
self.Out = None
self.scheduler = None
self.activejob = None
self.interuptjob = None
#debugging
self.jobsarrived = 0
self.jobsprocessed = 0
self.numfailures = 0
self.nummaint = 0
self.onzin = 0
def test_eq():
alpha = SortedSet(range(100))
alpha._reset(7)
beta = SortedSet(range(100))
beta._reset(17)
assert alpha == beta
assert alpha == beta._set
beta.add(101)
assert not (alpha == beta)
def __init__(self, nodes: Set[int]=None, r: int=1) -> None:
"""Make a new graph."""
if nodes is None:
self.nodes = SortedSet() # type: Set[int]
else:
self.nodes = nodes
self.radius = r
self.inarcs_by_weight = [defaultdict(SortedSet) for _ in range(self.radius)]
def test_ne():
alpha = SortedSet(range(100))
alpha._reset(7)
beta = SortedSet(range(99))
beta._reset(17)
assert alpha != beta
beta.add(100)
assert alpha != beta
assert alpha != beta._set
assert alpha != list(range(101))
def swapConnColors(c1, c2, ndx): #kempe chain connected new1 = SortedSet() new2 = SortedSet() for i in c1: swapped = False for j in ndx[i]: if j in c2: new1.add(j) new2.add(i) swapped = True if not swapped: new1.add(i) new2.add(j) #print c1, c2 #print new1, new2 return new1,new2
def __init__(self, seth=SortedSet([])):
'''
:param seth: set of items( string typically)
'''
self.set = seth
def find_data(ensemble_name, search_dir):
ensemble_info = sigmond.MCEnsembleInfo(ensemble_name, 'ensembles.xml')
# search for data files
file_list_infos = dict()
for root, dirs, files in os.walk(search_dir, topdown=True):
for filename in files:
base, ext = os.path.splitext(filename)
full_base = os.path.join(root, base)
suffix = int(ext[1:])
if full_base in file_list_infos:
if suffix < file_list_infos[full_base][0]:
file_list_infos[full_base][0] = suffix
elif suffix > file_list_infos[full_base][1]:
file_list_infos[full_base][1] = suffix
else:
file_list_infos[full_base] = [suffix, suffix]
file_list_infos = [sigmond.FileListInfo(stub, suffix[0], suffix[1], False)
for stub, suffix in file_list_infos.items()]
# create sigmond handlers
mcobs_xml = ET.Element("MCObservables")
corr_data_xml = ET.SubElement(mcobs_xml, "BLCorrelatorData")
for file_list_info in file_list_infos:
corr_data_xml.append(file_list_info.xml())
mcobs_xml_handler = sigmond.XMLHandler()
mcobs_xml_handler.set_from_string(ET.tostring(mcobs_xml))
sampling_info = sigmond.MCSamplingInfo()
bins_info = sigmond.MCBinsInfo(ensemble_info)
bins_info.addOmissions(defs.omissions[ensemble_name])
obs_get_handler = sigmond.MCObsGetHandler(mcobs_xml_handler, bins_info, sampling_info)
obs_handler = sigmond.MCObsHandler(obs_get_handler, False)
corr_handler = sigmond.BLCorrelatorDataHandler(file_list_infos, set(), set(), ensemble_info)
corrs = corr_handler.getCorrelatorSet()
# search through found data
operators = dict()
max_tsep = 0
time_seps = dict()
for corr in corrs:
keys = corr_handler.getOrderedKeys(corr)
tmin = keys[0].getTimeIndex()
tmax = keys[-1].getTimeIndex()
if tmax > max_tsep:
max_tsep = tmax
time_seps[corr] = (tmin, tmax)
op_snk = corr.getSink()
op_src = corr.getSource()
op_snk_bl = op_snk.getBasicLapH()
op_src_bl = op_src.getBasicLapH()
P = (op_src_bl.getXMomentum(), op_src_bl.getYMomentum(), op_src_bl.getZMomentum())
Psq = P[0]**2 + P[1]**2 + P[2]**2
irrep = op_src_bl.getLGIrrep()
irrep_row = op_src_bl.getLGIrrepRow()
averaged_channel = defs.AveragedChannel(Psq, irrep)
equivalent_frame = defs.EquivalentFrame(P, irrep_row)
if averaged_channel not in operators:
operators[averaged_channel] = dict()
if equivalent_frame not in operators[averaged_channel]:
operators[averaged_channel][equivalent_frame] = SortedSet()
operators[averaged_channel][equivalent_frame].add(op_snk)
operators[averaged_channel][equivalent_frame].add(op_src)
return operators
def extract_FL1L(self):
self.F_L1L = SortedSet()
cpt = 1
for transition1 in self.L1L:
A = SortedSet()
B = SortedSet()
for transition2 in self.T_pr:
if self.relations[transition2][
transition1] == Relations.RIGHT_CAUSALITY:
print("for transition ", transition1, " : ", transition2)
A.add(transition2)
if self.relations[transition1][
transition2] == Relations.RIGHT_CAUSALITY:
print("for transition ", transition1, " : ", transition2)
B.add(transition2)
'''
The solution to tackle length-one loops in sound SWF-nets focuses on the
pre- and post-processing phases of process mining. The key idea is to identify
the length-one-loop tasks and the single place to which each task should be
connected. Any length-one-loop task t can be identified by searching a loopcomplete
event log for traces containing the substring tt. To determine the correct
place p to which each t should be connected in the discovered net, we must check
which transitions are directed followed by t but do not direct follow t (i.e. p is an
output place of these transitions) and which transitions direct follow t but t does
not direct follow them (i.e. p is the input place of these transitions)
'''
print(len(A) == len(B))
place = 'p' + str(cpt)
for transition in A.difference(B):
# Add input places
transition_place = (transition1, place)
self.F_L1L.add(transition_place)
for transition in B.difference(A):
#Add output place
transition_place = (place, transition1)
self.F_L1L.add(transition_place)
cpt += 1
print(self.F_L1L)
pass
def factorSet(it=None): return SortedSet(iterable=it,key=lambda f:'{:04d}.'.format(f.nvar)+str(f.vars)[1:-1])
#def factorSet(it=None): return SortedListWithKey(iterable=it,key=lambda f:'{:04d}.'.format(f.nvar)+str(f.vars)[1:-1])
# So, fs = factorSet([list]) will make sure:
# fs[0] is a minimal factor (smallest # of variables) and fs[-1] is a maximal factor (largest # of variables)
class GraphModel(object):
def distinct_values(self, col_ix: ColumnIndex) -> List[str]:
return SortedSet(
str(self.fk_lookup(col=col_ix, val=row[col_ix]))
for row in self.visible_data)
# 차이가 가장 작은 수
## n개의 정수로 이루어진 수열에서 두 수를 골랐을 때, 그 차이가 m 이상이면서 제일 작은 경우의 그 차이를 구하는 프로그램을 작성해보세요.
### TreeSet 사용 이유 : 각 위치에 있을 떄, 가장 가깝게 크고, 가장 가깝게 작은 수를 빠르게 선택해야 되므로 사용
import sys
INT_MAX = sys.maxsize
from sortedcontainers import SortedSet
s = SortedSet() # treeset
cmd = list(map(int, input().split()))
result = INT_MAX
for i in range(cmd[0]):
num = int(input())
bigNum = num + cmd[1]
smallNum = num - cmd[1]
bigIdx = s.bisect_left(bigNum)
smallIdx = s.bisect_right(smallNum) - 1
if smallIdx < 0 and bigIdx == len(s):
s.add(num)
continue
elif bigIdx == len(s):
temp = num - s[smallIdx]
elif smallIdx < 0:
temp = s[bigIdx] - num
else:
smallTemp = num - s[smallIdx]
bigTemp = s[bigIdx] - num
temp = min(smallTemp, bigTemp)
if result > temp:
def dom_j_up(csp, queue):
return SortedSet(queue, key=lambda t: neg(len(csp.curr_domains[t[1]])))
class HyperLogLog(object):
"""
HyperLogLog cardinality counter
"""
__slots__ = ('alpha', 'p', 'm', 'M', 'k', 'k_len', 'error_rate')
def __init__(self, error_rate, minhash_counter_len=2**16):
"""
Implementes a HyperLogLog
error_rate = abs_err / cardinality
"""
if not (0 < error_rate < 1):
raise ValueError("Error_Rate must be between 0 and 1.")
# error_rate = 1.04 / sqrt(m)
# m = 2 ** p
# M(1)... M(m) = 0
p = int(math.ceil(math.log((1.04 / error_rate)**2, 2)))
self.alpha = get_alpha(p)
self.p = p
self.m = 1 << p
self.M = [0 for i in range(self.m)]
self.k = SortedSet(range(2**64, 2**64 + minhash_counter_len)
) #every register gets a unique placeholder value
self.k_len = minhash_counter_len
self.error_rate = error_rate
def __getstate__(self):
return dict([x, getattr(self, x)] for x in self.__slots__)
def __setstate__(self, d):
for key in d:
setattr(self, key, d[key])
def add(self, value):
"""
Adds the item to the HyperLogLog
"""
# h: D -> {0,1} ** 64
# x = h(v)
# j = <x_0x_1..x_{p-1}>
# w = <x_{p}x_{p+1}..>
# M[j] = max(M[j], rho(w))
x = mmh3.hash64(value, signed=False)[0]
j = x & (self.m - 1)
w = x >> self.p
self.M[j] = max(self.M[j], get_rho(w, 64 - self.p))
# add to minhash counter too (k):
if x < self.k[-1]:
self.k.add(x)
self.k.pop()
def update(self, *others):
"""
Merge other counters
"""
for item in others:
if self.m != item.m:
raise ValueError('Counters precisions should be equal')
self.M = [
max(*items)
for items in zip(*([item.M for item in others] + [self.M]))
]
self.k = SortedSet(
SortedSet([*self.k,
*[i for item in others for i in item.k]])[0:self.k_len])
def __eq__(self, other):
if self.m != other.m:
raise ValueError('Counters precisions should be equal')
return self.M == other.M
def __ne__(self, other):
return not self.__eq__(other)
def __len__(self):
return round(self.card())
def _Ep(self):
E = self.alpha * float(self.m**2) / sum(
math.pow(2.0, -x) for x in self.M)
return (E - estimate_bias(E, self.p)) if E <= 5 * self.m else E
def card(self):
"""
Returns the estimate of the cardinality
"""
#count number or registers equal to 0
V = self.M.count(0)
if V > 0:
H = self.m * math.log(self.m / float(V))
return H if H <= get_treshold(self.p) else self._Ep()
else:
return self._Ep()
def serialize(self):
'''
Serializes hll object as dictionary using compressed bytes string
'''
return zlib.compress(
pickle.dumps(dict([x, getattr(self, x)] for x in self.__slots__)))
@staticmethod
def deserialize(x):
'''
Get back the dictionary saved by the serialize method
'''
return pickle.loads(zlib.decompress(x))
@staticmethod
def jaccard(ks):
'''
Gets jaccard index of several minhash counters
'''
ks = [set(i) for i in ks]
return len(set.intersection(*ks)) / len(set.union(*ks))
@staticmethod
def get_min_card(x):
'''
Calculates min cardinality of several hlls
'''
return min([hll.card() for hll in x])
@staticmethod
def get_max_card(x):
'''
Calculates max cardinality of several hlls
'''
return max([hll.card() for hll in x])
@staticmethod
def get_corrected_ks(x):
'''
Returns the corrected minhash counters to reflect a "normalized" k density (i.e., hll.k_len / hll.card() )
'''
max_card = HyperLogLog.get_max_card(x)
k_len = x[0].k_len
return [[i for i in hll.k
if i < 2**64][0:int(k_len * hll.card() / max_card)]
for hll in x]
@staticmethod
def get_corrected_jaccard(x):
'''
Gets the jaccard index after correcting the minhash counter density
'''
return HyperLogLog.jaccard(HyperLogLog.get_corrected_ks(x))
@staticmethod
def get_intersection_card(x):
'''
Gets the cardinality of the intersection of multiple hlls
'''
hll_temp = HyperLogLog(x[0].error_rate)
[hll_temp.update(hll) for hll in x]
return int(HyperLogLog.get_corrected_jaccard(x) * hll_temp.card())
@staticmethod
def containment(x):
'''
Input: list of hll objects
Output: list of containment of all objects to each hll object
'''
int_card = HyperLogLog.get_intersection_card(x)
return [int_card / len(i) for i in x]
def __init__(self, startList=None):
"""Initialize set of integers.
"""
self._intSet = SortedSet()
if startList is not None: self.addIntegerList(startList)
def __init__(self, data, header=None, mode="complete", target=None):
self.base = 10
if len(data.shape) <= 1:
data = array([[data[i]] for i in range(data.shape[0])])
(n, m) = data.shape
if target is None:
target = [i for i in range(n)]
else:
target = target.tolist()
self.mode = mode
self.values = [SortedDict() for j in range(m)
] # Track the uniquely occuring values for each column
self.types = ['Numerical' for j in range(m)]
self.optional = [True for j in range(m)]
self.cardinalities = [None for j in range(m)]
self.defaults = [None for j in range(m)]
self.magnitudes = [None for j in range(m)]
self.radii = [None for j in range(m)]
self.radial_values = [
SortedDict() for j in range(m)
] # Track the uniquely occuring values for each column
self.names = header
if self.mode == "none":
self.headers = header
return
if self.mode == "tree":
from python.model.tree_encoder import TreeEncoder
(n, m) = data.shape
clf = RandomForestClassifier(n_estimators=10,
max_features=None,
random_state=0)
clf.fit(data, array([[i] for i in target]))
clf.feature_importances_
importance_index = argsort(clf.feature_importances_)[::-1]
def key(sample):
# print(tuple(sample[i] for i in importance_index))
return tuple(sample[i] for i in importance_index)
self.tree = TreeEncoder([tuple(data[i, :]) for i in range(n)], key)
# print("Unique Samples: {}, Tree Width: {} ".format(len(set(tuple(data[i,:]) for i in range(n))), self.tree.width()))
self.headers = [
"path[{}]".format(i) for i in range(self.tree.depth())
]
return
for j in range(m):
for i in range(n):
value = data[i, j]
if isnull(value) or value == '':
self.optional[j] = True
elif Encoder.__is_numeric__(value):
self.types[j] = 'Numerical'
if not value in self.values[j]:
self.values[j][value] = set()
self.values[j][value].add(target[i])
# Radial Analysis
mantissa = float(value)
magnitude = 0
if mantissa != 0:
while round(mantissa) != mantissa:
mantissa = mantissa * self.base
magnitude += 1
while round(mantissa / self.base) == (mantissa /
self.base):
mantissa = mantissa / self.base
magnitude -= 1
if self.magnitudes[j] is None:
self.magnitudes[j] = magnitude
else:
self.magnitudes[j] = max(self.magnitudes[j], magnitude)
else:
self.types[j] = 'Categorical'
if not value in self.values[j]:
self.values[j][value] = set()
self.values[j][value].add(target[i])
self.cardinalities[j] = len(self.values[j])
if self.types[j] == 'Numerical' and len(self.values[j]) > 1:
while len(
set(
int(value * pow(self.base, self.magnitudes[j]))
for value in self.values[j].keys())) == len(
set(
int(value *
pow(self.base, self.magnitudes[j] - 1))
for value in self.values[j].keys())):
self.magnitudes[j] -= 1
mantissa = int(
max(self.values[j].keys()) *
pow(self.base, self.magnitudes[j]))
self.radii[j] = ceil(log(mantissa, self.base))
values = list(
sorted(value * pow(self.base, self.magnitudes[j])
for value in self.values[j].keys()))
for k in range(self.radii[j]):
radial_values = SortedSet(
int(value / pow(self.base, k)) % self.base
for value in values)
if len(radial_values) > 1:
self.radial_values[j][k] = radial_values
self.encoders = [None for j in range(m)]
for j in range(m):
try:
if self.cardinalities[j] <= 1 and self.optional == False:
self.types[j] = 'Redundant'
elif self.cardinalities[j] <= 2:
self.types[j] = 'Binary'
elif self.cardinalities[j] <= 5 and all(
type(value) == type(0) for value in self.values[j]):
self.types[j] = 'Categorical'
elif self.mode == "radix" and sum(
len(values) for radial_index, values in
self.radial_values[j].items()) < self.cardinalities[j]:
self.types[j] = 'Radial'
if self.types[j] == 'Redundant':
self.encoders[j] = None
elif self.types[j] == 'Binary':
if self.optional[j]:
self.encoders[j] = [
{
'relation': operator.eq,
'reference': 0,
'value': 0
},
{
'relation': operator.eq,
'reference': 1,
'value': 1
},
]
else:
self.defaults[j] = min(self.values[j])
self.encoders[j] = [
{
'relation': operator.eq,
'reference': 1,
'value': 1
},
]
elif self.types[j] == 'Categorical':
if self.optional[j]:
self.encoders[j] = [{
'relation': operator.eq,
'reference': value,
'value': value
} for value in self.values[j]]
else:
self.defaults[j] = min(self.values[j])
self.encoders[j] = [{
'relation': operator.eq,
'reference': value,
'value': value
} for value in list(self.values[j].keys())[1:]]
elif self.types[j] == 'Radial':
if self.optional[j]:
self.defaults[j] = min(self.values[j])
self.encoders[j] = []
for k in range(self.radii[j]):
if k in self.radial_values[j] and len(
self.radial_values[j][k]) > 1:
for radial_value in self.radial_values[j][k]:
self.encoders[j].append({
'relation':
operator.ge,
'reference':
radial_value,
'value':
radial_value,
"radial_index":
k - self.magnitudes[j]
})
elif self.types[j] == "Numerical":
feature_values = list(self.values[j].keys())
base_value = feature_values[0]
references = [base_value]
for value in feature_values[1:]:
reference = (base_value + value) / 2.0
union = self.values[j][base_value].union(
self.values[j][value])
if self.mode == "bucketize" and len(union) <= 1:
references.append(None)
else:
references.append(reference)
base_value = value
if self.optional[j]:
self.encoders[j] = [{
'relation': operator.ge,
'reference': references[k],
'value': feature_values[k]
} for k in range(0, len(self.values[j]))
if not references[k] is None]
else:
self.defaults[j] = min(feature_values)
self.encoders[j] = [{
'relation': operator.ge,
'reference': references[k],
'value': feature_values[k]
} for k in range(1, len(self.values[j]))
if not references[k] is None]
except Exception as e:
print("Exception({}) at Column {}".format(e, j))
print("Cardinalities[j] = {}".format(self.cardinalities[j]))
print("Optional[j] = {}".format(self.optional[j]))
print("Type[j] = {}".format(self.types[j]))
print("Values[j] = {}".format(self.values[j]))
print([type(values) for values in self.values[j]])
raise e
# discrete = 0
# continuous = 0
# for j in range(m):
# if self.types[j] == 'Numerical':
# continuous += 1
# else:
# discrete += 1
# if not header is None:
# print("Index: {}, Name: {}, Type: {}, Cardinality: {}, Encoders: {}, Optional: {}, Radius: {}, Magnitude: {}".format(
# j, header[j], self.types[j], self.cardinalities[j], len(self.encoders[j]), self.optional[j], self.radii[j], self.magnitudes[j]))
# else:
# print("Index: {}, Name: {}, Type: {}, Cardinality: {}, Encoders: {}, Optional: {}, Radius: {}, Magnitude: {}".format(
# j, 'Unknown', self.types[j], self.cardinalities[j], len(self.encoders[j]), self.optional[j], self.radii[j], self.magnitudes[j]))
# print("Number of Discrete Columns: {}, Number of Continuous Columns: {}, Total Cardinality: {}\n".format(discrete, continuous, sum(self.cardinalities)))
offset = 0
self.headers = []
for j, encoders in enumerate(self.encoders):
if encoders != None:
if self.types[j] == "Radial":
self.headers += [
'{}{}{}'.format(
'x[i,{}][{}]'.format(j, encoder['radial_index'])
if header is None else '{}[{}]'.format(
header[j], encoder['radial_index']), '>='
if encoder['relation'] == operator.ge else '==',
encoder['reference']) for encoder in encoders
]
else:
self.headers += [
'{}{}{}'.format(
'x[i,{}]'.format(j)
if header is None else header[j], '>='
if encoder['relation'] == operator.ge else '==',
encoder['reference']) for encoder in encoders
]
def __init__(self, id=None):
if id is None:
id = generate_id()
self.id = id
self.predecessors = set()
self.successors = SortedSet(key=attrgetter('priority'))
def eliminationOrder(gm,
orderMethod=None,
nExtra=-1,
cutoff=inf,
priority=None,
target=None):
"""Find an elimination order for a graphical model
Args:
gm (GraphModel): A graphical model object
method (str): Heuristic method; one of {'minfill','wtminfill','minwidth','wtminwidth','random'}
nExtra (int): Randomly select eliminated variable from among the best plus nExtra; this adds
randomness to the order selection process. 0 => randomly from best; -1 => no randomness (default)
cutoff (float): Quit early if ``score`` exceeds a user-supplied cutoff value (returning ``target, cutoff``)
priority (list, optional): Optional list of variable priorities; lowest priority variables are
eliminated first. Useful for mixed elimination models, such as marginal MAP inference tasks.
target (list): If the identified order is better than cutoff, write it directly into passed ``target`` list
Returns:
list: The identified elimination order
float: The "score" of this ordering
Using ``target`` and ``cutoff`` one can easily search for better orderings by repeated calls:
>>> ord, score = eliminationOrder(model, 'minfill', nExtra=2, cutoff=score, target=ord)
"""
orderMethod = 'minfill' if orderMethod is None else orderMethod.lower()
priority = [1 for x in gm.X] if priority is None else priority
if orderMethod == 'minfill':
score = lambda adj, Xj: 0.5 * sum(
[len(adj[Xj] - adj[Xk]) for Xk in adj[Xj]])
elif orderMethod == 'wtminfill':
score = lambda adj, Xj: sum([(adj[Xj] - adj[Xk]).nrStatesDouble()
for Xk in adj[Xj]])
elif orderMethod == 'minwidth':
score = lambda adj, Xj: len(adj[Xj])
elif orderMethod == 'wtminwidth':
score = lambda adj, Xj: adj[Xj].nrStatesDouble()
elif orderMethod == 'random':
score = lambda adj, Xj: np.random.rand()
else:
raise ValueError('Unknown ordering method: {}'.format(orderMethod))
adj = [gm.markovBlanket(Xi) for Xi in gm.X] # build MRF
# initialize priority queue of scores using e.g. heapq or sort
reverse = [(priority[Xi], score(adj, Xi), Xi) for Xi in gm.X]
scores = SortedSet(reverse)
totalSize = 0.0
#_order = np.zeros((len(gm.X),)) #np.array([0 for Xi in gm.X])
_order = [0] * len(gm.X)
for idx in range(gm.nvar):
pick = 0
Pi, Si, Xi = scores[pick]
if nExtra >= 0:
mx = bisect.bisect_right(
scores,
(Pi, Si,
gm.X[-1])) # get one past last equal-priority & score vars
pick = min(mx + nExtra,
len(scores)) # then pick a random "near-best" variable
pick = np.random.randint(pick)
Pi, Si, Xi = scores[pick]
del scores[pick]
_order[idx] = Xi.label # write into order[idx] = Xi
totalSize += adj[Xi].nrStatesDouble()
if totalSize > cutoff:
return target, cutoff # if worse than cutoff, quit with no changes to "target"
fix = VarSet()
for Xj in adj[Xi]:
adj[Xj] |= adj[Xi]
adj[Xj] -= [
Xi
] # TODO adj[Xj].remove(Xi) slightly faster but still unsupported by cython version
fix |= adj[Xj] # shouldn't need to fix as much for min-width?
for Xj in fix:
Pj, Sj, Xj = reverse[Xj]
scores.remove(reverse[Xj])
reverse[Xj] = (Pj, score(adj, Xj), Xj)
scores.add(
reverse[Xj]
) # add (Pj,score(adj,Xj),Xj) to heap & update reverse lookup
if not (target is None):
target.extend([None for i in range(len(target), len(_order))
]) # make sure order is the right size
for idx in range(gm.nvar):
target[idx] = _order[
idx] # copy result if completed without quitting
return _order, totalSize
def factorSet(it=None):
return SortedSet(
iterable=it,
key=lambda f: '{:04d}.'.format(f.nvar) + str(f.vars)[1:-1])
def parse_time_sequence(filename, dtype=np.uint32):
reader = csv.DictReader(filename, delimiter="\t")
# TODO - check validity
time_headers = SortedSet(reader.fieldnames)
return np.array([row[header] for row in reader for header in time_headers],
dtype=dtype)
def channels(self):
return SortedSet(self._operators.keys())
class WattPadBook(RandomVariable, NamedEntity):
"""This class represents a WattPad book along with its associated snapshots.
Parameters
----------
url : str
The URL that uniquely identifies the WattPad book.
Attributes
----------
sample : sortedcontainers.SortedSet of WattPadBook.Snapshot
The associated snapshots.
"""
_samples = WeakValueDictionary()
def __init__(self, url):
self.url = url
if url in WattPadBook._samples:
self.sample = WattPadBook._samples[url]
else:
self.sample = SortedSet()
WattPadBook._samples[url] = self.sample
def _add(self, snapshot):
"""Associate a snapshot with the book.
Parameters
----------
shapshot : WattPadBook.Snapshot
The snapshot that will be associated with the book.
"""
assert isinstance(snapshot, WattPadBook.Snapshot)
self.sample.add(snapshot)
def getName(self):
if self.sample:
return self.sample[-1].title
else:
return "(unknown title)"
def __repr__(self):
return "%s(%s)" % (self.__class__.__name__,
("%s \"%s\"" % (self.url, self.getName()))
if self.sample else self.url)
def __hash__(self):
return hash(self.url)
def __getstate__(self):
return self.url
def __setstate__(self, url):
self.__init__(url)
def __getnewargs__(self):
return (self.url, )
class Snapshot(SampledIndividual):
"""This class represents a WattPad book snapshot.
Parameters
----------
book : WattPadBook or None
The WattPad book the snapshot belongs to. None if the snapshot belongs to a cluster. The
snapshot is associated with the book immediately after construction.
title : str or None
The title the book had at the time of the snapshot. None if the snapshot belongs to a
cluster.
date : datetime
The date, and time at which the snapshot was taken.
reads : int
The number of reads the book has received at the time of the snapshot.
votes : int
The number of votes the book has received at the time of the snapshot.
Attributes
----------
book : WattPadBook or None
The WattPad book the snapshot belongs to. None if the snapshot belongs to a cluster.
title : str
The title the book had at the time of the snapshot.
date : datetime
The date, and time at which the snapshot was taken.
reads : int
The number of reads the book has received at the time of the snapshot.
votes : int
The number of votes the book has received at the time of the snapshot.
votes / reads : float
The ratio between the number of votes, and the number of reads in percent if the number
of reads is non-zero and zero otherwise.
"""
def __init__(self, book, title, date, reads, votes):
assert isinstance(book, WattPadBook) or book is None
assert isinstance(title, str) or (title is None and book is None)
assert isinstance(date, datetime)
assert isinstance(reads, int)
assert isinstance(votes, int)
self.book = book
self.title = title
self.date = date
self.reads = reads
self.votes = votes
self.__dict__["votes / reads"] = fraction(votes, reads)
if self.book:
self.book._add(self)
def getDatetime(self):
return self.date
def __lt__(self, other):
return isinstance(other,
WattPadBook.Snapshot) and self.date < other.date
def __le__(self, other):
return isinstance(other,
WattPadBook.Snapshot) and self.date <= other.date
def __hash__(self):
return hash((self.book, self.date))
def __eq__(self, other):
return isinstance(other, WattPadBook.Snapshot) and self.book == other.book \
and self.date == other.date
def __repr__(self):
return "%s(%s)" % (self.__class__.__name__, self.__dict__)
def __add__(self, other):
assert isinstance(other, WattPadBook.Snapshot) or other == 0
return self if other == 0 else WattPadBook.Snapshot(
book=None,
title=None,
date=max(self.date, other.date),
reads=self.reads + other.reads,
votes=self.votes + other.votes)
def __getstate__(self):
return {
"book": self.book,
"title": self.title,
"date": self.date,
"reads": self.reads,
"votes": self.votes,
}
def __setstate__(self, state):
self.__init__(**state)
def __getnewargs__(self):
return (self.book, self.title, self.date, self.reads, self.votes)
@staticmethod
def from_html(book, date, f):
"""Constructs a WattPad book snapshot from an HTML dump.
Parameters
----------
book : WattPadBook or None
The WattPad book the snapshot belongs to.
date : datetime
The date, and time at which the dump was taken.
f : file-like readable object
The HTML dump.
Returns
-------
WattPadBook.Snapshot
The snapshot constructed from the HTML dump.
"""
document = BeautifulSoup(f, "html.parser")
title = document.find("h1").text.strip()
reads = parse_human_readable_int(
document.find("span", {
"data-toggle": "tooltip"
},
text=compile(r".* Reads")).text)
votes = parse_human_readable_int(
document.find("span", {
"data-toggle": "tooltip"
},
text=compile(r".* Votes")).text)
return WattPadBook.Snapshot(book, title, date, reads, votes)
def operators(self):
return SortedSet().union(*self._operators.values())
def load_word_set(file_path: str) -> SortedSet:
"""Load a word set and return it as a set rule."""
with open(file_path, encoding='utf-8') as file:
return SortedSet(line.strip() for line in file)
class ShotSelector(pg.LayoutWidget):
valueChanged = pyqtSignal()
selectionChanged = pyqtSignal()
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.nshots = 1
self.setFixedHeight(100)
self.current_idx_le = QLineEdit(self)
self.current_idx_le.setMaximumWidth(30)
self.current_idx_le.setValidator(QtGui.QIntValidator())
self.current_idx_le.setText(str(-1))
self.addWidget(self.current_idx_le)
self.slider = QSlider(self)
self.slider.setPageStep(1)
self.slider.setOrientation(Qt.Horizontal)
self.addWidget(self.slider, colspan=2)
self.nextRow()
self.addWidget(QLabel('index selector'))
self.idx_select_le = QLineEdit(self)
self.idx_select_le.setText(':')
self.addWidget(self.idx_select_le)
self.warning = QLabel()
self.warning.setMargin(5)
self.update_warning()
self.addWidget(self.warning)
self.update_nshots(self.nshots)
self.idx_select_le.editingFinished.connect(self.update_selection)
self.current_idx_le.editingFinished.connect(self.setSliderValue)
self.slider.valueChanged.connect(self.setLabelValue)
def update_nshots(self, nshots):
self.nshots = nshots
self.idx = np.arange(self.nshots)
self.slider.setRange(0, self.nshots - 1)
self.update_selection()
self.setSliderValue()
def update_warning(self, warning=''):
if warning == '':
self.warning.setStyleSheet("background-color: lightgreen")
warning = 'all good'
else:
self.warning.setStyleSheet("background-color: red")
self.warning.setText(warning)
def update_selection(self):
self.update_warning()
slice_text = self.idx_select_le.text()
slices = slice_text.split(',')
self.idx_selected = SortedSet([])
for s in slices:
try:
scope = locals()
select = eval('self.idx[' + s + ']', scope)
if isinstance(select, np.ndarray):
for idx in select:
self.idx_selected.add(idx)
else:
self.idx_selected.add(select)
except:
self.update_warning('problem in selected indeces')
return 0
self.slider.setRange(0, len(self.idx_selected) - 1)
if int(self.current_idx_le.text()
) % self.nshots not in self.idx_selected:
self.current_idx_le.setText(self.idx_selected[-1])
self.update_warning(
'last index not in selection <br> -> setting last selected')
self.selectionChanged.emit()
def setLabelValue(self, value):
newval = self.idx_selected[value]
if newval != self.get_current_index():
self.current_idx_le.setText(str(newval))
self.valueChanged.emit()
def setSliderValue(self):
self.update_warning()
value = int(self.current_idx_le.text())
try:
value_sl = self.idx_selected.index(value % len(self.idx))
self.slider.setValue(value_sl)
except ValueError:
self.update_warning('set index not in selection <br> ignore')
def get_current_index(self):
return int(self.current_idx_le.text()) % self.nshots
def get_selected_indices(self):
return (np.array(self.idx_selected), )
def save_word_set(file_path: str, words: Iterable[str]) -> None:
"""Load a word set and return it as a set rule."""
with open(file_path, 'w', encoding='utf-8') as file:
for word in SortedSet(words):
file.write(word)
file.write('\n')
class Substitute(object):
"""
Substitute algorithm from Yamada et al. 2010.
Original Paper:
Yamada, T. Kataoka, S. Watanabe, K.
"Listing all the minimum spanning trees in an undirected graph".
International Journal of Computer Mathematics. Vol 87, No. 14. pp.
3175 - 3185. November 2010.
Attributes:
graph (nx.Graph): undirected graph.
tree (nx.Graph): minimum spanning tree of `graph`
fixed_edges (set): set of fixed edges as described in Yamata et al. 2010.
restricted_edges (set): set of restricted edges as described in Yamata
et al. 2010.
source_node (boolean): source node for post-ordered tree. Defaults to
last lexigraphically sorted node.
postorder_nodes (dict, int:int): dictionary mapping original nodes to
their postorder index. Instantiated during `substitute()` call.
descendants (dict, int:list[int]): dictionary mapping nodes to their
postordered descendants. Descendants referenced by their postorder
index. Instantiated during `subsitute()` call.
directed (nx.Graph): directed graph of `tree` respecting postordered
nodes. Instantiated during `substitute()` call.
quasi_cutes (SortedSet, (w, u, v)): ordered sets of possible substitute
edges. Sorted by w, u, and then v. Instantiated during
`substitute()` call.
"""
def __init__(self, graph, tree, fixed_edges, restricted_edges):
"""
Substitute algorithm from Yamada et al. 2010.
Args:
graph (nx.Graph): undirected graph.
tree (nx.Graph): minimum spanning tree of `graph`
fixed_edges (set): set of fixed edges as described in Yamata et al.
2010.
restricted_edges (set): set of restricted edges as described in
Yamata et al. 2010.
"""
check_input_graph(graph)
self.graph = graph
check_input_tree(tree, graph)
self.tree = tree
self.fixed_edges = fixed_edges
self.restricted_edges = restricted_edges
self.source_node = list(graph.nodes)[-1]
self.instantiate_substitute_variables() # step 1 in Substitute
def instantiate_substitute_variables(self):
"""
Instantiate variables for postordering nodes and quasi cuts.
Represents `step 1` in `Substitute(F, R, T)` from original paper.
Instantiates directed, posterorder_nodes, descendants, and quasi_cuts
instance variables.
"""
self.directed = self.tree.to_directed() # directed graph for postorder
self.postorder_nodes, self.descendants = self.postorder_tree()
# set Q in original paper
self.quasi_cuts = SortedSet(key=lambda x: (x[0], x[1], x[2]))
@staticmethod
def check_edge_set_membership(edge, edge_set):
"""
Check whether an edge is in a set of edges.
Check whether an edge tuple (u, v) is in a set of edges by checking
both forward and reverse directions.
Args:
edge (tuple): tuple representing an edge between nodes `u` and node
`v`. Formatted `(u, v)`.
edge_set (set, tuple): set of edge tuples.
"""
return edge in edge_set or edge[::-1] in edge_set
def find_incident_edges(self, node):
"""
Find all incident edges for a given node not in the current minimum
spanning tree nor a set of restricted edges.
Args:
node (int): node of interest.
Return:
(set): set of weighted incident edges to `node` not contained in
`restricted_edges`.
"""
incident_set = set()
for neighbor in nx.neighbors(self.graph, node):
edge = (node, neighbor)
restricted = self.check_edge_set_membership(
edge, self.restricted_edges)
if not restricted and edge not in self.tree.edges():
w_edge = (self.graph.get_edge_data(*edge)['weight'], *edge)
incident_set.add(w_edge)
return incident_set
def postorder_tree(self):
"""
Invoke postorder ordering on all nodes and edges within the tree given
a source node.
Reorders a tree in a postorder fashion to retrieve descendants and order
mappings for all nodes within a tree. Source node is defined by
self.source_node and defaults to the lexigraphically last node.
Returns:
(dict, dict): tuple of dictionaries. First dictionary maps each node
in the graph to its postorder position. The second dictionary
serves as a look up for postorder descendants for each node.
"""
nodes = nx.dfs_postorder_nodes(self.directed, self.source_node)
postorder_nodes = OrderedDict()
# map nodes to their postorder position and remove child edges
child_edges = []
for i, node in enumerate(nodes):
postorder_nodes[node] = i + 1
# remove directed edges not already logged in dictionary
# --> higher postorder, won't be descendant
for neighbor in nx.neighbors(self.directed, node):
if neighbor not in postorder_nodes:
# neighbor has higher postorder, remove node -> neighbor
# edge, but keep neighbor -> node edge
child_edges.append((node, neighbor))
self.directed.remove_edges_from(child_edges)
# map nodes to their postordered descendants
descendants = {}
for each in postorder_nodes:
descendants[each] = [postorder_nodes[each]]
for child in nx.descendants(self.directed, each):
descendants[each].append(postorder_nodes[child])
descendants[each].sort()
return (postorder_nodes, descendants)
def postordered_edges(self):
"""Return postorded, weighted edges."""
edges = []
for u, v in self.tree.edges():
# ensure edges are orders (u, v) such that u has the lowest
# postorder
n1_idx = np.argmin(
(self.postorder_nodes[u], self.postorder_nodes[v]))
n2_idx = np.argmax(
(self.postorder_nodes[u], self.postorder_nodes[v]))
(n1, n2) = (u, v)[n1_idx], (u, v)[n2_idx]
w = self.graph.get_edge_data(*(u, v))['weight']
edges.append((w, n1, n2))
# order edge list by post order of first node, and then post order of
# second node
edges = sorted(
edges,
key=lambda x:
(self.postorder_nodes[x[1]], self.postorder_nodes[x[2]]))
return edges
def equal_weight_descendant(self, weighted_edge):
"""
Find an equal-weight descendant of the origin node for a provided edge.
Finds a edge (x, y, z) in `self.quasi_cuts` such that the starting node,
`y`, is a postorder descendant of a the starting node, `u`, in the
provided edge (w, u, v) and x == w.
Args:
weighted_edge (tuple, (int, int, int)): tuple representation of a
weighted edge with the form (w, u, v): `w` is the weight of the
edge, `u` is the starting node of the edge, and `v` is the final
node of the edge.
Returns:
tuple (int, int, int): returns tuple representation of the first
discovered equal-weighted descendant edge. Returns None if no
such edge exists.
"""
weight, node = weighted_edge[0:2]
for cut_edge in self.quasi_cuts:
related = self.postorder_nodes[
cut_edge[1]] in self.descendants[node]
if related and cut_edge[0] == weight:
return (cut_edge)
return (None)
def _create_substitute_dict(self, ordered_edges):
"""
Create dictionary linking edges to their substitutes.
Args:
ordered_edges (list, tuple (u, v)): list of postordered edges.
Returns:
(OrderedDict): dictionary linking edges to their substitutes.
"""
substitute_dict = OrderedDict()
for e in ordered_edges:
substitute_dict[e[1:]] = None
return substitute_dict
def substitute(self):
"""
Finds all substitute edges for a minimum spanning tree.
Returns:
(dict, (tuple(u, v): [(x, y)]): dictionary mapping edges in `tree`
to list of possible substitute edges.
"""
# step 1
substitute_dict = None
# step 2
ordered_edges = self.postordered_edges()
for n_edge in ordered_edges:
incident_edges = self.find_incident_edges(n_edge[1])
# step 2.1
for i_edge in incident_edges:
reversed_edge = (i_edge[0], *i_edge[1:][::-1])
# step 2.1.a
if self.postorder_nodes[i_edge[2]] < self.descendants[
i_edge[1]][0]:
if reversed_edge in self.quasi_cuts:
self.quasi_cuts.remove(reversed_edge)
self.quasi_cuts.add(i_edge)
# step 2.1.b
if self.postorder_nodes[i_edge[2]] in self.descendants[
i_edge[1]]:
if reversed_edge in self.quasi_cuts:
self.quasi_cuts.remove(reversed_edge)
# step 2.1.c
if self.postorder_nodes[i_edge[2]] > self.descendants[
i_edge[1]][-1]:
self.quasi_cuts.add(i_edge)
# step 2.2
if not self.check_edge_set_membership(n_edge[1:3],
self.fixed_edges):
# step 2.2.a
cut_edge = self.equal_weight_descendant(n_edge)
while cut_edge is not None:
# step 2.2.b
if self.postorder_nodes[cut_edge[2]] in\
self.descendants[n_edge[1]]:
self.quasi_cuts.remove(cut_edge)
# back to step 2.2.a
cut_edge = self.equal_weight_descendant(n_edge)
# step 2.2.c
else:
if substitute_dict is None:
substitute_dict = self._create_substitute_dict(
ordered_edges)
substitute_dict[n_edge[1:]] = cut_edge[1:]
cut_edge = None
return (substitute_dict)
class Geography:
""" Generate complete set of terrain data. """
tiles: List[List[Optional[Tile]]] = []
open_set_sorted = SortedSet()
enviroment = Enviroment()
def __init__(self) -> None:
for _ in range(TILE_COUNT):
self.tiles.append([None] * TILE_COUNT)
self._starting_points()
self.heights_algorithm()
self.drainage_algorithm()
def get_tile(self, x: int, y: int) -> Tile:
""" Retrieve a tile at specified coordinates. """
tile = self.tiles[x][y]
if tile is None:
tile = Tile((x, y), self.enviroment)
self.tiles[x][y] = tile
return tile
def heights_algorithm(self) -> None:
""" Populate all tiles with height data. Also set the sealevel. """
while self.open_set_sorted:
height, tile = self.open_set_sorted.pop(0)
for neighbour in self.get_neighbours_filtered(tile):
new_height = height + random.randint(1, JITTER)
neighbour.height = new_height
self.open_set_sorted.add((new_height, neighbour))
if new_height > self.enviroment.highest_point:
self.enviroment.highest_point = new_height
self.enviroment.sealevel = self.enviroment.highest_point / random.uniform(
1.5, 4)
def drainage_algorithm(self) -> None:
""" Calculate the number of uphill tiles draining into each tile on the
map. High tile.dampness values indicate a river runs through that tile.
"""
self.open_set_sorted.clear()
for x in range(TILE_COUNT):
for y in range(TILE_COUNT):
tile = self.get_tile(x, y)
if tile.height > self.enviroment.sealevel:
self.open_set_sorted.add((tile.height, tile))
self.enviroment.dampest = 0
while self.open_set_sorted:
height, tile = self.open_set_sorted.pop()
lowest_neighbour = None
for neighbour in self.get_neighbours(tile):
if neighbour.height < height:
if lowest_neighbour is None or lowest_neighbour.height > neighbour.height:
lowest_neighbour = neighbour
assert lowest_neighbour
lowest_neighbour.dampness += tile.dampness
if tile.dampness > self.enviroment.dampest:
self.enviroment.dampest = tile.dampness
def get_neighbours(self, tile: Tile) -> Generator[Tile, Tile, None]:
""" Get all tiles adjacent to the specified tile. """
if tile.x > 0:
neighbour = self.get_tile(tile.x - 1, tile.y)
yield neighbour
if tile.y > 0:
neighbour = self.get_tile(tile.x - 1, tile.y - 1)
yield neighbour
if tile.y < TILE_COUNT - 1:
neighbour = self.get_tile(tile.x - 1, tile.y + 1)
yield neighbour
if tile.x < TILE_COUNT - 1:
neighbour = self.get_tile(tile.x + 1, tile.y)
yield neighbour
if tile.y > 0:
neighbour = self.get_tile(tile.x + 1, tile.y - 1)
yield neighbour
if tile.y < TILE_COUNT - 1:
neighbour = self.get_tile(tile.x + 1, tile.y + 1)
yield neighbour
if tile.y > 0:
neighbour = self.get_tile(tile.x, tile.y - 1)
yield neighbour
if tile.y < TILE_COUNT - 1:
neighbour = self.get_tile(tile.x, tile.y + 1)
yield neighbour
def get_neighbours_filtered(self,
tile: Tile) -> Generator[Tile, Tile, None]:
""" Get all tiles adjacent to the specified tile that have not had their
height populated yet. """
for neighbour in self.get_neighbours(tile):
if neighbour.height < 0:
yield neighbour
def _starting_points(self) -> None:
""" Set seed heights on map to start the height generation algorithm at.
"""
for x in range(TILE_COUNT):
start = self.get_tile(x, 0)
start.height = 0
if (start.height, start) not in self.open_set_sorted:
self.open_set_sorted.add((0, start))
start = self.get_tile(x, TILE_COUNT - 1)
start.height = 0
if (start.height, start) not in self.open_set_sorted:
self.open_set_sorted.add((0, start))
for y in range(TILE_COUNT):
start = self.get_tile(0, y)
start.height = 0
if (start.height, start) not in self.open_set_sorted:
self.open_set_sorted.add((0, start))
start = self.get_tile(TILE_COUNT - 1, y)
start.height = 0
if (start.height, start) not in self.open_set_sorted:
self.open_set_sorted.add((0, start))
# And a few away from the edges...
for _ in range(random.randint(0, 30)):
start = self.get_tile(random.randint(0, TILE_COUNT - 1),
random.randint(0, TILE_COUNT - 1))
start.height = 0
if (start.height, start) not in self.open_set_sorted:
self.open_set_sorted.add((0, start))
def __init__(self):
self._container = SortedSet()
self._lock = asyncio.Lock()
def has_three_sum_distinct(A, t):
from sortedcontainers import SortedSet
A = SortedSet(A)
return any(has_two_sum_distinct(A, t - a) for a in A)
def __init__(self, tool_id, **kwds):
self.tool_id = tool_id
self.tool_versions = SortedSet(key=packaging.version.parse)
def add(l, col, bb):
coldict[bb] = col
if l not in bb_per_line:
# noinspection PyArgumentList
bb_per_line[l] = SortedSet(key=lambda x: coldict[x])
bb_per_line[l].add(bb)
class Alpha_plus(AlphaMiner):
def __init__(self, Traces):
super().__init__(Traces)
#traces within an event log
self.traces = Traces
# set of transitions that appear in loop of length one
self.L1L = None
# T' , traces minus L1L
self.T_pr = None
self.F_L1L = None
self.Wm_L1L = SortedDict()
self.alphaObject = None
def extractRelations(self):
#Extract non repetitive traces, alpha dont take care about frequencies !
nnrep_traces = SortedSet()
for trace in self.traces.values():
nnrep_traces.add("".join(trace))
print(nnrep_traces)
#Extract relations between each transitions
# generate Footprint
for transition1 in self.transitions:
self.relations[transition1] = SortedDict()
for transition2 in self.transitions:
concat = transition1 + transition2
concat_symetric_1 = transition1 + transition2 + transition1
concat_symetric_2 = transition2 + transition1 + transition2
print(concat)
print(concat_symetric_1)
print(concat_symetric_2)
relation = None
for trace in nnrep_traces:
if relation == None:
if trace.find(concat) >= 0:
relation = Relations.RIGHT_CAUSALITY
elif trace.find(concat[::-1]) >= 0:
relation = Relations.LEFT_CAUSALITY
else:
if trace.find(concat) >= 0:
if relation == Relations.LEFT_CAUSALITY:
if trace.find(
concat_symetric_1) <= 0 and trace.find(
concat_symetric_2) <= 0:
relation = Relations.PARALLEL
elif trace.find(concat[::-1]) >= 0:
if relation == Relations.RIGHT_CAUSALITY:
if trace.find(
concat_symetric_1) <= 0 and trace.find(
concat_symetric_2) <= 0:
relation = Relations.PARALLEL
if relation == None:
relation = Relations.CHOICES
self.relations[transition1][transition2] = relation
return self.relations
def extract_L1L(self):
#extract length one loop
self.L1L = SortedSet()
super().getTransitions()
#compute footprint and extract all transitions that have a causality relations with himself
print(self.transitions)
self.extractRelations()
print(self.relations)
for transition in self.transitions:
if self.relations[transition][
transition] == Relations.RIGHT_CAUSALITY:
self.L1L.add(transition)
return self.L1L
def extract_Tprime(self):
# T' := T \ L1L
self.T_pr = self.transitions.difference(self.L1L)
return self.T_pr
def extract_FL1L(self):
self.F_L1L = SortedSet()
cpt = 1
for transition1 in self.L1L:
A = SortedSet()
B = SortedSet()
for transition2 in self.T_pr:
if self.relations[transition2][
transition1] == Relations.RIGHT_CAUSALITY:
print("for transition ", transition1, " : ", transition2)
A.add(transition2)
if self.relations[transition1][
transition2] == Relations.RIGHT_CAUSALITY:
print("for transition ", transition1, " : ", transition2)
B.add(transition2)
'''
The solution to tackle length-one loops in sound SWF-nets focuses on the
pre- and post-processing phases of process mining. The key idea is to identify
the length-one-loop tasks and the single place to which each task should be
connected. Any length-one-loop task t can be identified by searching a loopcomplete
event log for traces containing the substring tt. To determine the correct
place p to which each t should be connected in the discovered net, we must check
which transitions are directed followed by t but do not direct follow t (i.e. p is an
output place of these transitions) and which transitions direct follow t but t does
not direct follow them (i.e. p is the input place of these transitions)
'''
print(len(A) == len(B))
place = 'p' + str(cpt)
for transition in A.difference(B):
# Add input places
transition_place = (transition1, place)
self.F_L1L.add(transition_place)
for transition in B.difference(A):
#Add output place
transition_place = (place, transition1)
self.F_L1L.add(transition_place)
cpt += 1
print(self.F_L1L)
pass
def diff(self, first, second):
second = set(second)
return [item for item in first if item not in second]
def extract_WmL1L(self):
l1l = self.L1L
print('###l1l', l1l)
for trace_key, trace in self.traces.items():
trace_pr = trace
print('###trace', trace_pr)
trace_pr = self.diff(trace_pr, l1l)
print('#difference', trace_pr)
self.Wm_L1L[trace_key] = trace_pr
print('Wm_L1L ', self.Wm_L1L)
def run_alphaMiner(self):
Alph = AlphaMiner(self.Wm_L1L)
Alph.getInitialTransitions()
Alph.getFinalTransitions()
print(Alph.getTransitions())
Alph.extractRelations()
Alph.computePairs()
Alph.extract_maximal_pairs()
Alph.add_places()
Alph.extract_PetriNet()
self.alphaObject = Alph
def extract_PetriNet(self):
for transition in self.L1L:
self.alphaObject.PetriNet.add_transition(Transition(transition))
for element in self.F_L1L:
if element[0].startswith('p'):
# is a place
place = element[0]
transition_output = element[1]
self.alphaObject.PetriNet.add_output(place, transition_output,
Value(dot))
else:
place = element[1]
transition_input = element[0]
self.alphaObject.PetriNet.add_input(place, transition_input,
Value(dot))
def show(self, model=None):
if model == "petrinet":
def draw_place(place, attr):
attr['label'] = place.name.upper()
attr['color'] = '#FF0000'
def draw_transition(trans, attr):
if str(trans.guard) == 'True':
attr['label'] = trans.name
else:
attr['label'] = '%s\n%s' % (trans.name, trans.guard)
self.alphaObject.PetriNet.draw(',net-with-colors.png',
place_attr=draw_place,
trans_attr=draw_transition)
import pygame
pygame.init()
size = width, height = 1200, 682
WHITE = (255, 255, 255)
screen = pygame.display.set_mode(size)
screen.fill(WHITE)
pygame.display.set_caption("petri net alphaminer")
petri_net = pygame.image.load(",net-with-colors.png").convert()
surf = pygame.transform.rotate(petri_net, 90)
screen.blit(surf, (50, 0))
pygame.display.flip()
while True:
for e in pygame.event.get():
if e.type == pygame.QUIT or (e.type == pygame.KEYDOWN
and e.key == pygame.K_ESCAPE):
done = True
break
def _add_operator(self, operator):
if operator.channel not in self._operators:
self._operators[operator.channel] = SortedSet()
self._operators[operator.channel].add(operator)
class Store:
def __repr__(self):
return f"<Store Size: {len(self._reducer_set)}>"
def __init__(self,
reducer_list: List[Type[Reducer]] = None,
init_full_state=True,
cleaner_period=1.0):
self._reducer_list = set(reducer_list or [])
self._reducer_set = dict()
self._observer_list = defaultdict(dict)
self._initialize_full_state = init_full_state
self._idle_set = SortedSet()
self._idle_cleaner = False
self.cleaner_period = float(cleaner_period)
self._initialize_lock = asyncio.Lock()
def __getitem__(self, item) -> Optional[Dict[str, Any]]:
if type(item) is not str:
raise TypeError
reducer_cell = self._reducer_set.get(item, None)
if reducer_cell is None:
return None
return reducer_cell.get_state()
def __contains__(self, item):
return item in self._reducer_set
def insert_reducer_type(self, reducer: Type[Reducer]):
self._reducer_list.add(reducer)
def remove_reducer_type(self, reducer: Type[Reducer]):
if reducer in self._reducer_list:
self._reducer_list.remove(reducer)
def find_reducer_type_by_prefix(self, key) -> Option:
for reducer in self._reducer_list:
if reducer.key_prefix is None:
continue
if isinstance(reducer.key_prefix, str):
if key.startswith(reducer.key_prefix):
return Option(reducer)
return Option.none()
def find_reducer_list_by_type(self, reducer_type: Type) -> List[Reducer]:
result = []
for reducer in self._reducer_set.values():
if type(reducer) == reducer_type:
result.append(reducer)
return result
def set_idle_key(self, reducer: Reducer):
if isinstance(reducer.recycle_option, IdleTimeoutRecycleOption):
key = reducer.recycle_option.create_key(reducer)
reducer.last_idle_key = key
self._idle_set.add(key)
self.try_start_idle_cleaner()
def remove_idle_key(self, reducer: Reducer):
if isinstance(reducer.recycle_option, IdleTimeoutRecycleOption):
if reducer.last_idle_key in self._idle_set:
self._idle_set.remove(reducer.last_idle_key)
def try_start_idle_cleaner(self):
if not self._idle_cleaner:
self._idle_cleaner = True
asyncio.ensure_future(self.idle_cleaner())
async def get_or_create_cell(self,
key,
reducer_type: Optional[Type] = None
) -> Option:
if key not in self._reducer_set:
if reducer_type is None:
return Option.none()
reducer: Reducer = reducer_type()
try:
await self._initialize_lock.acquire()
reducer.store = self
if not await reducer.initialize(key):
return Option.none()
reducer.enable = True
self._reducer_set[key] = reducer
except Exception as e:
return Option(ReduxError(e, traceback.format_exc()))
finally:
self._initialize_lock.release()
else:
reducer = self._reducer_set[key]
return Option(reducer)
def pop_reducer_by_key(self, key):
if key not in self._reducer_set:
return
reducer = self._reducer_set.pop(key)
reducer.enable = False
if reducer.listener_dict:
for listener in reducer.listener_dict.values():
listener()
reducer.listener_dict.clear()
if reducer.enable_call_shutdown:
asyncio.ensure_future(reducer.shutdown())
def enable_create_reducer(self, reducer_type):
return True #isinstance(reducer_type.recycle_option, UnsubscribeRecycleOption)
def enable_set_up_idle_key(self, reducer_type: Type[Reducer], action):
option = reducer_type.recycle_option
return isinstance(
option,
IdleTimeoutRecycleOption) and option.timeout and not action.soft
async def dispatch(self, key: str, action: Action) -> bool:
try:
if key is None:
return False
if key in self:
reducer = self._reducer_set[key]
elif action.soft:
return True
else:
reducer_type_opt = self.find_reducer_type_by_prefix(key)
if reducer_type_opt.is_none:
return False
reducer_type = reducer_type_opt.unwrap()
if self.enable_create_reducer(reducer_type):
reducer_opt = await self.get_or_create_cell(
key, reducer_type)
reducer = reducer_opt.unwrap(
) if reducer_opt.is_some else None
else:
return False
if not reducer:
return False
if self.enable_set_up_idle_key(type(reducer), action):
self.remove_idle_key(reducer)
self.set_idle_key(reducer)
if await self._combine_block(reducer, action):
await self._dispatch(reducer, action)
if reducer.is_new and isinstance(
reducer.recycle_option, IdleTimeoutRecycleOption
) and not reducer.recycle_option.timeout:
self.pop_reducer_by_key(key)
reducer.is_new = False
except Exception as e:
traceback.print_exc()
return False
return True
async def _combine_block(self, reducer, action):
action_type = action.type
for cb in list(reducer.combine_message_list):
combine_message: CombineMessage = cb
if action_type in combine_message.message_type_list:
combine_message.message_type_list.remove(action_type)
if not combine_message.message_type_list and not combine_message.future.done(
):
reducer.combine_message_list.remove(combine_message)
combine_message.future.set_result(True)
return combine_message.keep_origin
return True
async def _dispatch(self, reducer: Reducer, action: Action):
try:
key = reducer.key
await reducer.locker.acquire()
changed_state = await reducer.reduce(action)
except Exception as e:
raise e
finally:
reducer.locker.release()
if changed_state:
await self._call_listeners(key, changed_state, self[key])
async def _call_listeners(self, key: str, changed_state: Dict[str, Any],
state: Dict[str, Any]):
listeners = self._observer_list.get(key, dict()).values()
for listener in list(listeners):
try:
await listener.call_state_changed(changed_state, state)
except Exception:
self.unsubscribe(key, listener.listener)
async def subscribe(self, key: str, listener: Listener) -> Option:
reducer_type_opt = self.find_reducer_type_by_prefix(key)
if reducer_type_opt.is_none:
return Option.none()
reducer_type = reducer_type_opt.unwrap()
listener_wrapper = ListenerStateWrapper(listener,
self._initialize_full_state)
self._observer_list[key].setdefault(listener, listener_wrapper)
listener.is_binding = True
listener.store = self
listener.key = key
reducer_opt = await self.get_or_create_cell(key, reducer_type)
if not reducer_opt.is_some:
return Option.none()
reducer = reducer_opt.unwrap()
self.remove_idle_key(reducer)
await self._dispatch(reducer, Action.no_op_command())
reducer.is_new = False
state = self[key]
if state:
await listener_wrapper.call_state_changed(state, state)
def unsubscribe():
self.unsubscribe(key, listener)
return Option(unsubscribe)
def unsubscribe(self, key: str, listener: Listener):
if listener not in self._observer_list[key]:
return
del self._observer_list[key][listener]
listener.is_binding = False
listener.store = None
listener.key = None
if not len(self._observer_list[key]):
del self._observer_list[key]
option = self._reducer_set[key].recycle_option
if isinstance(option, IdleTimeoutRecycleOption) and option.timeout:
self.set_idle_key(self._reducer_set[key])
else:
self.pop_reducer_by_key(key)
async def idle_cleaner(self):
period = self.cleaner_period
if not self._idle_set:
asyncio.get_event_loop().call_later(
period, lambda: asyncio.ensure_future(self.idle_cleaner()))
return
timestamp, reducer = self._idle_set[0]
now_timestamp = time.time()
if now_timestamp < timestamp:
sleep_time = max(period, timestamp - now_timestamp)
asyncio.get_event_loop().call_later(
sleep_time, lambda: asyncio.ensure_future(self.idle_cleaner()))
return
while self._idle_set:
timestamp, reducer = self._idle_set.pop(0)
now_timestamp = time.time()
if now_timestamp >= timestamp:
self.pop_reducer_by_key(reducer.key)
else:
break
asyncio.get_event_loop().call_soon(
lambda: asyncio.ensure_future(self.idle_cleaner()))
class PersonalDataFilter(QVBoxLayout):
def __init__(self,
db,
tags,
selector,
rating=c.default_rating,
tab_size=c.tab_size):
### Layout Initializing
super().__init__()
self.addStretch(2)
self.setSpacing(0)
### Major Connections
self.db = db
self.tags = tags
self.selector = selector
### Internal Variables
self.persons = dict(
) # Persons Dictionary (person:{'filters':0,'old name':name})
self.filtered_tags = set(
) # List of filtered tags names for evaluating filtering after adding a person
self.rating = c.default_rating # Current Rating Filter
self.rating_filtered = set() # List of persons filtered by rating
self.names_set = SortedSet() # Names set used for sorting
self.names_filters = SortedSet(
) # List of filters by names and persons
self.name_filtered = set() # List of persons filtered by name
self.current_name_filter = "" # Name filter to update persons with
### Loading Persons From Database
for name in self.db.listNames():
self.addPerson(name)
self.updateWidths()
self.filterRating(rating)
#########################
# Convenience Functions #
#########################
def listPersons(self):
return set(self.persons.keys())
def addFilter(self, person, n=1):
if 'filters' not in self.persons[person].keys():
self.persons[person]['filters'] = 0
if 0 >= self.persons[person]['filters'] >= -n + 1:
person.hide()
self.selector.disablePerson(person)
self.persons[person]['filters'] += n
def removeFilter(self, person, n=1):
self.persons[person]['filters'] -= n
if 0 >= self.persons[person]['filters'] >= -n + 1:
person.show()
self.selector.enablePerson(person)
############################
# Name Filtering Functions #
############################
def filterName(self, name):
self.current_name_filter = name
l = list(self.names_filters)
persons = set(
filter.person
for filter in l[bisect(l, NameFilter(name)
):bisect(l, NameFilter(name, endian=True))])
for person in persons:
if person in self.name_filtered:
self.name_filtered.remove(person)
self.removeFilter(person)
for person in self.listPersons() - persons:
if person not in self.name_filtered:
self.name_filtered.add(person)
self.addFilter(person)
def updateNameFiltering(self, person):
if any(
NameFilter(self.current_name_filter) < NameFilter(
name_filter, person) < NameFilter(self.current_name_filter,
endian=True)
for name_filter in person.getName().split(' ')):
if person in self.name_filtered:
name_filtered.remove(person)
self.removeFilter(person)
else:
if person not in self.name_filtered:
name_filtered.add(person)
self.addFilter(person)
############################
# Tags Filtering Functions #
############################
def filterTag(self, tag, mode):
### Regulating if tag is being filtered
if (tag in self.filtered_tags) ^ (not mode):
return
self.filtered_tags.add(tag) if mode else self.filtered_tags.remove(tag)
### Filtering persons not having a tag by tag
for person in self.listPersons() - self.tags.listPersons(tag):
self.addFilter(person) if mode else self.removeFilter(person)
##############################
# Rating Filtering Functions #
##############################
def filterRating(self, rating):
### Updating Rating Value
self.rating = rating
### Filtering Persons By Rating
for person in self.listPersons():
self.updateRating(person)
def updateRating(self, person):
### Filter On
if person.getRating(
) < self.rating and person not in self.rating_filtered:
self.addFilter(person)
self.rating_filtered.add(person)
### Filter Off
if person.getRating(
) >= self.rating and person in self.rating_filtered:
self.rating_filtered.remove(person)
self.removeFilter(person)
#################
# Adding Person #
#################
def addPerson(self, name, new=False):
### Creating new PersonalData object
if new:
if self.db.addName(name) == None:
return
person = PersonalData(name, self.db, self.tags, self,
self.selector, self.rating)
else:
person = PersonalData(name, self.db, self.tags, self,
self.selector)
### Updating person's name
self.names_set.add(person.getSortableName())
self.persons[person] = dict()
self.persons[person]['old name'] = name
self.persons[person]['old sortable name'] = person.getSortableName()
self.names_filters.add(NameFilter(name, person))
if new:
self.updateWidths()
### Inserting PersonalData in proper place
self.insertWidget(
bisect(list(self.names_set), person.getSortableName()) - 1, person)
### Updating person's filters
if person.getRating() < self.rating:
self.rating_filtered.add(person)
self.addFilter(
person,
len(self.filtered_tags - self.tags.listTags(person)) +
(person in self.rating_filtered))
### Edit if a new person
if new:
person.edit()
############################
# Updating Persons' Widths #
############################
def updateWidths(self):
widths = tuple(
max(
tuple(person.getWidths()[i]
for person in self.persons) + (0, )) for i in range(5))
for person in self.persons:
person.updateWidths(widths)
################################
# Disabling Persons' Text Edit #
################################
def hideTextEdits(self):
for person in self.persons:
person.hideTextEdit()
###################
# Removing Person #
###################
def deletePerson(self, person):
### Confirmation with message box
message_box = FancyMessageBox(
[FancyMessageBox.Ok, FancyMessageBox.Cancel], c.question_icon_file,
"", "Are you sure you want to delete {person}?".format(
person=person.getName()))
if not message_box.exec():
return
### Deleting person's data
name = self.persons[person]['old name']
sortable_name = self.persons[person]['old sortable name']
self.db.deleteName(name)
self.tags.removePerson(person)
self.selector.removePerson(person)
self.persons.pop(person)
self.names_set.remove(sortable_name)
self.removeWidget(person)
self.names_filters.remove(NameFilter(name, person))
self.updateWidths()
person.deleteLater()
##########################
# Updating Person's Name #
##########################
def updateName(self, person):
### Checking if action is necessary
if person.getName() == self.persons[person]['old name']:
return
### Executing action
self.names_filters.remove(
NameFilter(self.persons[person]['old name'], person))
self.names_filters.add(NameFilter(person.getName(), person))
self.updateNameFiltering(person)
self.names_set.remove(self.persons[person]['old sortable name'])
self.names_set.add(person.getSortableName())
self.removeWidget(person)
self.insertWidget(
bisect(list(self.names_set), person.getSortableName()) - 1, person)
self.persons[person]['old name'] = person.getName()
self.persons[person]['old sortable name'] = person.getSortableName()
