def test_copy():
alpha = SortedList(range(100))
alpha._reset(7)
beta = alpha.copy()
alpha.add(100)
assert len(alpha) == 101
assert len(beta) == 100
class Episode_scores(object):
def __init__(self, options):
self.maxlen = options.score_averaging_length
self.threshold = options.score_highest_ratio
self.episode_scores = deque()
self.episode_scores.append(0) # to avoid 0-div in first averaging
self.episode_scores_sum = 0
self.sorted_scores = SortedList()
self.sorted_scores.add(0) # align to episode_scores
self.num_episode = 0
self.options = options
def add(self, n, global_t, thread_index):
self.episode_scores_sum += n
self.episode_scores.append(n)
self.sorted_scores.add(-n) # trick to use SortedList in reverse order
if len(self.episode_scores) > self.maxlen:
oldest = self.episode_scores.popleft()
self.sorted_scores.remove(-oldest)
self.episode_scores_sum -= oldest
self.num_episode += 1
if self.num_episode % self.options.average_score_log_interval == 0:
print("@@@ Average Episode score = {:.6f}, s={:9d},th={}".format(self.average(), global_t, thread_index))
def average(self):
return self.episode_scores_sum / len(self.episode_scores)
def is_highscore(self, n):
sorted_scores = self.sorted_scores
num_scores = len(sorted_scores)
sorted_scores.add(-n)
index = sorted_scores.index(-n)
highest_ratio = (index + 1) / num_scores
sorted_scores.remove(-n)
return highest_ratio <= self.threshold
def extract_collocations(self, metric_class):
assert issubclass(metric_class, Metric)
metric = metric_class()
collocations = SortedList(key=lambda x: -x[0])
unigram_counts = self.language_model.get_unigrams()
bigram_counts = self.language_model.get_bigrams()
for (first, last), freq_bigram in bigram_counts.items():
if self.exclude_punctuation:
if first in self.PUNCT or last in self.PUNCT:
continue
if self.exclude_conj:
if first in self.CONJ_RU or last in self.CONJ_RU:
continue
if self.exclude_props:
if first in self.PROPOSITIONS_RU or last in self.PROPOSITIONS_RU:
continue
freq_first, freq_last = unigram_counts[first], unigram_counts[last]
metric_val = metric.evaluate(freq_first, freq_last, freq_bigram,
self.language_model.get_vocab_size())
collocations.add((metric_val, freq_first,
freq_last, freq_bigram,
first, last))
return collocations
def predict(self, X):
y = np.zeros(len(X))
for i,x in enumerate(X): # test points
sl = SortedList(load=self.k) # stores (distance, class) tuples
for j,xt in enumerate(self.X): # training points
diff = x - xt
d = diff.dot(diff)
if len(sl) < self.k:
# don't need to check, just add
sl.add( (d, self.y[j]) )
else:
if d < sl[-1][0]:
del sl[-1]
sl.add( (d, self.y[j]) )
# print "input:", x
# print "sl:", sl
# vote
votes = {}
for _, v in sl:
# print "v:", v
votes[v] = votes.get(v,0) + 1
# print "votes:", votes, "true:", Ytest[i]
max_votes = 0
max_votes_class = -1
for v,count in votes.iteritems():
if count > max_votes:
max_votes = count
max_votes_class = v
y[i] = max_votes_class
return y
def iana_rir_gen_ip_list(user_rir_list):
# generates a list of networks that can be blocked by RIR
# we use a SortedList so that elements are inserted in order. This allows cidr_merge to work
rir_slash_eight_list = SortedList()
with open('iana') as iana_file:
iana_csv = csv.reader(iana_file)
for line in iana_csv:
for rir in user_rir_list:
# case in which the whois line from our csv contains the RIR
if rir in line[3]:
network = line[0].lstrip('0')
rir_slash_eight_list.add(netaddr.IPNetwork(network))
# if we find a match, there is no reason to see if the other RIRs are on the same line
break
# run cidr_merge to summarize
rir_slash_eight_list = netaddr.cidr_merge(rir_slash_eight_list)
return rir_slash_eight_list
def test_copy_copy():
import copy
alpha = SortedList(range(100), load=7)
beta = copy.copy(alpha)
alpha.add(100)
assert len(alpha) == 101
assert len(beta) == 100
def score_of_a_vacated_people(self, universo, work='translations'):
factor = math.sqrt(len(universo))
scores = SortedList(load=round(factor))
for (people, score) in self.__scores__().items():
if people in universo:
scores.add(TranslatorScore(people, score[work]))
return scores.pop(0)
def find_latest(self):
sorted = SortedList()
for i in self.bucket.list(prefix=self.db_name):
parts = i.name.split('/')
if len(parts) == 3:
d = datetime.datetime.strptime(parts[1], "%m%d%Y").date()
sorted.add(d)
return sorted[len(sorted)-1].strftime('%m%d%Y')
class InMemoryBackend(object):
"""
The backend that keeps the results in the memory.
"""
def __init__(self, *args, **kwargs):
def get_timestamp(result):
return timestamp_parser.parse(result['timestamp'])
self._results = dict()
self._sorted = SortedList(key=get_timestamp)
def disconnect(self):
return succeed(None)
def store(self, result):
"""
Store a single benchmarking result and return its identifier.
:param dict result: The result in the JSON compatible format.
:return: A Deferred that produces an identifier for the stored
result.
"""
id = uuid4().hex
self._results[id] = result
self._sorted.add(result)
return succeed(id)
def retrieve(self, id):
"""
Retrive a result by the given identifier.
"""
try:
return succeed(self._results[id])
except KeyError:
return fail(ResultNotFound(id))
def query(self, filter, limit=None):
"""
Return matching results.
"""
matching = []
for result in reversed(self._sorted):
if len(matching) == limit:
break
if filter.viewitems() <= result.viewitems():
matching.append(result)
return succeed(matching)
def delete(self, id):
"""
Delete a result by the given identifier.
"""
try:
result = self._results.pop(id)
self._sorted.remove(result)
return succeed(None)
except KeyError:
return fail(ResultNotFound(id))
def read_rirs(country_list, permit, rir_list=RIR_NAMES):
# list containing our file objects
file_list = []
# we use a SortedList so that elements are inserted in order. This allows cidr_merge to work
rir_ips = SortedList()
# Open the files we downloaded earlier and store the file object
for rir in rir_list:
file_list.append(open(rir))
for f in file_list:
for line in f:
curr_line = line.split('|')
try:
# we want only the ipv4 lines that are for a specific country
# also only want countries that we are going to block
if (curr_line[2] == "ipv4" and curr_line[1] != "*") and \
((permit and curr_line[1] not in country_list) or
(not permit and curr_line[1] in country_list)):
country_code = curr_line[1]
network_id = curr_line[3]
wildcard = int(curr_line[4])-1
try:
# Add network to list, if the number of IPs was not a
# power of 2 (wildcard is not valid).
# AddrFormatError is thrown
rir_ips.add(netaddr.IPNetwork(network_id + "/" + str(netaddr.IPAddress(wildcard))))
# Handle case in where our mask is invalid by rounding DOWN
except netaddr.AddrFormatError:
print "rounded network " + network_id + " with " + str(wildcard) + \
" hosts up to nearest power of 2"
wildcard = next_power_of_2(wildcard) - 1
print wildcard + 1
rir_ips.add(netaddr.IPNetwork(network_id + "/" + str(netaddr.IPAddress(wildcard))))
# IndexErrors only occur when parsing columns we don't need
except IndexError:
pass
f.close()
# cidr_merge takes our list of IPs and summarizes subnets where possible
# this greatly decreases the number of ACL entries
rir_ips = netaddr.cidr_merge(rir_ips)
return rir_ips
def dir(self, file_pattern):
attrs = self.sftp.listdir_attr(self.remote_dir)
filtered = SortedList()
for attr in attrs:
if hasattr(attr, "filename"):
filename = attr.filename
if re.match(file_pattern, filename):
remote_file = RemoteFile(filename, attr.st_mtime)
filtered.add(remote_file)
return filtered
def test_count():
slt = SortedList(load=7)
assert slt.count(0) == 0
for iii in range(100):
for jjj in range(iii):
slt.add(iii)
slt._check()
for iii in range(100):
assert slt.count(iii) == iii
def arrayRDP(arr, epsilon=0.0, n=None):
"""
This is a slightly modified version of the _aRDP function, that accepts
as arguments the tolerance in the distance and the maximum number of points
the algorithm can select.
**Note:** The results of this algoritm should be identical to the arrayRDP
function if the *n* parameter is not specified. In that case, the
performance is slightly worse, although the asymptotic complexity is the
same. For this reason, this function internally delegates the solution in
that function if the *n* parameter is missing.
Parameters
----------
arr:
Array of values of consecutive points.
epsilon:
Maximum difference allowed in the simplification process.
n:
Maximum number of points of the resulted simplificated array.
Returns
-------
out:
Array of indices of the selected points.
"""
if n is None:
return _aRDP(arr, epsilon)
if epsilon <= 0.0:
raise ValueError('Epsilon must be > 0.0')
n = n or len(arr)
if n < 3:
return arr
fragments = SortedDict()
#We store the distances as negative values due to the default order of
#sorteddict
dist, idx = max_vdist(arr, 0, len(arr) - 1)
fragments[(-dist, idx)] = (0, len(arr) - 1)
while len(fragments) < n-1:
(dist, idx), (first, last) = fragments.popitem(last=False)
if -dist <= epsilon:
#We have to put again the last item to prevent loss
fragments[(dist, idx)] = (first, last)
break
else:
#We have to break the fragment in the selected index
dist, newidx = max_vdist(arr, first, idx)
fragments[(-dist, newidx)] = (first, idx)
dist, newidx = max_vdist(arr, idx, last)
fragments[(-dist, newidx)] = (idx, last)
#Now we have to get all the indices in the keys of the fragments in order.
result = SortedList(i[0] for i in fragments.itervalues())
result.add(len(arr) - 1)
return np.array(result)
def test_getitem():
random.seed(0)
slt = SortedList(load=17)
lst = list()
for rpt in range(100):
val = random.random()
slt.add(val)
lst.append(val)
lst.sort()
assert all(slt[idx] == lst[idx] for idx in range(100))
assert all(slt[idx - 99] == lst[idx - 99] for idx in range(100))
def get_3_most_ambiguous(self, X, Y):
P = self.predict_proba(X)
N = len(X)
sl = SortedList(load=3) # stores (distance, sample index) tuples
for n in xrange(N):
p = P[n]
dist = np.abs(p - 0.5)
if len(sl) < 3:
sl.add( (dist, n) )
else:
if dist < sl[-1][0]:
del sl[-1]
sl.add( (dist, n) )
indexes = [v for k, v in sl]
return X[indexes], Y[indexes]
class _WaitingTasksQueue(object):
"""A FIFO queue of tasks that keeps track of RAM limits.
This class serves two purposes:
1) It is a FIFO queue for tasks that also supports fast deletion.
2) It keeps RAM requirements of all tasks in the queue in sorted order.
This is necessary for the implementation of worker blocking,
see ``PrioritingScheduler._getNumberOfBlockedAnyCpuWorkers()``
for more details.
"""
def __init__(self):
self._dict = OrderedDict()
self._tasks_required_ram = SortedList()
def __contains__(self, task):
return task in self._dict
def __len__(self):
return len(self._dict)
def __nonzero__(self):
return bool(self._dict)
__bool__ = __nonzero__ # for Python 2/3 compatibility
def add(self, task):
self._dict[task] = True
self._tasks_required_ram.add(task.required_ram_mb)
def remove(self, task):
del self._dict[task]
self._tasks_required_ram.discard(task.required_ram_mb)
def left(self):
if self._dict:
return six.next(six.iteritems(self._dict))[0]
else:
return None
def popleft(self):
task = self._dict.popitem(False)[0]
self._tasks_required_ram.discard(task.required_ram_mb)
return task
def getTasksRequiredRam(self):
return self._tasks_required_ram
def load_waveforms(annfile):
"""Obtains a sorted list of waveform objects from an annotations file"""
waveforms = SortedList()
anns = mit.read_annotations(annfile)
for i in xrange(len(anns)):
a = anns[i]
if mit.is_qrs_annotation(a):
wf = WaveForm()
wf.type = mit.ECGCodes.NORMAL
wf.peak = a.time
start = next(anns[j].time for j in xrange(i, -1, -1)
if anns[j].code is mit.ECGCodes.WFON)
end = next(anns[j].time for j in xrange(i, len(anns))
if anns[j].code is mit.ECGCodes.WFOFF)
wf.interval = Iv(start, end)
waveforms.add(wf)
return waveforms
def test_stress(repeat=1000):
slt = SortedList((random.random() for rpt in range(1000)))
slt._reset(23)
for rpt in range(repeat):
action = random.choice(actions)
action(slt)
slt._check()
fourth = int(len(slt) / 4)
count = 0 if fourth == 0 else random.randrange(-fourth, fourth)
while count > 0:
slt.add(random.random())
count -= 1
while count < 0:
pos = random.randrange(len(slt))
del slt[pos]
count += 1
while len(slt) > 2000:
# Shorten the sortedlist. This maintains the "jaggedness"
# of the sublists which helps coverage.
pos = random.randrange(len(slt._maxes))
del slt._maxes[pos]
del slt._lists[pos]
slt._len = sum(len(sublist) for sublist in slt._lists)
slt._index = []
slt._check()
slt._check()
slt._check()
stress_update(slt)
while len(slt) > 0:
pos = random.randrange(len(slt))
del slt[pos]
slt._check()
class Store:
def __init__(self,N=10):
self.store = SortedList()
self.N = N
def add(self,item):
self.store.add(item)
if len(self.store) > self.N: self.store.pop(0)
def pop(self,i):
self.store.pop(i)
def __len__(self):
return len(self.store)
def __getitem__(self,i):
return self.store[i]
def __str__(self):
return str(self.store)
def _getOpcodeAddrs(self):
addrs = SortedList()
funcs = self.getFunctions()
for f in funcs:
print('addrs for', f['name'])
ofs = f['offset']
sz = f['size']
end = ofs + sz
self.seek(ofs)
cur = ofs
while cur < end:
addrs.add(cur)
self.cmd('so')
cur = self.tell()
return addrs
def ToEdgeClusters(edgeListPath, clustersPath):
'''
Convert a node clusters file to an edge clusters path.
'''
# Load the edge list and clusters as a *sorted* list of lists of tuples
edgeList = LoadNumberFile(edgeListPath)
edgeClusters = SortedList()
with open(clustersPath, 'r') as f:
for l in f:
nodes = [int(x) for x in l.strip().split()]
cluster = SortedList()
for e in combinations(sorted(nodes), 2):
if edgeList.count([e]) is not 0:
cluster.add(e)
edgeClusters.add(cluster)
with open(clustersPath, 'w') as f:
for c in edgeClusters:
for t in c:
f.write(str(t[0]) + " " + str(t[1]) + " ")
f.write("\n")
def collect_matches():
initial_summoner_name = "GustavEnk"
region = "EUW"
summoner = Summoner(name=initial_summoner_name, region=region)
patch = Patch.from_str("8.9", region=region)
unpulled_summoner_ids = SortedList([summoner.id])
pulled_summoner_ids = SortedList()
unpulled_match_ids = SortedList()
pulled_match_ids = SortedList()
while unpulled_summoner_ids:
# Get a random summoner from our list of unpulled summoners and pull their match history
new_summoner_id = random.choice(unpulled_summoner_ids)
new_summoner = Summoner(id=new_summoner_id, region=region)
matches = filter_match_history(new_summoner, patch)
unpulled_match_ids.update([match.id for match in matches])
unpulled_summoner_ids.remove(new_summoner_id)
pulled_summoner_ids.add(new_summoner_id)
while unpulled_match_ids:
# Get a random match from our list of matches
new_match_id = random.choice(unpulled_match_ids)
new_match = Match(id=new_match_id, region=region)
for participant in new_match.participants:
if participant.summoner.id not in pulled_summoner_ids and participant.summoner.id not in unpulled_summoner_ids:
unpulled_summoner_ids.add(participant.summoner.id)
# The above lines will trigger the match to load its data by iterating over all the participants.
# If you have a database in your datapipeline, the match will automatically be stored in it.
unpulled_match_ids.remove(new_match_id)
pulled_match_ids.add(new_match_id)
def test_getitem_slice():
random.seed(0)
slt = SortedList()
slt._reset(17)
lst = list()
for rpt in range(100):
val = random.random()
slt.add(val)
lst.append(val)
lst.sort()
assert all(slt[start:] == lst[start:]
for start in [-75, -25, 0, 25, 75])
assert all(slt[:stop] == lst[:stop]
for stop in [-75, -25, 0, 25, 75])
assert all(slt[::step] == lst[::step]
for step in [-5, -1, 1, 5])
assert all(slt[start:stop] == lst[start:stop]
for start in [-75, -25, 0, 25, 75]
for stop in [-75, -25, 0, 25, 75])
assert all(slt[:stop:step] == lst[:stop:step]
for stop in [-75, -25, 0, 25, 75]
for step in [-5, -1, 1, 5])
assert all(slt[start::step] == lst[start::step]
for start in [-75, -25, 0, 25, 75]
for step in [-5, -1, 1, 5])
assert all(slt[start:stop:step] == lst[start:stop:step]
for start in [-75, -25, 0, 25, 75]
for stop in [-75, -25, 0, 25, 75]
for step in [-5, -1, 1, 5])
def _get_split_point_candidates(self):
splits = SortedList()
min_value = math.inf
max_value = -math.inf
for class_val, att_estimator in self._class_lookup.items():
min_val_observed_for_class_val = self._min_val_observed_per_class.get(class_val, None)
if min_val_observed_for_class_val is not None:
if min_val_observed_for_class_val < min_value:
min_value = min_val_observed_for_class_val
max_val_observed_for_class_val = self._max_val_observed_per_class.get(class_val)
if max_val_observed_for_class_val > max_value:
max_value = max_val_observed_for_class_val
if min_value < math.inf:
new_bin = max_value - min_value
new_bin /= (self._num_bins + 1)
for i in range(self._num_bins):
split = min_value + (new_bin * (i + 1))
if split > min_value and split < max_value:
splits.add(split)
return splits
class Rational:
def __init__(self):
self.numerator = SortedList()
self.denominator = SortedList()
def multiply_by(self,f):
self.numerator.update(primefac.primefac(f))
def divide_by(self,d):
self.denominator.update(primefac.primefac(d))
def value(self):
if len(self.numerator) == 0 or len(self.denominator) == 0:
return None
numerator_index = 0
denominator_index = 0
while numerator_index < len(self.numerator) and denominator_index < len(self.denominator):
if self.numerator[numerator_index] == self.denominator[denominator_index]:
del self.numerator[numerator_index]
del self.denominator[denominator_index]
elif self.numerator[numerator_index] < self.denominator[denominator_index]:
numerator_index += 1
else:
denominator_index += 1
self.numerator.add(1)
self.denominator.add(1)
num_product = reduce(lambda x, y: mpfr(x)*y, self.numerator)
den_product = reduce(lambda x, y: mpfr(x)*y, self.denominator)
if num_product <= 0 or den_product <= 0:
return 0
val = num_product/den_product
if val > 1:
return 1
return val
def __core_distance(self):
if len(self.neighbour_coordinates()) < min_pts:
return -1
else:
stop_on_next_it = False
for search_radius in range(1, eps + 2):
visited = SortedList([])
x_range = range(max(0, self.coordinates[0] - search_radius), min(mask.shape[0], self.coordinates[0] + search_radius))
y_range = range(max(0, self.coordinates[1] - search_radius), min(mask.shape[1], self.coordinates[1] + search_radius))
z_range = range(max(0, self.coordinates[2] - search_radius), min(mask.shape[2], self.coordinates[2] + search_radius))
for x in x_range:
for y in y_range:
for z in z_range:
if mask_data[(x,y,z)] > 0:
if m.fabs(data[self.coordinates] - data[(x,y,z)]) <= max_difference:
visited.add(distance(self.coordinates, (x,y,z)))
if stop_on_next_it:
return visited[min_pts - 1]
if len(visited) >= min_pts:
stop_on_next_it = True
class ChatThread(object):
"""
Represents a chat thread between the owner of the history
and a list of participants. Messages are stored in sorted
order.
"""
def __init__(self, participants):
self.participants = list(participants)
self.participants.sort()
self.messages = SortedList()
def add_message(self, message):
"""
Adds a message to the chat thread.
message -- the message to add
"""
self.messages.add(message)
def __lt__(self, other):
return len(self.messages) < len(other.messages)
def __len__(self):
return len(self.messages)
def test_add():
random.seed(0)
slt = SortedList()
for val in range(1000):
slt.add(val)
slt._check()
slt = SortedList()
for val in range(1000, 0, -1):
slt.add(val)
slt._check()
slt = SortedList()
for val in range(1000):
slt.add(random.random())
slt._check()
def test_copy():
alpha = SortedList(range(100), load=7)
beta = alpha.copy()
alpha.add(100)
assert len(alpha) == 101
assert len(beta) == 100
class Solver(object):
def __init__(self, start: Tree, goal: Tree, rules=DEFAULT_RULES):
self.heuristic = SimpleNodeAmountHeuristic
self.start = start
self.goal = goal
cost = 0.0
h = self.heuristic.estimate(start, goal)
self.nr_steps_made: int = 0
solution = SolutionStep(start, cost, h, self.nr_steps_made)
self.steps = SortedList([solution])
self.solution_hash_dict = {solution.root_hash: solution}
self.rules = sorted(
rules,
key=lambda x: x.cost,
)
self.stats = {
"operations_checked": 0,
"operations_applied": 0,
}
self.history: List[Tuple[str, str, float, float, float]] = []
def solve(self) -> Tree:
if self.start == self.goal:
return self.start
while True:
current_step, current_rule, current_nid = self._suggest_next_step()
if current_step is None:
raise UnableToFindSolutionCalculatorError(
"We tried all possible operations and could not find solution."
)
current_tree = current_step.tree
self.stats["operations_applied"] += 1
current_hash = hash(current_tree.root)
new_tree, operation_cost, typical_mistakes = current_rule.apply(current_tree, current_nid)
new_tree_cost = current_step.cost + operation_cost
assert hash(current_tree.root) == current_hash # Added to ensure that trees are not mutable
self._update(new_tree, new_tree_cost, current_tree, current_rule, current_nid, typical_mistakes)
if new_tree.root.equals(self.goal.root):
return new_tree
def get_best_path(self, tree: Tree) -> List[PathStep]:
"""get path (as lit of operations) from beginning to a given tree in the forest"""
step = self.solution_hash_dict[hash(tree.root)]
operations: List[PathStep] = []
while step.step_before:
tree_before = step.step_before
assert step.rule_before is not None
assert step.nid_before is not None
operations.append(
PathStep(
step.tree,
tree_before.tree,
step.rule_before.__class__,
step.nid_before,
step.cost,
step.h,
step.typical_mistakes,
)
)
step = tree_before
operations.reverse()
return operations
def get_solution_steps(self, tree: Tree) -> SimplificationResult:
path = self.get_best_path(tree)
steps: List[SimplificationResultStep] = []
for step in path:
typical_mistakes = [
SimplificationResultMistake(type=m.name, value=printer.do_print(m.full_error_expr))
for m in step.typical_mistakes
]
steps.append(
SimplificationResultStep(
rule_name=step.rule.__name__,
formula=printer.do_print(step.tree_current.root),
typical_mistakes=typical_mistakes,
)
)
return SimplificationResult(steps=steps)
def _suggest_next_step(
self,
) -> Tuple[
SolutionStep, BaseSolverRule, int
]: # TODO: change to prefer nodes around nodes that were modified in the previous step
for step in self.steps:
if step.fully_visited:
continue
for rule in self.rules:
rule_name = type(rule).__name__
if rule_name not in step.visited_operations:
step.visited_operations[rule_name] = []
else:
continue
for nid, node in step.tree.expand_tree():
node_type = type(node)
if node_type == rule.root_operation and nid not in step.visited_operations[rule_name]:
step.visited_operations[rule_name].append(nid)
self.stats["operations_checked"] = self.stats["operations_checked"] + 1
if rule.check_condition(step.tree, nid):
return step, rule, nid
step.fully_visited = True
raise UnableToFindSolutionCalculatorError("We tried all possible operations and could not find solution.")
def _update(
self,
new_tree: Tree,
new_tree_cost: float,
tree_before: Tree,
rule_before: BaseSolverRule,
nid_before: int,
typical_mistakes: List[TypicalMistake],
) -> None:
"""Add new calculated element to the path or, if it already exists update the cost"""
root_hash = hash(new_tree.root)
h = self.heuristic.estimate(new_tree, self.goal)
if root_hash in self.solution_hash_dict:
existing_step = self.solution_hash_dict[root_hash]
if existing_step.priority <= new_tree_cost + h:
return None
raise UnableToFindSolutionCalculatorError("Finding solution was too difficult.")
history_entry = (
printer.do_print(new_tree.root),
rule_before.__class__.__name__,
new_tree_cost,
h,
new_tree_cost + h,
)
self.history.append(history_entry)
step_before = self.solution_hash_dict[hash(tree_before.root)]
self.nr_steps_made += 1
step = SolutionStep(
new_tree, new_tree_cost, h, self.nr_steps_made, step_before, rule_before, nid_before, typical_mistakes
)
self.steps.add(step)
self.solution_hash_dict[step.root_hash] = step
class ColorTransferFunctionViewer(DataViewer):
def __init__(self, data: ColorTransferFunction = None):
super().__init__(data)
self.chart = ColorTransferFunctionChart(data)
# self.chart.legend().hide()
self.chart_rect_f = QRectF()
self.axis_x = QtCharts.QValueAxis()
# self.axis_x.setLabelFormat('%d')
self.axis_x.setLabelFormat('%.1f')
self.axis_x.setTitleText('Intensity')
self.chart.addAxis(self.axis_x, Qt.AlignBottom)
self.axis_y = QtCharts.QValueAxis()
# self.axis_y.setTickCount(10)
self.axis_y.setLabelFormat('%.2f')
# self.axis_y.setTitleText('Magnitude')
self.chart.addAxis(self.axis_y, Qt.AlignLeft)
self.axis_x.setRange(0, self.data.points[-1].x)
self.axis_y.setRange(0, 1)
# Add an empty series, else |chart.mapToPosition| will no work
self.series = self.add_series()
self.chart_view = QtCharts.QChartView(self.chart)
# self.chart_view.setRubberBand(QtCharts.QChartView.RectangleRubberBand)
self.chart_view.setRenderHint(QPainter.Antialiasing)
self.scene = self.chart_view.scene()
self._interval_views = SortedList()
self._point_views = []
if self.data is not None:
self._add_interval_views()
self._add_point_views()
self.data.point_added.connect(self._on_point_added)
grid_layout = QGridLayout()
grid_layout.setContentsMargins(0, 0, 0, 0)
grid_layout.addWidget(self.chart_view)
self.setLayout(grid_layout)
def _add_interval_views(self):
for point, next_point in pairwise(self.data.points):
if next_point is not None:
self._add_interval_view(point, next_point)
def _add_interval_view(self, begin_point: ColorTransferFunctionPoint, end_point: ColorTransferFunctionPoint) \
-> ColorTransferFunctionIntervalView:
interval_view = ColorTransferFunctionIntervalView(
self, begin_point, end_point, self.chart)
interval_view.setZValue(10) # To display item on top of chart grid
self.scene.addItem(interval_view)
self._interval_views.add(interval_view)
return interval_view
def _add_point_views(self):
for point in self.data.points:
self._add_point_view(point)
def _add_point_view(
self, point: ColorTransferFunctionPoint
) -> ColorTransferFunctionPointView:
point_view = ColorTransferFunctionPointView(self, point, self.chart)
point_view.setZValue(
11) # To display item on top of chart grid and interval views
self.scene.addItem(point_view)
self._point_views.append(point_view)
return point_view
def _update_chart_size(self):
top_left_pos = self.chart.mapToPosition(
QPointF(self.axis_x.min(), self.axis_y.max()))
bottom_right_pos = self.chart.mapToPosition(
QPointF(self.axis_x.max(), self.axis_y.min()))
self.chart_rect_f = QRectF(top_left_pos, bottom_right_pos)
def _on_point_added(self, point: ColorTransferFunctionPoint):
self._add_point_view(point)
# Update existed interval views
point_index = self.data.points.index(point)
if 0 < point_index < len(
self.data.points
) - 1: # Only if point was added between other existed points
# Exclude cases, where point was added before first and after last points
before_point_index = point_index - 1
before_point_interval = self._interval_views[before_point_index]
before_point_interval.end_point = point
before_point_interval.update()
# Add new interval view
if point_index == len(
self.data.points
) - 1: # If was added last point (point after last interval)
new_interval_view = self._add_interval_view(
self.data.point_before(point), point)
new_interval_view.update()
else:
new_interval_view = self._add_interval_view(
point, self.data.point_after(point))
new_interval_view.update()
def resizeEvent(self, resize_event: QResizeEvent):
self._update_chart_size()
# min_tick_count = self.axis_x.max() - self.axis_x.min() + 1
# tick_count = min(min_tick_count, self.width() / 50)
tick_count = self.chart_rect_f.width() / 50
self.axis_x.setTickCount(round(tick_count))
self.axis_y.setTickCount(round(self.chart_rect_f.height() / 20))
for interval_view in self._interval_views:
interval_view.update()
for point_view in self._point_views:
point_view.update_size()
point_view.update_pos()
def add_series(self):
series = QtCharts.QLineSeries()
series.setName('Color Transfer Function')
self.chart.addSeries(series)
series.attachAxis(self.axis_x)
series.attachAxis(self.axis_y)
return series
class Book:
"""
Represent bid / ask book.
* Book only allows one user order at a time
* User order will not affect book statistics like quote and volume
"""
def __init__(self, side: str, key_func: Optional[Callable[[int], int]]) -> None:
self.side = side
self.key_func = key_func if key_func else lambda x: x
# We need this because price levels follow price priority not time priority (which dict alone can provides)
self.prices = SortedList(key=key_func) # Sorted prices
self.price_levels: Dict[int, PriceLevel] = {} # Price to level map
self.order_pool: Dict[int, PriceLevel] = {} # Order ID to level map
# Store order price and PriceLevel. We do not need ID since there is only one order
self.user_order_info: Optional[Tuple[int, PriceLevel]] = None
self._front_idx: Optional[int] = None
def reset(self):
self.prices.clear()
self.price_levels.clear()
self.order_pool.clear()
self.user_order_info = None
self._front_idx = None
# ========== Order Operations ==========
def add_limit_order(self, order: LimitOrder) -> None:
""" Add limit order to the correct price level """
if order.id in self.order_pool:
raise RuntimeError(f'LimitOrder {order.id} already exists')
self.order_pool[order.id] = self._get_price_level(order.price, force_index=True).add_limit_order(order)
def match_limit_order(self, market_order: MarketOrder) -> Tuple[bool, Optional[Execution]]:
""" Match environment order against limit order. Remove empty price level where needed """
# Sometime environment order may not follow time priority. We should follow the referenced order ID in this case
user_order = None
target_price_level = self.order_pool[market_order.id]
# User orders may create price levels that do not exist in the real market. Need to match against those first
if target_price_level.price != self.prices[0]:
top_level = self.price_levels[self.prices[0]]
if top_level.shares > 0:
# Shares > 0 means that there are real LimitOrder exists in the top level
raise RuntimeError('Market order being matched against levels not in the front')
user_order = top_level.pop_user_order()
self._remove_price_level_if_empty(top_level)
# Now get the user orders that are in front of the matched real LimitOrder
price_level, exhausted, executed_order = target_price_level.match_limit_order(market_order)
self._remove_price_level_if_empty(price_level)
# It can be that both order are None
if executed_order is not None:
user_order = executed_order
# Whether the matching limit order is already exhausted
if exhausted:
del self.order_pool[market_order.id]
# Update user order pool and return executions
return exhausted, self._handle_matched_user_limit_order(user_order) if user_order else None
def cancel_order(self, order: CancelOrder) -> None:
""" Cancel (partial) shares of a LimitOrder """
self.order_pool[order.id].cancel_order(order)
def delete_order(self, order: DeleteOrder):
""" Delete the whole LimitOrder """
price_level = self.order_pool[order.id].delete_order(order)
del self.order_pool[order.id]
self._remove_price_level_if_empty(price_level)
# ========== User Order Operation ==========
def add_user_limit_order(self, order: UserLimitOrder) -> None:
"""
Add user limit order to the correct price level
* Remove the old user order if exists
* We do not want to deal with time priority because
* This simplifies the flow
* Last action's effect will spill over to the current one
"""
if self.user_order_info:
original_id, price_level = self.user_order_info
price_level.pop_user_order() # Only one user order is allowed
self._remove_price_level_if_empty(price_level)
self.user_order_info = order.price, self._get_price_level(order.price).add_user_limit_order(order)
def match_limit_order_for_user(self, order: UserMarketOrder) -> Execution:
""" Match LimitOrder for UserMarketOrder """
if self.user_order_info:
raise RuntimeError('Cannot execute MarketOrder on the side that also has user LimitOrder')
total_value = 0
shares = 0
# Recall that we are not actually matching the LimitOrders. No need to remove the executed LimitOrder.
for price in self.prices:
executed = order.shares - self.price_levels[price].match_limit_order_for_user(order)
total_value += price * executed
shares += executed
if order.shares == 0:
break
if order.shares > 0:
raise RuntimeError('User market order cannot be fully executed')
return Execution(order.id, int(total_value / shares), shares if order.side == 'B' else -shares)
def delete_user_order(self):
""" Remove user order """
if self.user_order_info:
_, price_level = self.user_order_info
price_level.pop_user_order()
self._remove_price_level_if_empty(price_level)
self.user_order_info = None
def resolve_book_crossing_on_user_order(self, price: int) -> Optional[Execution]:
"""
User orders may be placed inside the real market, in which case the newly added real order may cross with the
user orders. When this happens, we assume that the user orders are executed
"""
signed_price = self.key_func(price)
quote = self.quote
if quote and self.key_func(quote) <= signed_price:
raise RuntimeError('Real order crosses real order')
if self.user_order_info and self.key_func(self.user_order_info[0]) <= signed_price:
price_level = self.price_levels[self.user_order_info[0]]
execution = self._handle_matched_user_limit_order(price_level.pop_user_order())
# Must be empty
self._remove_price_level_if_empty(price_level)
return execution
return None
# ========== Private Methods ==========
def _get_price_level(self, price: int, force_index=False) -> PriceLevel:
""" Return price level indicated by price. Price level will be added if not already exists """
level = self.price_levels.get(price, None)
# shares == 0 means that the PriceLevel was previously occupied by user order only
if level is None:
self.prices.add(price)
level = PriceLevel(price)
self.price_levels[price] = level
# force_index is used when we are adding a new price level for real order. Order is not added at this point
# and shares will be 0. Therefore, we need to force it
# On the other hand, we still need to run update_front_index for user order because it may change the
# ordering
self._update_front_index(force_index, price)
elif level.shares == 0:
self._update_front_index(force_index, price)
return level
def _remove_price_level_if_empty(self, price_level: PriceLevel):
""" Remove PriceLevel if empty """
if price_level.empty:
del self.price_levels[price_level.price]
# "remove" will raise ValueError if not exists
self.prices.remove(price_level.price)
if price_level.shares == 0:
# Separate from the logic above because we run be in the situation where real orders are exhausted
# but at least one user order is waiting. In this case, this price level is technically gone
self._update_front_index()
def _update_front_index(self, force_index=False, target_price=None) -> None:
""" Find out the first price level that has real order """
if not self.prices:
self._front_idx = None
else:
price = self.prices[0]
if self.price_levels[price].shares > 0 or (force_index and price == target_price):
self._front_idx = 0
else:
self._front_idx = 1 if len(self.prices) > 1 else None
def _handle_matched_user_limit_order(self, order: UserLimitOrder) -> Execution:
""" Book-keeping actions for UserLimitOrder execution """
self.user_order_info = None
return Execution(order.id, order.price, order.shares if self.side == 'B' else -order.shares)
# ========== Properties ==========
# These statistics should not include user orders. Otherwise, we may end up being our own market
@property
def quote(self) -> Optional[int]:
""" Return the front price without user orders """
if self._front_idx is not None:
return self.prices[self._front_idx]
return None
@property
def volume(self) -> Optional[int]:
""" Return the volume at the front without user orders """
if self._front_idx is not None:
return self.price_levels[self.quote].shares
return None
def get_depth(self, num_levels: int) -> List[Tuple[int, int]]:
""" Return the top n price levels without user orders """
if self._front_idx is not None:
return [(price, self.price_levels[price].shares)
for price in self.prices[self._front_idx: self._front_idx + num_levels]]
return []
@property
def empty(self) -> bool:
if len(self.order_pool) == 0:
if self.user_order_info is None:
return len(self.prices) == 0
return len(self.prices) == 1 and self.price_levels[self.prices[0]].shares == 0
return False
@property
def user_order_price(self) -> Optional[int]:
if self.user_order_info:
return self.user_order_info[0]
return None
# Read Previous Results, if any
if AUTH_OUT == AUTH_PUBS:
AuthCount = 0
PubCount = 0
try:
with fileinput.input(files=(AUTH_PUBS),
openhook=fileinput.hook_encoded("utf-8")) as f:
print("Processing Previous Results")
for line in f:
pub = json.loads(line)
PubCount += 1
if pub['auth_sid'] not in AuthIDs:
AuthCount += 1
if not (AuthCount % 1000):
print("Loaded {} Authors".format(AuthCount))
AuthIDs.add(pub['auth_sid'])
print("{} Authors Processed".format(AuthCount))
print("{} Publications Processed".format(PubCount))
except:
pass
PubOut = Output(AUTH_OUT)
# Fetch Publications for a given Author ID. This can be called in parallel for multiple IDs
def fetch_pubs(auth):
results = []
pubs = auth.get_pubs(view='STANDARD')
for pub in pubs:
results.append({
"auth_sid": auth.SID,
def test_len():
slt = SortedList()
for val in range(10000):
slt.add(val)
assert len(slt) == (val + 1)
#stop_date = datetime.strptime(row['Stoptime'], '%m/%d/%y %H:%M')
start_date = parser.parse(row['Starttime'])
stop_date = parser.parse(row['Stoptime'])
start_day = start_date.date()
stop_day = stop_date.date()
# store the start time of each bike in sorted order
if bike_id in dates_time:
new_list = [from_sta_id, to_sta_id, start_day]
dates_time[bike_id].update({stop_date: new_list})
else:
dates_time[bike_id] = {
stop_date: [from_sta_id, to_sta_id, start_day]
}
if start_day not in date_as_column:
date_as_column.add(start_day)
if stop_day not in date_as_column:
date_as_column.add(stop_day)
# print(day,date)
# check if the from station_id is valid
# print(type(from_sta_id),from_sta_id)
if not math.isnan(from_sta_id):
if from_sta_id in station_records:
if start_day in station_records[from_sta_id]:
station_records[from_sta_id][start_day]['fromCNT'] += 1
else:
new_day = {'fromCNT': 1, 'toCNT': 0, 'rebalCNT': 0}
station_records[from_sta_id][start_day] = new_day
else:
station_records[from_sta_id] = {
start_day: {
def choose_sense(self, sense_actions: List[Square], move_actions: List[chess.Move], seconds_left: float) -> \
Optional[Square]:
allSqr = chess.SquareSet(chess.BB_ALL)
for square in allSqr:
if self.board.piece_at(square) is not None and self.board.piece_at(
square).color == self.color:
self.lastScanned[square] = 0
else:
self.lastScanned[square] = self.lastScanned[square] + 1
self.moveNum = self.moveNum + 1
if self.panic:
for square, piece in self.board.piece_map().items():
if piece.color == self.color:
sense_actions.remove(square)
return random.choice(sense_actions)
if self.interestingTile:
print("Scanning interesting tile")
return self.interestingTile
if self.expectedEnemy is not None:
if self.board.piece_at(self.expectedEnemy.to_square) is None:
print("Scanning expected move")
return self.expectedEnemy.to_square
m_king_square = self.board.king(self.color)
if m_king_square:
attackers = self.board.attackers(not self.color, m_king_square)
if not attackers:
potentialThreats = SortedList(key=lambda x: x[1])
tB = self.board.copy()
bfs = []
bfs.append((tB, None, 0, -1))
while len(bfs) > 0:
cS = bfs.pop(0)
if cS[0].turn == self.color:
cS[0].push(chess.Move.null())
m_king_square = cS[0].king(self.color)
if m_king_square:
attackers = cS[0].attackers(not self.color,
m_king_square)
if attackers:
potentialThreats.add(
(cS[1], self.lastScanned[cS[1]]))
continue
for mv in cS[0].legal_moves:
if cS[2] >= 3:
continue
if cS[2] >= cS[3] and cS[3] != -1:
continue
if cS[0].piece_at(mv.to_square) is not None:
continue
if self.lastScanned[mv.from_square] <= 1:
continue
nB = cS[0].copy()
nB.push(mv)
if cS[3] != -1:
bfs.append(
(nB,
cS[1] if cS[1] is not None else mv.to_square,
cS[2] + 1,
min(cS[3],
cS[2] + self.lastScanned[mv.to_square])))
else:
bfs.append(
(nB,
cS[1] if cS[1] is not None else mv.to_square,
cS[2] + 1, self.lastScanned[mv.from_square]))
if len(potentialThreats) > 0:
pot = potentialThreats.pop(-1)
print("Scanning threat: ", chess.SQUARE_NAMES[pot[0]])
return pot[0]
self.nextMove = self.choose_move(move_actions, seconds_left)
if self.nextMove is not None:
print("Scanning next move")
return self.nextMove.to_square
for square, piece in self.board.piece_map().items():
if piece.color == self.color:
sense_actions.remove(square)
print("Scanning random")
return random.choice(sense_actions)
def test(filename):
"""
run bentley ottmann
"""
adjuster, segments = load_segments(filename)
#segments = sorted(segments, key=lambda x: min(x.endpoints[0].coordinates[1],
# x.endpoints[0].coordinates[1]))
events = SortedList()
for s in segments:
events.add((min(s.endpoints), "in", s))
events.add((max(s.endpoints), "out", s))
sweep = SweepLines()
result = []
if DEBUG:
print("Events (init):", events)
print("\n========\n LOOP \n========\n\n ")
while True:
try:
current, event_type, segment = events.pop(0)
Segment.point = current
if DEBUG:
print("Current:", current, event_type, segment)
if DEBUG:
print("Events:", events)
if DEBUG:
print("SL:", len(sweep), sweep)
tmp_sweep = SweepLines()
for node in sweep:
tmp_sweep.put(node.value)
sweep = tmp_sweep
if DEBUG:
print("SL:", len(sweep), sweep)
if event_type == "in":
node = sweep.put(segment)
left = node.predecessor()
if left:
left = left.value
intrsctn = segment.intersection_with(left)
if intrsctn is not None:
intrsctn = adjuster.hash_point(intrsctn)
if intrsctn.coordinates[1] <= current.coordinates[
1] and intrsctn.coordinates[
0] != current.coordinates[0]:
events.add((intrsctn, "x", (left, segment)))
right = node.successor()
if right:
right = right.value
intrsctn = segment.intersection_with(right)
if intrsctn is not None:
intrsctn = adjuster.hash_point(intrsctn)
if intrsctn.coordinates[1] <= current.coordinates[
1] and intrsctn.coordinates[
0] != current.coordinates[0]:
events.add((intrsctn, "x", (segment, right)))
elif event_type == "out":
node = sweep.search(segment)
left = node.predecessor()
right = node.successor()
if left and right:
left = left.value
right = right.value
intrsctn = left.intersection_with(right)
if intrsctn is not None:
intrsctn = adjuster.hash_point(intrsctn)
if intrsctn.coordinates[1] <= current.coordinates[
1] and intrsctn.coordinates[
0] != current.coordinates[0]:
events.add((intrsctn, "x", (left, right)))
sweep.delete(segment)
else: #event_type == "x"
result.append(current)
u = sweep.search(segment[0])
right = u.successor()
if right:
intrsctn = u.value.intersection_with(right.value)
if intrsctn is not None:
intrsctn = adjuster.hash_point(intrsctn)
if intrsctn.coordinates[1] <= current.coordinates[
1] and intrsctn.coordinates[
0] != current.coordinates[0]:
events.add((intrsctn, "x", (u.value, right.value)))
v = sweep.search(segment[1])
left = v.predecessor()
if left:
intrsctn = v.value.intersection_with(left.value)
if intrsctn is not None:
intrsctn = adjuster.hash_point(intrsctn)
if intrsctn.coordinates[1] <= current.coordinates[
1] and intrsctn.coordinates[
0] != current.coordinates[0]:
events.add((intrsctn, "x", (left.value, v.value)))
if DEBUG:
print("Events:", events)
if DEBUG:
print("SL:", len(sweep), sweep)
#tycat(segments, result, current)
#input("Press [ENTER] to continue...\n")
except IndexError:
break
print("\n\n=========\n THE END\n=========")
if DEBUG:
print("Events:", events)
print("SL:", sweep)
print("IL:", result)
tycat(segments, result)
def bentley_ottmann(filename, nodisp=False, noinfo=False):
"""
Fonction principale de notre projet
"""
global COUPE
COUPE = 0
y_cour = None
adjuster, segments = load_segments(filename)
actifs = SortedList()
evenements = [
] #liste de nos evenements, valeurs des y, que lon transformera en Tas ensuite
pt_inter = {
} #dictionnaire que lon retournera a la fin, associant les segments interseptés
index = 0
cache_inters = {} #cache qui nous dira si on a deja compare 2 seg
intersections = [] #liste contenant tous nos points dintersections
for seg in segments:
#initialisation de nos evenements
(x_0, y_0) = seg.endpoints[0].coordinates
(x_1, y_1) = seg.endpoints[1].coordinates
Segment.y_cour = [x_0, y_0]
if y_0 < y_1: #Segments croissant suivant les y
evenements.append([y_0, -x_0, seg, 'D'])
evenements.append([y_1, -x_1, seg, 'F'])
elif y_0 > y_1: #Segments decroissant suivant les y:
evenements.append([y_0, -x_0, seg, 'F'])
evenements.append([y_1, -x_1, seg, 'D'])
else: #Cas d'un segment horizontal
evenements.append([y_1, -min(x_0, x_1), seg, max(x_0, x_1)])
pt_inter[seg] = [] #Initialisation du dictionnaire
cache_inters[seg] = []
heapify(
evenements
) #Tas des evenement,3 types, 'D' 'F' 'I': Debut, fin, intersection
#trié en fonction des y croissant, puis des x décroissants.
def indice(seg):
"""
Retourne l'indice de seg dans la liste actifs, None si le segment n'est
pas présent. Cette fonction auxiliaire est implémentée suite aux
problèmes majeurs rencontrés avec la méthode index de la classe
SortedList
"""
for i, elmt in enumerate(actifs):
if seg is elmt:
return i
def intersection(seg, seg_2):
"""
Fonction qui va légitimer et gérer l'intersection entre 2 segments
donnés.
"""
global COUPE
if seg_2 not in cache_inters[seg]: #On ne compare pas deux segments
#déja comparés
intersection = seg.intersection_with(seg_2)
cache_inters[seg].append(seg_2)
cache_inters[seg_2].append(seg)
if intersection is not None:
intersection = adjuster.hash_point(intersection) #Ajustement
if intersection not in seg.endpoints or intersection not in seg_2.endpoints:
#Le point nest pas lextrémitié des deux segments
pt_inter[seg].append(seg_2)
pt_inter[seg_2].append(seg)
heappush(evenements, [
intersection.coordinates[1],
-intersection.coordinates[0], seg, 'I', seg_2
])
#L'ordre dans le tuple est important: il permet de savoir
#qui est à gauche ou à droite
if intersection not in intersections:
intersections.append(intersection)
COUPE += 1
return
while evenements: #Boucle traitant tous les évènements tant que notre tas
#n'est pas vide.
y_cour = heappop(evenements)
if y_cour[3] == 'D': #evenement de debut de segment
Segment.y_cour = [-y_cour[1], y_cour[0]]
actifs = SortedList(actifs) #Mise à jour de actifs
seg = y_cour[2]
actifs.add(seg) #Ajout du nouveau segment aux actifs
if len(actifs
) > 1: #Si un seul segment dans actifs: on ne fait rien
try:
index = actifs.index(seg)
except ValueError:
index = indice(seg)
if index != len(actifs) - 1:
seg_2 = actifs[index + 1]
intersection(seg, seg_2)
if index != 0:
seg_2 = actifs[index - 1]
intersection(seg_2, seg)
elif y_cour[3] == 'F': #evenement de fin de segment
Segment.y_cour = [-y_cour[1], y_cour[0]]
actifs = SortedList(actifs) #Mise à jour de actifs
seg = y_cour[2]
try:
index = actifs.index(seg)
except ValueError:
index = indice(seg)
actifs.pop(index)
actifs = SortedList(actifs) #Mise à jour de actifs
if len(actifs) > 1:
if 0 < index < len(actifs): #On n'enleve pas le seg le plus à
#droite/gauche
seg = actifs[index]
seg_2 = actifs[index - 1]
intersection(seg, seg_2)
elif y_cour[3] == 'I': #evenement de point d'intersection
seg, seg_2 = y_cour[2], y_cour[4]
try:
actifs.remove(seg)
except ValueError:
index = indice(seg)
if index is not None: #Renvoie parfois une erreur:
#"segment not in actifs"
del actifs[index]
try:
actifs.remove(seg_2)
except ValueError:
index_2 = indice(seg_2)
if index_2 is not None:
del actifs[index_2]
Segment.y_cour = [-y_cour[1], y_cour[0] + 0.00000000001]
#Cf. convention: A une intersection, on se situe
#au dessus de l'intersection
actifs = SortedList(actifs) #Mise à jour de actifs
actifs.add(seg) #Une fois changés de place l'intersection passée,
#on remet nos deux segments dans actifs
actifs.add(seg_2)
try:
index = actifs.index(seg) #Indice du seg a droite une fois
#l'intersection faite
except ValueError:
index = indice(seg)
if len(actifs
) > 2: #On teste les nouvelles intersections possibles
if index < len(
actifs) - 1: #Cas de l'extrémité droite de actifs
seg_2 = actifs[index + 1]
intersection(seg, seg_2)
if index - 1 != 0: #Cas de l'extrémité gauche
seg_2 = actifs[index - 2]
intersection(seg_2, y_cour[4])
else: #Cas dun segment horizontal
seg_h = y_cour[2]
for seg in actifs:
inter = seg_h.intersection_with(seg)
if inter:
inter = adjuster.hash_point(inter)
if inter not in seg_h.endpoints or inter not in seg.endpoints:
#Le point n'est pas l'extrémité ds deux segments
pt_inter[seg_h].append(seg)
pt_inter[seg].append(seg_h)
if inter not in intersections:
intersections.append(inter)
COUPE += 1
if nodisp and noinfo:
return pt_inter, intersections
if noinfo:
tycat(segments, intersections)
return pt_inter, intersections
if nodisp:
print(
"Le nombre d'intersections (= le nombre de points differents) est : ",
len(intersections))
print("Le nombre de coupes est : ", COUPE)
return pt_inter, intersections
print(
"le nombre d'intersections (= le nombre de points differents) est : ",
len(intersections))
print("le nombre de coupes est : ", COUPE)
def choose_move(self, move_actions: List[chess.Move],
seconds_left: float) -> Optional[chess.Move]:
self.approxTime = seconds_left
self.response = None
if self.nextMove is not None:
mv = self.nextMove
self.nextMove = None
return mv
if self.board.turn is not self.color:
self.board.push(chess.Move.null())
e_king_square = self.board.king(not self.color)
if e_king_square:
attackers = self.board.attackers(self.color, e_king_square)
for attack in attackers:
m = chess.Move(attack, e_king_square)
if m in move_actions:
return m
# m_king_square = self.board.king(self.color)
# if m_king_square:
# attackers = self.board.attackers(not self.color, m_king_square)
# if attackers:
# legalMoves = self.board.legal_moves
# for move in legalMoves:
# tempBoard = self.board.copy()
# tempBoard.push(move)
# attackers = tempBoard.attackers(not self.color, tempBoard.king(self.color))
# if (attackers is None or len(attackers) is 0) and move in move_actions:
# return move
class State:
def __init__(self, board, iM):
self.brd = board
self.score = 0
self.initialMove = iM
self.responses = SortedList(key=lambda x: x.score)
def eval_board(board) -> float:
score = 0
for square, piece in board.piece_map().items():
mult = 1
attackers = board.attackers(not piece.color, square)
defenders = board.attackers(piece.color, square)
val = 0
if piece.color != self.color:
mult = -1
if piece.piece_type == chess.PAWN:
val = 5
elif piece.piece_type == chess.KNIGHT:
val = 10
elif piece.piece_type == chess.BISHOP:
val = 10
elif piece.piece_type == chess.ROOK:
val = 10
elif piece.piece_type == chess.QUEEN:
val = 20
elif piece.piece_type == chess.KING:
val = 100
#if len(attackers) != 0 and len(defenders) == 0 and piece.color == self.color:
# val = val / 10
#if len(defenders) != 0 and piece.piece_type != chess.KING and piece.color != self.color:
# val = val * 2
score = score + mult * val
return score
self.score = eval_board(self.board)
trialsPer = 20
depth = 10
wasKing = False
if self.board.king(not self.color):
wasKing = True
self.panic = False
else:
wasKing = False
self.panic = True
potentials = SortedList(key=lambda x: (x.score))
for mv in self.board.legal_moves:
startState = State(self.board.copy(), mv)
startState.brd.push(mv)
count = 0
for resp in startState.brd.legal_moves:
res = State(startState.brd.copy(), resp)
res.brd.push(resp)
scr = 0
for i in range(trialsPer):
tempBoard = res.brd.copy()
tscr = 0
for j in range(depth):
if tempBoard.turn == self.color:
e_king_square = tempBoard.king(not self.color)
if e_king_square:
attackers = tempBoard.attackers(
self.color, e_king_square)
if attackers:
tscr = 1000
break
m_king_square = tempBoard.king(self.color)
if m_king_square:
attackers = tempBoard.attackers(
not self.color, m_king_square)
if attackers:
tscr = -1000
break
if tempBoard.turn != self.color:
m_king_square = tempBoard.king(self.color)
if m_king_square:
attackers = tempBoard.attackers(
not self.color, m_king_square)
if attackers:
tscr = -1000
break
e_king_square = tempBoard.king(not self.color)
if e_king_square:
attackers = tempBoard.attackers(
self.color, e_king_square)
if attackers:
tscr = 1000
break
potMoves = []
for pM in tempBoard.legal_moves:
potMoves.append(pM)
if len(potMoves) != 0:
tempBoard.push(random.choice(potMoves))
else:
tscr = -1000
break
if tscr == 0:
tscr = eval_board(tempBoard)
scr = scr + tscr
scr = scr / trialsPer
res.score = scr
startState.responses.add(res)
startState.score = startState.score + scr
count = count + 1
if count != 0:
startState.score = startState.score / count
else:
startState.score = eval_board(startState.brd)
potentials.add(startState)
while len(potentials) > 0:
top = potentials.pop(-1)
if top.initialMove in move_actions:
if len(top.responses) > 0:
self.expectedEnemy = top.responses.pop(0).initialMove
self.expectedScore = top.score
return top.initialMove
return random.choice(move_actions + [None])
def _all_segment_intersections_no_horizontal(segments): # noqa
# Must be unique
assert len(set(segments)) == len(segments)
segments = list(segments)
# Must not be degenerate
for segment in segments:
assert segment[0] != segment[1]
# Use the convention from the book: sweep on Y axis
def event_key(pt):
return (pt[1], pt[0])
# From point to list of segments
event_queue = SortedDict(event_key)
def add_event(pt, segment_key=None):
if pt not in event_queue:
event_queue[pt] = []
if segment_key is not None:
event_queue[pt].append(segment_key)
for i, segment in enumerate(segments):
if event_key(segment[0]) < event_key(segment[1]):
add_event(segment[0], _SweepKey(segment, segment[0]))
add_event(segment[1], None)
else:
add_event(segment[0], None)
add_event(segment[1], _SweepKey(segment, segment[1]))
active = SortedList()
y = -math.inf
while len(event_queue) > 0:
v = event_queue.popitem(0)
pt, segstarts = v
# Can't be > since while there are no horizontal segments,
# there can still be points in horizontal relation to one another
assert pt[1] >= y
y = pt[1]
# Find all segments within the event point
fake_segment = ((pt[0], pt[1]), (pt[0], pt[1] + 1))
fake_key = _SweepKey(fake_segment, pt)
touches = []
# The next lower / higher keys, respectively, to enter new events for
neighbours = []
if _extra_checks:
_assert_fully_sorted(list(active), y)
# Iterate on both sides
for it in (
active.irange(None,
fake_key,
inclusive=(True, True),
reverse=True),
active.irange(fake_key, None, inclusive=(False, True)),
):
neighbour = None
for sweep_key in it:
if sweep_key.at_y(y) != pt[0]:
neighbour = sweep_key
break
touches.append(sweep_key)
neighbours.append(neighbour)
# Remove the old sweep keys
for touch in touches:
active.remove(touch)
segments_at_pt = [
sweep_key.segment for sweep_key in touches + segstarts
]
if len(segments_at_pt) > 1:
yield (pt, tuple(segments_at_pt))
# Create new _SweepKeys, automatically sorts
# according to order after point
sweep_keys = []
for segment in segments_at_pt:
# Is this segment still relevant?
if max(segment[0][1], segment[1][1]) <= pt[1]:
continue
sweep_keys.append(_SweepKey(segment, pt))
sweep_keys = list(sorted(sweep_keys))
# Add new events for neighbours
if len(sweep_keys) == 0:
# If we just removed stuff, the neighbours might now meet...
if neighbours[0] is not None and neighbours[1] is not None:
ipt = _nonparallel_intersection_point(neighbours[0].segment,
neighbours[1].segment)
if ipt and ipt[1] > pt[1]:
add_event(ipt)
continue
if neighbours[0] is not None:
ipt = _nonparallel_intersection_point(sweep_keys[0].segment,
neighbours[0].segment)
# hyp.note(fstr('IPTL', ipt, pt))
if ipt and ipt[1] > pt[1]:
add_event(ipt)
if neighbours[1] is not None:
ipt = _nonparallel_intersection_point(sweep_keys[-1].segment,
neighbours[1].segment)
# hyp.note(fstr('IPTR', ipt, pt))
if ipt and ipt[1] > pt[1]:
add_event(ipt)
# Add them in and continue
for sweep_key in sweep_keys:
active.add(sweep_key)
class SweepStatus:
def __init__(self):
self.lines_list = SortedList()
self.num_lines = 0
# all lines as seen by the sweep line
def find_line(self, l):
# can replace this with a binary search
# return binary_search(self.lines_list,SweepStatusEntry(l,-1))
for i in range(len(self.lines_list)):
if self.lines_list[i].l == l:
return i
return None
def check_intersection(self, l_index, direction):
res = []
print("Looking for intersection at index", l_index, " direction =",
direction)
cur = l_index #the index of the line which we are checking
if direction == "left":
cur = l_index - 1
while cur >= 0:
p = intersection_point(self.lines_list[l_index].l,
self.lines_list[cur].l)
if p is not None:
res.append(
IntersectionPoint(p, self.lines_list[cur].l,
self.lines_list[l_index].l))
else:
break
cur -= 1
else: # direction is "right"
cur = l_index + 1
while cur <= len(self.lines_list) - 1:
p = intersection_point(self.lines_list[l_index].l,
self.lines_list[cur].l)
if p is not None:
res.append(
IntersectionPoint(p, self.lines_list[l_index].l,
self.lines_list[cur].l))
else:
break
cur += 1
s = "Found Intersection Point(s): "
for l in res:
s += (str(l) + '\n')
s += "%%%%%%%%%%"
print(s)
return res
def add_line(self, new_l): # return an iterable
print("Adding line ", new_l)
res = []
# insert new one in place
new_entry = SweepStatusEntry(new_l, -1)
new_index = bisect.bisect(self.lines_list, new_entry)
print(new_index)
new_key = self.num_lines + 1
self.num_lines += 1
self.lines_list.add(SweepStatusEntry(new_l, new_key))
# search is independent of the key
# need the key to keep the line rooted in place
print("lines list after adding")
for i in self.lines_list:
print(i)
print()
res.extend(self.check_intersection(new_index, "left"))
res.extend(self.check_intersection(new_index, "right"))
return res
def remove_line(self, l):
print("Removing line ", l)
res = []
l_index = self.find_line(l)
if l_index is None:
return res
# remove the line found
self.lines_list.pop(l_index)
# check for new intersections
if (len(self.lines_list) != 0):
res.extend(self.check_intersection(l_index - 1, "right"))
# new l_index is the one that was on the right
return res
def intersection_point(self, ipt):
print("Reached intersection point ", ipt)
res = []
l1_index = self.find_line(ipt.l1)
if (l1_index == None):
return res
l2_index = self.find_line(ipt.l2)
if (l2_index == None):
return res
print("Swapping indices: ", l1_index, l2_index)
self.lines_list[l1_index].l = ipt.l2
self.lines_list[l2_index].l = ipt.l1
print("Lines after swapping: ")
print(self.lines_list[l1_index])
print(self.lines_list[l2_index])
if l1_index > l2_index:
temp = l1_index
l1_index = l2_index
l2_index = temp
# now l1_index is the lower index
res.extend(self.check_intersection(l1_index, "left"))
res.extend(self.check_intersection(l2_index, "right"))
return res
def __str__(self):
s = "--- Sweep Line Status: ---\n"
for i in self.lines_list:
s += str(i)
s += " ;\n"
s += "-----------------"
return s
return ''.join(chars)
def pron_strs(prons):
return ' '.join(map(lambda x: pron_str(x), prons))
for pron1 in prons:
closest_prons = []
dists = SortedList()
for pron2 in prons:
if pron1 != pron2:
dist = Levenshtein.distance(pron1, pron2)
if len(dists) > 0 and dist < dists[0]:
closest_prons = []
dists.add(dist)
if dist == dists[0]:
closest_prons.append(pron2)
best1_dists.append(dists[0])
best5_dists.append(np.mean(dists[:5]))
print("%s - %s (%.2f)" %
(pron_str(pron1), pron_strs(closest_prons), dists[0]))
best1_dists = np.array(best1_dists)
best5_dists = np.array(best5_dists)
print()
print("For %.1f %% of pronuncations the shortest Levenshtein distance is 1" %
(100 * (best1_dists == 1).sum() / len(dists)))
print("1-best average Levenshtein distance: %.2f" % best1_dists.mean())
print("5-best average Levenshtein distance: %.2f" % best5_dists.mean())
class UrlClusterizedQueue:
def __init__(self):
self._features_count_list = SortedList() # list of pairs (feature_count, feature_name)
self._features_count_dict = dict()
self._clusterizer = DBSCAN(metric='jaccard')
self._min_freq = 0.1
self._max_freq = 0.9
self._urls = dict()
self._urls_keys = []
self._index = -1
self._min_urls_count = 50
self._subqueue = Queue()
self._subqueue_len = 4
# constants
self._i_is_used = 1
self._i_features = 0
self._i_list_for_unused = 0
self._i_list_for_used = 1
self._i_list_total = 2
def _return_url(self, url):
self._urls[url][1] = True
return url
def _next_queue_fallback(self):
self._index += 1
url = self._urls_keys[self._index]
return self._return_url(url)
def _run_clustering(self):
print("try to use clustering")
# here we need to run clustering
# first of all we need to choose features
max_feature_count = int(self._max_freq * len(self._urls))
min_feature_count = int(self._min_freq * len(self._urls))
start_index = bisect.bisect_left(self._features_count_list, (min_feature_count, ''))
end_index = bisect.bisect_right(self._features_count_list, (max_feature_count, 'ZZZ'))
if start_index >= end_index:
print("not enough features")
return self._next_queue_fallback()
chosen_features = SortedSet()
for i in range(start_index, end_index):
chosen_features.add(self._features_count_list[i][1])
# then we need to build features matrix
X = np.empty((len(self._urls), len(chosen_features)))
for i in range(len(self._urls)):
features = self._urls[self._urls_keys[i]][self._i_features]
for j, fname in enumerate(chosen_features):
if fname in features:
X[i][j] = 1
else:
X[i][j] = 0
# now we can run clustering
y = self._clusterizer.fit_predict(X)
# and we need to create uniform distributed queue
def get_list_of_2_sets():
return [set(), set(), 0] # 0 is for used urls, # 1 is for unused # 3 is for total count
url_in_cluster = defaultdict(get_list_of_2_sets)
for i in range(len(y)):
url = self._urls_keys[i]
if self._urls[url][self._i_is_used]:
url_in_cluster[y[i]][self._i_list_for_used].add(url)
else:
url_in_cluster[y[i]][self._i_list_for_unused].add(url)
url_in_cluster[y[i]][self._i_list_total] += 1
limit = self._subqueue_len
cluster_keys = SortedKeyList(url_in_cluster.keys(), key=lambda x: -len(url_in_cluster[x][self._i_list_for_used]))
while limit > 0: # Todo: optimize
if len(cluster_keys) > 0:
less_index = cluster_keys.pop()
unused_urls = url_in_cluster[less_index][self._i_list_for_unused]
if len(unused_urls) > 0:
url = unused_urls.pop()
self._subqueue.put(url)
limit -= 1
if len(unused_urls) > 0:
url_in_cluster[less_index][self._i_list_for_used].add(url)
cluster_keys.add(less_index)
else:
break
async def _run_and_wait_clustering(self):
t = threading.Thread(target=UrlClusterizedQueue._run_clustering, args=(self,))
t.run()
while t.is_alive():
await asyncio.sleep(0.3)
async def get(self):
if len(self._urls) < self._min_urls_count:
return self._next_queue_fallback()
else:
if self._subqueue.empty():
await self._run_and_wait_clustering()
# if self._subqueue.qsize() == 1:
# asyncio.create_task(self._run_and_wait_clustering())
return self._return_url(self._subqueue.get())
async def empty(self):
if self._index + 1 >= len(self._urls):
return True
else:
return False
async def put(self, url):
if url in self._urls:
return
features = url_features.extract(url)
for fname in features:
if fname in self._features_count_dict:
fcount = self._features_count_dict[fname]
del self._features_count_list[self._features_count_list.index((fcount, fname))]
else:
fcount = 0
fcount += 1
self._features_count_dict[fname] = fcount
self._features_count_list.add((fcount, fname))
self._urls[url] = [features, False] # False is for used
self._urls_keys.append(url)
def test_len():
slt = SortedList()
for val in range(10000):
slt.add(val)
assert len(slt) == (val + 1)
class SegmentedBuffer:
__slots__ = ('buffer_size', 'buffer_bits', 'data_bits', 'data_size',
'method', 'sorted_data_count', 'remaining_bits',
'remaining_size', 'remaining')
def __init__(self):
self.buffer_bits: int = 0
self.buffer_size: int = 0
self.data_bits: int = 0
self.data_size: int = 0
self.method: Method = Method.UNDEFINED
self.sorted_data_count: SortedList[DataCount] = SortedList()
self.remaining_bits: int = 0
self.remaining_size: int = 0
self.remaining: DataType = 0
# check & print values
def check_values(self, file: AnyFile = sys.stderr):
max_data = bits_mask(self.data_bits)
for data_count in self.sorted_data_count:
if data_count.data > max_data:
print(f'!\tinvalid data: {data_count.data:0>b}', file=file)
def print_values(self, file: AnyFile = sys.stdout):
_print_bits_size('buffer',
self.buffer_bits,
self.buffer_size,
file=file)
_print_bits_size('data', self.data_bits, self.data_size, file=file)
if self.method == Method.INT:
def print_data_count(item: DataCount):
return print(
f'-\t{item.data:0>{self.data_bits}b}: {item.count}',
file=file)
else:
raise ValueError(f'unsupported method: {self.method}')
for data_count in self.sorted_data_count:
print_data_count(data_count)
_print_bits_size('remaining',
self.remaining_bits,
self.remaining_size,
file=file)
if self.method == Method.INT:
print(f'-\t{self.remaining:0>{self.remaining_bits}b}')
# write, read
def _write_sorted_data_count(self, wrapper: FileWrapper,
size_of_size: int):
wrapper.write_unsigned_int(len(self.sorted_data_count), size_of_size)
if self.method == Method.INT:
for data_count in self.sorted_data_count:
wrapper.write_unsigned_int(data_count.data, self.data_size)
wrapper.write_unsigned_int(data_count.count, size_of_size)
else:
raise ValueError(f'unsupported method: {self.method}')
def _read_sorted_data_count(self, wrapper: FileWrapper, size_of_size: int):
length = wrapper.read_unsigned_int(size_of_size)
self.sorted_data_count = SortedList()
if self.method == Method.INT:
for step in range(length):
data = wrapper.read_unsigned_int(self.data_size)
count = wrapper.read_unsigned_int(size_of_size)
self.sorted_data_count.add(DataCount(data, count))
else:
raise ValueError(f'unsupported method: {self.method}')
return SortedList()
def _write_remaining(self, wrapper: FileWrapper, size_of_size: int):
if self.method == Method.INT:
wrapper.write_unsigned_int(self.remaining, self.remaining_size)
elif self.method == Method.BYTE:
wrapper.write_bytes(self.remaining)
else:
raise ValueError(f'unsupported method: {self.method}')
def _read_remaining(self, wrapper: FileWrapper):
if self.method == Method.INT:
self.remaining = wrapper.read_unsigned_int(self.remaining_size)
elif self.method == Method.BYTE:
self.remaining = wrapper.read_bytes(self.remaining_size)
else:
raise ValueError(f'unsupported method: {self.method}')
def write(self, wrapper: FileWrapper, size_of_size: int = 4):
wrapper.write_unsigned_int(size_of_size, 1) # size of size
wrapper.write_unsigned_int(self.buffer_bits,
size_of_size) # buffer bits
wrapper.write_unsigned_int(self.buffer_size,
size_of_size) # buffer size
wrapper.write_unsigned_int(self.data_bits, size_of_size) # data bits
wrapper.write_unsigned_int(self.data_size, size_of_size) # data size
wrapper.write_unsigned_int(self.remaining_bits,
size_of_size) # remaining bits
wrapper.write_unsigned_int(self.remaining_size,
size_of_size) # remaining size
self.method.write(wrapper) # method
# sorted data count & remaining
self._write_sorted_data_count(wrapper, size_of_size)
self._write_remaining(wrapper, size_of_size)
def read(self, wrapper: FileWrapper):
size_of_size = wrapper.read_unsigned_int(1) # size of size
self.buffer_bits = wrapper.read_unsigned_int(
size_of_size) # buffer bits
self.buffer_size = wrapper.read_unsigned_int(
size_of_size) # buffer size
self.data_bits = wrapper.read_unsigned_int(size_of_size) # data bits
self.data_size = wrapper.read_unsigned_int(size_of_size) # data size
self.remaining_bits = wrapper.read_unsigned_int(
size_of_size) # remaining bits
self.remaining_size = wrapper.read_unsigned_int(
size_of_size) # remaining size
self.method = Method.read(wrapper) # method
# sorted data count & remaining
self._read_sorted_data_count(wrapper, size_of_size)
self._read_remaining(wrapper)
@staticmethod
def static_read(wrapper: FileWrapper):
instance = SegmentedBuffer()
instance.read(wrapper)
return instance
@staticmethod
def static_write(wrapper: FileWrapper,
instance: 'SegmentedBuffer',
size_of_size: int = 4):
return instance.write(wrapper, size_of_size=size_of_size)
# scan & sort data count
def _scan_data_count(self, buffer: bytes):
if self.method == Method.BYTE:
if self.data_bits % 8 == 0:
max_index = self.buffer_size - self.remaining_size
count_dict = defaultdict(lambda: 1)
for index in range(0, max_index, self.data_size):
count_dict[buffer[index:index + self.data_size]] += 1
return count_dict, buffer[max_index:]
else:
raise ValueError(
f'{self.data_bits} data bits is not supported in {self.method}'
)
elif self.method == Method.INT:
count_dict = defaultdict(lambda: 0)
if self.data_bits == 8:
for value in buffer:
count_dict[value] += 1
return count_dict, 0
elif self.data_bits % 8 == 0:
max_index = self.data_size - self.remaining_size
for index in range(0, max_index, self.data_size):
count_dict[from_bytes(buffer[index:index +
self.data_size])] += 1
return count_dict, from_bytes(buffer[max_index:])
else:
bits_io = BitsIO(buffer, self.data_bits)
for value in bits_io:
count_dict[value] += 1
return count_dict, bits_io.remaining()
@staticmethod
def _sort_count_dict(count_dict: Dict):
return SortedList(
(DataCount(key, value) for key, value in count_dict.items()),
key=lambda item: item.count)
@staticmethod
def scan_buffer(data_bits: int,
buffer: bytes,
method: Method = Method.INT):
if data_bits <= 1:
raise ValueError(f'invalid data bits: {data_bits}')
elif data_bits > 1024:
raise ValueError(f'data bits is too big: {data_bits}')
result = SegmentedBuffer()
result.buffer_size = len(buffer)
result.buffer_bits = result.buffer_size * 8
result.data_bits = data_bits
result.data_size = get_bytes_per_bits(data_bits)
result.remaining_bits = result.buffer_bits % result.data_bits
result.remaining_size = get_bytes_per_bits(result.remaining_bits)
result.method = method
count_dict, result.remaining = result._scan_data_count(buffer)
result.sorted_data_count = SegmentedBuffer._sort_count_dict(count_dict)
return result
def convert(self, data_bits: int):
if self.data_bits % data_bits != 0:
raise ValueError()
result = SegmentedBuffer()
result.buffer_size = self.buffer_size
result.buffer_bits = self.buffer_bits
result.data_bits = data_bits
result.data_size = get_bytes_per_bits(data_bits)
result.remaining_bits = result.buffer_bits % result.data_bits
result.remaining_size = get_bytes_per_bits(result.remaining_bits)
raise BaseException('incomplete code!')
class PlatformBatchLightSystem:
"""Batch light system for platforms."""
__slots__ = [
"dirty_lights", "dirty_schedule", "clock", "is_sequential_function",
"update_task", "update_callback", "sort_function", "update_hz",
"max_batch_size"
]
# pylint: disable-msg=too-many-arguments
def __init__(self, clock, sort_function, is_sequential_function,
update_callback, update_hz, max_batch_size):
"""Initialise light system."""
self.dirty_lights = SortedSet(
key=sort_function) # type: Set[PlatformBatchLight]
self.dirty_schedule = SortedList(
key=lambda x: x[0] + sort_function(x[1]))
self.is_sequential_function = is_sequential_function
self.sort_function = sort_function
self.update_task = None
self.clock = clock
self.update_callback = update_callback
self.update_hz = update_hz
self.max_batch_size = max_batch_size
def start(self):
"""Start light system."""
self.update_task = self.clock.loop.create_task(self._send_updates())
self.update_task.add_done_callback(self._done)
def stop(self):
"""Stop light system."""
if self.update_task:
self.update_task.cancel()
self.update_task = None
@staticmethod
def _done(future):
try:
future.result()
except asyncio.CancelledError:
pass
async def _send_updates(self):
while True:
while self.dirty_schedule and self.dirty_schedule[0][
0] <= self.clock.get_time():
self.dirty_lights.add(self.dirty_schedule[0][1])
del self.dirty_schedule[0]
sequential_lights = []
for light in list(self.dirty_lights):
if not sequential_lights:
# first light
sequential_lights = [light]
elif self.is_sequential_function(sequential_lights[-1], light):
# lights are sequential
sequential_lights.append(light)
else:
# sequence ended
await self._send_update_batch(sequential_lights)
# this light is a new sequence
sequential_lights = [light]
if sequential_lights:
await self._send_update_batch(sequential_lights)
self.dirty_lights.clear()
await asyncio.sleep(.001, loop=self.clock.loop)
async def _send_update_batch(self, sequential_lights):
sequential_brightness_list = []
common_fade_ms = None
current_time = self.clock.get_time()
for light in sequential_lights:
brightness, fade_ms, done = light.get_fade_and_brightness(
current_time)
if not done:
self.dirty_schedule.add(
(current_time + (fade_ms / 1000), light))
if common_fade_ms is None:
common_fade_ms = fade_ms
if common_fade_ms == fade_ms and len(
sequential_brightness_list) < self.max_batch_size:
sequential_brightness_list.append(
(light, brightness, common_fade_ms))
else:
await self.update_callback(sequential_brightness_list)
# start new list
current_time = self.clock.get_time()
common_fade_ms = fade_ms
sequential_brightness_list = [(light, brightness,
common_fade_ms)]
if sequential_brightness_list:
await self.update_callback(sequential_brightness_list)
def mark_dirty(self, light: "PlatformBatchLight"):
"""Mark as dirty."""
self.dirty_lights.add(light)
self.dirty_schedule = SortedList(
[x for x in self.dirty_schedule if x[1] != light],
key=lambda x: x[0] + self.sort_function(x[1]))
class PriorityDict(MutableMapping):
"""
A PriorityDict provides the same methods as a dict. Additionally, a
PriorityDict efficiently maintains its keys in value sorted order.
Consequently, the keys method will return the keys in value sorted order,
the popitem method will remove the item with the highest value, etc.
"""
def __init__(self, *args, **kwargs):
"""
A PriorityDict provides the same methods as a dict. Additionally, a
PriorityDict efficiently maintains its keys in value sorted order.
Consequently, the keys method will return the keys in value sorted
order, the popitem method will remove the item with the highest value,
etc.
If the first argument is the boolean value False, then it indicates
that keys are not comparable. By default this setting is True and
duplicate values are tie-breaked on the key. Using comparable keys
improves the performance of the PriorityDict.
An optional *iterable* argument provides an initial series of items to
populate the PriorityDict. Each item in the sequence must itself
contain two items. The first is used as a key in the new dictionary,
and the second as the key's value. If a given key is seen more than
once, the last value associated with it is retained in the new
dictionary.
If keyword arguments are given, the keywords themselves with their
associated values are added as items to the dictionary. If a key is
specified both in the positional argument and as a keyword argument, the
value associated with the keyword is retained in the dictionary. For
example, these all return a dictionary equal to ``{"one": 2, "two":
3}``:
* ``SortedDict(one=2, two=3)``
* ``SortedDict({'one': 2, 'two': 3})``
* ``SortedDict(zip(('one', 'two'), (2, 3)))``
* ``SortedDict([['two', 3], ['one', 2]])``
The first example only works for keys that are valid Python
identifiers; the others work with any valid keys.
Note that this constructor mimics the Python dict constructor. If
you're looking for a constructor like collections.Counter(...), see
PriorityDict.count(...).
"""
self._dict = dict()
if len(args) > 0 and isinstance(args[0], bool):
if args[0]:
self._list = SortedList()
else:
self._list = SortedListWithKey(key=lambda tup: tup[0])
else:
self._list = SortedList()
self.iloc = _IlocWrapper(self)
self.update(*args, **kwargs)
def clear(self):
"""Remove all elements from the dictionary."""
self._dict.clear()
self._list.clear()
def clean(self, value=0):
"""
Remove all items with value less than or equal to `value`.
Default `value` is 0.
"""
_list, _dict = self._list, self._dict
pos = self.bisect_right(value)
for key in (key for value, key in _list[:pos]):
del _dict[key]
del _list[:pos]
def __contains__(self, key):
"""Return True if and only if *key* is in the dictionary."""
return key in self._dict
def __delitem__(self, key):
"""
Remove ``d[key]`` from *d*. Raises a KeyError if *key* is not in the
dictionary.
"""
value = self._dict[key]
self._list.remove((value, key))
del self._dict[key]
def __getitem__(self, key):
"""
Return the priority of *key* in *d*. Raises a KeyError if *key* is not
in the dictionary.
"""
return self._dict[key]
def __iter__(self):
"""
Create an iterator over the keys of the dictionary ordered by the value
sort order.
"""
return iter(key for value, key in self._list)
def __reversed__(self):
"""
Create an iterator over the keys of the dictionary ordered by the
reversed value sort order.
"""
return iter(key for value, key in reversed(self._list))
def __len__(self):
"""Return the number of (key, value) pairs in the dictionary."""
return len(self._dict)
def __setitem__(self, key, value):
"""Set `d[key]` to *value*."""
if key in self._dict:
old_value = self._dict[key]
self._list.remove((old_value, key))
self._list.add((value, key))
self._dict[key] = value
def copy(self):
"""Create a shallow copy of the dictionary."""
result = PriorityDict()
result._dict = self._dict.copy()
result._list = self._list.copy()
result.iloc = _IlocWrapper(result)
return result
def __copy__(self):
"""Create a shallow copy of the dictionary."""
return self.copy()
@classmethod
def fromkeys(cls, iterable, value=0):
"""
Create a new dictionary with keys from `iterable` and values set to
`value`. The default *value* is 0.
"""
return PriorityDict((key, value) for key in iterable)
def get(self, key, default=None):
"""
Return the value for *key* if *key* is in the dictionary, else
*default*. If *default* is not given, it defaults to ``None``,
so that this method never raises a KeyError.
"""
return self._dict.get(key, default)
def has_key(self, key):
"""Return True if and only in *key* is in the dictionary."""
return key in self._dict
def pop(self, key, default=_NotGiven):
"""
If *key* is in the dictionary, remove it and return its value,
else return *default*. If *default* is not given and *key* is not in
the dictionary, a KeyError is raised.
"""
if key in self._dict:
value = self._dict[key]
self._list.remove((value, key))
return self._dict.pop(key)
else:
if default == _NotGiven:
raise KeyError
else:
return default
def popitem(self, index=-1):
"""
Remove and return item at *index* (default: -1). Raises IndexError if
dict is empty or index is out of range. Negative indices are supported
as for slice indices.
"""
value, key = self._list.pop(index)
del self._dict[key]
return key, value
def setdefault(self, key, default=0):
"""
If *key* is in the dictionary, return its value. If not, insert *key*
with a value of *default* and return *default*. *default* defaults to
``0``.
"""
if key in self._dict:
return self._dict[key]
else:
self._dict[key] = default
self._list.add((default, key))
return default
def elements(self):
"""
Return an iterator over elements repeating each as many times as its
count. Elements are returned in value sort-order. If an element’s count
is less than one, elements() will ignore it.
"""
values = (repeat(key, value) for value, key in self._list)
return chain.from_iterable(values)
def most_common(self, count=None):
"""
Return a list of the `count` highest priority elements with their
priority. If `count` is not specified, `most_common` returns *all*
elements in the dict. Elements with equal counts are ordered by key.
"""
_list, _dict = self._list, self._dict
if count is None:
return [(key, value) for value, key in reversed(_list)]
end = len(_dict)
start = end - count
return [(key, value) for value, key in reversed(_list[start:end])]
def subtract(self, elements):
"""
Elements are subtracted from an iterable or from another mapping (or
counter). Like dict.update() but subtracts counts instead of replacing
them. Both inputs and outputs may be zero or negative.
"""
self -= Counter(elements)
def tally(self, *args, **kwargs):
"""
Elements are counted from an iterable or added-in from another mapping
(or counter). Like dict.update() but adds counts instead of replacing
them. Also, the iterable is expected to be a sequence of elements, not a
sequence of (key, value) pairs.
"""
self += Counter(*args, **kwargs)
@classmethod
def count(self, *args, **kwargs):
"""
Consume `args` and `kwargs` with a Counter and use that mapping to
initialize a PriorityDict.
"""
return PriorityDict(Counter(*args, **kwargs))
def update(self, *args, **kwargs):
"""
Update the dictionary with the key/value pairs from *other*, overwriting
existing keys.
*update* accepts either another dictionary object or an iterable of
key/value pairs (as a tuple or other iterable of length two). If
keyword arguments are specified, the dictionary is then updated with
those key/value pairs: ``d.update(red=1, blue=2)``.
"""
_list, _dict = self._list, self._dict
if len(args) == 1 and len(kwargs) == 0 and isinstance(args[0], Mapping):
items = args[0]
else:
items = dict(*args, **kwargs)
if (10 * len(items)) > len(_dict):
_dict.update(items)
_list.clear()
_list.update((value, key) for key, value in iteritems(_dict))
else:
for key, value in iteritems(items):
old_value = _dict[key]
_list.remove((old_value, key))
_dict[key] = value
_list.add((value, key))
def index(self, key):
"""
Return the smallest *i* such that `d.iloc[i] == key`. Raises KeyError
if *key* is not present.
"""
value = self._dict[key]
return self._list.index((value, key))
def bisect_left(self, value):
"""
Similar to the ``bisect`` module in the standard library, this returns
an appropriate index to insert *value* in PriorityDict. If *value* is
already present in PriorityDict, the insertion point will be before (to
the left of) any existing entries.
"""
return self._list.bisect_left((value,))
def bisect(self, value):
"""Same as bisect_left."""
return self._list.bisect((value,))
def bisect_right(self, value):
"""
Same as `bisect_left`, but if *value* is already present in
PriorityDict, the insertion point will be after (to the right
of) any existing entries.
"""
return self._list.bisect_right((value, _Biggest))
def __iadd__(self, that):
"""Add values from `that` mapping."""
_list, _dict = self._list, self._dict
if len(_dict) == 0:
_dict.update(that)
_list.update((value, key) for key, value in iteritems(_dict))
elif len(that) * 3 > len(_dict):
_list.clear()
for key, value in iteritems(that):
if key in _dict:
_dict[key] += value
else:
_dict[key] = value
_list.update((value, key) for key, value in iteritems(_dict))
else:
for key, value in iteritems(that):
if key in _dict:
old_value = _dict[key]
_list.remove((old_value, key))
value = old_value + value
_dict[key] = value
_list.add((value, key))
return self
def __isub__(self, that):
"""Subtract values from `that` mapping."""
_list, _dict = self._list, self._dict
if len(_dict) == 0:
_dict.clear()
_list.clear()
elif len(that) * 3 > len(_dict):
_list.clear()
for key, value in iteritems(that):
if key in _dict:
_dict[key] -= value
_list.update((value, key) for key, value in iteritems(_dict))
else:
for key, value in iteritems(that):
if key in _dict:
old_value = _dict[key]
_list.remove((old_value, key))
value = old_value - value
_dict[key] = value
_list.add((value, key))
return self
def __ior__(self, that):
"""Or values from `that` mapping (max(v1, v2))."""
_list, _dict = self._list, self._dict
if len(_dict) == 0:
_dict.update(that)
_list.update((value, key) for key, value in iteritems(_dict))
elif len(that) * 3 > len(_dict):
_list.clear()
for key, value in iteritems(that):
if key in _dict:
old_value = _dict[key]
_dict[key] = old_value if old_value > value else value
else:
_dict[key] = value
_list.update((value, key) for key, value in iteritems(_dict))
else:
for key, value in iteritems(that):
if key in _dict:
old_value = _dict[key]
_list.remove((old_value, key))
value = old_value if old_value > value else value
_dict[key] = value
_list.add((value, key))
return self
def __iand__(self, that):
"""And values from `that` mapping (min(v1, v2))."""
_list, _dict = self._list, self._dict
if len(_dict) == 0:
_dict.clear()
_list.clear()
elif len(that) * 3 > len(_dict):
_list.clear()
for key, value in iteritems(that):
if key in _dict:
old_value = _dict[key]
_dict[key] = old_value if old_value < value else value
_list.update((value, key) for key, value in iteritems(_dict))
else:
for key, value in iteritems(that):
if key in _dict:
old_value = _dict[key]
_list.remove((old_value, key))
value = old_value if old_value < value else value
_dict[key] = value
_list.add((value, key))
return self
def __add__(self, that):
"""Add values from this and `that` mapping."""
result = PriorityDict()
_list, _dict = result._list, result._dict
_dict.update(self._dict)
for key, value in iteritems(that):
if key in _dict:
_dict[key] += value
else:
_dict[key] = value
_list.update((value, key) for key, value in iteritems(_dict))
return result
def __sub__(self, that):
"""Subtract values in `that` mapping from this."""
result = PriorityDict()
_list, _dict = result._list, result._dict
_dict.update(self._dict)
for key, value in iteritems(that):
if key in _dict:
_dict[key] -= value
_list.update((value, key) for key, value in iteritems(_dict))
return result
def __or__(self, that):
"""Or values from this and `that` mapping."""
result = PriorityDict()
_list, _dict = result._list, result._dict
_dict.update(self._dict)
for key, value in iteritems(that):
if key in _dict:
old_value = _dict[key]
_dict[key] = old_value if old_value > value else value
else:
_dict[key] = value
_list.update((value, key) for key, value in iteritems(_dict))
return result
def __and__(self, that):
"""And values from this and `that` mapping."""
result = PriorityDict()
_list, _dict = result._list, result._dict
_dict.update(self._dict)
for key, value in iteritems(that):
if key in _dict:
old_value = _dict[key]
_dict[key] = old_value if old_value < value else value
_list.update((value, key) for key, value in iteritems(_dict))
return result
def __eq__(self, that):
"""Compare two mappings for equality."""
if isinstance(that, PriorityDict):
that = that._dict
return self._dict == that
def __ne__(self, that):
"""Compare two mappings for inequality."""
if isinstance(that, PriorityDict):
that = that._dict
return self._dict != that
def __lt__(self, that):
"""Compare two mappings for less than."""
if isinstance(that, PriorityDict):
that = that._dict
_dict = self._dict
return (_dict != that and self <= that)
def __le__(self, that):
"""Compare two mappings for less than equal."""
if isinstance(that, PriorityDict):
that = that._dict
_dict = self._dict
return (len(_dict) <= len(that) and
all(_dict[key] <= that[key] if key in that else False
for key in _dict))
def __gt__(self, that):
"""Compare two mappings for greater than."""
if isinstance(that, PriorityDict):
that = that._dict
_dict = self._dict
return (_dict != that and self >= that)
def __ge__(self, that):
"""Compare two mappings for greater than equal."""
if isinstance(that, PriorityDict):
that = that._dict
_dict = self._dict
return (len(_dict) >= len(that) and
all(_dict[key] >= that[key] if key in _dict else False
for key in that))
def isdisjoint(self, that):
"""
Return True if no key in `self` is also in `that`.
This doesn't check that the value is greater than zero.
To remove keys with value less than or equal to zero see *clean*.
"""
return not any(key in self for key in that)
def items(self):
"""
Return a list of the dictionary's items (``(key, value)``
pairs). Items are ordered by their value from least to greatest.
"""
return list((key, value) for value, key in self._list)
def iteritems(self):
"""
Return an iterable over the items (``(key, value)`` pairs) of the
dictionary. Items are ordered by their value from least to greatest.
"""
return iter((key, value) for value, key in self._list)
@not26
def viewitems(self):
"""
In Python 2.7 and later, return a new `ItemsView` of the dictionary's
items. Beware iterating the `ItemsView` as items are unordered.
In Python 2.6, raise a NotImplementedError.
"""
if hexversion < 0x03000000:
return self._dict.viewitems()
else:
return self._dict.items()
def keys(self):
"""
Return a list of the dictionary's keys. Keys are ordered
by their corresponding value from least to greatest.
"""
return list(key for value, key in self._list)
def iterkeys(self):
"""
Return an iterable over the keys of the dictionary. Keys are ordered
by their corresponding value from least to greatest.
"""
return iter(key for value, key in self._list)
@not26
def viewkeys(self):
"""
In Python 2.7 and later, return a new `KeysView` of the dictionary's
keys. Beware iterating the `KeysView` as keys are unordered.
In Python 2.6, raise a NotImplementedError.
"""
if hexversion < 0x03000000:
return self._dict.viewkeys()
else:
return self._dict.keys()
def values(self):
"""
Return a list of the dictionary's values. Values are
ordered from least to greatest.
"""
return list(value for value, key in self._list)
def itervalues(self):
"""
Return an iterable over the values of the dictionary. Values are
iterated from least to greatest.
"""
return iter(value for value, key in self._list)
@not26
def viewvalues(self):
"""
In Python 2.7 and later, return a `ValuesView` of the dictionary's
values. Beware iterating the `ValuesView` as values are unordered.
In Python 2.6, raise a NotImplementedError.
"""
if hexversion < 0x03000000:
return self._dict.viewvalues()
else:
return self._dict.values()
def __repr__(self):
"""Return a string representation of PriorityDict."""
return 'PriorityDict({0})'.format(repr(dict(self)))
def _check(self):
self._list._check()
assert len(self._dict) == len(self._list)
assert all(key in self._dict and self._dict[key] == value
for value, key in self._list)
def create_group_diagram(entities, entities_times, all_time_stamps, d,
time_shift, first, last, birds, save_path):
print "this code is used"
# Initialize kml-file to store the GD representatives
kml = simplekml.Kml()
# find common start and end time stamps of the trajectories
start_times = SortedList()
end_times = SortedList()
for entityTime in entities_times:
start_times.add(entityTime[0])
end_times.add(entityTime[-1])
common_start_time = start_times[-1]
common_end_time = end_times[0]
all_common_time_stamps = SortedList(all_time_stamps[all_time_stamps.index(
common_start_time):all_time_stamps.index(common_end_time)])
# COMPUTING ALL SPLIT AND MERGE EVENTS
event_count = 0
event_times = SortedList()
print "number of all common time stamps:", len(all_common_time_stamps)
# loop through all time stamps and compute the events
for time_index, timestamp in enumerate(all_common_time_stamps[:-1]):
for index1 in range(0, len(entities)):
start_segment_1 = entities_times[index1].index(timestamp)
end_segment_1 = start_segment_1 + 1
for index2 in range(0, len(entities)):
start_segment_2 = entities_times[index2].index(timestamp)
end_segment_2 = start_segment_2 + 1
new_events = compare_two_segments(
entities[index1][start_segment_1][1],
entities[index1][end_segment_1][1],
entities[index2][start_segment_2][1],
entities[index2][end_segment_2][1],
entities[index1][start_segment_1][2],
entities[index1][end_segment_1][2],
entities[index2][start_segment_2][2],
entities[index2][end_segment_2][2], time_index + first,
all_common_time_stamps, d)
# print new_events
if len(new_events) > 0:
for event in new_events:
event_times.add(event)
event_count += 1
# print event_count
# propagate event time stamps to all trajectories
adding_count = 0
print "Number of events:", len(event_times)
for timestamp in event_times:
if timestamp not in all_common_time_stamps:
all_common_time_stamps.add(timestamp)
adding_count += 1
for index, entity in enumerate(entities):
length = len(entities_times[index])
# check if the current entity has this timestamp, if not add it and interpolate the
# coordinates but only if it is in the range of the trajectory. Otherwise interpolation is not possible
if timestamp not in entities_times[index] and entities_times[
index].bisect(timestamp) > 0 and entities_times[
index].bisect(timestamp) < length:
entity.add(
interpolate(entity, entities_times[index], timestamp))
entities_times[index].add(timestamp)
# solve set cover instance for every time stamp, return the segments of the solution and draw them
universe = set(range(0, len(entities)))
cover_size = []
cover_numbers = []
# for each entity initialize
all_together = 0
all_together_time = 0
number_of_representations = 0
start = time.time()
for time_index, timestamp in enumerate(all_common_time_stamps[first:last]):
if timestamp in event_times or timestamp == all_common_time_stamps[
first]:
set_list = []
start_segment_common = all_common_time_stamps.index(timestamp)
end_segment_common = start_segment_common + 1
for index in range(0, len(entities)):
current_set = set()
start_segment = entities_times[index].index(timestamp)
end_segment = start_segment + 1
segment = [[
entities[index][start_segment][1],
entities[index][start_segment][2]
],
[
entities[index][end_segment][1],
entities[index][end_segment][2]
]]
for number, entity in enumerate(entities):
if within_alpha_similar_time_distance(
segment, timestamp,
all_common_time_stamps[end_segment_common], entity,
entities_times[number], time_shift, d, 0):
current_set.add(number)
set_list.append(current_set)
result = set_cover(universe, set_list)
# print time_index, result
cover_size.append(len(result[0]))
number_of_representations += len(result[0])
if len(result[0]) == 1:
all_together += 1
all_together_time += all_common_time_stamps[time_index +
1] - timestamp
cover_numbers.append(result[1])
for indexSet, entity in enumerate(result[1]):
points = [
(entities[entity][entities_times[entity].index(timestamp)][1],
entities[entity][entities_times[entity].index(timestamp)][2]),
(entities[entity][entities_times[entity].index(timestamp) +
1][1],
entities[entity][entities_times[entity].index(timestamp) +
1][2])
]
lin = kml.newlinestring(coords=points)
# needs to be adjusted for groups with more than 4 entities
if entity == 0:
lin.style.linestyle.color = simplekml.Color.navy
elif entity == 1:
lin.style.linestyle.color = simplekml.Color.darkgreen
elif entity == 2:
lin.style.linestyle.color = simplekml.Color.darkorange
elif entity == 3:
lin.style.linestyle.color = simplekml.Color.darkred
lin.style.linestyle.width = result[2][indexSet] * 2
end = time.time()
print "time to construct and solve Set Cover", d, time_shift, end - start
all_together = 1.0 * all_together / (len(all_common_time_stamps))
number_of_representations = 1.0 * number_of_representations / (
len(all_common_time_stamps))
size = os.path.join(
save_path, "outputSize_" + str(d) + '_' + str(time_shift) + ".txt")
numbers = os.path.join(
save_path,
"outputNumber_" + birds + str(d) + '_' + str(time_shift) + ".txt")
kml_save = os.path.join(
save_path,
'groupDiagram2D' + birds + str(d) + '_' + str(time_shift) + '.kml')
file1 = open(size, "w")
simplejson.dump(cover_size, file1)
file1.close()
file2 = open(numbers, 'w')
simplejson.dump(cover_numbers, file2)
file2.close()
kml.save(kml_save)
return all_together, number_of_representations
class BarGridKernel(Kernel):
'''
Store a kernel of n dimensions as a list of bars in the space of dimension
(n-1). A bar is given by its start and its end coordinates, and corresponds
to the hull of the viable points in the last dimension for each coordinates
in the space of dimension (n-1). The order of the dimensions may have been
changed, and the last dimension of these data may not correspond to the
last dimension of the viability problem. Therefore the attribute
``permutation`` give a matrix describing the permutation of the dimension.
'''
def __init__(self, originCoords, oppositeCoords, intervalNumberperaxis, permutation=None, kernelMinPoint=None, kernelMaxPoint=None,data=[], metadata={}):
super(BarGridKernel, self).__init__(metadata)
self.originCoords = np.array(originCoords, float)
self.oppositeCoords = np.array(oppositeCoords, float)
self.intervalNumberperaxis = np.array(intervalNumberperaxis, int)
self.bars = SortedList(data)
if permutation is None:
self.permutation = np.eye(len(originCoords),dtype = int)
else:
self.permutation = permutation
if kernelMinPoint is None:
self.kernelMinPoint = []
for i in range(len(originCoords)):
self.kernelMinPoint.append(intervalNumberperaxis[i])
else :
self.kernelMinPoint = kernelMinPoint
if kernelMaxPoint is None:
self.kernelMaxPoint = [-1] * len(originCoords)
else :
self.kernelMaxPoint = kernelMaxPoint
self.metadata.update(self.getDataAttributes())
@staticmethod
def getFormatCode():
return "bars"
def getDataAttributes(self):
da = super(BarGridKernel, self).getDataAttributes()
da['origin'] = self.originCoords
da['opposite'] = self.oppositeCoords
da['intervals'] = self.intervalNumberperaxis
da['permutation']= self.permutation
da['maxPoint']= self.kernelMaxPoint
da['minPoint']= self.kernelMinPoint
return da
@classmethod
def initFromHDF5(cls, metadata, attrs, data):
'''
Create an object of class BarGridKernel from attributes and data loaded
from an HDF5 file. This method is intended to be used by the method
hdf5common.readKernel
'''
return cls(
originCoords=attrs['origin'],
oppositeCoords=attrs['opposite'],
intervalNumberperaxis=attrs['intervals'],
permutation=attrs['permutation'],
kernelMinPoint=attrs['minPoint'],
kernelMaxPoint=attrs['maxPoint'],
data=data.tolist(),
metadata=metadata
)
def getData(self):
return np.array(list(self.bars), dtype='int64')
def getIntervalSizes(self):
'''
Give the coordinates of the point of the grid with minimal coordinates
'''
intervalsizes = []
intervalsizes = list((self.oppositeCoords-self.originCoords)/self.intervalNumberperaxis)
return intervalsizes
def getMinFrameworkBounds(self):
return list(self.originCoords-np.array(self.getIntervalSizes())/2)
def getMaxFrameworkBounds(self):
return list(self.oppositeCoords+np.array(self.getIntervalSizes())/2)
def getMinBounds(self):
'''
Give the coordinates of the point of the vino with minimal coordinates
'''
minbounds = []
intervalSizes = np.array(self.getIntervalSizes())
permutOriginCoords = np.dot(self.permutation, self.originCoords)
permutOppositeCoords = np.dot(self.permutation, self.oppositeCoords)
permutIntervalNumberperaxis = np.dot(self.permutation, self.intervalNumberperaxis)
minbounds = list(np.dot(np.transpose(self.permutation),permutOriginCoords+(permutOppositeCoords-permutOriginCoords)*self.kernelMinPoint/permutIntervalNumberperaxis))
minbounds = minbounds - intervalSizes/2
return minbounds
def getMaxBounds(self):
'''
Give the coordinates of the point of the vino with maximal coordinates
'''
maxbounds = []
intervalSizes = np.array(self.getIntervalSizes())
permutOriginCoords = np.dot(self.permutation, self.originCoords)
permutOppositeCoords = np.dot(self.permutation, self.oppositeCoords)
permutIntervalNumberperaxis = np.dot(self.permutation, self.intervalNumberperaxis)
maxbounds = list(np.dot(np.transpose(self.permutation),permutOriginCoords+(permutOppositeCoords-permutOriginCoords)*self.kernelMaxPoint/permutIntervalNumberperaxis))
maxbounds = maxbounds + intervalSizes/2
return maxbounds
def getDataToPlot(self):
data = []
permutOriginCoords = np.dot(self.permutation, self.originCoords)
permutOppositeCoords = np.dot(self.permutation, self.oppositeCoords)
permutIntervalNumberperaxis = np.dot(self.permutation, self.intervalNumberperaxis)
for i in range(len(self.bars)):
data.append([
permutOriginCoords+(permutOppositeCoords-permutOriginCoords)*np.array(self.bars[i][:-1])/permutIntervalNumberperaxis,
permutOriginCoords[-1]+(permutOppositeCoords[-1]-permutOriginCoords[-1])*self.bars[i][-1]/permutIntervalNumberperaxis[-1]
])
perm = np.dot(self.permutation,np.arange(len(self.originCoords)))
data = [self.getMinFrameworkBounds()+self.getMaxFrameworkBounds()+self.getIntervalSizes()+[perm], data]
return data
def getTotalPointNumber(self):
return sum([elt[-1] - elt[-2] + 1 for elt in self.bars])
def toRegularGridKernel(self):
'''
Convert the kernel to the regular grid representation.
Returns an instance of RegularGridKernel.
The returned grid is trimed to not include empty portion of grid.
'''
minPoint = np.array(self.kernelMinPoint)
maxPoint = np.array(self.kernelMaxPoint)
dimensionsExtents = maxPoint - minPoint + 1
grid = RegularGridKernel(self.originCoords, self.intervalNumberperaxis,
dimensionsExtents, metadata=self.metadata)
for bar in self.bars:
barPosition = (bar[:-2]-minPoint[:-1]).tolist()
grid.grid[tuple(barPosition)].put(list(range(bar[-2], bar[-1] + 1)), True)
return grid
def intersectionwithBarGridKernel(self,othergrid):
'''
Returns an instance of BarGridKernel which is the intersection of two BarGridKernels
with the same underlying grid characteristics
'''
data = []
grid = BarGridKernel(self.originCoords,self.oppositeCoords,self.intervalNumberperaxis,self.permutation,None,None,data,self.metadata)
barsindex = 0
otherbarsindex = 0
while (barsindex < len(self.bars)) and (otherbarsindex < len(othergrid.bars)):
actualbarposition = self.bars[barsindex][:-2]
# print("actualbarposition[0] ::%d " %actualbarposition[0])
while (otherbarsindex < len(othergrid.bars)) and (othergrid.bars[otherbarsindex][:-2] < self.bars[barsindex][:-2]):
otherbarsindex = otherbarsindex + 1
while (barsindex < len(self.bars)) and (otherbarsindex < len(othergrid.bars)) and (othergrid.bars[otherbarsindex][:-2] == self.bars[barsindex][:-2]):
if othergrid.bars[otherbarsindex][-1] < self.bars[barsindex][-2]:
otherbarsindex = otherbarsindex + 1
elif othergrid.bars[otherbarsindex][-2] > self.bars[barsindex][-1]:
barsindex = barsindex + 1
elif othergrid.bars[otherbarsindex][-1] > self.bars[barsindex][-1]:
grid.addBar(self.bars[barsindex][:-2], max(othergrid.bars[otherbarsindex][-2], self.bars[barsindex][-2]), self.bars[barsindex][-1])
barsindex = barsindex + 1
elif othergrid.bars[otherbarsindex][-1] < self.bars[barsindex][-1]:
grid.addBar(self.bars[barsindex][:-2], max(othergrid.bars[otherbarsindex][-2], self.bars[barsindex][-2]), othergrid.bars[otherbarsindex][-1])
otherbarsindex = otherbarsindex + 1
else:
grid.addBar(self.bars[barsindex][:-2], max(othergrid.bars[otherbarsindex][-2], self.bars[barsindex][-2]), othergrid.bars[otherbarsindex][-1])
otherbarsindex = otherbarsindex + 1
barsindex = barsindex + 1
while (barsindex < len(self.bars)) and (otherbarsindex < len(othergrid.bars)) and (othergrid.bars[otherbarsindex][:-2] > self.bars[barsindex][:-2]):
barsindex = barsindex + 1
return grid
def MinusBarGridKernel(self,othergrid):
'''
Returns an instance of BarGridKernel which is the element of the BarGridKernels
which are not in the other one. The Bargridkernel have the same underlying grid characteristics
'''
data = []
grid = BarGridKernel(self.originCoords,self.oppositeCoords,self.intervalNumberperaxis,self.permutation,None,None,data,self.metadata)
barsindex = 0
otherbarsindex = 0
while (barsindex < len(self.bars)) and (otherbarsindex < len(othergrid.bars)):
while (otherbarsindex < len(othergrid.bars)) and (othergrid.bars[otherbarsindex][:-2] < self.bars[barsindex][:-2]):
otherbarsindex = otherbarsindex + 1
alreadycut = False
while (barsindex < len(self.bars)) and (otherbarsindex < len(othergrid.bars)) and (othergrid.bars[otherbarsindex][:-2] == self.bars[barsindex][:-2]):
if othergrid.bars[otherbarsindex][-1] < self.bars[barsindex][-2]:
otherbarsindex = otherbarsindex + 1
elif othergrid.bars[otherbarsindex][-1] >= self.bars[barsindex][-1]:
if othergrid.bars[otherbarsindex][-2] > self.bars[barsindex][-2]:
if alreadycut:
grid.addBar(self.bars[barsindex][:-2], remember, othergrid.bars[otherbarsindex][-2]-1)
else :
grid.addBar(self.bars[barsindex][:-2], self.bars[barsindex][-2], othergrid.bars[otherbarsindex][-2]-1)
barsindex = barsindex + 1
alreadycut = False
else :
if othergrid.bars[otherbarsindex][-2] > self.bars[barsindex][-2]:
if alreadycut:
grid.addBar(self.bars[barsindex][:-2], remember, othergrid.bars[otherbarsindex][-2]-1)
else :
grid.addBar(self.bars[barsindex][:-2], self.bars[barsindex][-2], othergrid.bars[otherbarsindex][-2]-1)
alreadycut = True
remember = othergrid.bars[otherbarsindex][-1]+1
otherbarsindex = otherbarsindex + 1
while (barsindex < len(self.bars)) and (otherbarsindex < len(othergrid.bars)) and (othergrid.bars[otherbarsindex][:-2] > self.bars[barsindex][:-2]):
if alreadycut:
grid.addBar(self.bars[barsindex][:-2], remember, self.bars[barsindex][-1])
barsindex = barsindex + 1
alreadycut = False
else :
grid.addBar(self.bars[barsindex][:-2], self.bars[barsindex][-2], self.bars[barsindex][-1])
barsindex = barsindex + 1
if (otherbarsindex >= len(othergrid.bars)):
while (barsindex < len(self.bars)):
if alreadycut:
grid.addBar(self.bars[barsindex][:-2], remember, self.bars[barsindex][-1])
barsindex = barsindex + 1
alreadycut = False
else :
grid.addBar(self.bars[barsindex][:-2], self.bars[barsindex][-2], self.bars[barsindex][-1])
barsindex = barsindex + 1
return grid
def toBarGridKernel(self, newOriginCoords, newOppositeCoords, newIntervalNumberperaxis):
'''
Convert a BarGridKernel to another BarGridKernel with another underlying grid.
Returns an instance of BarGridKernel.
'''
dimension = len(self.originCoords)
actualbarposition = np.zeros(dimension-1,int)
barsindex = 0
# converting to numpy arrays
newOriginCoords = np.array(newOriginCoords,float)
newIntervalNumberperaxis = np.array(newIntervalNumberperaxis,float)
permutnewOriginCoords = np.dot(self.permutation, newOriginCoords)
# permuting coordinates
permutnewIntervalNumberperaxis = np.dot(self.permutation, newIntervalNumberperaxis)
permutnewpas = np.dot(self.permutation,(np.array(newOppositeCoords,float) - newOriginCoords) / newIntervalNumberperaxis)
permutOriginCoords = np.dot(self.permutation, self.originCoords)
permutinversepas = np.dot(self.permutation, self.intervalNumberperaxis / (self.oppositeCoords - self.originCoords))
data = []
grid = BarGridKernel(newOriginCoords,newOppositeCoords,newIntervalNumberperaxis,self.permutation,None,None,data,self.metadata)
# oups = 0
# while (oups < 1) :
# oups = 1
while(actualbarposition[0]<permutnewIntervalNumberperaxis[0]+1):
realpoint = permutnewOriginCoords[:-1] + actualbarposition * permutnewpas[:-1]
intpoint = (realpoint-permutOriginCoords[:-1]) * permutinversepas[:-1]
intpoint = [int(e+0.5) for e in intpoint]
while (barsindex < len(self.bars)) and (self.bars[barsindex][:2] < intpoint):
barsindex = barsindex+1
barinprocess = False
# print intpoint
while (barsindex < len(self.bars)) and (self.bars[barsindex][:-2] == intpoint):
inf = self.bars[barsindex][-2]
realinf = inf/permutinversepas[-1] +permutOriginCoords[-1]
intinf = int((realinf-permutnewOriginCoords[-1])/permutnewpas[-1]+0.5)
sup = self.bars[barsindex][-1]
realsup = sup/permutinversepas[-1] +permutOriginCoords[-1]
intsup = int((realsup-permutnewOriginCoords[-1])/permutnewpas[-1]+0.5)
# print realinf
# print realsup
# print intinf
# print intsup
if (intinf<permutnewIntervalNumberperaxis[-1]+1) or (intsup >=0):
if barinprocess == True :
if intinf == grid.bars[-1][-1]:
grid.bars[-1][-1] = min(intsup,permutnewIntervalNumberperaxis[-1]+1)
else :
grid.addBar(actualbarposition.tolist(), max(intinf,0), min(intsup,permutnewIntervalNumberperaxis[-1]+1))
else :
grid.addBar(actualbarposition.tolist(), max(intinf,0), min(intsup,permutnewIntervalNumberperaxis[-1]+1))
barinprocess = True
barsindex = barsindex+1
for i in range(dimension-1):
if ((i == dimension - 2) or (actualbarposition[dimension-2-i]<permutnewIntervalNumberperaxis[dimension-2-i])):
actualbarposition[dimension-2-i] = actualbarposition[dimension-2-i]+1
break
else :
actualbarposition[dimension-2-i] = 0
return grid
def addBar(self, coords, inf, sup):
# First, we collect the bars already present at the position 'coords'
# and we merge the bar to add with the existing ones
# two bars will be merged if at least they touch themselves
# "touch" means that a lower bound of one bar is equals to the (upper bound of the other one) + 1
insertion_point = self.bars.bisect(coords)
merged = False
mergedBarsToRemove=[]
rightExpanded = False
while insertion_point<len(self.bars) and self.bars[insertion_point][:-2]==coords:
if inf > self.bars[insertion_point][-1] + 1:
# the new bar doesn't touch the right of the current one
# we should test if it isn't equals to the upper bound + 1 to ensure that it doesn't touch
insertion_point += 1
continue;
if rightExpanded:
# a previous bar has been modified, we check that it doesn't cross or touch the current bar
if self.bars[insertion_point][-2] <= self.bars[insertion_point-1][-1] + 1:
# the previous bar now intersects the lower bound of the current one
# so let's merge the two bars
self.bars[insertion_point][-2] = self.bars[insertion_point-1][-2]
if self.bars[insertion_point][-2] <= self.bars[insertion_point-1][-2]:
# the previous bar completly overlaps the current one
self.bars[insertion_point][-2] = self.bars[insertion_point-1][-2]
mergedBarsToRemove.append(insertion_point-1)
rightExpanded = True
elif inf >= self.bars[insertion_point][-2] and inf <= self.bars[insertion_point][-1] + 1:
# the lower bound of the inserted bar is inside the current one
merged = True
if sup > self.bars[insertion_point][-1]:
# the upper bound is outside the current bar, so we update the upper bound
self.bars[insertion_point][-1] = sup
rightExpanded = True
elif inf < self.bars[insertion_point][-2]:
# the lower bound of the inserted bar is before the current one
if sup >= self.bars[insertion_point][-2] - 1:
# the inserted bar crosses or touches the current bar, so we update the lower bound
self.bars[insertion_point][-2] = inf
merged = True
if sup > self.bars[insertion_point][-1]:
# the inserted bound is globally bigger than the current one, so we update also the upper bound
self.bars[insertion_point][-1] = sup
rightExpanded = True
insertion_point += 1
for index in reversed(mergedBarsToRemove):
del self.bars[index]
if not merged:
self.bars.add(coords[:] + [inf,sup])
self.kernelMinPoint[:-1] = [min(x) for x in zip(self.kernelMinPoint[:-1],coords)]
self.kernelMinPoint[-1] = min(self.kernelMinPoint[-1], inf)
self.kernelMaxPoint[:-1] = [max(x) for x in zip(self.kernelMaxPoint[:-1],coords)]
self.kernelMaxPoint[-1] = max(self.kernelMaxPoint[-1], sup)
def getBars(self):
return self.bars
def isInSet(self, point):
'''
Returns if point belongs to the BarGridKernel.
This method will find the cell where to lookup a bar containing the point. If not,
the point is not considered in the set.
Technical details:
In a BarGrid, each cell of the (n-1) dimensional space is stored by its
index inside a matrix between two opposite points and the number of cells in each dimension.
First, this method will compute the index(es) of the cell where to lookup for bars,
then it will look for the last dimension to check if the point is inside a cell
covered by a bar in the selected cell of the (n-1) dimensional space.
Note that if the point is exactly between several cells (on a vertex or on an edge),
the method will check the bars of all the touching cells.
Let's consider s(i) the size of a cell (the step size) on the dimension i,
p(i) the coordinates of the point requested on the dimension i, and
c(i) the coordinates of the center of a cell of the BarGrid in the dimension i,
thus we consider that the point p is inside the cell c if:
- p(i)>=c(i)-s(i)/2 (after the left side of the cell)
- p(i)<=c(i)+s(i)/2 (before the right side of the cell)
- for i belongs to [0;n-1]
'''
result = False
point = np.array(point,float)
# first we need to project the point into the cells coordinate system
point_int = self.intervalNumberperaxis * (point - self.originCoords)/(self.oppositeCoords - self.originCoords)
point_int = np.dot(self.permutation, np.transpose(point_int))
points = [point_int]
for i,coord in enumerate(point_int):
new_points = []
for p in points:
if (coord%1)==0.5: # the point is exactly between two cells on the current dimension
left = [pp for pp in p]
left[i] = int(math.floor(left[i]))
right = [pp for pp in p]
right[i] = int(math.ceil(right[i]))
new_points.extend([left, right])
else:
# we just need to round to the nearest cell center
new_point = [pp for pp in p]
new_point[i] = int(round(new_point[i]))
new_points.append(new_point)
points=new_points
l = len(point)
# we will look at each bar if they are positioned in the coordinates
# in (n-1) dimensions space than our point
for point in points:
candidateBar = False
for bar in self.bars:
if point[:-1] == bar[:-2]:
# we have reached the interesting zone
candidateBar = True
# is our point in the bar?
if (point[l-1] >= bar[l-1]-0.5) and (point[l-1] <= bar[l] + 0.5):
result = True
break
elif candidateBar:
# we have passed the position in (n-1) dimensions space, so we can't find candidates anymore
break
return result
def permute(self,permutation):
'''
Create a BarGrid corresponding to the same data as the initial one but with a different permutation of the variables : np.dot(np.transpose(permutation),self.permutation)
instead of self.permutation
'''
griddata = []
unitbars = []
dimension = len(self.originCoords)
matid = np.identity(dimension,dtype = int)
b = False
for i in range(dimension):
for j in range(dimension):
if permutation[i][j] != matid[i][j]:
b = True
if b:
permutegrid = BarGridKernel(self.originCoords,self.oppositeCoords,self.intervalNumberperaxis,np.dot(np.transpose(permutation),self.permutation),np.dot(np.transpose(permutation),self.kernelMinPoint),np.dot(np.transpose(permutation),self.kernelMaxPoint),griddata,self.metadata)
if permutation[dimension-1][dimension-1] == 0:
barposition = [0]*(dimension-1)
increment = [0]*len(barposition)
increment.append(1)
oldincrement = list(np.dot(permutation,np.array(increment,int)))[:-1]
oldindex = oldincrement.index(1)
newincrement = list(np.dot(np.transpose(permutation),np.array(increment,int)))[:-1]
newindex = newincrement.index(1)
NmaxUsefullBars = list(np.dot(self.permutation, self.intervalNumberperaxis))[oldindex]
barposition.append(0)
permutnewIntervalNumberperaxis = np.dot(permutegrid.permutation, permutegrid.intervalNumberperaxis)
if (newindex == 0):
indexbar = 1
else :
indexbar = 0
if dimension == 2:
# print "barposition"
# print barposition
usefuloldbars = []
newbars = []
oldbarposition = list(np.dot(permutation,np.array(barposition,int)))[:-1]
for i in range(NmaxUsefullBars+1):
oldbarposition[oldindex] = i
# print oldbarposition
insertion_point = self.bars.bisect(oldbarposition)
while insertion_point<len(self.bars) and self.bars[insertion_point][:-2]==oldbarposition:
usefuloldbars.append(self.bars[insertion_point])
insertion_point = insertion_point+1
# print usefuloldbars
for oldbar in usefuloldbars:
# print "newoldbar"
# print oldbar
level = oldbar[oldindex]
unitbar = barposition[:-1] + [level,level]
mini = oldbar[-2]
maxi = oldbar[-1]
newbartoupdateindex = mini
if newbars:
k = 0
while k <len(newbars):
newbar = newbars[k]
# print "ole"
# print newbars
# print newbar
# print k
if (newbar[newindex] > maxi):
break
elif (newbar[newindex] >= mini):
# print newbar[newindex]
if (newbar[newindex] > newbartoupdateindex):
for l in range(newbartoupdateindex,newbar[newindex]):
unitbar[newindex] = l
newbars.insert(k,copy.copy(unitbar))
k=k+1
newbartoupdateindex = newbar[newindex]
if (newbar[-1] == level-1):
newbar[-1] = level
newbartoupdateindex = newbar[newindex]+1
k = k+1
for l in range(newbartoupdateindex,maxi+1):
unitbar[newindex] = l
newbars.insert(k,copy.copy(unitbar))
k = k+1
else :
for l in range(mini,maxi+1):
unitbar[newindex] = l
newbars.append(copy.copy(unitbar))
# print "newbars"
# print newbars
permutegrid.bars.update(newbars)
else:
while(barposition[indexbar]<permutnewIntervalNumberperaxis[indexbar]+1):
# print "barposition"
# print barposition
usefuloldbars = []
newbars = []
oldbarposition = list(np.dot(permutation,np.array(barposition,int)))[:-1]
for i in range(NmaxUsefullBars+1):
oldbarposition[oldindex] = i
# print oldbarposition
insertion_point = self.bars.bisect(oldbarposition)
while insertion_point<len(self.bars) and self.bars[insertion_point][:-2]==oldbarposition:
usefuloldbars.append(self.bars[insertion_point])
insertion_point = insertion_point+1
# print usefuloldbars
for oldbar in usefuloldbars:
# print "newoldbar"
# print oldbar
level = oldbar[oldindex]
unitbar = barposition[:-1] + [level,level]
mini = oldbar[-2]
maxi = oldbar[-1]
newbartoupdateindex = mini
if newbars:
k = 0
while k <len(newbars):
newbar = newbars[k]
# print "ole"
# print newbars
# print newbar
# print k
if (newbar[newindex] > maxi):
break
elif (newbar[newindex] >= mini):
# print newbar[newindex]
if (newbar[newindex] > newbartoupdateindex):
for l in range(newbartoupdateindex,newbar[newindex]):
unitbar[newindex] = l
newbars.insert(k,copy.copy(unitbar))
k=k+1
newbartoupdateindex = newbar[newindex]
if (newbar[-1] == level-1):
newbar[-1] = level
newbartoupdateindex = newbar[newindex]+1
k = k+1
for l in range(newbartoupdateindex,maxi+1):
unitbar[newindex] = l
newbars.insert(k,copy.copy(unitbar))
k = k+1
else :
for l in range(mini,maxi+1):
unitbar[newindex] = l
newbars.append(copy.copy(unitbar))
# print "newbars"
# print newbars
permutegrid.bars.update(newbars)
for i in range(dimension-1):
if ((dimension-2-i) != newindex):
if ((i == dimension - 2- indexbar) or (barposition[dimension-2-i]<permutnewIntervalNumberperaxis[dimension-2-i])):
barposition[dimension-2-i] = barposition[dimension-2-i]+1
break
else :
barposition[dimension-2-i] = 0
else:
for bar in self.bars :
tpermutation = np.transpose(permutation)
permutegrid.bars.add(list(np.dot(tpermutation,bar[:-1]))+[bar[-1]])
else:
permutegrid = BarGridKernel(self.originCoords,self.oppositeCoords,self.intervalNumberperaxis,np.dot(np.transpose(permutation),self.permutation),np.dot(np.transpose(permutation),self.kernelMinPoint),np.dot(np.transpose(permutation),self.kernelMaxPoint),list(self.bars),self.metadata)
return permutegrid
def buildNewBars(self,barposition,permutation,data):
newdata = []
unitbar = []
increment = [0]*len(barposition)
increment.append(1)
newincrement = list(np.dot(np.transpose(permutation),np.array(increment,int)))[:-1]
newindex = newincrement.index(1)
oldincrement = list(np.dot(permutation,np.array(increment,int)))[:-1]
oldindex = oldincrement.index(1)
NmaxNewBars = list(np.dot(self.permutation, self.intervalNumberperaxis))[-1]
# print newincrement
# print newindex
# print oldincrement
# print oldindex
# print NmaxNewBars
for oldbar in data:
level = oldbar[oldindex]
unitbar = barposition + [level,level]
mini = oldbar[-2]
maxi = oldbar[-1]
newbartoupdateindex = mini
if newdata:
for k in range(len(newdata)):
# print "ole"
# print newdata
# print k
newbar = newdata[k]
if (newbar[newindex] >= mini) and (newbar[newindex] <= maxi):
# print newbar[newindex]
if (newbar[newindex] > newbartoupdateindex):
for l in range(newbartoupdateindex,newbar[newindex]):
unitbar[newindex] = l
newdata.insert(k,copy.copy(unitbar))
k=k+1
if (newbar[-1] == level-1):
newbar[-1] = level
newbartoupdateindex = newbar[newindex]+1
for l in range(newbartoupdateindex,maxi+1):
unitbar[newindex] = l
newdata.insert(len(newdata),copy.copy(unitbar))
else :
for l in range(mini,maxi+1):
unitbar[newindex] = l
newdata.append(copy.copy(unitbar))
return newdata
def findUsefullBars(self,barposition,permutation):
data = []
increment = [0]*len(barposition)
barposition.append(0)
increment.append(1)
oldincrement = list(np.dot(permutation,np.array(increment,int)))[:-1]
index = oldincrement.index(1)
oldbarposition = list(np.dot(permutation,np.array(barposition,int)))[:-1]
NmaxUsefullBars = list(np.dot(self.permutation, self.intervalNumberperaxis))[index]
# print oldbarposition
# print oldincrement
# print index
# print NmaxUsefullBars
for i in range(NmaxUsefullBars+1):
oldbarposition[index] = i
insertion_point = self.bars.bisect(oldbarposition)
# print oldbarposition
# print insertion_point
while insertion_point<len(self.bars) and self.bars[insertion_point][:-2]==oldbarposition:
data.append(self.bars[insertion_point])
insertion_point = insertion_point+1
return data
sig_buf.add_signal_fragment(record.signal[i], record.leads[i])
annots = mit.read_annotations(args.r + '.' + args.a)
#Delineation of every QRS annotation.
for ann in (a for a in annots if mit.is_qrs_annotation(a)):
#Temporal window
btime = ann.time
beg = btime-C.QRS_BANN_DMAX
end = btime+C.QRS_EANN_DMAX
#Path simplification and signal characterization
siginfo = _characterize_signal(beg, end)
try:
#QRS delineation
qrs = delineate_qrs(siginfo)
#Output annotations
bann = mit.MITAnnotation()
bann.code = mit.ECGCodes.WFON
bann.time = int(beg+qrs.start)
peak = mit.MITAnnotation()
peak.code = mit.ECGCodes.NORMAL
peak.time = int(beg+qrs.peak)
eann = mit.MITAnnotation()
eann.code = mit.ECGCodes.WFOFF
eann.time = int(beg+qrs.end)
output.add(bann)
output.add(peak)
output.add(eann)
except InconsistencyError:
warnings.warn('Annotation [{0}] could not be delineated'.format(
ann))
mit.save_annotations(output, args.r + '.' + args.o)
class Chain:
def __init__(self, anchor):
self.anchors = SortedList([anchor])
def align_locally_to_graph(self,
graphs,
sequence_graphs,
linear_ref_nodes,
read_sequence,
n_mismatches_allowed=7,
k=21,
print_debug=False):
##logging.debug("Trying to align chain %s locally" % self)
# Attempts to align this chain locally to the graph by aligning from first anchor hit in both directions
numeric_sequence = letter_sequence_to_numeric(read_sequence)
first_anchor = self.anchors[0]
sequence_after_first_anchor = read_sequence[
first_anchor.read_offset + k -
1:] #numeric_sequence[first_anchor.read_offset+k-1:]
# Forward
chromosome = first_anchor.chromosome
graph = graphs[str(chromosome)]
sequence_graph = sequence_graphs[str(chromosome)]
linear_ref_nodes_this_chr = linear_ref_nodes[str(chromosome)]
#print("ALIGNING ANCHOR with node %d, offset %d" % (first_anchor.node, first_anchor.offset))
"""
aligner = SingleSequenceAligner(graph, sequence_graph, first_anchor.node,
first_anchor.offset, sequence_after_first_anchor,
n_mismatches_allowed=n_mismatches_allowed, print_debug=debug_read)
"""
#aligner = LocalGraphAligner(graph, sequence_graph, sequence_after_first_anchor, linear_ref_nodes_this_chr, first_anchor.node, first_anchor.offset)
#aligner = Aligner(graph, sequence_graph, int(first_anchor.node), sequence_after_first_anchor)
# Align whole sequence at one
if print_debug:
logging.debug("First anchor read offset: %d" %
first_anchor.read_offset)
aligner = Aligner(graph,
sequence_graph,
int(first_anchor.node),
read_sequence,
n_bp_to_traverse_left=first_anchor.read_offset + 32,
n_bp_to_traverse_right=len(read_sequence) + 20)
alignment_after, score_after = aligner.align()
if print_debug:
logging.debug("Alignment after: %s, score: %d" %
(alignment_after, score_after))
#print(alignment_after, score_after)
if not alignment_after:
return Alignment([], [], 0, False, chromosome,
first_anchor.position)
alignment_before = []
score_before = 0
return Alignment(alignment_before, alignment_after,
score_before + score_after, True, chromosome,
first_anchor.position)
def add_anchor(self, anchor):
self.anchors.add(anchor)
def anchor_fits(self, anchor):
if self.anchors[-1].fits_to_left_of(anchor):
return True
else:
##logging.debug(" Does not fit to left of %s" % self.anchors[-1])
return False
def anchor_is_smaller(self, anchor):
return anchor < self.anchors[-1]
def __len__(self):
return len(self.anchors)
def __lt__(self, other):
return self.anchors[-1] < other.anchors[-1]
def __compare__(self, other):
return self.anchors[-1].__compare__(other)
def __str__(self):
return "-".join(str(anchor) for anchor in self.anchors)
def __repr__(self):
return self.__str__()
def __eq__(self, other):
print("Running eq %s == %s" % (self, other))
return str(self) == str(other)
class Image(object):
"""Manage images.
An image is a collection of :class:`Section`s and meta-data.
Parameters
----------
sections: iteratable (typically `list`or `tuple`
The sections the image should initialized with.
meta: object
Arbitrary meta-data.
"""
def __init__(self, sections=None, join=True, meta=None):
if meta is None:
meta = {}
if not sections:
sections = []
elif isinstance(sections, Section) or hasattr(sections, "__iter__"):
sections = list(sections)
else:
raise TypeError(
"Argument section is of wrong type '{}'".format(sections))
self._sections = SortedList(sections, key=attrgetter("start_address"))
self._join = join
if join:
self.join_sections()
_validate_sections(self._sections)
self.address = 0
#if meta and not isinstance(meta, MetaRecord):
# raise TypeError("meta-data must be of instance 'MetaRecord'")
self.meta = meta
def __repr__(self):
result = []
for segment in self.sections:
result.append(repr(segment))
return '\n'.join(result)
__str__ = __repr__
def __len__(self):
return len(self.sections)
def __iter__(self):
return iter(self.sections)
def __getitem__(self, idx):
return self.sections[idx]
def __eq__(self, other):
if len(self.sections) == len(other.sections):
return all(eq(l, r) for l, r in zip(self.sections, other.sections))
else:
return False
def __ne__(self, other):
return not (self == other)
def __contains__(self, addr):
return any(addr in sec for sec in self.sections)
def hexdump(self, fp=sys.stdout):
"""
"""
for idx, section in enumerate(self.sections):
print("\nSection #{0:04d}".format(idx), file=fp)
print("-" * 13, file=fp)
section.hexdump(fp)
def _call_address_function(self, func_name, addr, *args):
for section in self.sections:
if addr in section:
func = getattr(section, func_name)
return func(addr, *args)
raise InvalidAddressError(
"Address 0x{:08x} not in range.".format(addr))
def read(self, addr, length):
"""Read bytes from image.
Parameters
----------
addr: int
Startaddress.
length: int
Number of bytes to read.
Returns
-------
bytes
Raises
------
:class:`InvalidAddressError`
if `addr` is out of range
Note
----
if `addr` + `len` is out of range, result is silently truncated, i.e. without raising an exception.
"""
return self._call_address_function("read", addr, length)
def write(self, addr, length, data):
"""Write bytes to image.
Parameters
----------
addr: int
Startaddress.
length: int
Number of bytes to write.
data: bytes
Raises
------
:class:`InvalidAddressError`
if `addr` is out of range
Note
----
if `addr` + `len` is out of range, result is silently truncated, i.e. without raising an exception.
"""
self._call_address_function("write", addr, length, data)
def read_numeric(self, addr, dtype):
"""
"""
return self._call_address_function("readNumeric", addr, dtype)
def write_numeric(self, addr, value, dtype):
"""
"""
self._call_address_function("writeNumeric", addr, value, dtype)
def read_string(self, addr, encoding="latin1", length=-1):
"""
"""
return self._call_address_function("readString", addr, encoding,
length)
def write_string(self, addr, value, encoding="latin1"):
"""
"""
self._call_address_function("writeString", addr, value, encoding)
def _address_contained(self, address, length):
"""Check if address space exists.
Parameters
----------
address: int
length: int
Returns
-------
bool
"""
return address in self or (address + length - 1) in self
def insert_section(self, data, start_address=None, dont_join=False):
"""Insert/add a new section to image.
Parameters
----------
data: convertible to bytearray() -- s. :func:`_data_converter`.
Bytes making up the section.
start_address: int
dont_join: bool
Don't join/merge adjacent section.
Raises
------
:class:`InvalidAddressError`
Notes
-----
Overlapping sections are not supported.
To relace a section use :meth:`update_section`.
"""
start_address = start_address if start_address is not None else self.address # If Address omitted, create continuous address space.
if self._address_contained(start_address, len(data)):
raise InvalidAddressError("Overlapping address-space")
if isinstance(data, str):
data = [ord(x) for x in data] # array.array('B',data)
self._sections.add(Section(start_address, data))
if self._join:
self.join_sections()
self.address = start_address + len(data)
@property
def sections(self):
return self._sections
def get_section(self, address):
"""Get :class:`Section` containing `address`.
Parameters
----------
address: int
Returns
-------
:class:`Section`
Raises
------
:class:`InvalidAddressError`
"""
if not address in self:
raise InvalidAddressError("Address not in range")
result = self._sections.bisect_right(SearchType(address))
return self._sections[result - 1]
def update_section(self, data, start_address=None):
"""
"""
if not self._address_contained(start_address, len(data)):
raise InvalidAddressError("Address-space not in range")
def delete_section(self, start_address=None):
"""
"""
def join_sections(self):
"""
"""
self._sections = join_sections(self._sections)
def split(self, at=None, equal_parts=None, remap=None):
print("SPLIT-IMAGE", at, equal_parts, remap)
def from_scratch(self):
doc = SortedList()
doc.add(self.empty_start)
doc.add(self.empty_end)
return doc
avg_j = np.mean(list(ratings_j.values()))
dev_j = {
user: (rating - avg_j)
for user, rating in ratings_j.items()
}
dev_j_values = np.array(list(dev_j.values()))
sigma_j = np.sqrt(dev_j_values.dot(dev_j_values))
# calculate correlation coefficient
numerator = sum(dev_i[m] * dev_j[m] for m in common_users)
w_ij = numerator / (sigma_i * sigma_j)
# insert into sorted list and truncate
# negate weight, because list is sorted ascending
# maximum value (1) is "closest"
sl.add((-w_ij, j))
if len(sl) > K:
del sl[-1]
# store the neighbors
neighbors.append(sl)
# print out useful things
if i % 1 == 0:
print(i)
# using neighbors, calculate train and test MSE
def predict(i, u):
# calculate the weighted sum of deviations
class TTLCache(object):
"""A key/value cache implementation where each entry has its own TTL"""
def __init__(self, cache_name, timer=time.time):
# map from key to _CacheEntry
self._data = {}
# the _CacheEntries, sorted by expiry time
self._expiry_list = SortedList()
self._timer = timer
def set(self, key, value, ttl):
"""Add/update an entry in the cache
:param key: Key for this entry.
:param value: Value for this entry.
:param paramttl: TTL for this entry, in seconds.
:type paramttl: float
"""
expiry = self._timer() + ttl
self.expire()
e = self._data.pop(key, SENTINEL)
if e != SENTINEL:
self._expiry_list.remove(e)
entry = _CacheEntry(expiry_time=expiry, key=key, value=value)
self._data[key] = entry
self._expiry_list.add(entry)
def get(self, key, default=SENTINEL):
"""Get a value from the cache
:param key: The key to look up.
:param default: default value to return, if key is not found. If not
set, and the key is not found, a KeyError will be raised.
:returns a value from the cache, or the default.
"""
self.expire()
e = self._data.get(key, SENTINEL)
if e == SENTINEL:
if default == SENTINEL:
raise KeyError(key)
return default
return e.value
def get_with_expiry(self, key):
"""Get a value, and its expiry time, from the cache
:param key: key to look up
:returns The value from the cache, and the expiry time.
:rtype: Tuple[Any, float]
Raises:
KeyError if the entry is not found
"""
self.expire()
try:
e = self._data[key]
except KeyError:
raise
return e.value, e.expiry_time
def pop(self, key, default=SENTINEL):
"""Remove a value from the cache
If key is in the cache, remove it and return its value, else return default.
If default is not given and key is not in the cache, a KeyError is raised.
:param key: key to look up
:param default: default value to return, if key is not found. If not
set, and the key is not found, a KeyError will be raised
:returns a value from the cache, or the default
"""
self.expire()
e = self._data.pop(key, SENTINEL)
if e == SENTINEL:
if default == SENTINEL:
raise KeyError(key)
return default
self._expiry_list.remove(e)
return e.value
def __getitem__(self, key):
return self.get(key)
def __delitem__(self, key):
self.pop(key)
def __contains__(self, key):
return key in self._data
def __len__(self):
self.expire()
return len(self._data)
def expire(self):
"""Run the expiry on the cache. Any entries whose expiry times are due will
be removed
"""
now = self._timer()
while self._expiry_list:
first_entry = self._expiry_list[0]
if first_entry.expiry_time - now > 0.0:
break
del self._data[first_entry.key]
del self._expiry_list[0]
# 315. Count of Smaller Numbers After Self # https://leetcode.com/problems/count-of-smaller-numbers-after-self from sortedcontainers import SortedList class Solution: def countSmaller(self, nums: List[int]) -> List[int]: n = len(nums) res = [0] * n sl = SortedList([nums[-1]]) for i in range(n - 2, -1, -1): index = sl.bisect(nums[i] - 1) res[i] = index sl.add(nums[i]) return res
def match_candidates(segment_kwargs, directory=None, match_all=False):
for s in streams.segments:
if not original_video(s, directory):
continue
if s.youtube and s.official and not match_all:
continue
if not s.official and segment_kwargs.get('official') is False:
print(f'Skipping segment {s.hash} '
'(both videos are unofficial)',
file=sys.stderr)
continue
subrefs = SortedList(
[subref for ref in s.references for subref in ref.subrefs],
key=lambda x: x.abs_start)
if len(subrefs) == 0:
print(f'Skipping segment {s.hash} (no subrefs)', file=sys.stderr)
continue
tmp_subref = None
if subrefs[0].abs_start != 0:
tmp_subref = SubReference(name='Начало', parent=s.references[0])
subrefs.add(tmp_subref)
tmp_stream = Stream(data=[], key=s.stream.twitch)
tmp_segment = Segment(stream=tmp_stream, **segment_kwargs)
for subref in subrefs:
tmp_segment.offset = subref.abs_start
covered, partial, _ = refs_coverage(s.stream, tmp_segment)
c_refs = SortedList([
subref for ref in flat([covered, partial])
for subref in ref.subrefs if subref._coverage >= 50
],
key=lambda x: x.abs_start)
if len(c_refs) == 0:
print(
f'Skipping segment {s.hash} '
'(would not cover any subrefs)',
file=sys.stderr)
continue
time_range = Timecode(c_refs[0].abs_start)
time_range.end = c_refs[-1].abs_end
scan_range = Timecode(time_range) - s.offset()
# Expand scan range to cover all possible offsets
overscan = tmp_segment.duration - time_range.duration
if overscan > 0:
scan_range.start -= overscan
scan_range.end += overscan
# Cut the end of the scan range that can't be matched anyway
scan_range.end -= tmp_segment.duration - MATCH_OFFSET - MATCH_CHUNK
# Keep range in stream bounds
if scan_range.start < 0:
scan_range.start = T + 0
if scan_range.end < 0:
scan_range.end = T + MATCH_OFFSET + MATCH_CHUNK
yield s, time_range, scan_range
if tmp_subref:
tmp_subref.parent.subrefs.remove(tmp_subref)
def longestRepeating(self, s: str, queryCharacters: str, queryIndices: List[int]) -> List[int]:
from sortedcontainers import SortedList
A = SortedList()
B = SortedList()
sz, last = 1, 0
ss = list((ord(x) - ord('a')) for x in s)
for i in range(1, len(ss)):
if ss[i] == ss[last]:
sz += 1
else:
A.add((last, last + sz - 1, ss[last]))
B.add(sz)
last = i
sz = 1
A.add((last, last + sz - 1, ss[last]))
B.add(sz)
# very very error-prone. quite hard to write correct code.
def update(p, c):
# locate which range `p` is in.
index = A.bisect((p,))
if index >= len(A):
index -= 1
x, y, code = A[index]
if x > p:
index -= 1
x, y, code = A[index]
size = (y - x + 1)
assert code == ss[p]
assert x <= p <= y
# split that range to two segments.
del A[index]
B.remove(size)
t0 = x, p - 1, ss[p]
t1 = p + 1, y, ss[p]
insert(t0)
insert(t1)
# generate new segment.
t = p, p, c
insert(t)
def insert(t):
x, y, code = t
size = (y - x + 1)
if size == 0:
return
index = A.bisect((x,))
index -= 1
# try to merge with neighbours.
del_vec = []
# previous segment.
if 0 <= index < len(A) and A[index][-1] == code:
z0, z1, _ = A[index]
# z0, z1, x, y
if (z1 + 1) == x:
x = z0
del_vec.append(A[index])
# next segment.
if 0 <= (index + 1) < len(A) and A[index + 1][-1] == code:
z0, z1, _ = A[index + 1]
# x, y, z0, z1
if (y + 1) == z0:
y = z1
del_vec.append(A[index + 1])
if del_vec:
for t in del_vec:
z0, z1, _t = t
size = (z1 - z0 + 1)
A.remove(t)
B.remove(size)
A.add((x, y, code))
size = (y - x + 1)
B.add(size)
ans = []
for c, i in zip(queryCharacters, queryIndices):
c = ord(c) - ord('a')
if c != ss[i]:
update(i, c)
ss[i] = c
# print(A, B)
sz = B[-1]
ans.append(sz)
return ans
class Kutana:
"""
Main class for kutana application
:ivar ~.storage: Storage for things like states, e.t.c.
:ivar ~.config: Application's configuration
"""
def __init__(
self,
concurrent_handlers_count=3000,
storage=None,
loop=None,
):
self._plugins = []
self._backends = []
self._loop = loop or asyncio.new_event_loop()
self._sem = asyncio.Semaphore(value=concurrent_handlers_count,
loop=self._loop)
self._routers = None
self.storage = storage
self.config = {
"prefixes": (".", "/"),
}
def get_loop(self):
"""Return application's asyncio loop."""
return self._loop
def add_plugin(self, plugin):
"""Add plugin to the application."""
if plugin in self._plugins:
raise RuntimeError("Plugin already added")
self._plugins.append(plugin)
def add_plugins(self, plugins):
"""Add every plugin in passed iterable to the application."""
for plugin in plugins:
self.add_plugin(plugin)
def get_plugins(self):
"""Return list of added plugins."""
return self._plugins
def add_backend(self, backend):
"""Add backend to the application."""
if backend in self._backends:
raise RuntimeError("Backend already added")
self._backends.append(backend)
def get_backends(self):
return self._backends
async def _on_start(self, queue):
for backend in self._backends:
await backend.on_start(self)
async def perform_updates_request(backend):
def submit_update(update):
return queue.put((update, backend))
while True:
await backend.perform_updates_request(submit_update)
await asyncio.sleep(0)
asyncio.ensure_future(perform_updates_request(backend),
loop=self._loop)
for plugin in self._plugins:
if plugin._on_start:
await plugin._on_start(self)
async def _main_loop(self):
if self.storage is None:
self.storage = NaiveMemory()
queue = asyncio.Queue(maxsize=32, loop=self._loop)
await self._on_start(queue)
while True:
await self._sem.acquire()
update, backend = await queue.get()
ctx = await Context.create(
app=self,
config=self.config,
update=update,
backend=backend,
)
backend.prepare_context(ctx)
task = asyncio.ensure_future(self._handle_update_with_logger(
update, ctx),
loop=self._loop)
task.add_done_callback(lambda t: self._sem.release())
def _init_routers(self):
self._routers = SortedList([], key=lambda r: -r.priority)
def _add_router(new_router):
for router in self._routers:
if router.alike(new_router):
router.merge(new_router)
return
self._routers.add(new_router)
for plugin in self._plugins:
for router in plugin._routers:
_add_router(router)
async def _handle_update_with_logger(self, update, ctx):
logger.debug("Processing update %s", update)
try:
return await self._handle_update(update, ctx)
except asyncio.CancelledError:
pass
except Exception as exc:
logger.exception("Exception while handling the update")
for plugin in self._plugins:
if plugin._on_exception:
await plugin._on_exception(update, ctx, exc)
async def _handle_update(self, update, ctx):
if self._routers is None:
self._init_routers()
for plugin in self._plugins:
if plugin._on_before:
await plugin._on_before(update, ctx)
for router in self._routers:
if await router.handle(update, ctx) != hr.SKIPPED:
ctx._result = hr.COMPLETE
break
else:
ctx._result = hr.SKIPPED
for plugin in self._plugins:
if plugin._on_after:
await plugin._on_after(update, ctx, ctx._result)
return ctx._result
async def _shutdown(self):
logger.info("Gracecfully shutting application down...")
# Clean up
tasks = []
for backend in self._backends:
tasks.append(backend.on_shutdown(self))
for plugin in self._plugins:
if plugin._on_shutdown:
tasks.append(plugin._on_shutdown(self))
await asyncio.gather(*tasks, loop=self._loop, return_exceptions=True)
# Cancel everything else
tasks = []
for task in asyncio.Task.all_tasks(loop=self._loop):
if task is not asyncio.Task.current_task():
task.cancel()
tasks.append(task)
await asyncio.gather(*tasks, loop=self._loop, return_exceptions=True)
self._loop.stop()
def run(self):
"""Start the application."""
logger.info("Starting application...")
try:
asyncio.ensure_future(self._main_loop(), loop=self._loop)
self._loop.run_forever()
except KeyboardInterrupt:
asyncio.ensure_future(self._shutdown(), loop=self._loop)
self._loop.run_forever()
finally:
self._loop.close()
logger.info("Stopped application")
def stop(self):
"""Stop the application."""
return asyncio.ensure_future(self._shutdown(), loop=self._loop)
def create_group_diagram_water_land(entities, entities_times, all_time_stamps,
d, time_shift, first, last, birds,
save_path):
gmap = gmplot.GoogleMapPlotter(47, 9, 6)
over_water = 0
over_land = 0
over_water_rep = 0
over_land_rep = 0
all_together_water = 0
all_together_land = 0
# start and end with same time
start_times = SortedList()
end_times = SortedList()
for entityTime in entities_times:
start_times.add(entityTime[0])
end_times.add(entityTime[-1])
common_start_time = start_times[-1]
common_end_time = end_times[0]
all_common_time_stamps = SortedList(all_time_stamps[all_time_stamps.index(
common_start_time):all_time_stamps.index(common_end_time)])
# computing all events
event_count = 0
event_times = SortedList()
for time_index, timestamp in enumerate(all_common_time_stamps[:-1]):
coordinates = [list() for i in xrange(len(entities))]
for index, entity in enumerate(entities):
start_segment = entities_times[index].index(timestamp)
end_segment = start_segment + 1
coordinates[index] = [
entities[index][start_segment][1],
entities[index][start_segment][2],
entities[index][end_segment][1],
entities[index][end_segment][2]
]
for index1 in range(
0,
len(coordinates)): # , coordinates1 in enumerate(coordinates):
for index2 in range(0, len(coordinates)):
new_events = compare_two_segments(
coordinates[index1][0], coordinates[index1][2],
coordinates[index2][0], coordinates[index2][2],
coordinates[index1][1], coordinates[index1][3],
coordinates[index2][1], coordinates[index2][3],
time_index + first, all_common_time_stamps, d)
if len(new_events) > 0:
for event in new_events:
event_times.add(event)
event_count += 1
print event_count
# propagate event time stamps to all trajectories
adding_count = 0
for timestamp in event_times:
if timestamp not in all_common_time_stamps:
all_common_time_stamps.add(timestamp)
adding_count += 1
for index, entity in enumerate(entities):
length = len(entities_times[index])
# check if the current entity has this timestamp, if not add it and interpolate the
# coordinates but only if it is in the range of the trajectory. Otherwise interpolation is not possible
if timestamp not in entities_times[index] and entities_times[
index].bisect(timestamp) > 0 and entities_times[
index].bisect(timestamp) < length:
entity.add(
interpolate_surface(entity, entities_times[index],
timestamp))
entities_times[index].add(timestamp)
# solve set cover instance for every time stamp, return the segments of the solution and draw them
universe = set(range(0, len(entities)))
cover_size = []
cover_numbers = []
color_of_entities = ['red', 'blue', 'green', 'orange', 'black']
# for each entity initialize
average_representation_strength = [0 for i in xrange(len(entities))]
all_together = 0
number_of_representations = 0
for time_index, timestamp in enumerate(all_common_time_stamps[:-200]):
set_list = []
start_segment_common = all_common_time_stamps.index(timestamp)
end_segment_common = start_segment_common + 1
for index in range(0, len(entities)):
current_set = set()
start_segment = entities_times[index].index(timestamp)
end_segment = start_segment + 1
segment = [[
entities[index][start_segment][1],
entities[index][start_segment][2]
],
[
entities[index][end_segment][1],
entities[index][end_segment][2]
]]
for number, entity in enumerate(entities):
if within_alpha_similar_time_distance(
segment, timestamp,
all_common_time_stamps[end_segment_common], entity,
entities_times[number], time_shift, d, 0):
current_set.add(number)
set_list.append(current_set)
average_representation_strength[index] += len(current_set)
result = set_cover(universe, set_list)
number_of_representations += len(result[0])
if len(result[0]) == 1:
all_together += 1
cover_numbers.append(result[1])
landmarks_start = [
entities[i][entities_times[i].index(timestamp)][3]
for i in range(0, len(entities))
]
landmarks_end = [
entities[i][entities_times[i].index(timestamp)][3]
for i in range(0, len(entities))
]
if 210 in landmarks_start or 210 in landmarks_end:
water = True
else:
water = False
cover_size.append([(len(result[0])), water])
# get
if water:
over_water += 1
over_water_rep += len(result[0])
if len(result[0]) == 1:
all_together_water += 1
else:
over_land += 1
over_land_rep += len(result[0])
if len(result[0]) == 1:
all_together_land += 1
for indexSet, entity in enumerate(result[1]):
# print entities[3][entities_times[3].index(timestamp)][2], entities[3][entities_times[3].index(timestamp)][1]
# print entities[3][entities_times[3].index(timestamp)][3]
# print color
# print "\n"
color = color_of_entities[entity]
gmap.plot([
entities[entity][entities_times[entity].index(timestamp)][2],
entities[entity][entities_times[entity].index(timestamp) +
1][2]
], [
entities[entity][entities_times[entity].index(timestamp)][1],
entities[entity][entities_times[entity].index(timestamp) +
1][1]
],
color=color,
edge_width=result[2][indexSet])
all_together = 1.0 * all_together / (len(all_common_time_stamps) - 200
) # ((last + adding_count) - first)
number_of_representations = 1.0 * number_of_representations / (
len(all_common_time_stamps) - 200) # ((last + adding_count) - first)
print "Together Test:", d, time_shift, all_together
print "Average Number Representatives:", d, time_shift, number_of_representations
# average_representation_strength = [1.0 * entry / ((last + len(event_times)) - first) - 1 for entry in
# average_representation_strength]
average_representation_strength = [
1.0 * entry / (len(all_common_time_stamps) - 200) - 1
for entry in average_representation_strength
]
size = os.path.join(
save_path, "outputSize_" + str(d) + '_' + str(time_shift) + ".csv")
numbers = os.path.join(
save_path,
"outputNumber_" + birds + str(d) + '_' + str(time_shift) + ".txt")
representation = os.path.join(
save_path, "averageRepresentationStrength" + birds + str(d) + '_' +
str(time_shift) + ".txt")
html = os.path.join(
save_path, 'groupDiagram_water_land' + birds + str(d) + '_' +
str(time_shift) + '.html')
# file1 = open(size, "w")
# simplejson.dump(cover_size, file1)
# file1.close()
with open(size, "wb") as f:
writer = csv.writer(f)
writer.writerows(cover_size)
file2 = open(numbers, 'w')
simplejson.dump(cover_numbers, file2)
file2.close()
file3 = open(representation, 'w')
simplejson.dump(average_representation_strength, file3)
file3.close()
gmap.draw(html)
print "average representatives over land:", 1.0 * over_land_rep / over_land
print "average representatives over water:", 1.0 * over_water_rep / over_water
print "time together over land:", 1.0 * all_together_land / over_land
print "time together over water:", 1.0 * all_together_water / over_water
return {
d: [
all_together, number_of_representations,
1.0 * all_together_land / over_land,
1.0 * over_land_rep / over_land,
1.0 * all_together_water / over_water,
1.0 * over_water_rep / over_water
]
}
