def test_eq():
alpha = SortedSet(range(100), load=7)
beta = SortedSet(range(100), load=17)
assert alpha == beta
assert alpha == beta._set
beta.add(101)
assert not (alpha == beta)
def get_links(names, html):
"""
Return a SortedSet of computer scientist names that are linked from this
html page. The return set is restricted to those people in the provided
set of names. The returned list should contain no duplicates.
Params:
names....A SortedSet of computer scientist names, one per filename.
html.....A string representing one html page.
Returns:
A SortedSet of names of linked computer scientists on this html page, restricted to
elements of the set of provided names.
>>> get_links({'Gerald_Jay_Sussman'},
... '''<a href="/wiki/Gerald_Jay_Sussman">xx</a> and <a href="/wiki/Not_Me">xx</a>''')
SortedSet(['Gerald_Jay_Sussman'], key=None, load=1000)
"""
pagenames = SortedSet()
for link in BeautifulSoup(html, "html.parser", parse_only=SoupStrainer('a')):
if link.has_attr('href'):
name = link['href'].split('/')[-1]
if name in names:
pagenames.add(name)
return pagenames
def test_ne():
alpha = SortedSet(range(100), load=7)
beta = SortedSet(range(99), load=17)
assert alpha != beta
beta.add(100)
assert alpha != beta
assert alpha != beta._set
def get_links(names, html):
"""
Return a SortedSet of computer scientist names that are linked from this
html page. The return set is restricted to those people in the provided
set of names. The returned list should contain no duplicates.
Params:
names....A SortedSet of computer scientist names, one per filename.
html.....A string representing one html page.
Returns:
A SortedSet of names of linked computer scientists on this html page, restricted to
elements of the set of provided names.
>>> get_links({'Gerald_Jay_Sussman'},
... '''<a href="/wiki/Gerald_Jay_Sussman">xx</a> and <a href="/wiki/Not_Me">xx</a>''')
SortedSet(['Gerald_Jay_Sussman'], key=None, load=1000)
"""
soup = BeautifulSoup(html,"html.parser")
list = [l['href'] for l in soup.find_all('a') if l.get('href')]
res = SortedSet()
for l in list:
if l.startswith('/wiki/'):
tokens = l.split('/')
if tokens[2] in names:
res.add(tokens[2])
return res
def test_add():
temp = SortedSet(range(100))
temp._reset(7)
temp.add(100)
temp.add(90)
temp._check()
assert all(val == temp[val] for val in range(101))
def test_count():
temp = SortedSet(range(100), load=7)
assert all(temp.count(val) == 1 for val in range(100))
assert temp.count(100) == 0
assert temp.count(0) == 1
temp.add(0)
assert temp.count(0) == 1
temp._check()
def test_eq():
alpha = SortedSet(range(100))
alpha._reset(7)
beta = SortedSet(range(100))
beta._reset(17)
assert alpha == beta
assert alpha == beta._set
beta.add(101)
assert not (alpha == beta)
class DictGraph(Graph):
"""Graph that supports nonconsecutive vertex ids."""
def __init__(self, nodes: Set[int]=None, r: int=1) -> None:
"""Make a new graph."""
if nodes is None:
self.nodes = SortedSet() # type: Set[int]
else:
self.nodes = nodes
self.radius = r
self.inarcs_by_weight = [defaultdict(SortedSet) for _ in range(self.radius)]
def __len__(self):
"""len() support."""
return len(self.nodes)
def __iter__(self):
"""Iteration support."""
return iter(self.nodes)
def __contains__(self, v):
"""Support for `if v in graph`."""
return v in self.nodes
def add_node(self, u: int):
"""Add a new node."""
self.nodes.add(u)
def arcs(self, weight: int=None):
"""
Return all the arcs in the graph.
restrict to a given weight when provided
"""
if weight:
return [(x, y) for x in self.nodes
for y in self.inarcs_by_weight[weight-1][x]]
else:
return [(x, y, w+1) for w, arc_list in
enumerate(self.inarcs_by_weight)
for x in self.nodes
for y in arc_list[x]]
def remove_isolates(self) -> List[int]:
"""
Remove all isolated vertices and return a list of vertices removed.
Precondition: the graph is bidirectional
"""
isolates = [v for v in self if self.in_degree(v, 1) == 0]
for v in isolates:
self.nodes.remove(v)
for N in self.inarcs_by_weight:
if v in N:
del N[v]
return isolates
def test_ne():
alpha = SortedSet(range(100))
alpha._reset(7)
beta = SortedSet(range(99))
beta._reset(17)
assert alpha != beta
beta.add(100)
assert alpha != beta
assert alpha != beta._set
assert alpha != list(range(101))
def get_links(names, html):
"""
Return a SortedSet of computer scientist names that are linked from this
html page. The return set is restricted to those people in the provided
set of names. The returned list should contain no duplicates.
Params:
names....A SortedSet of computer scientist names, one per filename.
html.....A string representing one html page.
Returns:
A SortedSet of names of linked computer scientists on this html page, restricted to
elements of the set of provided names.
>>> get_links({'Gerald_Jay_Sussman'},
... '''<a href="/wiki/Gerald_Jay_Sussman">xx</a> and <a href="/wiki/Not_Me">xx</a>''')
SortedSet(['Gerald_Jay_Sussman'], key=None, load=1000)
"""
###TODO
#remove BeautifulSoap - later
listofHrefs = []
listofHrefTexts = []
FinalSortedSet = SortedSet()
splice_char = '/'
#getting all the tags using the BeautifulSoup
#soup = BeautifulSoup(html, "html.parser")
#fectching all the links in anchor tags
#for link in soup.find_all('a'):
#listofHrefs.append(link.get('href'))
for i in range(0,len(listofHrefs)):
value = listofHrefs[i][6:]
listofHrefTexts.append(value)
listofHrefTexts = re.findall(r'href="([^"]*)', html)
#print(listofHrefTexts)
for i in listofHrefTexts:
#print(i)
value = i[6:]
listofHrefs.append(value)
#print(listofHrefs)
listofHrefs = list(set(listofHrefs))
#print(len(listofHrefs))
for href in listofHrefs:
for name in names:
#windows OS handling
if(name == "Guy_L._Steele,_Jr"):
names.remove(name)
names.add("Guy_L._Steele,_Jr.")
if(href == name):
FinalSortedSet.add(name)
return FinalSortedSet
pass
def compute_domset(graph: Graph, radius: int):
"""
Compute a d-dominating set using Dvorak's approximation algorithm
for dtf-graphs (see `Structural Sparseness and Complex Networks').
Graph needs a distance-d dtf augmentation (see rdomset() for usage).
"""
domset = SortedSet()
infinity = float('inf')
# minimum distance to a dominating vertex, obviously infinite at start
domdistance = defaultdict(lambda: infinity) # type: Dict[int, float]
# counter that keeps track of how many neighbors have made it into the
# domset
domcounter = defaultdict(int) # type: Dict[int, int]
# cutoff for how many times a vertex needs to have its neighbors added to
# the domset before it does. We choose radius^2 as a convenient "large"
# number
c = (2*radius)**2
# Sort the vertices by indegree so we take fewer vertices
order = sorted([v for v in graph], key=lambda x: graph.in_degree(x),
reverse=True)
# vprops = [(v,graph.in_degree(v)) for v in nodes]
# vprops.sort(key=itemgetter(1),reverse=False)
# order = map(itemgetter(0),vprops)
for v in order:
# look at the in neighbors to update the distance
for r in range(1, radius + 1):
for u in graph.in_neighbors(v, r):
domdistance[v] = min(domdistance[v], r+domdistance[u])
# if v is already dominated at radius, no need to work
if domdistance[v] <= radius:
continue
# if v is not dominated at radius, put v in the dominating set
domset.add(v)
domdistance[v] = 0
# update distances of neighbors of v if v is closer if u has had too
# many of its neighbors taken into the domset, include it too.
for r in range(1, graph.radius + 1):
for u in graph.in_neighbors(v, r):
domcounter[u] += 1
domdistance[u] = min(domdistance[u], r)
if domcounter[u] > c and u not in domset:
# add u to domset
domset.add(u)
domdistance[u] = 0
for x, rx in graph.in_neighbors(u):
domdistance[x] = min(domdistance[x], rx)
# only need to update domdistance if u didn't get added
return domset
class Server:
_ids = 0
def __init__(self):
self.queue = SortedSet(key = lambda job: job.arrivalTime)
self.numServers = 1
Server._ids +=1
self.busyServers = 0
self.serviceTimeDistribution = None
self.name = "Server {}".format(Server._ids)
self.In = None
self.Out = None
self.scheduler = None
def receive(self, m):
if m.event == "end": # end of service
self.send(m.job)
if len(self.queue) > 0:
assert self.busyServers == self.numServers
job = self.queue.pop(0)
self.startService(job)
else:
assert self.busyServers > 0
self.busyServers -= 1
#self.departureStats()
else: # receive new job
assert "job" in m.event
job = m.job
job.setArrivalTime(self.scheduler.now())
serviceTime = self.serviceTimeDistribution.rvs()
job.setServiceTime(serviceTime)
job.log(self.scheduler.now(), "a", self.busyServers + len(self.queue))
if self.busyServers < self.numServers:
self.busyServers += 1
self.startService(job)
else:
self.queue.add(job)
def startService(self, job):
job.log(self.scheduler.now(), "s", len(self.queue))
t = self.scheduler.now() + job.serviceTime
m = Event(self, self, t, job = job, event = "end")
self.scheduler.add(m)
def send(self, job): # job departure
job.log(self.scheduler.now(), "d", len(self.queue))
m = Event(self, self.Out, self.scheduler.now(), job = job, event = "job")
self.scheduler.add(m)
def setServiceTimeDistribution(self, distribution):
self.serviceTimeDistribution = distribution
def euler():
fracs = SortedSet()
limit = 10**6
for d in range(5, limit+1):
target = d*3/7
for n in range(int(floor(target)), int(ceil(target))+1):
if gcd(n, d) == 1:
fracs.add(Fraction(n, d))
result = 0
compare = Fraction(3, 7)
for i, f in enumerate(fracs):
if f == compare:
result = i-1
break
print(fracs[result])
class FileReference:
"""A class that manages n-triple files.
This class stores inforamtation about the location of a n-triple file and is
able to add and delete triples to that file.
"""
def __init__(self, path, content):
"""Initialize a new FileReference instance.
Args:
filelocation: A string of the filepath.
filecontentinmem: Boolean to decide if local filesystem should be used to
or if file content should be kept in memory too . (Defaults false)
Raises:
ValueError: If no file at the filelocation, or in the given directory + filelocation.
"""
if isinstance(content, str):
new = []
for line in content.splitlines():
new.append(' '.join(line.split()))
content = new
self._path = path
self._content = SortedSet(content)
self._modified = False
@property
def path(self):
return self._path
@property
def content(self):
return "\n".join(self._content) + "\n"
def add(self, data):
"""Add a triple to the file content."""
self._content.add(data)
def extend(self, data):
"""Add triples to the file content."""
self._content.extend(data)
def remove(self, data):
"""Remove trple from the file content."""
try:
self._content.remove(data)
except KeyError:
pass
def euler50():
limit = 100
primes = SortedSet(sieve.primerange(1, limit))
prime_sums = SortedSet()
prime_sums.add(0)
consecutive = 0
for p in primes:
prime_sums.add(prime_sums[-1] + p)
for i in range(consecutive, len(prime_sums)):
for j in range(i - consecutive - 1, -1, -1):
if prime_sums[i] - prime_sums[j] > limit:
break
if isprime(prime_sums[i] - prime_sums[j]):
consecutive = i - j
result = prime_sums[i] - prime_sums[j]
print(result)
def reduce(stdin):
current_key = None
neighbors = SortedSet()
for line in stdin:
#bye bye whitespace
line = line.strip()
# parse the mapper input
key, friend = line.split('\t', 1)
if current_key == key:
neighbors.add(friend)
else:
if current_key:
yield '%s\t%s' % (current_key, '\t'.join(neighbors))
neighbors = SortedSet()
current_key = key
neighbors.add(friend)
#make sure to include the last entry
yield '%s\t%s' % (current_key, '\t'.join(neighbors))
def get_top_pageranks(inlinks, outlinks, b, n=50, iters=20):
"""
>>> inlinks = SortedDict({'a': ['c'], 'b': set(['a']), 'c': set(['a', 'b'])})
>>> outlinks = SortedDict({'a': ['b', 'c'], 'b': set(['c']), 'c': set(['a'])})
>>> res = get_top_pageranks(inlinks, outlinks, b=.5, n=2, iters=1)
>>> len(res)
2
>>> res[0] # doctest:+ELLIPSIS
('a', 0.6666...
"""
urls = SortedSet()
for value in inlinks:
urls.add(value)
for value in outlinks:
urls.add(value)
Score = compute_pagerank(urls, inlinks, outlinks, b, iters)
return sorted(Score.items(),key=lambda x:x[1],reverse=True)[:n]
def get_members(num_pages=20):
"""
Goes to the top 20 pages of member listings, and get's em all
:param num_pages:
:return:
"""
all_members = SortedSet()
for page_num in range(1, num_pages):
this_page = urllib.urlopen(root + route_people + '&page=' + str(page_num)).read()
this_page_soup = BeautifulSoup(this_page, 'lxml')
members_on_this_page = this_page_soup.find_all('div', class_='member')
# -- Goes over all members in the page, and adds em to the set -- #
for member in members_on_this_page:
member_url = member.find('div', class_='pic').a['href']
member_obj = get_member_with_url(str(member_url))
print member_obj.username
all_members.add(member_obj)
if len(all_members) % 10 == 0:
write_to_file(list(all_members[:-10]), 'all_members.txt', _MemberEncoder, False)
return all_members
def take_first_resources(available_res, nb_res_to_take):
"""Take the first nb_res_to_take resources into available_res."""
if nb_res_to_take > len(available_res):
raise Exception('Invalid schedule. Want to use {} resources whereas '
'only {} are available({})'.format(nb_res_to_take,
len(available_res),
available_res))
allocation = SortedSet()
l = list(available_res)
for i in range(nb_res_to_take):
allocation.add(l[i])
available_res.difference_update(allocation)
if len(available_res) > 0:
min_available_res = min(available_res)
for res in allocation:
if res >= min_available_res:
raise Exception('Invalid sort')
return (available_res, allocation)
def get_links(names, html):
"""
Return a SortedSet of computer scientist names that are linked from this
html page. The return set is restricted to those people in the provided
set of names. The returned list should contain no duplicates.
Params:
names....A SortedSet of computer scientist names, one per filename.
html.....A string representing one html page.
Returns:
A SortedSet of names of linked computer scientists on this html page, restricted to
elements of the set of provided names.
>>> get_links({'Gerald_Jay_Sussman'},
... '''<a href="/wiki/Gerald_Jay_Sussman">xx</a> and <a href="/wiki/Not_Me">xx</a>''')
SortedSet(['Gerald_Jay_Sussman'], key=None, load=1000)
"""
###TODO
my_set = SortedSet()
soup = BeautifulSoup(html,'lxml')
for link in soup.find_all('a'):
if link.get('href'):
count = 0
for page in names:
if count == 0:
if '/wiki/'+page == link['href']:
count = 1
my_set.add(page)
# to early terminate
elif count == 1:
break
###
return my_set
def containsNearbyAlmostDuplicate(self, nums, k, t):
"""
:type nums: List[int]
:type k: int
:type t: int
:rtype: bool
"""
if k < 1 or t < 0 or nums == None or len(nums) < 2:
return False
treeset = SortedSet()
for i in xrange(len(nums)):
# Solution 1
subset = [x for x in treeset.irange(nums[i] - t, nums[i] + t)]
if len(subset) > 0:
return True
treeset.add(nums[i])
if i >= k:
treeset.discard(nums[i - k])
return False
def test_stress(repeat=1000):
sst = SortedSet(range(1000))
for rpt in range(repeat):
action = random.choice(actions)
action(sst)
try:
sst._check()
except AssertionError:
print(action)
raise
start_len = len(sst)
while len(sst) < 500:
sst.add(random.randrange(0, 2000))
while len(sst) > 2000:
del sst[random.randrange(0, len(sst))]
if start_len != len(sst):
sst._check()
def revealed2sortedset(revealed: List[Union[tuple, slice]]) -> SortedSet:
""" Converts a list of included pairs to a sorted set of integers in O(n), n = size of @slices.
Every number from every slice is added to the sorted set, except 0.
"""
# 10, [] -> 10, []
# 10, [(0, 10)] -> 10, [10]
# 10, [(0, 7)] -> 10, [7]
# 10, [(7, 10)] -> 10, [7, 10]
# 10, [(3, 7)] -> 10, [3, 7]
# 10, [(0, 3), (7, 10)] -> 10, [3, 7, 10]
# 10, [(0, 1), (2, 3), (4, 5), (6, 7), (8, 9)] -> 10, [1, 2, 3, 4, 5, 6, 7, 8, 9]
try:
#intervals = SortedSet(a for a, _ in revealed).union(b for _, b in revealed)
intervals = SortedSet()
for a, b in revealed:
intervals.add(a)
intervals.add(b)
except TypeError: # slice
intervals = SortedSet(sl.start for sl in revealed).union(sl.stop for sl in revealed)
if 0 in intervals:
intervals.remove(0)
return intervals
def swapConnColors(c1, c2, ndx): #kempe chain connected new1 = SortedSet() new2 = SortedSet() for i in c1: swapped = False for j in ndx[i]: if j in c2: new1.add(j) new2.add(i) swapped = True if not swapped: new1.add(i) new2.add(j) #print c1, c2 #print new1, new2 return new1,new2
def get_ready_steps(deps):
ready = SortedSet()
for step in deps:
if not deps[step]:
ready.add(step)
return ready
first_repo = True
print "Searching stash project[" + project["name"] + "]"
for repo in stash.projects[project["key"]].repos.list():
name = repo["name"].lower()
if None == repositories or name in repositories:
if first_repo:
print ""
first_repo = False
print "Found repository[" + name + "]"
repositories.remove(name)
if os.path.exists(args.clone_directory + "/" + name):
print name + " already cloned, continuing..."
print ""
repositories_already_cloned.add(name)
continue
for url in repo["links"]["clone"]:
if (url["name"] == "ssh"):
os.system("git clone " + url["href"] + " " + args.clone_directory + "/" + name)
repositories_cloned.add(name)
break
print ""
print "\n\n*******************"
print "SUMMARY"
print "*******************"
print "Repositories Cloned:"
for s in repositories_cloned:
def fetch_us_surnames(url_1990: str,
url_2010: str,
filename: str,
freq_csv_filename: str = "",
min_cumfreq_pct: float = 0,
max_cumfreq_pct: float = 100,
min_word_length: int = 1,
show_rejects: bool = False) -> None:
"""
Fetches US surnames from the 1990 and 2010 census data. Writes them to a
file.
Args:
url_1990:
URL for 1990 US census data
url_2010:
URL for 2010 US census data
filename:
text filename to write names to (one name per line)
freq_csv_filename:
optional CSV to write "name, frequency" pairs to, one name per line
min_cumfreq_pct:
minimum cumulative frequency (%): 0 for no limit, or above 0 to
exclude common names
max_cumfreq_pct:
maximum cumulative frequency (%): 100 for no limit, or below 100 to
exclude rare names
min_word_length:
minimum word length; all words must be at least this long
show_rejects:
report rejected words to the Python debug log
"""
nameinfo_p1 = (
gen_name_info_via_min_length(
gen_sufficiently_frequent_names(
gen_us_surname_1990_info(
gen_lines_from_binary_files(
gen_binary_files_from_urls([url_1990])
)
),
min_cumfreq_pct=min_cumfreq_pct,
max_cumfreq_pct=max_cumfreq_pct,
show_rejects=show_rejects
),
min_name_length=min_word_length
)
)
nameinfo_p2 = (
gen_name_info_via_min_length(
gen_sufficiently_frequent_names(
gen_us_surname_2010_info(
gen_rows_from_csv_binfiles(
gen_files_from_zipfiles(
gen_binary_files_from_urls([url_2010],
on_disk=True), # a zip
# The zip file contains a .CSV and a .XLS
filespec="*.csv",
on_disk=True
),
skip_header=True
)
),
min_cumfreq_pct=min_cumfreq_pct,
max_cumfreq_pct=max_cumfreq_pct,
show_rejects=show_rejects
),
min_name_length=min_word_length
)
)
pipeline = itertools.chain(nameinfo_p1, nameinfo_p2)
names = SortedSet()
freq = {} # type: Dict[str, float]
for nameinfo in pipeline:
name = nameinfo.name
if name not in names:
names.add(nameinfo.name)
freq[name] = nameinfo.freq_p
write_words_to_file(filename, names)
if freq_csv_filename:
log.info(f"Writing to: {freq_csv_filename}")
with open(freq_csv_filename, "wt") as f:
csvwriter = csv.writer(f)
for name in names:
csvwriter.writerow([name, freq[name]])
log.info(f"... finished writing to: {freq_csv_filename}")
def readConfig(self, **task_options):
"""readConfig function for AverageCorrelators
YAML:
file_mode: overwrite # optional
off_diagonal: false
hermitian: false
# used for giving more details in averaged op name
use_spatial_info: true
use_irrep_info: true
write_operators: false
plot_info:
corrname: standard # optional
timestep: 3 # optional
rescale: 1.0 # optional
symbol_color: blue # optional
symbol_type: circle # optional
max_relative_error: 2.0 # optional
effective_energy_type: time_forward # optional
coefficients: # optional
- operator: op_string
coefficient: -92
- ...
...
# list of operators to exclude from averaging
excluded_operators:
- kbar S=1 P=(0,0,1) A2 k 0
- ...
averaged_channels: # optional
- isospin: quintet
strangeness: 0
momentum: [0,0,0]
irrep: A1gp
- ...
...
"""
if 'averaged_channels' in task_options:
averaged_channels = SortedSet()
for averaged_channel in task_options.pop('averaged_channels'):
averaged_channels.add(
operator_info.channel.Channel(**averaged_channel))
task_options['averaged_channels'] = averaged_channels
if 'coefficients' in task_options:
coefficients = dict()
for coefficient in task_options.pop('coefficients'):
try:
operator = operator_info.operator.Operator(
coefficient.get('operator'))
coefficient = float(coefficient.get('coefficient'))
except KeyError as err:
logging.error(
f"coefficients item missing required key '{err}'")
coefficients[operator] = coefficient
task_options['coefficients'] = coefficients
if 'file_mode' in task_options:
try:
task_options['file_mode'] = sigmond.WriteMode.create(
task_options['file_mode'])
except ValueError:
logging.error(f"Invalid file_mode in '{self.task_name}'")
excluded_operators = set()
for excluded_operator in task_options.pop('excluded_operators', []):
excluded_operators.add(
operator_info.operator.Operator(excluded_operator))
task_options['excluded_operators'] = excluded_operators
task_options[
'plot_info'] = sigmond_info.sigmond_info.PlotInfo.createFromConfig(
task_options)
self.initiliaze(**task_options)
class SocialNetwork(object):
ID = 0
strategies = [COOP, DEFE]
def __init__(self,
fluct,
rep,
nt_seed,
nt_desc,
nt_randomseed,
coop_prob = JUST_COOPERATORS,
randomseed = None,
b=1,
n_per_gen=10,
e_per_gen=2,
epsilon = 0.99,
max=1000,
tourn=0.01,
X=0.025,
K=sys.maxsize,
X2= 0.025):
# this is for identification of the network
self.id = self.__class__.ID
self.__class__.ID += 1
self.fluct = fluct
self.rep = rep
self.nt_desc = nt_desc
self.nt_randomseed = nt_randomseed
self.coop_prob = coop_prob
# set the PD game
self.T = b
self.R = 1
self.P = 0
self.S = 0
# seed for the network, this is useful to replicate exactly the same
# experiment, particularly useful for debugging
if randomseed == None:
self.randomseed = time.time()
else:
print("WARNING: random seed is not null. Are you sure?")
self.randomseed = randomseed
random.seed(self.randomseed)
# main parameters
self.b = b
self.n_per_gen = n_per_gen
self.e_per_gen = e_per_gen
if (epsilon >= 1.0):
raise ValueError("""Epsilon cannot be bigger or equal to 1.0.
You can use epsilon that are similar to 1.0,
e.g 0.999999999 """)
else:
self.epsilon = epsilon
self.max = max
self.tourn = tourn
self.X = X
self.K = K
self.X2 = X2
# counters
self.gen = 0
self.count = 0
self.cooperators = 0
self.removed_nodes = 0
self.total_fit = 0
self.total_efit = 0
self.degrees = 0
self.size = 0
g = self.g = nx.Graph()
# crate auxiliary network structures to increase efficiency
self._max = max+n_per_gen
self.eps_fitness = np.empty(self._max)
self.degrees = np.empty(self._max)
self.fitness = np.empty(self._max)
self.fitness_of = np.empty(self._max, dtype=np.int_)
self.free_indexes = []
self.node_set = SortedSet()
# initialize the auxiliary structures
for i in range(0, self._max):
self.degrees[i] = 0
self.fitness_of[i] = -1
self.free_indexes.append(i)
# create the network
self.__create_from_seed(nt_seed, coop_prob)
# define the game the nodes are going to play
self.game = PD(b, self.fitness)
self.treatment = '_'.join(str(x) for x in (self.nt_desc,
self.coop_prob,
self.fluct, self.b,
self.X, self.K, self.X2))
self.signature = str(self.id) + '_' + \
str(self.rep) + '(' + self.treatment + ')'
def __create_from_seed(self, seed, coop_prob):
""" This method use the networks structure that comes in the parameter
seed as a template for the graph. It adds the necessary attributes to
run the algorithm, such as which nodes are cooperators and defectors
based on the coop_prob parameter. A value from 0 to 1 indicating a
probability of any node of being a cooperators.
Assumes that it is called from the constructor. So it assumes a new
SocialNetwork.
"""
self.count = -1
g = self.g
# add nodes from the seed to the network
for node in seed.nodes_iter(data = True):
# define the attributes of the node
id = node[0]
if coop_prob == 1 or random.uniform(0,1) < coop_prob:
st = COOP
self.cooperators += 1
else:
st = DEFE
r_index = self.free_indexes.pop()
# add the node
g.add_node(id, st=st, nst=st, r_index=r_index)
self.node_set.add(id)
self.fitness_of[r_index] = id
self.fitness[r_index] = 0
# update parameters of the graph
if id > self.count:
self.count = id
self.size += 1
self.count += 1
# add edges from the seed to the network
for e0, e1 in seed.edges_iter():
g.add_edge(e0, e1)
self.__remove_isolated_nodes()
def __remove_isolated_nodes(self):
g = self.g
to_remove = []
for n, adj in g.adj.items():
if (len(adj) == 0):
to_remove.append(n)
for n in to_remove:
r_index = g.node[n]['r_index']
self.fitness_of[r_index] = -1
self.free_indexes.append(r_index)
self.node_set.discard(n)
g.remove_node(n)
self.size -= 1
def add_node(self, st):
""" Add a node to the network
"""
# calculate rest of the node attributes
id = self.count
r_index = self.free_indexes.pop()
# add node
self.g.add_node(id, st=st, nst=st, r_index=r_index, gen=self.gen)
# update network structures
self.node_set.add(id)
self.fitness_of[r_index] = id
self.fitness[r_index] = 0
self.degrees[r_index] = 0
# update network parameters
if st == COOP:
self.cooperators += 1
self.size += 1
self.count += 1
return id
def play_games_and_remove_isolated_nodes(self):
g = self.g
node = g.node
node_set = self.node_set
adjacency = self.g.adj
f = self.fitness
ef = self.eps_fitness
eps = self.epsilon
degrees = self.degrees
f.fill(0)
total_fit = 0
total_efit = 0
total_degrees = 0
to_remove=[]
for n1 in node_set:
adj = adjacency[n1]
len_adj = len(adj)
# make sure to remove the nodes that has no more edges
if (len_adj == 0):
to_remove.append(n1)
self.removed_nodes += 1
else:
att1 = node[n1]
r_index1 = att1['r_index']
#update the strategy
n1_e = att1['st'] = att1['nst']
# play against all the neighbors
for n2 in adj.keys():
# make sure to play just once, nodes should be in order
# make sure all the adjacent nodes are in order
if (n2 > n1):
att2 = node[n2]
if n1_e == att2['nst']:
if n1_e == COOP:
f[r_index1] += self.R
f[att2['r_index']] += self.R
total_fit += self.R + self.R
else:
f[r_index1] += self.P
f[att2['r_index']] += self.P
total_fit += self.P + self.P
else:
if n1_e == COOP:
f[r_index1] += self.S
f[att2['r_index']] += self.T
total_fit += self.S + self.T
else:
f[r_index1] += self.T
f[att2['r_index']] += self.S
total_fit += self.T + self.S
# this epsilon is important to give some of the nodes
# some chance to cooperate
ef[r_index1] = 1 - eps + eps * f[r_index1]
total_efit += ef[r_index1]
# keep the degrees updates for PA
degrees[r_index1] = len_adj
total_degrees += degrees[r_index1]
# set the class attribute
self.total_fit = total_fit
self.total_efit = total_efit
self.total_degrees = total_degrees
# population will collapse
if self.size - len(to_remove) < self.e_per_gen:
print ("population collapsed with",
count_coop(sn), "cooperators and",
self.size - count_coop(sn), "defectors" )
# remove nodes that didn't have any edges
for n in to_remove:
r_index = g.node[n]['r_index']
self.fitness_of[r_index] = -1
self.free_indexes.append(r_index)
self.node_set.discard(n)
g.remove_node(n)
self.size -= 1
def update_strategies(self):
g = self.g
self.gen += 1
cooperators = 0
degrees = self.degrees
for n1 in g.nodes_iter(data = True):
neighbors_n1 = g.neighbors(n1[0])
r_index1 = n1[1]['r_index']
n2_index = random.choice(neighbors_n1)
n2 = g.node[n2_index]
# check that the strategies are actually different
if n1[1]['st'] != n2['st']:
r_n1 = self.fitness[r_index1]
r_n2 = self.fitness[n2['r_index']]
# Look to see if difference is less than a millionth of
# largest value and then assume equivalence
epsilon_fitness = max(r_n2,r_n1) / 1000000
# if the neighbor has a bigger accumulated fitness
if r_n2 > r_n1 + epsilon_fitness:
# probP = (neighbour_fitness - focal_node_fitness)
# ----------------------------------------
# b * max[k_focal_node, k_neighbour]
if random.random() < \
(1.0 * (r_n2 - r_n1)) / \
(self.b * max(len(neighbors_n1), \
len(g.neighbors(n2_index)))):
# update the strategy to a temporary vector
n1[1]['nst'] = n2['st']
"""
Poncela´s Formula gives to much weight to the number
of nodes, this is an alternate version that would be
worth to test:
probability P = neighbour_fitness focal_node_fitness
------------------ - -----------------
b * k_neighbour b * k_focal_node
if random.random() < (1.0 * r_n2) / \
(self.b*len(g.neighbors(n2_index)))-\
(1.0 * r_n1) / \
(self.b*len(neighbors_n1)):
n1[1]['nst'] = n2['st']
"""
# update cooperators counter
if n1[1]['nst'] == COOP:
cooperators += 1
self.cooperators = cooperators
def growth_initial(self, growth):
""" This method make sure that the first growth completes the nodes
necessary to get to a consistent increment of 10 per generation. It
just applies for starting networks that are smaller than self.n_per_gen
"""
if self.size < self.n_per_gen:
temp = self.n_per_gen
self.n_per_gen = self.n_per_gen - self.count
growth(self)
self.n_per_gen = temp
def attrition(self, selection_method):
g = self.g
# it should be call losers
winners = selection_method(self)
# remove the winning nodes
for winner in winners:
# remove the node from the graph and update fitness arrays
r_index = g.node[winner]['r_index']
self.fitness_of[r_index] = -1
self.free_indexes.append(r_index)
self.node_set.discard(winner)
g.remove_node(winner)
self.size -= 1
# I have moved the removal of nodes with no edges to the play_games
# phase to save optimize the code. The auxiliary method remove_isolated
# has been created in order to produce real results.
def remove_isolated(self, select_winners):
g = self.g
to_remove = []
for n, adj in g.adj.items():
if (len(adj) == 0):
to_remove.append(n)
self.removed_nodes += 1
if self.size - len(to_remove) < self.e_per_gen:
print ("population collapsed with",
self.count_coop(), "cooperators and",
self.size - self.count_coop(), "defectors" )
for n in to_remove:
r_index = g.node[n]['r_index']
self.fitness_of[r_index] = -1
self.free_indexes.append(r_index)
self.node_set.discard(n)
g.remove_node(n)
self.size -= 1
def generate_workload(input_swf,
output_json,
computation_speed,
job_walltime_factor=2,
given_walltime_only=False,
job_grain=1,
platform_size=None,
indent=None,
translate_submit_times=False,
keep_only=False,
verbose=False,
quiet=False,
job_size_function_string='1*nb_res'):
"""Generate a Batsim workload from a SWF trace."""
element = '([-+]?\d+(?:\.\d+)?)'
r = re.compile('\s*' + (element + '\s+') * 17 + element + '\s*')
current_id = 0
version = 0
# Let a job be a tuple (job_id, nb_res, run_time, submit_time, profile,
# walltime)
jobs = []
profiles = SortedSet()
nb_jobs_discarded = 0
minimum_observed_submit_time = float('inf')
# Let's loop over the lines of the input file
for line in input_swf:
res = r.match(line)
if res:
job_id = (int(float(res.group(SwfField.JOB_ID.value))))
nb_res = int(
float(res.group(SwfField.ALLOCATED_PROCESSOR_COUNT.value)))
run_time = float(res.group(SwfField.RUN_TIME.value))
submit_time = max(0, float(res.group(SwfField.SUBMIT_TIME.value)))
walltime = max(job_walltime_factor * run_time,
float(res.group(SwfField.REQUESTED_TIME.value)))
# nb_res may be changed by calling a user-given function
nb_res = eval(job_size_function_string)
if given_walltime_only:
walltime = float(res.group(SwfField.REQUESTED_TIME.value))
if nb_res > 0 and walltime > run_time and run_time > 0 and submit_time >= 0:
profile = int(((run_time // job_grain) + 1) * job_grain)
profiles.add(profile)
job = (current_id, nb_res, run_time, submit_time, profile,
walltime)
current_id = current_id + 1
minimum_observed_submit_time = min(
minimum_observed_submit_time, submit_time)
jobs.append(job)
else:
nb_jobs_discarded += 1
if verbose:
print('Job {} has been discarded'.format(job_id))
# Export JSON
# Let's generate a list of dictionaries for the jobs
djobs = list()
for (job_id, nb_res, run_time, submit_time, profile, walltime) in jobs:
use_job = True
if keep_only is not None:
use_job = eval(keep_only)
if use_job:
djobs.append({
'id': job_id,
'subtime': submit_time - minimum_observed_submit_time,
'walltime': walltime,
'res': nb_res,
'profile': str(profile)
})
# Let's generate a dict of dictionaries for the profiles
dprofs = {}
for profile in profiles:
dprofs[str(profile)] = {
'type': 'msg_par_hg',
'cpu': profile * computation_speed,
'com': 0.0
}
biggest_job = max([
nb_res
for (job_id, nb_res, run_time, submit_time, profile, walltime) in jobs
])
if platform_size is not None:
if platform_size < 1:
raise Exception(
'Invalid input: platform size must be strictly positive')
if platform_size < biggest_job:
print('Warning: platform size {size} is smaller than '
'the biggest job ({big})'.format(size=platform_size,
big=biggest_job))
else:
platform_size = biggest_job
data = {
'version': version,
'command': ' '.join(sys.argv[:]),
'date': datetime.datetime.now().isoformat(' '),
'description': 'this workload had been automatically generated',
'nb_res': platform_size,
'jobs': djobs,
'profiles': dprofs
}
try:
outFile = open(output_json, 'w')
json.dump(data, outFile, indent=indent, sort_keys=True)
if not quiet:
print('{} jobs and {} profiles had been created'.format(
len(djobs), len(dprofs)))
print('{} jobs have been discarded'.format(nb_jobs_discarded))
if keep_only:
print('{} jobs have been removed by keep_only'.format(
len(jobs) - len(djobs)))
except IOError:
print('Cannot write file', output_json)
def execute(self):
self.controller.signal_algo_start()
dist = []
vis = []
INF = 9999
for i in range(self.DS.n):
dist.append(INF)
vis.append(0)
dist[self.DS.start] = 0
ss = SortedSet()
ss.add((0, self.DS.start))
finish = []
while len(ss) > 0:
(cost, nod) = ss.pop(0)
if vis[nod] == 1:
continue
vis[nod] = 1
finish.append(nod)
self.controller.wait_for_next_step()
file = open(self.DS.filename, "w")
file.write(
str(self.DS.n) + " " + str(self.DS.m) + " " + str(self.DS.tp) +
'\n')
self.DS.nodes[nod]['color'] = (255, 0, 0)
file.write(str(self.DS.nodes) + '\n')
file.write(str(self.DS.edges) + '\n')
file.close()
self.controller.signal_step_done()
update = []
if nod in self.DS.graph.keys():
for nxt in self.DS.graph[nod]:
if vis[nxt[0]] == 0 and dist[nxt[0]] > dist[nod] + nxt[1]:
update.append(nxt[0])
dist[nxt[0]] = dist[nod] + nxt[1]
ss.add((dist[nxt[0]], nxt[0]))
self.controller.wait_for_next_step()
file = open("test.txt", "w")
file.write(str(self.DS.n) + "\n")
for k in range(self.DS.n):
if k in finish:
file.write("{'content':" + str(dist[k]) +
", 'color':(255,0,0)}\n")
elif k in update:
file.write("{'content':" + str(dist[k]) +
", 'color':(0,255,100)}\n")
else:
file.write("{'content':" + str(dist[k]) +
", 'color':(100,100,100)}\n")
file.close()
self.controller.signal_step_done()
self.controller.wait_for_next_step()
for i in range(self.DS.n):
self.DS.nodes[i]['color'] = (112, 112, 112)
file = open(self.DS.filename, "w")
file.write(
str(self.DS.n) + " " + str(self.DS.m) + " " + str(self.DS.tp) +
'\n')
file.write(str(self.DS.nodes) + '\n')
file.write(str(self.DS.edges) + '\n')
file.close()
self.controller.signal_step_done()
self.controller.wait_for_next_step()
file = open("test.txt", "w")
file.write(str(self.DS.n) + "\n")
for k in range(self.DS.n):
file.write("{'content':" + str(dist[k]) +
", 'color':(100,100,100)}\n")
file.close()
self.controller.signal_step_done()
self.controller.signal_algo_done()
def _merge_slot_iteration(
self,
slot_values: "SlotValues",
previously_updated: Iterable[TemplateSlot],
slot_list: List[TemplateSlot],
slot_indices: Dict[TemplateSlot, int],
relative_similarity_threshold: float,
):
updated = SortedSet()
length = len(slot_values.keys())
# Go over all slots
# TODO: make it only go over updated
# for i_key in previously_updated:
# i = slot_indices[i_key]
for i in range(len(slot_list)):
i_key = slot_list[i]
i_changed = False
# If this slot is not already getting replaced
if i_key not in slot_values.get_replacements().keys():
i_vals = set(slot_values[i_key])
# Check for all other slots that are ordinally later
for j in range(i + 1, length):
j_key = slot_list[j]
j_vals = slot_values[j_key]
ij_changed, i_vals = _absorb_slot_if_similar(
i_key,
i_vals,
j_key,
j_vals,
slot_values,
relative_similarity_threshold,
)
if ij_changed and j_key not in updated:
updated.add(j_key)
i_changed = i_changed or ij_changed
# Check if i values contain the j slot on itself (thus being a superset of j)
i_changed, i_vals = _remove_values_if_containg_slot_already_maps(
i_changed, i, i_vals, slot_indices, slot_values)
# Check if some slots it refers to are contained by the content of all other slots
i_changed, i_vals = _remove_unnecessary_slot_references(
i_changed, i_vals, slot_values)
# Check if i has replacable slots: rename these template slots to the new ones
i_changed, i_vals = _replace_all_replacable_slots(
i_changed, i_vals, slot_values)
# Check if i contains itself
i_changed, i_vals = _filter_out_self(i_changed, i_key, i_vals)
# Check if i only contains a single value
if len(i_vals) == 1:
single = next(iter(i_vals))
# Check if the value it maps to is a pure slot, and if so, mark as replaced
if (len(single.get_elements()) == 1
and single.get_elements()[0].is_slot()):
slot_values.add_replacement(i_key,
single.get_elements()[0])
i_changed = True
# Otherwise, just replace all occurrences of this slot with the new values
else:
pass # TODO: add slot removal
# new_slot_values.
# Store all newly found i_vals
slot_values[i_key] = i_vals
# Replacement exists
else:
# Update replacement to purest replacement
i_changed = i_changed or _update_to_most_specific_replacement(
i_key, slot_values)
if i_changed and i_key not in updated:
updated.add(i_key)
return updated
def plotBoxes(data,
col,
ylabel,
minticks,
fname,
plotXLabels=False,
plotLegend=False,
figsize=(6, 3.6)):
fig = figure(figsize=figsize)
ax = axes()
hold(True)
maxval = 0
batch = 0
alldims = natsorted(data.keys(), alg=ns.IGNORECASE)
labels = SortedSet()
for dims in alldims:
print(dims)
expsdata = data[dims]
bpdata = numpy.zeros((50, 0))
for method in natsorted(expsdata.keys(), alg=ns.IGNORECASE):
labels.add(method)
mdata = expsdata[method]
print(method)
networkerror = numpy.asarray(mdata)[:, col].reshape(50, 1)
if numpy.max(networkerror) > maxval:
maxval = numpy.max(networkerror)
bpdata = numpy.append(bpdata, networkerror, axis=1)
#bpdata = numpy.log(bpdata)
# print(bpdata)
positions = numpy.asarray([1, 2, 3, 4]) + (5 * batch)
box = boxplot(bpdata + 1, positions=positions, widths=0.85)
batch += 1
colors = ['b', 'g', 'k', 'r']
for elem in ['boxes', 'caps', 'whiskers', 'medians']:
elems = len(box[elem])
i = 1
if elems > 4:
i = 2
for e in range(elems):
setp(box[elem][e], linewidth=3)
setp(box[elem][e], color=colors[floor(e / i)])
plt.ylabel(ylabel, fontsize=18)
if plotXLabels:
ax.set_xticklabels(map(
lambda dim: "n=" + dim.split("_")[0] + "\nd=" + dim.split("_")[1],
alldims),
multialignment='left',
fontsize=16)
ax.set_xticks(numpy.asarray([2.5, 7.5, 12.5, 17.5, 22.5]))
else:
ax.xaxis.set_major_formatter(NullFormatter())
xlim(0, batch * 5)
#ylim(-maxval * 0.1, maxval * 1.1)
for b in range(batch):
ax.axvline(x=5 * b, c="k", linewidth=0.5, zorder=0)
ax.set_yscale("log")
#ylim(-10000.0,10.0)
#ax.set_yticks(numpy.arange(maxval, maxval*0.1)*2, minor=True)
#ax.set_yticks(numpy.arange(0, maxval*1.1, minticks), minor=True)
ax.yaxis.grid(True, which='major', linestyle='--', color='k')
ax.yaxis.grid(True, which='minor', linestyle='--', color='#DBDBDB')
[line.set_zorder(3) for line in ax.lines]
if plotLegend:
leg = legend(list(
map(
lambda l: l.replace("_", " ").upper().replace("XMRF", "LPGM").
replace("GLMPTEST ", "PSPN p="), labels)),
bbox_to_anchor=(0.54, 1.02))
ax.add_artist(leg)
ltext = leg.get_texts()
setp(ltext[0], fontsize=14, color='b')
setp(ltext[1], fontsize=14, color='g')
setp(ltext[2], fontsize=14, color='k')
setp(ltext[3], fontsize=14, color='r')
for line, txt in zip(leg.get_lines(), leg.get_texts()):
line.set_linewidth(10)
line.set_color(txt.get_color())
for lh in leg.legendHandles:
lh.set_dashes((None, None))
[tick.label.set_fontsize(16) for tick in ax.yaxis.get_major_ticks()]
savefig(fname, bbox_inches='tight', dpi=600)
def oddEvenJumps(self, arr: List[int]) -> int:
# Time Complexity: O(N log N)
# Space Complexity: O(N)
nearest_larger = []
nearest_smaller = []
sorted_set = SortedSet()
for i in range(len(arr) - 1, -1, -1):
val = arr[i]
if sorted_set:
ind = sorted_set.bisect_left((val, -i))
if ind == 0:
nearest_smaller.append(len(arr))
else:
nearest_smaller.append(-sorted_set[ind - 1][1])
else:
nearest_smaller.append(len(arr))
sorted_set.add((val, -i))
nearest_smaller = nearest_smaller[::-1]
sorted_set = SortedSet()
for i in range(len(arr) - 1, -1, -1):
val = arr[i]
if sorted_set:
ind = sorted_set.bisect_left((-val, -i))
if ind == 0:
nearest_larger.append(len(arr))
else:
nearest_larger.append(-sorted_set[ind - 1][1])
else:
nearest_larger.append(len(arr))
sorted_set.add((-val, -i))
nearest_larger = nearest_larger[::-1]
dp = {}
def reachable(index: int, is_even: bool):
if (index, is_even) in dp:
return dp[(index, is_even)]
if index == len(arr) - 1:
return True
next_arr = nearest_smaller if is_even else nearest_larger
ret = next_arr[index] < len(arr) and reachable(
next_arr[index], not is_even)
dp[(index, is_even)] = ret
return ret
ret = 0
for i in range(len(arr)):
ret += int(reachable(i, False))
return ret
def __get_column_info(self, table: str) -> SortedSet:
columns = SortedSet()
for row in self.engine.execute("PRAGMA table_info(%s)" % table):
columns.add(row["name"])
return columns
class EventManager:
def __init__(self, resource_manager, dispatcher, additional_data,
**kwargs):
"""
This class coordinates events submission, queueing and ending.
:param resource_manager: Resource manager instance
:param \*\*kwargs: nothing for the moment.
"""
assert (isinstance(
resource_manager,
ResourceManager)), 'Wrong type for the resource_manager argument.'
self.resource_manager = resource_manager
self.dispatcher = dispatcher
self.additional_data = additional_data
self.constants = CONSTANT()
# self.debug = debug
# Stats
self.first_time_dispatch = None
self.last_run_time = None
self.slowdowns = []
self.wtimes = []
self.current_time = None
self.time_points = SortedSet()
self.events = SortedDict()
self.loaded = SortedDict()
self.queued = []
self.real_ending = SortedDict()
self.running = []
self.finished = 0
self._logger = logging.getLogger('accasim')
self._writers = []
if self.constants.SCHEDULING_OUTPUT:
_sched_fp = path.join(
self.constants.RESULTS_FOLDER_PATH,
self.constants.SCHED_PREFIX + self.constants.WORKLOAD_FILENAME)
self._sched_writer = AsyncWriter(
path=_sched_fp,
pre_process_fun=EventManager._schd_write_preprocessor)
self._writers.append(self._sched_writer)
if self.constants.PPRINT_OUTPUT:
_pprint_fp = path.join(
self.constants.RESULTS_FOLDER_PATH,
self.constants.PPRINT_PREFIX +
self.constants.WORKLOAD_FILENAME)
self._pprint_writer = AsyncWriter(
path=_pprint_fp,
pre_process_fun=EventManager._schd_pprint_preprocessor)
self._writers.append(self._pprint_writer)
for ad in self.additional_data:
ad.set_event_manager(self)
def load_events(self, es):
"""
Jobs are loaded to the system. This is the first step for a job simulation.
:param es: List of jobs. Jobs must be subclass of event class.
"""
for ad in self.additional_data:
ad.exec_before_submission(es)
if isinstance(es, list):
for e in es:
assert (isinstance(
e, Event)), 'Only subclasses of event can be simulated.'
self.load_event(e)
else:
assert (isinstance(
es, Event)), 'Only subclasses of event can be simulated.'
self.load_event(es)
for ad in self.additional_data:
ad.exec_after_submission()
def load_event(self, e):
"""
Internal method for job submission.
:param e: Single job (event subclass).
"""
assert (isinstance(e,
Event)), 'Using {}, expecting a single {}'.format(
e.__class__, Event.__name__)
if not self.current_time:
self.current_time = e.queued_time - 1
self.time_points.add(self.current_time)
if self.current_time == e.queued_time:
self.submit_event(e.id)
elif self.current_time < e.queued_time:
if e.queued_time not in self.loaded:
self.loaded[e.queued_time] = []
self.loaded[e.queued_time].append(e.id)
self.time_points.add(e.queued_time)
else:
raise EventException(
'Time sync problem, the actual event {} was loaded after ({}) the real submit time ({}). This a programming error, must be checked.'
.format(e.id, self.current_time, e.queued_time))
def move_to_finished(self, events_dict):
"""
There are two time points for a job could ends, the expected one and the real one.
The job must run until the real one is reached, then if a job is waiting to finish but is less than the
real ending time, this value must be updated with the real one.
:param events_dict: Actual Loaded, queued and running jobs in a dictionary {id: job object}
:return: Array of completed jobs
"""
_es = []
for e_id in self.real_ending.pop(self.current_time, []):
if e_id in self.running:
self.running.remove(e_id)
e = events_dict[e_id]
self.finish_event(e)
_es.append(e_id)
if (not self.loaded and not self.running and not self.queued) and _es:
self.last_run_time = self.current_time
return _es
def finish_event(self, e):
"""
Internal method for Job's completion. This method sets the ending time, and make some standard calculations for statistics, such as slowdown, waiting time.
Finally it calls the methods for output.
:param e: Job to be completed.
"""
e.end_time = self.current_time
e.running_time = e.end_time - e.start_time
e.waiting_time = e.start_time - e.queued_time
e.slowdown = float(
"{0:.2f}".format(
(e.waiting_time + e.running_time) / e.running_time)
) if e.running_time != 0 else e.waiting_time if e.waiting_time != 0 else 1.0
self.slowdowns.append(e.slowdown)
self.wtimes.append(e.waiting_time)
self.finished += 1
e.end_order = self.finished
if self.constants.SCHEDULING_OUTPUT:
self._sched_writer.push(e)
if self.constants.PPRINT_OUTPUT:
self._pprint_writer.push(e)
def dispatch_event(self, _job, _time, _time_diff, _nodes):
"""
Internal method for Job's dispatching. This method updates the related attributes for allocation of the job.
:param _job: Job object
:param _time: Time of dispatching
:param _time_diff: Time used if dispatching processing _time must be considered.
:param _nodes: Nodes to be allocated.
:return: True if the allocation must be performed, false otherwise. False for jobs that have duration equal to 0
"""
_id = _job.id
start_time = _time + _time_diff
#=======================================================================
# started, started_ended, postponed
# state = (0, 0, 0)
#=======================================================================
assert (self.current_time == start_time
), 'Start _time is different to the current _time'
# Try to allocate
done, msg = self.resource_manager.allocate_event(_job, _nodes)
if not done:
self._logger.error(
'{} Must be postponed. Reason: {}. If you see this message many times, probably you have to check your allocation heuristic.'
.format(_id, msg))
self.submit_event(_id)
return (0, 0, 1)
# Continue with a successfully allocated job
# Update job info
_job.start_time = start_time
_job.assigned_nodes = _nodes
# Initialize the output writers
if self.first_time_dispatch == None:
self.first_time_dispatch = start_time
for writer in self._writers:
writer.start()
ans = (1, 0, 0)
real_end_time = _job.start_time + _job.duration
if _job.duration == 0:
self._logger.trace(
'{}: {} Dispatched and Finished at the same moment. Job Lenght 0'
.format(self.current_time, _id))
ans = (0, 1, 0)
else:
# Include real ending time int time points
self.time_points.add(real_end_time)
# Move to running jobs
self.running.append(_id)
# Relate ending time with job id
if real_end_time not in self.real_ending:
self.real_ending[real_end_time] = []
self.real_ending[real_end_time].append(_id)
return ans
def submit_event(self, _id):
"""
Internal method for Job's queueing.
"""
self.queued.append(_id)
def next_events(self):
"""
Return the jobs that belongs to the next time point.
:return: Array of jobs recently submitted + queued available at current time.
"""
if len(self.time_points) > 0:
self.current_time = self.time_points.pop(0)
else:
self._logger.trace(
'No more time points... but there still jobs in the queue')
self.current_time += 1
submitted = self.loaded.pop(self.current_time, [])
new_queue = self.queued + submitted
self._logger.trace(
'{} Next events: \n-Recently submited: {}\n-Already queued: {}'.
format(self.current_time, submitted, self.queued))
self.queued.clear()
return new_queue
def has_events(self):
"""
:return: True if are loaded, queued or running jobs. False otherwise.
"""
return (self.loaded or self.queued or self.running)
def call_dispatcher(self, event_dict, events):
"""
Call the defined dispatcher. In addition, before the dispatcher call, the exec_before_dispatching method of
AdditionalData objects is called passing the job dictionary and current queue job ids. After the dispatcher call,
it calls the exec_after_dispatching and pass the job dictionary, the dispatching tuple and the rejected list.
:return: Return a tuple with a list of jobs to dispatch and to reject.
"""
for ad in self.additional_data:
ad.exec_before_dispatching(event_dict, events)
to_dispatch, rejected = self.dispatcher.schedule(
self.current_time, event_dict, events)
for ad in self.additional_data:
_tmp = ad.exec_after_dispatching(event_dict, to_dispatch, rejected)
if _tmp:
to_dispatch, rejected = _tmp
return to_dispatch, rejected
def dispatch_events(self,
event_dict,
to_dispatch,
time_diff,
omit_timediff=True):
"""
Internal method for processing the job's dispatching. Jobs are started if start time is equals to current time.
:param event_dict: Actual Loaded, queued and running jobs in a dictionary {id: job object}
:param to_dispatch: A tuple which contains the (start time, job id, nodes)
:param time_diff: Time which takes the dispatching processing time.
:param omit_timediff: If True the time spent in generating a decision is considered as 0. False this time is considered, and all events in \
in [current time, current time + time diff] are moved to the new current time (current time + time diff). The latter isn't implemented.
:return return a tuple of (#dispatched, #Dispatched + Finished (0 duration), #postponed)
"""
if omit_timediff:
time_diff = 0
n_disp = 0
n_disp_finish = 0
n_post = 0
for (_time, _id, _nodes) in to_dispatch:
assert (
isinstance(_id, str)
), 'Please check your return tuple in your Dispatching method. _id must be a str type. Received wrong type: {}'.format(
_id.__class__)
assert (_time is None or
_time >= self.current_time), 'Receiving wrong schedules.'
#===================================================================
# Time must be equal or later than current time.
# Equals will be dispatched in the momement, instead later ones, which will be requeued with expected ending time of the job that release the resources.
# If the expected ending is surpass, because the job takes more time to finish, the time tuple\'s element must be None. '
#===================================================================
if not _nodes:
# For blocked jobs
if _time is not None and _time != self.current_time:
self.time_points.add(_time)
#==============================================================
# Maintaining the event in the queue
#==============================================================
self.submit_event(_id)
n_post += 1
continue
#=======================================================================
# started, started_ended, postponed
#=======================================================================
dispatched, ended, postponed = self.dispatch_event(
event_dict[_id], _time, time_diff, _nodes)
n_disp += dispatched
n_disp_finish += ended
n_post += postponed
if n_disp_finish > 0:
self.release_ended_events(event_dict)
return (n_disp, n_disp_finish, n_post)
def release_ended_events(self, event_dict):
"""
Internal method for completed jobs. Removes from the dictionary finished jobs.
:param event_dict: Actual Loaded, queued and running jobs in a dictionary {id: job object}
:return: return Array list of jobs objects.
"""
for ad in self.additional_data:
ad.exec_before_completion()
_es = self.move_to_finished(event_dict)
if _es:
_removed_jobs = []
for _e in _es:
self.resource_manager.remove_event(_e)
# Freeing mem (finished events)
_removed_jobs.append(event_dict.pop(_e))
for ad in self.additional_data:
ad.exec_after_completion(_removed_jobs)
return _es
def simulated_status(self):
"""
Show the current state of the system in terms of loaded, queued, running and finished jobs.
:return: String including the system info.
"""
return (
'Loaded {}, Queued {}, Running {}, and Finished {} Jobs'.format(
len(self.loaded), len(self.queued), len(self.running),
self.finished))
def availability(self):
"""
Current availability of the system.
:return: Return the availability of the system.
"""
return self.resource_manager.current_availability
def usage(self):
"""
Current usage of the system
:return: Return the usage of the system
"""
return self.resource_manager.current_usage
def simulated_current_time(self):
"""
Current time
:return: Return the current simulated time
"""
return self.current_time
def stop_writers(self):
"""
Stops the output writer threads and closes the file streams
"""
for i in range(len(self._writers)):
self._writers[i].stop()
self._writers[i] = None
for ad in self.additional_data:
ad.stop()
@staticmethod
def _schd_write_preprocessor(event):
"""
To be used as a pre-processor for AsyncWriter objects applied to event schedules.
Pre-processes an event object and converts it to a String representation.
It uses the format specified in the SCHEDULE_OUTPUT constant.
:param event: The event to be written to output
"""
constants = CONSTANT()
_dict = constants.SCHEDULE_OUTPUT
_attrs = {}
for a, av in _dict['attributes'].items():
try:
_attrs[a] = locate(av[-1])(*event.subattr(event, av[:-1]))
except ValueError:
_attrs[a] = 'NA'
output_format = _dict['format']
return output_format.format(**_attrs) + '\n'
@staticmethod
def _schd_pprint_preprocessor(event):
"""
To be used as a pre-processor for AsyncWriter objects applied to pretty-print event schedules.
Pre-processes an event object and converts it to a String representation.
It uses the format specified in the PPRINT_SCHEDULE_OUTPUT constant.
:param event: The event to be written to output
"""
constants = CONSTANT()
_dict = constants.PPRINT_SCHEDULE_OUTPUT
_order = _dict['order']
_attrs = {}
for a, av in _dict['attributes'].items():
try:
_attrs[a] = locate(av[-1])(*event.subattr(event, av[:-1]))
except ValueError:
_attrs[a] = 'NA'
output_format = _dict['format']
values = [_attrs[k] for k in _order]
if event.end_order == 1:
return (output_format.format(*_order) + '\n',
output_format.format(*values) + '\n')
else:
return output_format.format(*values) + '\n'
def __str__(self):
"""
Str representation of the event job_mapper
:return: Return the current system info.
"""
return 'Loaded: {}\nQueued: {}\nRunning: {}\nReal job finish on: {},\nFinished: {}\nNext time events: {}'.format(
self.loaded, self.queued, self.running, self.real_ending,
self.finished, self.time_points)
class PlatformBatchLightSystem:
"""Batch light system for platforms."""
__slots__ = [
"dirty_lights", "dirty_schedule", "clock", "update_task",
"update_callback", "update_hz", "max_batch_size", "scheduler_task",
"schedule_changed", "dirty_lights_changed"
]
# pylint: disable-msg=too-many-arguments
def __init__(self, clock, update_callback, update_hz, max_batch_size):
"""Initialise light system."""
self.dirty_lights = SortedSet() # type: Set[PlatformBatchLight]
self.dirty_lights_changed = asyncio.Event(loop=clock.loop)
self.dirty_schedule = SortedList()
self.schedule_changed = asyncio.Event(loop=clock.loop)
self.update_task = None
self.scheduler_task = None
self.clock = clock
self.update_callback = update_callback
self.update_hz = update_hz
self.max_batch_size = max_batch_size
def start(self):
"""Start light system."""
self.update_task = self.clock.loop.create_task(self._send_updates())
self.update_task.add_done_callback(Util.raise_exceptions)
self.scheduler_task = self.clock.loop.create_task(
self._schedule_updates())
self.scheduler_task.add_done_callback(Util.raise_exceptions)
def stop(self):
"""Stop light system."""
if self.scheduler_task:
self.scheduler_task.cancel()
self.scheduler_task = None
if self.update_task:
self.update_task.cancel()
self.update_task = None
async def _schedule_updates(self):
while True:
run_time = self.clock.get_time()
self.schedule_changed.clear()
while self.dirty_schedule and self.dirty_schedule[0][0] <= run_time:
self.dirty_lights.add(self.dirty_schedule[0][1])
del self.dirty_schedule[0]
self.dirty_lights_changed.set()
if self.dirty_schedule:
await asyncio.wait([self.schedule_changed.wait()],
loop=self.clock.loop,
timeout=self.dirty_schedule[0][0] -
run_time,
return_when=asyncio.FIRST_COMPLETED)
else:
await self.schedule_changed.wait()
async def _send_updates(self):
poll_sleep_time = 1 / self.update_hz
max_fade_tolerance = int(poll_sleep_time * 1000)
while True:
await self.dirty_lights_changed.wait()
self.dirty_lights_changed.clear()
sequential_lights = []
for light in list(self.dirty_lights):
if not sequential_lights:
# first light
sequential_lights = [light]
elif light.is_successor_of(sequential_lights[-1]):
# lights are sequential
sequential_lights.append(light)
else:
# sequence ended
await self._send_update_batch(sequential_lights,
max_fade_tolerance)
# this light is a new sequence
sequential_lights = [light]
if sequential_lights:
await self._send_update_batch(sequential_lights,
max_fade_tolerance)
self.dirty_lights.clear()
await asyncio.sleep(poll_sleep_time, loop=self.clock.loop)
async def _send_update_batch(self,
sequential_lights: List[PlatformBatchLight],
max_fade_tolerance):
sequential_brightness_list = [
] # type: List[Tuple[LightPlatformInterface, float, int]]
common_fade_ms = None
current_time = self.clock.get_time()
for light in sequential_lights:
brightness, fade_ms, done = light.get_fade_and_brightness(
current_time)
if not done:
schedule_time = current_time + (fade_ms / 1000)
if not self.dirty_schedule or self.dirty_schedule[0][
0] > schedule_time:
self.schedule_changed.set()
self.dirty_schedule.add((schedule_time, light))
if common_fade_ms is None:
common_fade_ms = fade_ms
if -max_fade_tolerance < common_fade_ms - fade_ms < max_fade_tolerance and \
len(sequential_brightness_list) < self.max_batch_size:
sequential_brightness_list.append(
(light, brightness, common_fade_ms))
else:
await self.update_callback(sequential_brightness_list)
# start new list
current_time = self.clock.get_time()
common_fade_ms = fade_ms
sequential_brightness_list = [(light, brightness,
common_fade_ms)]
if sequential_brightness_list:
await self.update_callback(sequential_brightness_list)
def mark_dirty(self, light: "PlatformBatchLight"):
"""Mark as dirty."""
self.dirty_lights.add(light)
self.dirty_lights_changed.set()
self.dirty_schedule = SortedList(
[x for x in self.dirty_schedule if x[1] != light])
class Agenda(object):
def __init__(self, parse):
self._parse = parse
self._skipMC = False
self._skipCompose = False
self._mc_neighs = dict()
self._compose_cnt = dict()
self._agendaToScore = set()
self._clustIdx_agenda = dict()
self._inactiveAgenda_score = dict()
self._activeAgenda_score = dict()
self._scoreActiveAgenda = SortedSet() # (float, SearchOp)
self._minAbsCntObserved = ParseParams.minAbsCnt \
* (ParseParams.minAbsCnt-1)/2
# self.logc = open("/Users/ben_ryan/Documents/DARPA ASKE/usp-code/genia_full/create_agenda.log", "a+")
# self.logp = open("/Users/ben_ryan/Documents/DARPA ASKE/usp-code/genia_full/proc_agenda.log", "a+")
def save_agenda(self, path):
'''
Save all objects necessary to recreate the current state of Agenda
'''
with open(path, 'wb') as f:
pickle.dump({'saved_agenda': self}, f)
return None
def load_agenda(path, prs):
'''
Given a Parse object, load the saved state of an Agenda and
attach it, returning the updated Parse object.
'''
with open(path, 'rb') as f:
sav = pickle.load(f)
prs.agenda = sav['saved_agenda']
prs.agenda._parse = prs
return prs
def createAgenda(self, verbose=False):
if verbose:
clust_cnt = len(Part.getClustPartRootNodeIds())
milestones = set([x for x in range(1, 10, 1)])
i = 0
for clust_id in Part.getClustPartRootNodeIds():
clust = Clust.getClust(clust_id)
if clust.getType() != 'C':
continue
elif clust.isStop():
continue
# # self.logc.write("Adding to agenda for cluster {}\n".format(clust_id))
self.addAgendaForNewClust(clust_id, verbose)
if verbose:
i += 1
done = math.floor(i * 10 / clust_cnt)
if done in milestones:
milestones.remove(done)
print("{}% complete.".format(done * 10))
# self.logc.close()
return None
def addAgendaForNewClust(self, newClustIdx, verbose=False):
part_node_ids = Part.getClustPartRootNodeIds()[newClustIdx]
num_parts = len(part_node_ids)
# if verbose:
# print("Updating agenda: {} possible operations.".format(num_parts*(num_parts-1)))
if len(part_node_ids) > 1:
for node_id in part_node_ids:
part_1 = Part.getPartByRootNodeId(node_id)
for node_id2 in part_node_ids:
if node_id <= node_id2:
break
part_2 = Part.getPartByRootNodeId(node_id2)
# self.logc.write("\tAdding parts {} and {} to agenda for cluster {}\n".format(node_id, node_id2, newClustIdx))
self.addAgendaAfterMergeClust(part_1, part_2)
return None
def addAgendaAfterMergeClust(self, part_1, part_2):
# First, check that these parts belong to the same cluster
assert part_1._clustIdx == part_2._clustIdx
clustIdx = part_1._clustIdx
# If they have parents, check whether the parents are in the same cluster
# If not, look at merging their clusters, and if so, look at composing
# the clusters for part_1 and its parent.
if part_1.getParPart() is not None and part_2.getParPart() is not None:
clustIdx1 = part_1.getParPart()._clustIdx
clustIdx2 = part_2.getParPart()._clustIdx
if clustIdx1 != clustIdx2:
self.addAgendaMC(clustIdx1, clustIdx2, 2 * clustIdx + 1)
else:
self.addAgendaAbs(clustIdx1, clustIdx)
# Next, get the arguments (children) of each part
# Compare each argument in A) with each argument in B) - if they have
# different clusters, look at merging them, and if they have the same
# look at composing the clusters for part_1 and its argument(s).
kids_1 = part_1.getArguments()
kids_2 = part_2.getArguments()
# # self.logc.write("\tAdding to agenda for kids of {} and {} in {}\n".format(part_1.getRelTreeRoot().getId(),
# part_2.getRelTreeRoot().getId(),
# clustIdx))
for kid1 in kids_1.values():
clustIdx1 = kid1._argPart._clustIdx
for kid2 in kids_2.values():
clustIdx2 = kid2._argPart._clustIdx
if clustIdx1 != clustIdx2:
#print("Add agenda - Merge Clusters {} and {}".format(clustIdx1, clustIdx2))
self.addAgendaMC(clustIdx1, clustIdx2, 2 * clustIdx + 1)
else:
#print("Add agenda - Compose Clusters {} and {}".format(clustIdx, clustIdx1))
self.addAgendaAbs(clustIdx, clustIdx1)
return None
def addAgendaMC(self, clustIdx1, clustIdx2, neighType):
if not (self._skipMC or clustIdx1 == clustIdx2):
type1 = Clust.getClust(clustIdx1).getType()
type2 = Clust.getClust(clustIdx2).getType()
if type2 == 'C' and type1 == 'C':
op = SearchOp()
op._op = SearchOp.OP_MERGE_CLUST
op._clustIdx1 = min((clustIdx1, clustIdx2))
op._clustIdx2 = max((clustIdx1, clustIdx2))
if not self.moveAgendaToScore(op):
if op not in self._mc_neighs:
self._mc_neighs[op] = set()
if len(self._mc_neighs[op]) + 1 >= ParseParams.minMCCnt:
self._agendaToScore.add(op)
del self._mc_neighs[op]
else:
self._mc_neighs[op].add(neighType)
## self.logc.write("\t\tMerge Op: {}; mc_neighs: {}, agendaToScore: {}\n".format(op, len(self._mc_neighs), len(self._agendaToScore)))
return None
def addAgendaAbs(self, parClustIdx, chdClustIdx):
if not self._skipCompose:
op = SearchOp()
op._op = SearchOp.OP_COMPOSE
op._parClustIdx = parClustIdx
op._chdClustIdx = chdClustIdx
if not self.moveAgendaToScore(op):
if op not in self._compose_cnt:
self._compose_cnt[op] = 1
if self._compose_cnt[op] + 1 >= self._minAbsCntObserved:
self._agendaToScore.add(op)
del self._compose_cnt[op]
else:
self._compose_cnt[op] += 1
## self.logc.write("\t\tCompose Op: {}; compose_cnt: {}, agendaToScore: {}\n".format(op, len(self._compose_cnt), len(self._agendaToScore)))
return None
def moveAgendaToScore(self, op):
#assert op in self._activeAgenda_score or op in self._inactiveAgenda_score
if op in self._agendaToScore:
return True
if op in self._activeAgenda_score:
score = self._activeAgenda_score[op]
self._scoreActiveAgenda.discard((score, op))
del self._activeAgenda_score[op]
self._agendaToScore.add(op)
return True
elif op in self._inactiveAgenda_score:
del self._inactiveAgenda_score[op]
self._agendaToScore.add(op)
return True
return False
def procAgenda(self, verbose=False):
if verbose:
print("Processing agenda with {} operations in queue.".format(
len(self._agendaToScore)))
ttlAgendaScored, ttlExecMC, ttlExecAbs = (0, 0, 0)
i = 1
while True:
As = 0
for op in self._agendaToScore:
score = self._parse.scorer.scoreOp(op)
if verbose:
print("<SCORE> {} score={}".format(op, score))
As += 1
if score < -200:
continue
if verbose:
print("<Add Agenda> {} score={}".format(op, score))
self.addAgenda(op, score)
self._agendaToScore.clear()
ttlAgendaScored = As + 1
if len(self._scoreActiveAgenda) == 0:
break
score, op = next(reversed(self._scoreActiveAgenda))
if verbose:
print("Executing: {}, score={}".format(op, score))
newClustIdx = self._parse.executor.executeOp(op)
self.updateAgendaAfterExec(op, newClustIdx, verbose)
if op._op == SearchOp.OP_COMPOSE:
ttlExecAbs += 1
elif op._op == SearchOp.OP_MERGE_CLUST:
ttlExecMC += 1
if verbose:
print("Total op_compose: {}, Total op_merge_clust: {}".format(
ttlExecAbs, ttlExecMC))
i += 1
if verbose and i % 10 == 0:
print("{} Processing agenda: {} loops".format(
datetime.now(), i))
return None
def addAgenda(self, op, score):
ci1, ci2 = (-1, -1)
if op._op == SearchOp.OP_MERGE_CLUST:
ci1 = op._clustIdx1
ci2 = op._clustIdx2
elif op._op == SearchOp.OP_COMPOSE:
ci1 = op._parClustIdx
ci2 = op._chdClustIdx
if ci1 not in self._clustIdx_agenda:
self._clustIdx_agenda[ci1] = set()
self._clustIdx_agenda[ci1].add(op)
if ci2 not in self._clustIdx_agenda:
self._clustIdx_agenda[ci2] = set()
self._clustIdx_agenda[ci2].add(op)
if score < ParseParams.priorCutOff:
self._inactiveAgenda_score[op] = score
else:
self._activeAgenda_score[op] = score
self._scoreActiveAgenda.add((score, op))
return None
def updateAgendaAfterExec(self, op, newClustIdx, verbose=False):
self.removeAgenda(op)
if newClustIdx >= 0:
if op._op == SearchOp.OP_MERGE_CLUST:
self.updateAgendaAfterExecMC(op, newClustIdx, verbose)
elif op._op == SearchOp.OP_COMPOSE:
self.updateAgendaAfterExecAbs(op, newClustIdx, verbose=verbose)
return None
def addAgendaToScore(self, op):
self._agendaToScore.add(op)
return None
def updateAgendaAfterExecMC(self, op, newClustIdx, verbose=False):
assert op._op == SearchOp.OP_MERGE_CLUST
oldClustIdx = op._clustIdx2
if oldClustIdx == newClustIdx:
oldClustIdx = op._clustIdx1
while len(self._clustIdx_agenda[oldClustIdx]) > 0:
oop = next(iter(self._clustIdx_agenda[oldClustIdx]))
self.removeAgenda(oop)
if oop._op == SearchOp.OP_MERGE_CLUST:
ci1 = oop._clustIdx1
ci2 = oop._clustIdx2
if ci1 == oldClustIdx:
ci1 = newClustIdx
if ci2 == oldClustIdx:
ci2 = newClustIdx
if ci1 != ci2:
nop = oop
nop._clustIdx1 = min((ci1, ci2))
nop._clustIdx2 = max((ci1, ci2))
nop.genString()
self.addAgendaToScore(nop)
elif oop._op == SearchOp.OP_COMPOSE:
ci1 = oop._parClustIdx
ci2 = oop._chdClustIdx
if ci1 == oldClustIdx:
ci1 = newClustIdx
if ci2 == oldClustIdx:
ci2 = newClustIdx
nop = oop
nop._parClustIdx = ci1
nop._chdClustIdx = ci2
nop.genString()
self.addAgendaToScore(nop)
del self._clustIdx_agenda[oldClustIdx]
num_parts_old = len(Part.getClustPartRootNodeIds()[oldClustIdx])
num_parts_new = len(Part.getClustPartRootNodeIds()[newClustIdx])
if verbose:
print("Updating agenda: {} possible operations.".format(
num_parts_new * (num_parts_old)))
for prnid in Part.getClustPartRootNodeIds()[newClustIdx]:
p = Part.getPartByRootNodeId(prnid)
for prnid2 in Part.getClustPartRootNodeIds()[oldClustIdx]:
p2 = Part.getPartByRootNodeId(prnid2)
self.addAgendaAfterMergeClust(p, p2)
return None
def updateAgendaAfterExecAbs(self,
op,
newClustIdx,
oop=None,
verbose=False):
if op._op == SearchOp.OP_COMPOSE:
parClustIdx = op._parClustIdx
chdClustIdx = op._chdClustIdx
while len(self._clustIdx_agenda[parClustIdx]) > 0:
oop = next(iter(self._clustIdx_agenda[parClustIdx]))
self.removeAgenda(oop)
# oop.genString()
self.addAgendaToScore(oop)
while len(self._clustIdx_agenda[chdClustIdx]) > 0:
oop = next(iter(self._clustIdx_agenda[chdClustIdx]))
self.removeAgenda(oop)
# oop.genString()
self.addAgendaToScore(oop)
self.addAgendaForNewClust(newClustIdx, verbose)
# elif oop is not None:
# ci1, ci2 = (-1, -1)
# if oop._op == SearchOp.OP_MERGE_CLUST:
# ci1 = oop._clustIdx1
# ci2 = oop._clustIdx2
# elif oop._op == SearchOp.OP_COMPOSE:
# ci1 = oop._parClustIdx
# ci2 = oop._chdClustIdx
# if ci1 in (op._parClustIdx, op._chdClustIdx):
# ci1 = newClustIdx
# if ci2 in (op._parClustIdx, op._chdClustIdx):
# ci2 = newClustIdx
# if oop._op == SearchOp.OP_MERGE_CLUST:
# if ci1 != ci2:
# nop = SearchOp()
# nop._clustIdx1 = min((ci1, ci2))
# nop._clustIdx2 = max((ci1, ci2))
# nop._op = oop._op
# self.addAgendaToScore(nop)
# elif oop._op == SearchOp.OP_COMPOSE:
# nop = SearchOp()
# nop._parClustIdx = ci1
# nop._chdClustIdx = ci2
# nop._op = oop._op
# self.addAgendaToScore(nop)
return None
def removeAgenda(self, op):
# assert (op in self._activeAgenda_score or op in self._inactiveAgenda_score)
if op in self._activeAgenda_score:
score = self._activeAgenda_score[op]
self._scoreActiveAgenda.discard((score, op))
del self._activeAgenda_score[op]
elif op in self._inactiveAgenda_score:
del self._inactiveAgenda_score[op]
if op._op == SearchOp.OP_MERGE_CLUST:
self._clustIdx_agenda[op._clustIdx1].discard(op)
self._clustIdx_agenda[op._clustIdx2].discard(op)
elif op._op == SearchOp.OP_COMPOSE:
self._clustIdx_agenda[op._parClustIdx].discard(op)
self._clustIdx_agenda[op._chdClustIdx].discard(op)
return None
class PlatformBatchLightSystem:
"""Batch light system for platforms."""
__slots__ = ["dirty_lights", "dirty_schedule", "clock", "update_task", "update_callback",
"update_hz", "max_batch_size", "scheduler_task", "schedule_changed", "dirty_lights_changed",
"last_state"]
# pylint: disable-msg=too-many-arguments
def __init__(self, clock, update_callback, update_hz, max_batch_size):
"""Initialise light system."""
self.dirty_lights = SortedSet() # type: Set[PlatformBatchLight]
self.dirty_lights_changed = asyncio.Event()
self.dirty_schedule = SortedList()
self.schedule_changed = asyncio.Event()
self.update_task = None
self.scheduler_task = None
self.clock = clock
self.update_callback = update_callback
self.update_hz = update_hz
self.max_batch_size = max_batch_size
self.last_state = {}
def start(self):
"""Start light system."""
self.update_task = self.clock.loop.create_task(self._send_updates())
self.update_task.add_done_callback(Util.raise_exceptions)
self.scheduler_task = self.clock.loop.create_task(self._schedule_updates())
self.scheduler_task.add_done_callback(Util.raise_exceptions)
def stop(self):
"""Stop light system."""
if self.scheduler_task:
self.scheduler_task.cancel()
self.scheduler_task = None
if self.update_task:
self.update_task.cancel()
self.update_task = None
async def _schedule_updates(self):
while True:
run_time = self.clock.get_time()
self.schedule_changed.clear()
while self.dirty_schedule and self.dirty_schedule[0][0] <= run_time:
self.dirty_lights.add(self.dirty_schedule[0][1])
del self.dirty_schedule[0]
self.dirty_lights_changed.set()
if self.dirty_schedule:
try:
await asyncio.wait_for(self.schedule_changed.wait(), self.dirty_schedule[0][0] - run_time)
except asyncio.TimeoutError:
pass
else:
await self.schedule_changed.wait()
async def _send_updates(self):
poll_sleep_time = 1 / self.update_hz
max_fade_tolerance = int(poll_sleep_time * 1000)
while True:
await self.dirty_lights_changed.wait()
self.dirty_lights_changed.clear()
sequential_lights = []
for light in list(self.dirty_lights):
if not sequential_lights:
# first light
sequential_lights = [light]
elif light.is_successor_of(sequential_lights[-1]):
# lights are sequential
sequential_lights.append(light)
else:
# sequence ended
await self._send_update_batch(sequential_lights, max_fade_tolerance)
# this light is a new sequence
sequential_lights = [light]
if sequential_lights:
await self._send_update_batch(sequential_lights, max_fade_tolerance)
self.dirty_lights.clear()
await asyncio.sleep(poll_sleep_time)
async def _send_update_batch(self, sequential_lights: List[PlatformBatchLight], max_fade_tolerance):
sequential_brightness_list = [] # type: List[Tuple[LightPlatformInterface, float, int]]
common_fade_ms = None
current_time = self.clock.get_time()
for light in sequential_lights:
brightness, fade_ms, done = light.get_fade_and_brightness(current_time)
schedule_time = current_time + (fade_ms / 1000)
if not done:
if not self.dirty_schedule or self.dirty_schedule[0][0] > schedule_time:
self.schedule_changed.set()
self.dirty_schedule.add((schedule_time, light))
else:
# check if we realized this brightness earlier
last_state = self.last_state.get(light, None)
if last_state and last_state[0] == brightness and last_state[1] < schedule_time and \
not sequential_brightness_list:
# we already set the light to that color earlier. skip it
# we only skip this light if we are in the beginning of the list for now
# the reason for that is that we do not want to break fade chains when one color channel
# of an RGB light did not change
# this could become an option in the future
continue
self.last_state[light] = (brightness, schedule_time)
if common_fade_ms is None:
common_fade_ms = fade_ms
if -max_fade_tolerance < common_fade_ms - fade_ms < max_fade_tolerance and \
len(sequential_brightness_list) < self.max_batch_size:
sequential_brightness_list.append((light, brightness, common_fade_ms))
else:
await self.update_callback(sequential_brightness_list)
# start new list
current_time = self.clock.get_time()
common_fade_ms = fade_ms
sequential_brightness_list = [(light, brightness, common_fade_ms)]
if sequential_brightness_list:
await self.update_callback(sequential_brightness_list)
def mark_dirty(self, light: "PlatformBatchLight"):
"""Mark as dirty."""
self.dirty_lights.add(light)
self.dirty_lights_changed.set()
self.dirty_schedule = SortedList([x for x in self.dirty_schedule if x[1] != light])
class ReplayBuffer(object):
"""Buffer to store environment transitions."""
def __init__(self, obs_shape, action_shape, capacity, device,
normalize_obs):
self.obs_shape = obs_shape
self.action_shape = action_shape
self.capacity = capacity
self.device = device
self.pixels = len(obs_shape) > 1
self.empty_data()
self.done_idxs = SortedSet()
self.global_idx = 0
self.global_last_save = 0
self.normalize_obs = normalize_obs
if normalize_obs:
assert not self.pixels
self.welford = utils.Welford()
def __getstate__(self):
d = copy.copy(self.__dict__)
del d['obses'], d['next_obses'], d['actions'], d['rewards'], \
d['not_dones'], d['not_dones_no_max']
return d
def __setstate__(self, d):
self.__dict__ = d
# Manually need to re-load the transitions with load()
self.empty_data()
def empty_data(self):
obs_dtype = np.float32 if not self.pixels else np.uint8
obs_shape = self.obs_shape
action_shape = self.action_shape
capacity = self.capacity
self.obses = np.empty((capacity, *obs_shape), dtype=obs_dtype)
self.next_obses = np.empty((capacity, *obs_shape), dtype=obs_dtype)
self.actions = np.empty((capacity, *action_shape), dtype=np.float32)
self.rewards = np.empty((capacity, 1), dtype=np.float32)
self.not_dones = np.empty((capacity, 1), dtype=np.float32)
self.not_dones_no_max = np.empty((capacity, 1), dtype=np.float32)
self.idx = 0
self.full = False
self.payload = []
self.done_idxs = None
def __len__(self):
return self.capacity if self.full else self.idx
def get_obs_stats(self):
assert not self.pixels
MIN_STD = 1e-1
MAX_STD = 10
mean = self.welford.mean()
std = self.welford.std()
std[std < MIN_STD] = MIN_STD
std[std > MAX_STD] = MAX_STD
return mean, std
def add(self, obs, action, reward, next_obs, done, done_no_max):
# For saving
self.payload.append((obs.copy(), next_obs.copy(), action.copy(),
reward, not done, not done_no_max))
if self.normalize_obs:
self.welford.add_data(obs)
# if self.full and not self.not_dones[self.idx]:
if done:
self.done_idxs.add(self.idx)
elif self.full:
self.done_idxs.discard(self.idx)
np.copyto(self.obses[self.idx], obs)
np.copyto(self.actions[self.idx], action)
np.copyto(self.rewards[self.idx], reward)
np.copyto(self.next_obses[self.idx], next_obs)
np.copyto(self.not_dones[self.idx], not done)
np.copyto(self.not_dones_no_max[self.idx], not done_no_max)
self.idx = (self.idx + 1) % self.capacity
self.global_idx += 1
self.full = self.full or self.idx == 0
def sample(self, batch_size):
idxs = np.random.randint(0,
self.capacity if self.full else self.idx,
size=batch_size)
obses = self.obses[idxs]
next_obses = self.next_obses[idxs]
if self.normalize_obs:
mu, sigma = self.get_obs_stats()
obses = (obses - mu) / sigma
next_obses = (next_obses - mu) / sigma
obses = torch.as_tensor(obses, device=self.device).float()
actions = torch.as_tensor(self.actions[idxs], device=self.device)
rewards = torch.as_tensor(self.rewards[idxs], device=self.device)
next_obses = torch.as_tensor(next_obses, device=self.device).float()
not_dones = torch.as_tensor(self.not_dones[idxs], device=self.device)
not_dones_no_max = torch.as_tensor(self.not_dones_no_max[idxs],
device=self.device)
return obses, actions, rewards, next_obses, not_dones, not_dones_no_max
def sample_multistep(self, batch_size, T):
assert batch_size < self.idx or self.full
last_idx = self.capacity if self.full else self.idx
last_idx -= T
# raw here means the "coalesced" indices that map to valid
# indicies that are more than T steps away from a done
done_idxs_sorted = np.array(list(self.done_idxs) + [last_idx])
n_done = len(done_idxs_sorted)
done_idxs_raw = done_idxs_sorted - np.arange(1, n_done + 1) * T
samples_raw = npr.choice(
last_idx - (T + 1) * n_done,
size=batch_size,
replace=True # for speed
)
samples_raw = sorted(samples_raw)
js = np.searchsorted(done_idxs_raw, samples_raw)
offsets = done_idxs_raw[js] - samples_raw + T
start_idxs = done_idxs_sorted[js] - offsets
obses, actions, rewards = [], [], []
for t in range(T):
obses.append(self.obses[start_idxs + t])
actions.append(self.actions[start_idxs + t])
rewards.append(self.rewards[start_idxs + t])
assert np.all(self.not_dones[start_idxs + t])
obses = np.stack(obses)
actions = np.stack(actions)
rewards = np.stack(rewards).squeeze(2)
if self.normalize_obs:
mu, sigma = self.get_obs_stats()
obses = (obses - mu) / sigma
obses = torch.as_tensor(obses, device=self.device).float()
actions = torch.as_tensor(actions, device=self.device)
rewards = torch.as_tensor(rewards, device=self.device)
return obses, actions, rewards
def save_data(self, save_dir):
if self.global_idx == self.global_last_save:
return
if not os.path.exists(save_dir):
os.makedirs(save_dir)
path = os.path.join(
save_dir, f'{self.global_last_save:08d}_{self.global_idx:08d}.pt')
payload = list(zip(*self.payload))
payload = [np.vstack(x) for x in payload]
self.global_last_save = self.global_idx
torch.save(payload, path)
self.payload = []
def load_data(self, save_dir):
def parse_chunk(chunk):
start, end = [int(x) for x in chunk.split('.')[0].split('_')]
return (start, end)
self.idx = 0
chunks = os.listdir(save_dir)
chunks = filter(lambda fname: 'stats' not in fname, chunks)
chunks = sorted(chunks, key=lambda x: int(x.split('_')[0]))
self.full = self.global_idx > self.capacity
global_beginning = self.global_idx - self.capacity if self.full else 0
for chunk in chunks:
global_start, global_end = parse_chunk(chunk)
if global_start >= self.global_idx:
continue
start = global_start - global_beginning
end = global_end - global_beginning
if end <= 0:
continue
path = os.path.join(save_dir, chunk)
payload = torch.load(path)
if start < 0:
payload = [x[-start:] for x in payload]
start = 0
assert self.idx == start
obses = payload[0]
next_obses = payload[1]
self.obses[start:end] = obses
self.next_obses[start:end] = next_obses
self.actions[start:end] = payload[2]
self.rewards[start:end] = payload[3]
self.not_dones[start:end] = payload[4]
self.not_dones_no_max[start:end] = payload[5]
self.idx = end
self.last_save = self.idx
if self.full:
assert self.idx == self.capacity
self.idx = 0
last_idx = self.capacity if self.full else self.idx
self.done_idxs = SortedSet(np.where(1. - self.not_dones[:last_idx])[0])
def fetch_us_forenames(url: str,
filename: str,
freq_csv_filename: str = "",
freq_sex_csv_filename: str = "",
min_cumfreq_pct: float = 0,
max_cumfreq_pct: float = 100,
min_name_length: int = 1,
show_rejects: bool = False) -> None:
"""
Fetch US forenames and store them in a file, one per line.
Args:
url:
URL to fetch file from
filename:
filename to write to
freq_csv_filename:
optional CSV to write "name, frequency" pairs to, one name per line
freq_sex_csv_filename:
optional CSV to write "name, gender, frequency" rows to
min_cumfreq_pct:
minimum cumulative frequency (%): 0 for no limit, or above 0 to
exclude common names
max_cumfreq_pct:
maximum cumulative frequency (%): 100 for no limit, or below 100 to
exclude rare names
min_name_length:
minimum word length; all words must be at least this long
show_rejects:
report rejected words to the Python debug log
"""
pipeline = (
gen_name_info_via_min_length(
gen_sufficiently_frequent_names(
gen_us_forename_info(
gen_lines_from_binary_files(
gen_files_from_zipfiles(
gen_binary_files_from_urls([url], on_disk=True),
# The zip file contains a README and then a
# bunch of files named yob<year>.txt (e.g.
# yob1997.txt).
filespec="*.txt"
)
)
),
min_cumfreq_pct=min_cumfreq_pct,
max_cumfreq_pct=max_cumfreq_pct,
show_rejects=show_rejects
),
min_name_length=min_name_length,
)
)
names = SortedSet()
freq = {} # type: Dict[str, float]
for nameinfo in pipeline:
name = nameinfo.name
if name not in names:
names.add(name)
freq[name] = nameinfo.freq_p
write_words_to_file(filename, names)
if freq_csv_filename:
log.info(f"Writing to: {freq_csv_filename}")
with open(freq_csv_filename, "wt") as f:
csvwriter = csv.writer(f)
for name in names:
csvwriter.writerow([name, freq[name]])
log.info(f"... finished writing to: {freq_csv_filename}")
if freq_sex_csv_filename:
pipeline_by_sex = (
# As above, but by sex
gen_name_info_via_min_length(
gen_sufficiently_frequent_names(
gen_us_forename_info_by_sex(
gen_lines_from_binary_files(
gen_files_from_zipfiles(
gen_binary_files_from_urls([url], on_disk=True),
filespec="*.txt"
)
)
),
min_cumfreq_pct=min_cumfreq_pct,
max_cumfreq_pct=max_cumfreq_pct,
show_rejects=show_rejects
),
min_name_length=min_name_length,
)
)
name_sex_pairs = SortedSet()
sexfreq = {} # type: Dict[Tuple[str, str], float]
for nameinfo in pipeline_by_sex: # type: UsForenameInfo
name = nameinfo.name
sex = nameinfo.sex
name_sex = name, sex
if name_sex not in name_sex_pairs:
name_sex_pairs.add(name_sex)
sexfreq[name_sex] = nameinfo.freq_p
log.info(f"Writing to: {freq_sex_csv_filename}")
with open(freq_sex_csv_filename, "wt") as f:
csvwriter = csv.writer(f)
for name_sex in name_sex_pairs:
csvwriter.writerow([name_sex[0],
name_sex[1],
sexfreq[name_sex]])
log.info(f"... finished writing to: {freq_sex_csv_filename}")
def multidim_search_opt_3(xspace,
oracle,
epsilon=EPS,
delta=DELTA,
max_step=STEPS,
blocking=False,
sleep=0.0,
logging=True):
# type: (Rectangle, Oracle, float, float, float, bool, float, bool) -> ResultSet
# xspace is a particular case of maximal rectangle
# xspace = [min_corner, max_corner]^n = [0, 1]^n
# xspace.min_corner = (0,) * n
# xspace.max_corner = (1,) * n
# Dimension
n = xspace.dim()
# Set of comparable and incomparable rectangles, represented by 'alpha' indices
comparable = comp(n)
incomparable = incomp(n)
# comparable = [zero, one]
# incomparable = list(set(alpha) - set(comparable))
# with:
# zero = (0_1,...,0_n)
# one = (1_1,...,1_n)
# List of incomparable rectangles
# border = [xspace]
# border = SortedListWithKey(key=Rectangle.volume)
border = SortedSet([], key=Rectangle.volume)
border.add(xspace)
lattice_border_ylow = Lattice(dim=xspace.dim(), key=lambda x: x.min_corner)
lattice_border_yup = Lattice(dim=xspace.dim(), key=lambda x: x.max_corner)
lattice_border_ylow.add(xspace)
lattice_border_yup.add(xspace)
ylow = []
yup = []
# x_minimal = points from 'x' that are strictly incomparable (Pareto optimal)
ylow_minimal = []
yup_minimal = []
# oracle function
f = oracle.membership()
error = (epsilon,) * n
vol_total = xspace.volume()
vol_yup = 0
vol_ylow = 0
vol_border = vol_total
step = 0
RootSearch.logger.debug('xspace: {0}'.format(xspace))
RootSearch.logger.debug('vol_border: {0}'.format(vol_border))
RootSearch.logger.debug('delta: {0}'.format(delta))
RootSearch.logger.debug('step: {0}'.format(step))
RootSearch.logger.debug('incomparable: {0}'.format(incomparable))
RootSearch.logger.debug('comparable: {0}'.format(comparable))
# Create temporary directory for storing the result of each step
tempdir = tempfile.mkdtemp()
RootSearch.logger.info(
'Report\nStep, Ylow, Yup, Border, Total, nYlow, nYup, nBorder, BinSearch, nBorder dominated by Ylow, nBorder dominated by Yup')
while (vol_border >= delta) and (step <= max_step) and (len(border) > 0):
step = step + 1
# if RootSearch.logger.isEnabledFor(RootSearch.logger.DEBUG):
# RootSearch.logger.debug('border: {0}'.format(border))
# l.sort(key=Rectangle.volume)
xrectangle = border.pop()
lattice_border_ylow.remove(xrectangle)
lattice_border_yup.remove(xrectangle)
RootSearch.logger.debug('xrectangle: {0}'.format(xrectangle))
RootSearch.logger.debug('xrectangle.volume: {0}'.format(xrectangle.volume()))
RootSearch.logger.debug('xrectangle.norm: {0}'.format(xrectangle.norm()))
# y, segment
# y = search(xrectangle.diag(), f, epsilon)
y, steps_binsearch = binary_search(xrectangle.diag(), f, error)
RootSearch.logger.debug('y: {0}'.format(y))
# discovered_segments.append(y)
# b0 = Rectangle(xrectangle.min_corner, y.low)
# b1 = Rectangle(y.high, xrectangle.max_corner)
#
# ylow.append(b0)
# yup.append(b1)
#
# vol_ylow += b0.volume()
# vol_yup += b1.volume()
################################
# Every Border rectangle that dominates B0 is included in Ylow
b0_extended = Rectangle(xspace.min_corner, y.low)
# border_overlapping_b0 = [rect for rect in border if rect.overlaps(b0_extended)]
# border_overlapping_b0 = [rect for rect in border_overlapping_b0 if rect.overlaps(b0_extended)]
ylow_rectangle = Rectangle(y.low, y.low)
border_overlapping_b0 = lattice_border_ylow.less_equal(ylow_rectangle)
# border_intersecting_b0 = [b0_extended.intersection(rect) for rect in border_overlapping_b0]
## border_nondominatedby_b0 = [rect - b0_extended for rect in border_overlapping_b0]
# border_nondominatedby_b0 = []
# for rect in border_overlapping_b0:
# border_nondominatedby_b0 += list(rect - b0_extended)
list_idwc = (idwc(b0_extended, rect) for rect in border_overlapping_b0)
border_nondominatedby_b0 = set(itertools.chain.from_iterable(list_idwc))
# border_nondominatedby_b0 = Rectangle.fusion_rectangles(border_nondominatedby_b0)
# if 'rect' is completely dominated by b0_extended (i.e., rect is strictly inside b0_extended), then
# set(rect - b0_extended) == {rect}
# Therefore, 'rect' must be removed from 'non dominated' borders
border |= border_nondominatedby_b0
border -= border_overlapping_b0
lattice_border_ylow.add_list(border_nondominatedby_b0)
lattice_border_ylow.remove_list(border_overlapping_b0)
lattice_border_yup.add_list(border_nondominatedby_b0)
lattice_border_yup.remove_list(border_overlapping_b0)
# Every Border rectangle that is dominated by B1 is included in Yup
b1_extended = Rectangle(y.high, xspace.max_corner)
# border_overlapping_b1 = [rect for rect in border if rect.overlaps(b1_extended)]
# border_overlapping_b1 = [rect for rect in border_overlapping_b1 if rect.overlaps(b1_extended)]
yup_rectangle = Rectangle(y.high, y.high)
border_overlapping_b1 = lattice_border_yup.greater_equal(yup_rectangle)
# border_intersecting_b1 = [b1_extended.intersection(rect) for rect in border_overlapping_b1]
## border_nondominatedby_b1 = [rect - b1_extended for rect in border_overlapping_b1]
# border_nondominatedby_b1 = []
# for rect in border_overlapping_b1:
# border_nondominatedby_b1 += list(rect - b1_extended)
list_iuwc = (iuwc(b1_extended, rect) for rect in border_overlapping_b1)
border_nondominatedby_b1 = set(itertools.chain.from_iterable(list_iuwc))
# border_nondominatedby_b1 = Rectangle.fusion_rectangles(border_nondominatedby_b1)
# if 'rect' is completely dominated by b1_extended (i.e., rect is strictly inside b1_extended), then
# set(rect - b1_extended) == {rect}
# Therefore, 'rect' must be removed from 'non dominated' borders
border |= border_nondominatedby_b1
border -= border_overlapping_b1
lattice_border_ylow.add_list(border_nondominatedby_b1)
lattice_border_ylow.remove_list(border_overlapping_b1)
lattice_border_yup.add_list(border_nondominatedby_b1)
lattice_border_yup.remove_list(border_overlapping_b1)
db0 = Rectangle.difference_rectangles(b0_extended, ylow_minimal)
db1 = Rectangle.difference_rectangles(b1_extended, yup_minimal)
vol_db0 = sum(b0.volume() for b0 in db0)
vol_db1 = sum(b1.volume() for b1 in db1)
# rs = ResultSet([], border_intersecting_b0 + [b0], border_intersecting_b1 + [b1], Rectangle())
# vol_db0 = rs.volume_ylow() - rs.overlapping_volume_ylow()
# vol_db1 = rs.volume_yup() - rs.overlapping_volume_yup()
vol_ylow += vol_db0
vol_yup += vol_db1
ylow.extend(db0)
yup.extend(db1)
ylow_minimal.append(b0_extended)
yup_minimal.append(b1_extended)
RootSearch.logger.debug('b0: {0}'.format(db0))
RootSearch.logger.debug('b1: {0}'.format(db1))
RootSearch.logger.debug('ylow: {0}'.format(ylow))
RootSearch.logger.debug('yup: {0}'.format(yup))
################################
# Every rectangle in 'i' is incomparable for current B0 and for all B0 included in Ylow
# Every rectangle in 'i' is incomparable for current B1 and for all B1 included in Yup
################################
yrectangle = Rectangle(y.low, y.high)
i = irect(incomparable, yrectangle, xrectangle)
# i = pirect(incomparable, yrectangle, xrectangle)
# l.extend(i)
border |= i
RootSearch.logger.debug('irect: {0}'.format(i))
lattice_border_ylow.add_list(i)
lattice_border_yup.add_list(i)
# Remove boxes in the boundary with volume 0
boxes_null_vol = border[:border.bisect_key_left(0.0)]
border -= boxes_null_vol
lattice_border_ylow.remove_list(boxes_null_vol)
lattice_border_yup.remove_list(boxes_null_vol)
vol_border = vol_total - vol_yup - vol_ylow
RootSearch.logger.info('{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}, {10}'
.format(step, vol_ylow, vol_yup, vol_border, vol_total, len(ylow), len(yup), len(border),
steps_binsearch,
len(border_overlapping_b0), len(border_overlapping_b1)))
if sleep > 0.0:
rs = ResultSet(border, ylow, yup, xspace)
if n == 2:
rs.plot_2D_light(blocking=blocking, sec=sleep, opacity=0.7)
elif n == 3:
rs.plot_3D_light(blocking=blocking, sec=sleep, opacity=0.7)
if logging:
rs = ResultSet(border, ylow, yup, xspace)
name = os.path.join(tempdir, str(step))
rs.to_file(name)
return ResultSet(border, ylow, yup, xspace)
def _ls_group(proj, paths, files_only=True, checksum=False, json=False, id=False):
"""
Construct DataFrame with 'mc ls' results for paths. Also outputs
the sets of paths that are files or directories (either local or remote).
Arguments:
files_only: If True, only include files in output DataFrame
checksum: If True, calculate MD5 checksum of local files and compared to remote
Results:
(df, files, dirs, remotes):
df: pandas.DataFrame containing:
'l_mtime', 'l_size', 'l_type', 'r_mtime', 'r_size', 'r_type', 'eq', 'name'
files: set of local and remote paths that are files
dirs: set of local and remote paths that are directories
remotes: set of remote File and Directory objects
"""
path_data = []
columns = [
'l_mtime', 'l_size', 'l_type',
'r_mtime', 'r_size', 'r_type',
'eq', 'name', 'id']
data_init = {k:'-' for k in columns}
files = SortedSet()
dirs = SortedSet()
remotes = SortedSet()
for path in paths:
data = copy.deepcopy(data_init)
data['name'] = os.path.relpath(path, os.getcwd())
l_checksum = None
# locals
if os.path.exists(path):
data['l_mtime'] = time.strftime("%b %Y %d %H:%M:%S", time.localtime(os.path.getmtime(path)))
data['l_size'] = _humanize(os.path.getsize(path))
if os.path.isfile(path):
data['l_type'] = 'file'
if checksum:
with open(path, 'r') as f:
l_checksum = hashlib.md5(f.read()).hexdigest()
files.add(path)
elif os.path.isdir(path):
data['l_type'] = 'dir'
dirs.add(path)
# remotes
obj = proj.get_by_local_path(path)
if obj is not None:
data['r_size'] = _humanize(obj.size)
if isinstance(obj, mcapi.File):
if obj.mtime:
data['r_mtime'] = obj.mtime.strftime("%b %Y %d %H:%M:%S")
data['r_type'] = 'file'
if checksum and l_checksum is not None:
data['eq'] = (obj.checksum == l_checksum)
files.add(path)
remotes.add(obj)
elif isinstance(obj, mcapi.Directory):
if obj.mtime:
data['r_mtime'] = obj.time.strftime("%b %Y %d %H:%M:%S")
data['r_type'] = 'dir'
dirs.add(path)
if not files_only:
remotes.add(obj)
data['id'] = obj.id
if not files_only or ('file' in [data['l_type'], data['r_type']]):
path_data.append(data)
return (pandas.DataFrame.from_records(path_data, columns=columns).sort_values(by='name'), files, dirs, remotes)
def encode_text(self, text):
codes = SortedSet()
for word in text.split():
codes.add(self.word_to_code(word))
return codes
# 점 뺴기
## 2차 평면 위에 서로 다른 n개의 점이 주어집니다. 이후 m개의 질의가 주어지는데, 각 질의마다는 한 개의 숫자 k가 주어집니다. 각 질의에 대해 주어진 숫자 k보다 x값이 같거나 큰 점 중 x값이 가장 작은 점을 찾아 지우려고 합니다.
## 만약 x값이 가장 작은 점이 여러 개라면, 그 중 y값이 가장 작은 점을 지우면 됩니다. 각 질의에 대해 해당하는 점을 순서대로 출력하고 지우는 프로그램을 작성해보세요.
### TreeSet 사용 이유 : 큰 것중에 작은 것을 빼고, y 값 역시 동시에 비교해주기 위해 사용
from sortedcontainers import SortedSet
s = SortedSet()
cmd = list(map(int, input().split()))
for i in range(cmd[0]):
x, y = input().split()
s.add((int(x),int(y)))
for i in range(cmd[1]):
numX = int(input())
where = s.bisect_left((numX,0))
if where >= len(s):
print(-1, -1)
else:
x, y = s[where]
print(x, y)
s.remove(s[where])
# 자리 찾기
### TreeSet 사용 이유 : 의자 위치는 정렬이 필요하고, 계속 조회를 해야하므로 사용
from sortedcontainers import SortedSet
s = SortedSet() # treeset
cmd = list(map(int,input().split()))
peopleWant = list(map(int, input().split()))
peopleWant.sort(reverse=True)
chairSet = SortedSet()
chair = []
for i in range(cmd[1]):
chairSet.add(i+1)
resultCount = 0
for i in peopleWant:
idx = chairSet.bisect_right(i)-1
if i in chairSet:
chairSet.remove(i)
resultCount += 1
else:
if idx < 0:
#print(i)
break
def multidim_search_opt_1(xspace,
oracle,
epsilon=EPS,
delta=DELTA,
max_step=STEPS,
blocking=False,
sleep=0.0,
logging=True):
# type: (Rectangle, Oracle, float, float, float, bool, float, bool) -> ResultSet
# Xspace is a particular case of maximal rectangle
# Xspace = [min_corner, max_corner]^n = [0, 1]^n
# xspace.min_corner = (0,) * n
# xspace.max_corner = (1,) * n
# Dimension
n = xspace.dim()
# Set of comparable and incomparable rectangles, represented by 'alpha' indices
comparable = comp(n)
incomparable = incomp(n)
# comparable = [zero, one]
# incomparable = list(set(alpha) - set(comparable))
# with:
# zero = (0_1,...,0_n)
# one = (1_1,...,1_n)
# List of incomparable rectangles
# border = [xspace]
# border = SortedListWithKey(key=Rectangle.volume)
border = SortedSet([], key=Rectangle.volume)
border.add(xspace)
ylow = []
yup = []
# oracle function
f = oracle.membership()
error = (epsilon,) * n
vol_total = xspace.volume()
vol_yup = 0
vol_ylow = 0
vol_border = vol_total
step = 0
RootSearch.logger.debug('xspace: {0}'.format(xspace))
RootSearch.logger.debug('vol_border: {0}'.format(vol_border))
RootSearch.logger.debug('delta: {0}'.format(delta))
RootSearch.logger.debug('step: {0}'.format(step))
RootSearch.logger.debug('incomparable: {0}'.format(incomparable))
RootSearch.logger.debug('comparable: {0}'.format(comparable))
# Create temporary directory for storing the result of each step
tempdir = tempfile.mkdtemp()
RootSearch.logger.info(
'Report\nStep, Ylow, Yup, Border, Total, nYlow, nYup, nBorder, BinSearch, nBorder dominated by Ylow, nBorder dominated by Yup')
while (vol_border >= delta) and (step <= max_step) and (len(border) > 0):
step = step + 1
# if RootSearch.logger.isEnabledFor(RootSearch.logger.DEBUG):
# RootSearch.logger.debug('border: {0}'.format(border))
# l.sort(key=Rectangle.volume)
xrectangle = border.pop()
RootSearch.logger.debug('xrectangle: {0}'.format(xrectangle))
RootSearch.logger.debug('xrectangle.volume: {0}'.format(xrectangle.volume()))
RootSearch.logger.debug('xrectangle.norm: {0}'.format(xrectangle.norm()))
# y, segment
# y = search(xrectangle.diag(), f, epsilon)
y, steps_binsearch = binary_search(xrectangle.diag(), f, error)
RootSearch.logger.debug('y: {0}'.format(y))
# discovered_segments.append(y)
b0 = Rectangle(xrectangle.min_corner, y.low)
b1 = Rectangle(y.high, xrectangle.max_corner)
ylow.append(b0)
yup.append(b1)
vol_ylow += b0.volume()
vol_yup += b1.volume()
RootSearch.logger.debug('b0: {0}'.format(b0))
RootSearch.logger.debug('b1: {0}'.format(b1))
RootSearch.logger.debug('ylow: {0}'.format(ylow))
RootSearch.logger.debug('yup: {0}'.format(yup))
################################
# Every Border rectangle that dominates B0 is included in Ylow
# Every Border rectangle that is dominated by B1 is included in Yup
b0_extended = Rectangle(xspace.min_corner, y.low)
b1_extended = Rectangle(y.high, xspace.max_corner)
# Every cube in the boundary overlaps another cube in the boundary
# When cubes from the boundary are moved to ylow or yup, they may still have a complementary cube
# remaining in the boundary with a non-empty intersection.
border_overlapping_ylow = [r for r in ylow if r.overlaps(b0_extended)]
border_overlapping_yup = [r for r in yup if r.overlaps(b1_extended)]
border_overlapping_b0 = [rect for rect in border if rect.overlaps(b0_extended)]
# Warning: Be aware of the overlapping areas of the cubes in the border.
# If we calculate the intersection of b0_extended with all the cubes in the frontier, and two cubes
# 'a' and 'b' partially overlaps, then the volume of this overlapping portion will be counted twice
# border_dominatedby_b0 = [rect.intersection(b0_extended) for rect in border_overlapping_b0]
# Solution: Project the 'shadow' of the cubes in the border over b0_extended.
border_dominatedby_b0_shadow = Rectangle.difference_rectangles(b0_extended, border_overlapping_b0)
# The negative of this image returns a set of cubes in the boundary without overlapping.
# border_dominatedby_b0 will be appended to ylow.
# Remove the portion of the negative that overlaps any cube that is already appended to ylow
border_dominatedby_b0 = Rectangle.difference_rectangles(b0_extended,
border_dominatedby_b0_shadow + border_overlapping_ylow)
# border_nondominatedby_b0 = [rect - b0_extended for rect in border_overlapping_b0]
border_nondominatedby_b0 = []
for rect in border_overlapping_b0:
border_nondominatedby_b0 += list(rect - b0_extended)
# border_nondominatedby_b0 = set()
# for rect in border_overlapping_b0:
# border_nondominatedby_b0 |= set(rect - b0_extended)
# border_nondominatedby_b0 -= set(border_overlapping_b0)
# if 'rect' is completely dominated by b0_extended (i.e., rect is strictly inside b0_extended), then
# set(rect - b0_extended) == {rect}
# Therefore, 'rect' must be removed from 'non dominated' borders
# border -= border_overlapping_b0
border |= border_nondominatedby_b0
border -= border_overlapping_b0
border_overlapping_b1 = [rect for rect in border if rect.overlaps(b1_extended)]
# Warning: Be aware of the overlapping areas of the cubes in the border.
# If we calculate the intersection of b1_extended with all the cubes in the frontier, and two cubes
# 'a' and 'b' partially overlaps, then the volume of this overlapping portion will be considered twice
# border_dominatedby_b1 = [rect.intersection(b1_extended) for rect in border_overlapping_b1]
# Solution: Project the 'shadow' of the cubes in the border over b1_extended.
border_dominatedby_b1_shadow = Rectangle.difference_rectangles(b1_extended, border_overlapping_b1)
# The negative of this image returns a set of cubes in the boundary without overlapping.
# border_dominatedby_b1 will be appended to yup.
# Remove the portion of the negative that overlaps any cube that is already appended to yup
border_dominatedby_b1 = Rectangle.difference_rectangles(b1_extended,
border_dominatedby_b1_shadow + border_overlapping_yup)
# border_nondominatedby_b1 = [rect - b1_extended for rect in border_overlapping_b1]
border_nondominatedby_b1 = []
for rect in border_overlapping_b1:
border_nondominatedby_b1 += list(rect - b1_extended)
# border_nondominatedby_b1 = set()
# for rect in border_overlapping_b1:
# border_nondominatedby_b1 |= set(rect - b1_extended)
# border_nondominatedby_b1 -= set(border_overlapping_b1)
# if 'rect' is completely dominated by b1_extended (i.e., rect is strictly inside b1_extended), then
# set(rect - b1_extended) == {rect}
# Therefore, 'rect' must be removed from 'non dominated' borders
# border -= border_overlapping_b1
border |= border_nondominatedby_b1
border -= border_overlapping_b1
ylow.extend(border_dominatedby_b0)
yup.extend(border_dominatedby_b1)
vol_ylow += sum(b0.volume() for b0 in border_dominatedby_b0)
vol_yup += sum(b1.volume() for b1 in border_dominatedby_b1)
################################
# Every rectangle in 'i' is incomparable for current B0 and for all B0 included in Ylow
# Every rectangle in 'i' is incomparable for current B1 and for all B1 included in Yup
################################
yrectangle = Rectangle(y.low, y.high)
i = irect(incomparable, yrectangle, xrectangle)
# i = pirect(incomparable, yrectangle, xrectangle)
# l.extend(i)
border |= i
RootSearch.logger.debug('irect: {0}'.format(i))
# Remove boxes in the boundary with volume 0
border -= border[:border.bisect_key_left(0.0)]
vol_border = vol_total - vol_yup - vol_ylow
RootSearch.logger.info('{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}, {10}'
.format(step, vol_ylow, vol_yup, vol_border, vol_total, len(ylow), len(yup), len(border),
steps_binsearch,
len(border_overlapping_b0), len(border_overlapping_b1)))
if sleep > 0.0:
rs = ResultSet(border, ylow, yup, xspace)
if n == 2:
rs.plot_2D_light(blocking=blocking, sec=sleep, opacity=0.7)
elif n == 3:
rs.plot_3D_light(blocking=blocking, sec=sleep, opacity=0.7)
if logging:
rs = ResultSet(border, ylow, yup, xspace)
name = os.path.join(tempdir, str(step))
rs.to_file(name)
return ResultSet(border, ylow, yup, xspace)
adj[y].append(x)
graph['from'].append(x)
graph['to'].append(y)
p = []
p.append((N, N))
for i in range(1, n + 1):
p.append((len(adj[i]), i))
p.sort(reverse=True)
for i in range(1, n + 1):
x, y = p[i]
st = SortedSet()
for k in adj[y]:
st.add(col[k])
for j in range(1, n + 1):
if ((j in st) == False):
col[y] = j
break
ans = {}
for i in range(1, n + 1):
ans[str(i)] = aaa[col[i]]
results = {'answer': ans}
ans = str(ans)
ans = ans.replace("\'", "\"")
#sys.stdout.write(str(results))
print(ans)
class Feature(object):
def __init__(self, software_system, logical_name, size):
"""
Constructs a new feature specification, initially with no corresponding chunks in the system.
"""
self._logical_name = logical_name
self.software_system = software_system
self.size = size
self.chunks = SortedSet(key=lambda c: c.logical_name)
self.tests = SortedSet(key=lambda t: t.logical_name)
@property
def logical_name(self):
return self._logical_name
@property
def bugs(self):
bug_sets = map(lambda c: frozenset(c.bugs), self.chunks)
return reduce(lambda a, b: a.union(b), bug_sets,
SortedSet(key=lambda b: b.fully_qualified_name))
@property
def test_coverage(self):
covered_chunks = SortedSet(
reduce(lambda a, b: a.union(b),
map(lambda t: frozenset(t.chunk_indexes), self.tests),
set()))
return float(len(covered_chunks)) / self.size
@property
def tests_per_chunk_ratio(self):
return 1.0 * len(self.tests) / self.size
@property
def dependencies(self):
all_dependencies = reduce(
lambda a, b: a.union(b),
map(lambda c: frozenset(c.dependencies), self.chunks), set())
external_dependencies = filter(lambda c: c.feature != self,
all_dependencies)
return SortedSet(external_dependencies, lambda c: c.logical_name)
@property
def is_implemented(self):
return len(self.chunks) >= self.size
def add_chunk(self, logical_name, local_content=None):
chunk = Chunk(logical_name, self, local_content)
self.chunks.add(chunk)
return chunk
def extend(self, logical_name, random):
chunk = self.add_chunk(logical_name)
chunks_to_modify = random.sample_chunks(self.chunks)
chunks_to_modify.add(chunk)
for chunk_to_modify in chunks_to_modify:
chunk_to_modify.modify(random)
return chunk
def debug(self, random, detected_bug=None):
if detected_bug is None:
for chunk in self.chunks:
chunk.debug(random)
else:
detected_bugs = SortedSet(self.bugs & {detected_bug},
key=lambda b: b.fully_qualified_name)
for detected_bug in detected_bugs:
detected_bug.chunk.debug(random, detected_bug)
def refactor(self, random):
random.choose_chunk(self.chunks).refactor(random)
def add_test(self, logical_name):
test = Test(logical_name, self)
self.tests.add(test)
return test
def operate(self, random):
"""
Operates a random sample of the feature's implemented chunks if the feature has been implemented.
"""
if self.is_implemented:
for sampled_chunk in random.sample_chunks(self.chunks):
sampled_chunk.operate(random)
else:
raise InoperableFeatureException(self)
def exercise_tests(self):
for test in self.tests:
test.exercise()
def __str__(self):
chunk_strings = map(lambda chunk: str(chunk), self.chunks)
return "f_%d[%s]" % (self._logical_name, ",".join(chunk_strings))
def __repr__(self):
return "f_%s" % self._logical_name
def eliminationOrder(gm, orderMethod=None, nExtra=-1, cutoff=inf, priority=None, target=None):
"""Find an elimination order for a graphical model
Args:
gm (GraphModel): A graphical model object
method (str): Heuristic method; one of {'minfill','wtminfill','minwidth','wtminwidth','random'}
nExtra (int): Randomly select eliminated variable from among the best plus nExtra; this adds
randomness to the order selection process. 0 => randomly from best; -1 => no randomness (default)
cutoff (float): Quit early if ``score`` exceeds a user-supplied cutoff value (returning ``target, cutoff``)
priority (list, optional): Optional list of variable priorities; lowest priority variables are
eliminated first. Useful for mixed elimination models, such as marginal MAP inference tasks.
target (list): If the identified order is better than cutoff, write it directly into passed ``target`` list
Returns:
list: The identified elimination order
float: The "score" of this ordering
Using ``target`` and ``cutoff`` one can easily search for better orderings by repeated calls:
>>> ord, score = eliminationOrder(model, 'minfill', nExtra=2, cutoff=score, target=ord)
"""
orderMethod = 'minfill' if orderMethod is None else orderMethod.lower()
priority = [1 for x in gm.X] if priority is None else priority
if orderMethod == 'minfill': score = lambda adj,Xj: 0.5*sum([len(adj[Xj]-adj[Xk]) for Xk in adj[Xj]])
elif orderMethod == 'wtminfill': score = lambda adj,Xj: sum([(adj[Xj]-adj[Xk]).nrStatesDouble() for Xk in adj[Xj]])
elif orderMethod == 'minwidth': score = lambda adj,Xj: len(adj[Xj])
elif orderMethod == 'wtminwidth': score = lambda adj,Xj: adj[Xj].nrStatesDouble()
elif orderMethod == 'random': score = lambda adj,Xj: np.random.rand()
else: raise ValueError('Unknown ordering method: {}'.format(orderMethod))
adj = [ gm.markovBlanket(Xi) for Xi in gm.X ] # build MRF
# initialize priority queue of scores using e.g. heapq or sort
reverse = [ (priority[Xi],score(adj,Xi),Xi) for Xi in gm.X ]
scores = SortedSet( reverse );
totalSize = 0.0
#_order = np.zeros((len(gm.X),)) #np.array([0 for Xi in gm.X])
_order = [0]*len(gm.X)
for idx in range(gm.nvar):
pick = 0
Pi,Si,Xi = scores[pick]
if nExtra >= 0:
mx = bisect.bisect_right(scores, (Pi,Si,gm.X[-1])) # get one past last equal-priority & score vars
pick = min(mx+nExtra, len(scores)) # then pick a random "near-best" variable
pick = np.random.randint(pick)
Pi,Si,Xi = scores[pick]
del scores[pick]
_order[idx] = Xi.label # write into order[idx] = Xi
totalSize += adj[Xi].nrStatesDouble()
if totalSize > cutoff: return target,cutoff # if worse than cutoff, quit with no changes to "target"
fix = VarSet()
for Xj in adj[Xi]:
adj[Xj] |= adj[Xi]
adj[Xj] -= [Xi] # TODO adj[Xj].remove(Xi) slightly faster but still unsupported by cython version
fix |= adj[Xj] # shouldn't need to fix as much for min-width?
for Xj in fix:
Pj,Sj,Xj = reverse[Xj]
scores.remove(reverse[Xj])
reverse[Xj] = (Pj,score(adj,Xj),Xj)
scores.add(reverse[Xj]) # add (Pj,score(adj,Xj),Xj) to heap & update reverse lookup
if not (target is None):
target.extend([None for i in range(len(target),len(_order))]) # make sure order is the right size
for idx in range(gm.nvar): target[idx]=_order[idx] # copy result if completed without quitting
return _order,totalSize
class Chunk(object):
"""
Represents a chunk of code providing some useful functionality in the system.
"""
def __init__(self, logical_name, feature, local_content=None):
self.logical_name = logical_name
self.feature = feature
self.local_content = local_content
self.dependencies = SortedSet(key=lambda d: d.fully_qualified_name)
self.bugs = SortedSet(key=lambda b: b.logical_name)
self.bug_count = 0
def __eq__(self, other):
if self.local_content != other.local_content:
return False
elif self.bugs_logical_names != other.bugs_logical_names:
return False
elif self.dependency_logical_names != other.dependency_logical_names:
return False
else:
return True
def __ne__(self, other):
return not(self.__eq__(other))
@property
def probability_gain_feature_dependency(self):
return self.feature.software_system.probability_gain_feature_dependency
@property
def probability_lose_feature_dependency(self):
return self.feature.software_system.probability_lose_feature_dependency
@property
def probability_gain_system_dependency(self):
return self.feature.software_system.probability_gain_system_dependency
@property
def probability_lose_system_dependency(self):
return self.feature.software_system.probability_lose_system_dependency
@property
def probability_new_bug(self):
return self.feature.software_system.probability_new_bug
@property
def probability_debug_known(self):
return self.feature.software_system.probability_debug_known
@property
def probability_debug_unknown(self):
return self.feature.software_system.probability_debug_unknown
@property
def dependency_logical_names(self):
return map(lambda d: d.logical_name, self.dependencies)
@property
def bugs_logical_names(self):
return map(lambda b: b.logical_name, self.bugs)
@property
def bugs_in_dependencies(self):
chunk_bug_set = frozenset(map(lambda chunk: frozenset(chunk.bugs), self.dependencies))
return reduce(lambda bugs_a, bugs_b: bugs_a.union(bugs_b), chunk_bug_set, set())
@property
def tests(self):
return filter(lambda t: self in t.chunks, self.feature.tests)
def modify(self, random):
feature_chunks = self.feature.chunks - {self}
system_chunks = set(self.feature.software_system.chunks.difference(self.feature.chunks))
self._add_dependencies(random, system_chunks, self.probability_gain_system_dependency)
self._add_dependencies(random, feature_chunks, self.probability_gain_feature_dependency)
self.local_content = random.create_local_content()
self._insert_bugs(random)
def merge(self, source_chunk, random):
for dependency in source_chunk.dependencies:
working_copy_dependency = self.feature.software_system.get_chunk(dependency.fully_qualified_name)
self.dependencies.add(working_copy_dependency)
self.modify(random)
def overwrite_with(self, source_chunk):
self.local_content = source_chunk.local_content
self.bugs.clear()
for old_bug in source_chunk.bugs:
new_bug = self.get_bug(old_bug.logical_name)
if new_bug is None:
self.add_bug(old_bug.logical_name)
self.dependencies.clear()
for dependency in source_chunk.dependencies:
new_dependency = self.feature.software_system.get_chunk(dependency.fully_qualified_name)
self.dependencies.add(new_dependency)
def _add_dependencies(self, random, candidate_chunks, threshold):
for candidate in SortedSet(candidate_chunks, key=lambda c: c.logical_name):
if random.dependency_should_be_added(threshold):
self.add_dependency(candidate)
def add_dependency(self, candidate):
self.dependencies.add(candidate)
def _insert_bugs(self, random):
while random.a_bug_should_be_inserted(self):
self.add_bug(self.bug_count)
self.bug_count += 1
def add_bug(self, logical_name):
self.bugs.add(Bug(logical_name, self))
def get_bug(self, logical_name):
result = filter(lambda bug: bug.logical_name == logical_name, self.bugs)
if len(result) is 0:
return None
else:
return result[0]
def refactor(self, random):
to_remove = set()
for dependency in self.dependencies:
if random.dependency_should_be_removed(self, dependency):
to_remove.add(dependency)
self.dependencies.difference_update(to_remove)
def debug(self, random, bug=None):
if len(self.bugs) == 0:
return False
if bug is None or bug not in self.bugs:
if random.unknown_bug_should_be_removed(self):
bug = random.choose_bug(self)
self.bugs.remove(bug)
elif random.known_bug_should_be_removed(self):
self.bugs.remove(bug)
def operate(self, random):
for bug in self.bugs_in_dependencies.union(self.bugs):
bug.manifest(random)
def __str__(self):
def string_repr_set(iterable):
return ",".join(map(lambda e: repr(e), iterable))
feature_dependencies = string_repr_set(filter(lambda c: c.feature == self.feature, self.dependencies))
system_dependencies = string_repr_set(filter(lambda c: c.feature != self.feature, self.dependencies))
bugs = ", ".join(map(lambda bug: str(bug), self.bugs))
return "c_%s:[%s]:[%s]->(in[%s],ex[%s])" % \
(str(self.logical_name), self.local_content, bugs, feature_dependencies, system_dependencies)
@property
def fully_qualified_name(self):
return "%s.%s" % (str(self.feature.logical_name), str(self.logical_name))
def __repr__(self):
return "c%s" % str(self.fully_qualified_name)
count_not_bouncy = 0
count_bouncy = 0
increasing_with_n_digits = SortedSet()
decreasing_with_n_digits = SortedSet()
# both increasing and decreasing, e.g. 777
both_with_n_digits = SortedSet()
for n in range(1, digits_upper_threshold):
if n % 1000 == 0:
print("Done {}".format(n))
is_inc = is_increasing(n)
is_dec = is_decreasing(n)
if is_inc or is_dec:
count_not_bouncy += 1
if n >= digits_lower_threshold:
if is_inc and is_dec:
both_with_n_digits.add(n)
elif is_inc:
increasing_with_n_digits.add(n)
else:
decreasing_with_n_digits.add(n)
else:
count_bouncy += 1
print("n < {} (10^{}) num_not_bouncy: {} num bouncy {}".format(
digits_upper_threshold, digits_lower_threshold, count_not_bouncy,
count_bouncy))
# print("{}".format(not_bouncy))
len_all = len(increasing_with_n_digits) + len(decreasing_with_n_digits) + len(
both_with_n_digits)
# print("{}".format(all))
print("num of just len {} is {}".format(n + 1, len_all))
def test_add():
temp = SortedSet(range(100), load=7)
temp.add(100)
temp.add(90)
temp._check()
assert all(val == temp[val] for val in range(101))
class AuctionEngine:
def __init__(self, auction_manager, trader_address, strategy, frequency):
self.auction_manager = auction_manager
self.trader_address = trader_address
self.strategy = strategy
self.frequency = frequency
self._active_auctionlets = SortedSet()
self._process_lock = threading.Lock()
self._set_lock = threading.Lock()
def start(self):
self._bind_events()
self._discover_recent_auctionlets()
while True:
for auctionlet_id in self._active_auctionlets[:]:
self._process_auctionlet(auctionlet_id)
time.sleep(self.frequency)
def _bind_events(self):
self.auction_manager.on_new_auction(self._on_new_auctionlet)
self.auction_manager.on_bid(self._on_bid)
self.auction_manager.on_split(self._on_split)
def _discover_recent_auctionlets(self):
self.auction_manager.discover_recent_auctionlets(
self._on_recovered_auctionlet)
def _on_recovered_auctionlet(self, auctionlet_id):
self._register_auctionlet(auctionlet_id)
self._process_auctionlet(auctionlet_id)
def _on_new_auctionlet(self, auctionlet_id):
self._register_auctionlet(auctionlet_id)
self._process_auctionlet(auctionlet_id)
def _on_bid(self, auctionlet_id):
self._register_auctionlet(auctionlet_id)
self._process_auctionlet(auctionlet_id)
def _on_split(self, base_id, new_id, split_id):
self._register_auctionlet(new_id)
self._register_auctionlet(split_id)
self._process_auctionlet(new_id)
self._process_auctionlet(split_id)
def _register_auctionlet(self, auctionlet_id):
with self._set_lock:
self._active_auctionlets.add(auctionlet_id)
def _unregister_auctionlet(self, auctionlet_id):
with self._set_lock:
try:
self._active_auctionlets.remove(auctionlet_id)
except:
pass
def _process_auctionlet(self, auctionlet_id):
with self._process_lock:
auctionlet = self.auction_manager.get_auctionlet(auctionlet_id)
if auctionlet is not None:
self._print_auctionlet(auctionlet_id, auctionlet)
result = self.strategy.perform(
auctionlet,
StrategyContext(self.auction_manager.address,
self.trader_address))
self._print_auctionlet_outcome(auctionlet_id,
result.description)
if result.forget:
self._unregister_auctionlet(auctionlet_id)
else:
self._unregister_auctionlet(auctionlet_id)
def _print_auctionlet(self, auctionlet_id, auctionlet):
auction = auctionlet.get_auction()
heading = self._heading(auctionlet_id)
padding = ' ' * len(heading)
price = "{0:.8f}".format(auctionlet.sell_amount /
auctionlet.buy_amount)
logging.info(
f"{heading} [ selling: {str(auctionlet.sell_amount).rjust(25)} {auction.selling.name()}] [ creator: {auction.creator}]"
)
logging.info(
f"{padding} [ last_bid: {str(auctionlet.buy_amount).rjust(25)} {auction.buying.name()}] [ last_bidder: {auctionlet.last_bidder} (@ {auctionlet.last_bid_time})]"
)
logging.info(
f"{padding} [ price: {price.rjust(21)} {auction.buying.name()}/{auction.selling.name()}] [ parameters: min_incr={auction.min_increase}, min_decr={auction.min_decrease}, ttl={auction.ttl}, reversed={auction.reversed}, expired={auctionlet.expired}]"
)
def _print_auctionlet_outcome(self, auctionlet_id, result):
padding = ' ' * len(self._heading(auctionlet_id))
logging.info(f"{padding} [ outcome: {result}]")
logging.info("")
def _heading(self, auctionlet_id):
return f"Auctionlet #{auctionlet_id}:"
class AbstractReadyDecorator(AbstractGraphDecorator):
"""
Wraps a graph and keeps track of tasks that are `ready`.
A task is ready if and only if:
* For each input connection to certain ports: The connected output port has data set with `set_outut_data()`.
The input ports can be selected with the `port_filter` passed to `__init__`.
* The task is not a sync-point and all sync-point tasks with a lower tick have been executed.
* The task is a sync-point task and all tasks with a lower tick have been executed.
A task is considered to be `executed` once `set_output_data()` has been called.
"""
def __init__(self, g, prefix, port_filter, property_filter, syncpoint_run_last=True):
"""
:param prefix: Prefix for the task properties used by the implementation
to keep track of the state. If several decorators of this type
are wrapped around the same graph, then different prefixes must be used.
:param port_filter: Function to filter the relevant input ports. The function
takes three arguments:
* The graph (`self`)
* The tick of the task.
* The name of the port.
If the function returns `False` Then the port is ignored.
:param property_filter: Function that filters tasks which are collected.
The function takes three
* The graph (`self`)
* The tick of the task.
* The properties of the task (`graph.get_task_properties(tick)`).
Only tasks for which this function returns `True` are returned. The function
is reevaluated if the properties of a task change.
:param syncpoint_run_last: If `True` then tasks with syncpoints only
run once all tasks with a lower tick have completed.
"""
AbstractGraphDecorator.__init__(self, g)
self._prefix = prefix
self._port_filter = port_filter
self._property_filter = property_filter
self._syncpoint_run_last = syncpoint_run_last
#: ticks of all tasks that have data for all inputs.
#: That is, if every output port connected to each input port
#: had data set with `set_output_data`.
self._queue = SortedSet()
#: ticks of all tasks with
#: * `properties["syncpoint"] == True`
#: * `set_output_data` not yet called.
self._pending_syncpoints = SortedSet()
#: ticks of all tasks with
#: * `set_output_data` not yet called.
self._pending_ticks = SortedSet()
self._collected_prop = self._prefix + "_collected"
self._count_prop = self._prefix + "_count"
self._ready_prop = self._prefix + "_ready"
self.g.set_task_property(graph.FINAL_TICK, self._count_prop, 0)
self.g.set_task_property(graph.FINAL_TICK, self._ready_prop, 0)
self.g.set_task_property(graph.FINAL_TICK, self._collected_prop, False)
self._consider(graph.FINAL_TICK)
self._pending_ticks.add(graph.FINAL_TICK)
def past_all_syncpoints(self):
"""
Returns `True` if all sync-point tasks have been executed.
"""
return len(self._pending_syncpoints) == 0
def collect_tasks(self):
"""
Returns the ticks of all tasks which are ready.
"""
ticks = set()
while True:
tick = self.consume_ready_task()
if tick is not None:
ticks.add(tick)
else:
break
return ticks
def consume_ready_task(self):
"""
Returns the next task which is ready or `None`.
"""
def check_tick_against_sync_point(tick):
if not self._pending_syncpoints:
return True # no more sync points
next_sync_point = self._pending_syncpoints[0]
if tick < next_sync_point:
# `tick` is not a sync_point and must run
# before the next sync_point.
return True
elif tick == next_sync_point:
# `tick` is the sync_point
if self._syncpoint_run_last and self._pending_ticks[0] < next_sync_point:
# There are still unfinished tasks that have to run
# before it.
return False
else:
# It is time to run the sync_point.
return True
else: # tick > next_sync_point
# has to wait til the sync_point completed
return False
tick = next(iter(self._queue), None)
if tick is not None:
if not check_tick_against_sync_point(tick):
return None
props = self.g.get_task_properties(tick)
if props.get(self._collected_prop, False):
raise ValueError("Task %s became ready twice." % tick)
self.g.set_task_property(tick, self._collected_prop, True)
self._queue.remove(tick)
return tick
def was_collected(self, tick):
"""
Returns if the given tick was collected.
"""
props = self.g.get_task_properties(tick)
return props.get(self._collected_prop, False)
def add_task(self, tick, task, properties={}):
properties = dict(properties)
properties[self._count_prop] = 0
properties[self._ready_prop] = 0
properties[self._collected_prop] = False
self.g.add_task(tick, task, properties=properties)
self._consider(tick)
self._pending_ticks.add(tick)
if properties.get("syncpoint", False):
self._pending_syncpoints.add(tick)
def remove_task(self, tick):
self.g.remove_task(tick)
if tick in self._queue:
self._queue.remove(tick)
if tick in self._pending_syncpoints:
self._pending_syncpoints.remove(tick)
if tick in self._pending_ticks:
self._pending_ticks.remove(tick)
def connect(self, source, dest):
self.g.connect(source, dest)
if not self._port_filter(self, dest.tick, dest.port):
return
source_props = self.g.get_task_properties(source.tick)
dest_props = self.g.get_task_properties(dest.tick)
self.g.set_task_property(dest.tick, self._count_prop, dest_props[self._count_prop] + 1)
if source.port in source_props["out_data"]:
self.g.set_task_property(dest.tick, self._ready_prop, dest_props[self._ready_prop] + 1)
self._consider(dest.tick)
def disconnect(self, source, dest):
self.g.disconnect(source, dest)
if not self._port_filter(self, dest.tick, dest.port):
return
source_props = self.g.get_task_properties(source.tick)
dest_props = self.g.get_task_properties(dest.tick)
self.g.set_task_property(dest.tick, self._count_prop, dest_props[self._count_prop] - 1)
if source.port in source_props["out_data"]:
self.g.set_task_property(dest.tick, self._ready_prop, dest_props[self._ready_prop] - 1)
self._consider(dest.tick)
def set_task_property(self, tick, key, value):
retval = AbstractGraphDecorator.set_task_property(self, tick, key, value)
if key == "syncpoint":
if not value and tick in self._pending_syncpoints:
self._pending_syncpoints.remove(tick)
if value:
self._pending_syncpoints.add(tick)
self._consider(tick)
return retval
def set_output_data(self, tick, outputs):
self.g.set_output_data(tick, outputs)
if tick in self._pending_syncpoints:
self._pending_syncpoints.remove(tick)
if tick in self._pending_ticks:
self._pending_ticks.remove(tick)
for source, dest in self.g.get_out_connections(tick):
if source.port in outputs:
if not self._port_filter(self, dest.tick, dest.port):
continue
dest_props = self.get_task_properties(dest.tick)
self.set_task_property(dest.tick, self._ready_prop, dest_props[self._ready_prop] + 1)
self._consider(dest.tick)
def _consider(self, tick):
props = self.get_task_properties(tick)
if props.get(self._collected_prop, False):
# Already collected
return False
if props[self._count_prop] == props[self._ready_prop]:
if self._property_filter(self, tick, props):
should_be_in_queue = True
else:
should_be_in_queue = False
else:
should_be_in_queue = False
if should_be_in_queue:
self._queue.add(tick)
elif tick in self._queue:
self._queue.remove(tick)
# return False
#solution by idontknoooo, use SortedSet, time O(Nlog(min(N,k))), space O(min(N,k))
from sortedcontainers import SortedSet
# Create SortedSet. `n` is the size of sortedset, max value of `n` is `k` from input
ss, n = SortedSet(), 0
for i, num in enumerate(nums):
# index whose value is greater than or equal to `num`
ceiling_idx = ss.bisect_left(num)
# index whose value is smaller than `num`
floor_idx = ceiling_idx - 1
if ceiling_idx < n and abs(ss[ceiling_idx]-num) <= t: # check right neighbour
return True
if 0 <= floor_idx and abs(ss[floor_idx]-num) <= t: # check left neighbour
return True
ss.add(num)
n += 1
if i - k >= 0: # maintain the size of sortedset by finding & removing the earliest number in sortedset
ss.remove(nums[i-k])
n -= 1
return False
# #Right idea but time O(N^2), TLE
# htb = {}
# for i, num in enumerate(nums):
# for val in htb:
# if abs(val-num) <= t and i-htb[val] <= k:
# return True
# htb[num] = i
# return False
def eliminationOrder(gm, orderMethod=None, nExtra=-1, cutoff=inf, priority=None, target=None):
"""Find an elimination order for a graphical model
Args:
gm (GraphModel): A graphical model object
method (str): Heuristic method; one of {'minfill','wtminfill','minwidth','wtminwidth','random'}
nExtra (int): Randomly select eliminated variable from among the best plus nExtra; this adds
randomness to the order selection process. 0 => randomly from best; -1 => no randomness (default)
cutoff (float): Quit early if ``score`` exceeds a user-supplied cutoff value (returning ``target, cutoff``)
priority (list, optional): Optional list of variable priorities; lowest priority variables are
eliminated first. Useful for mixed elimination models, such as marginal MAP inference tasks.
target (list): If the identified order is better than cutoff, write it directly into passed ``target`` list
Returns:
list: The identified elimination order
float: The "score" of this ordering
Using ``target`` and ``cutoff`` one can easily search for better orderings by repeated calls:
>>> ord, score = eliminationOrder(model, 'minfill', nExtra=2, cutoff=score, target=ord)
"""
orderMethod = 'minfill' if orderMethod is None else orderMethod.lower()
priority = [1 for x in gm.X] if priority is None else priority
if orderMethod == 'minfill': score = lambda adj,Xj: 0.5*sum([len(adj[Xj]-adj[Xk]) for Xk in adj[Xj]])
elif orderMethod == 'wtminfill': score = lambda adj,Xj: sum([(adj[Xj]-adj[Xk]).nrStatesDouble() for Xk in adj[Xj]])
elif orderMethod == 'minwidth': score = lambda adj,Xj: len(adj[Xj])
elif orderMethod == 'wtminwidth': score = lambda adj,Xj: adj[Xj].nrStatesDouble()
elif orderMethod == 'random': score = lambda adj,Xj: np.random.rand()
else: raise ValueError('Unknown ordering method: {}'.format(orderMethod))
adj = [ gm.markovBlanket(Xi) for Xi in gm.X ] # build MRF
# initialize priority queue of scores using e.g. heapq or sort
reverse = [ (priority[Xi],score(adj,Xi),Xi) for Xi in gm.X ]
scores = SortedSet( reverse );
totalSize = 0.0
#_order = np.zeros((len(gm.X),)) #np.array([0 for Xi in gm.X])
_order = [0]*len(gm.X)
for idx in range(gm.nvar):
pick = 0
Pi,Si,Xi = scores[pick]
if nExtra >= 0:
mx = bisect.bisect_right(scores, (Pi,Si,gm.X[-1])) # get one past last equal-priority & score vars
pick = min(mx+nExtra, len(scores)) # then pick a random "near-best" variable
pick = np.random.randint(pick)
Pi,Si,Xi = scores[pick]
del scores[pick]
_order[idx] = Xi.label # write into order[idx] = Xi
totalSize += adj[Xi].nrStatesDouble()
if totalSize > cutoff: return target,cutoff # if worse than cutoff, quit with no changes to "target"
fix = VarSet()
for Xj in adj[Xi]:
adj[Xj] |= adj[Xi]
adj[Xj] -= [Xi] # TODO adj[Xj].remove(Xi) slightly faster but still unsupported by cython version
fix |= adj[Xj] # shouldn't need to fix as much for min-width?
for Xj in fix:
Pj,Sj,Xj = reverse[Xj]
scores.remove(reverse[Xj])
reverse[Xj] = (Pj,score(adj,Xj),Xj)
scores.add(reverse[Xj]) # add (Pj,score(adj,Xj),Xj) to heap & update reverse lookup
if not (target is None):
target.extend([None for i in range(len(target),len(_order))]) # make sure order is the right size
for idx in range(gm.nvar): target[idx]=_order[idx] # copy result if completed without quitting
return _order,totalSize
class Intervals:
def __init__(self, xl=[], regions=None, use_range_data=False, merge=1, is_int=None):
self.xl = SortedSet()
self.use_range_data = use_range_data
self.merge= merge
self.is_int = is_int
if regions is not None:
xl = [r.getRegion() for r in regions]
else:
xl = to_list(xl)
if len(xl) > 0 and not isinstance(xl[0], list) and not isinstance(xl[0], tuple):
xl = [xl]
for x in xl:
self.add(x)
glog.debug('Intervals >> %s', self.xl)
def add(self, *args, **kwargs):
kwargs = dict(kwargs)
if self.is_int is not None:
kwargs['is_int'] = self.is_int
if self.use_range_data:
elem = Range1DWithData(*args, **kwargs)
else:
elem = Range1D(*args, **kwargs)
if elem.empty: return
pos = self.xl.bisect_left(elem)
cur = None
next_pos = pos + 1
if pos > 0:
prev = self.xl[pos - 1]
if self.merge and prev.contains(elem.low - 1):
self.xl.pop(pos - 1)
ne = prev.union(elem)
self.xl.add(ne)
cur = ne
next_pos = pos
else:
assert elem.low >= prev.high
if cur is None:
cur = elem
self.xl.add(elem)
next = None
assert cur == self.xl.pop(next_pos - 1)
next_pos -= 1
while next_pos < len(self.xl):
next = self.xl[next_pos]
if cur.high < next.low:
break
self.xl.pop(next_pos)
cur = cur.union(next)
cur.high = max(cur.high, next.high)
self.xl.add(cur)
return self
def filter_dataset(self, dataset):
nx = []
ny = []
for i in range(dataset.n):
if self.should_keep(dataset.x[i]):
nx.append(dataset.x[i])
ny.append(dataset.y[i])
return dataset.make_new('filter', y=ny, x=nx)
def should_keep(self, v):
if self.xl is None:
return True
for xlow, xhigh in self.xl:
if xlow <= v < xhigh:
return True
return False
def split_data(self, dataset):
res = []
for v in self.xl:
res.append(dataset[max(0, v.low):v.high])
return res
@staticmethod
def FromIndices(vals, merge_dist=1):
last = None
res = Intervals()
for x in vals:
if last is None or last.high + merge_dist < x:
if last is not None:
res.xl.add(last)
last = Range1D(x, x, is_int=1)
else:
print('mergin here', x, last.low)
last.high = x
if last is not None:
res.xl.add(last)
return res
def complement(self, superset):
superset = Range1D(superset, is_int=1)
cur = Range1D(superset.low, 0, is_int=1)
res = Intervals()
for e in list(self.xl) + [Range1D(superset.high, 0, is_int=1)]:
cur.high = e.low - 1
res.xl.add(cur.clone())
cur.low = e.high + 1
return res
def shorten(self, val):
res = Intervals()
for x in self.xl:
res.add(Range1D(x.low + val, x.high - val, is_int=1))
return res
def expand(self, val):
res = Intervals()
for x in self.xl:
res.add(Range1D(x.low - val, x.high + val, is_int=1))
return res
def filter(self, func):
res = Intervals()
for x in self.xl:
if func(x):
res.add(x)
return res
def shift(self, p):
res = Intervals()
for x in self.xl:
res.add(Range1D(x.low + p, x.high + p))
return res
def query(self, q, closest=0):
pos = self.xl.bisect_left(Range1D(q, math.inf))
if pos == 0: return None
if closest or q <= self.xl[pos - 1].high: return self.xl[pos - 1]
return None
def query_data_do(self, q, action, fail_if_not=0):
obj = self.query(q)
assert obj is not None or not fail_if_not, hex(q)
if obj is None:
return None
return action(obj.data, q - obj.low)
def query_data_raw(self, q, **kwargs):
q = self.query(q, **kwargs)
if q is None: return None
return q.data
return found_range.get_data(qr)
def query_data(self, qr):
found_range = self.query(qr.low)
if found_range is None: return bytes([0] * (qr.length() + 1))
return found_range.get_data(qr)
def get_ordered_ranges(self):
return list(self.xl)
def intersection(self, other):
prim_ranges = get_primitive_ranges(self.get_ordered_ranges(), other.get_ordered_ranges())
res = Intervals()
for e in prim_ranges:
if self.query(e.low) and other.query(e.low):
res.add(e)
return res
def __str__(self):
s = io.StringIO()
s.write('Interval:\n')
for e in self.xl:
s.write(str(e) + '\n')
res = s.getvalue()
s.close()
return res
def group_by(self, tb, res=defaultdict(list)):
res = copy(res)
for pos, data in tb:
res[self.query(pos)].append(data)
return res
def openFileList(file): list = SortedSet([]) with open(file) as f: for l in f.read().splitlines(): list.add(l.lower()) return list
class Solver:
def __init__(self, var_count, clause_count):
self.var_count = var_count
self.clause_count = clause_count
self.clauses = []
self.unary_clauses = []
self.curr_level = 0 # Current depth of the decision tree
self.max_level = 0
# Since variables are 1-indexed, size of these lists if (var_count + 1)
# curr_assignment gives the latest assignment of a variable
self.curr_assignment = [LiteralState.L_UNASSIGNED] * (var_count + 1)
self.curr_literal_assignment = [LiteralState.L_UNASSIGNED
] * (2 * var_count + 1)
# prev_assignment is used in PHASE SAVING
self.prev_assignment = [-1] * (var_count + 1)
# The level the variable was assigned at (if at all)
self.assignment_level = [-1] * (var_count + 1)
# A stack of all assigned variables in current path, most recently assigned variables are at top
self.assigned_till_now = []
self.assignments_upto_level = [
0
] # How many assignments had happened upto a level?
self.conflicts_upto_level = [
0
] # How many conflicts hence clauses learned upto a level?
self.antecedent = [-1] * (var_count + 1)
self.score2var = SortedSet()
self.bcp_stack = []
# watch_map: literal -> list of clauses for which this literal is the watcher
self.watch_map = {}
# Used in MINISAT decision heuristic explained in decider()
self.increment_value = 1.0
self.activity = [0.0] * (var_count + 1)
# Used in restart optimisation explained in reset_state()
self.restart_threshold = CONSTANTS.RESTART_LOWER_BOUND
self.restart_upper_bound = CONSTANTS.RESTART_UPPER_BOUND_BASE
# Statistics
self.restart_count = 0
self.learnt_clauses_count = 0
self.decision_count = 0
self.assignments_count = 0
self.global_max_score = 0.0
def assign_variable(self, var: int, assignment: LiteralState):
self.curr_assignment[var] = assignment
if assignment != LiteralState.L_UNASSIGNED:
self.prev_assignment[var] = assignment
self.curr_literal_assignment[get_literal(var)] = assignment
neg_assignment = LiteralState.L_UNASSIGNED
if assignment == LiteralState.L_TRUE:
neg_assignment = LiteralState.L_FALSE
elif assignment == LiteralState.L_FALSE:
neg_assignment = LiteralState.L_TRUE
self.curr_literal_assignment[get_literal(-1 * var)] = neg_assignment
def bump_var_score(self, var: int, increment_value=0.0):
if increment_value > 0:
self.score2var.discard((self.activity[var], var))
self.activity[var] += increment_value
self.score2var.add((self.activity[var], var))
def print_clauses(self):
print("{} variables, {} clauses".format(self.var_count,
self.clause_count))
for clause_id, clause in enumerate(self.clauses):
clause.print(clause_id)
def print_curr_assignment(self):
assignment = "State: "
for var, state in enumerate(self.curr_assignment):
if (var == 0):
continue
assignment += ", {}: {}".format(var, state.value)
print(assignment)
# This fn is used to add the given clause to the watchlist of given literal
def watch_this_clause(self, lit, clause_id):
if lit in self.watch_map:
self.watch_map[lit].add(clause_id)
else:
self.watch_map[lit] = set([clause_id])
# Insert a new (input / learned) clause to the cnf
def insert_clause(self, clause: Clause, first_watch, second_watch):
self.clauses.append(clause)
# Setup the two-watch mechanism, both these literals are guaranteed to be unassigned currently
clause.first_watcher = first_watch
clause.second_watcher = second_watch
clause_id = len(self.clauses) - 1
self.watch_this_clause(clause.get_first_watcher(), clause_id)
self.watch_this_clause(clause.get_second_watcher(), clause_id)
# In MINISAT decision heusristic:
# Score of a varible is the number of clauses in it
# Since we are inserting a clause, increase the scores of variables in this literal
for literal in clause.literals:
var = get_variable(literal)
self.bump_var_score(var, self.increment_value)
# self.activity[var] += self.increment_value
# Function used to assign a literal TRUE in a unary clause
# These assignments are never reset hence not put in assigned_till_now[]
def assert_unary_literal(self, lit):
self.assignments_count += 1
var = get_variable(lit)
# Set state of the underlying variable
if is_negative(lit):
self.assign_variable(var, LiteralState.L_FALSE)
# self.curr_assignment[var] = LiteralState.L_FALSE
else:
self.assign_variable(var, LiteralState.L_TRUE)
# self.curr_assignment[var] = LiteralState.L_TRUE
self.assignment_level[var] = 0 # Always done at ground level
# Function used to assign a literal TRUE in a non-unary clause
# Note that current level is important here
def assert_nonunary_literal(self, lit):
self.assignments_count += 1
self.assigned_till_now.append(lit)
var = get_variable(lit)
if is_negative(lit):
self.assign_variable(var, LiteralState.L_FALSE)
# self.prev_assignment[var] = self.curr_assignment[var] = LiteralState.L_FALSE
else:
self.assign_variable(var, LiteralState.L_TRUE)
# self.prev_assignment[var] = self.curr_assignment[var] = LiteralState.L_TRUE
self.assignment_level[var] = self.curr_level
"""
Function to implement Boolean Constant Propagation using Two-watcher optimisation:
bcp_stack contains all the literals which have been assigned false in current search path.
Since we know these literals can now change the state of other clauses.
A naive approach of bcp would be iterate every clause to find a unit/unsatisifed clause.
If found, repreat the process again else, stop and start guessing some variables in decider()
"""
def bcp(self) -> (SolverState, int):
# print("Running BCP with stack", self.bcp_stack)
conflicting_clause_id = -1
while (self.bcp_stack):
# Got a literal with FALSE assignment
lit = self.bcp_stack.pop()
assert self.curr_literal_assignment[lit] == LiteralState.L_FALSE
# assert self.get_literal_status(lit) == LiteralState.L_FALSE
if lit not in self.watch_map:
self.watch_map[lit] = set()
new_watch_list = copy.copy(
self.watch_map[lit]) # Backup watch list of lit
# Traverse only the watchlist of that clause to save computation
for clause_id in self.watch_map[lit]:
clause = self.clauses[clause_id]
# This block determines which watcher (1st / 2nd) was lit
first_watch = clause.get_first_watcher()
second_watch = clause.get_second_watcher()
lit_is_first = (lit == first_watch)
other_watch = second_watch if lit_is_first else first_watch
# Now that we know lit has been assigned FALSE, we need to find another watcher
new_clause_state, new_watch_loc = clause.change_watch_location(
self, lit_is_first, other_watch)
# clause has one more literal FALSE, this might change a state
if (new_clause_state == ClauseState.C_SATISFIED):
pass
elif (new_clause_state == ClauseState.C_UNIT):
# If the clause had become unit, we have got another implication here
self.assert_nonunary_literal(other_watch)
var = get_variable(other_watch)
self.antecedent[var] = clause_id
self.bcp_stack.append(get_opposite_literal(other_watch))
elif (new_clause_state == ClauseState.C_CONFLICTING):
# All the literals of this clause became false, we have a conflict, need to backtrack
# If the conflict occured at ground level, we have a unsatisfiable cnf like (x) ^ (-x)
if self.curr_level == 0:
return SolverState.S_UNSATISFIED, conflicting_clause_id
conflicting_clause_id = clause_id
# Clear bcp_stack as a backtrack is coming, which will unassign several variables
# As such some information in bcp_state is likely to become stale
self.bcp_stack.clear()
break
elif (new_clause_state == ClauseState.C_UNRESOLVED):
# The clause is still unresolved as we have found another watcher
# Remove this clause from watch list of current lit
new_watch_list.remove(clause_id)
new_watcher = clause.literals[new_watch_loc]
self.watch_this_clause(new_watcher, clause_id)
# new_watch_list contains the clauses for which lit is still the watcher
# Note that in case of backtrack, we dot need to revert the watchers in two-watcher method
# since in backtracking, some variables will be unassigned, enforcing the two-watch invariant
self.watch_map[lit].clear()
self.watch_map[lit] = new_watch_list
if (conflicting_clause_id >= 0):
return SolverState.S_CONFLICT, conflicting_clause_id
return SolverState.S_UNRESOLVED, conflicting_clause_id
"""
This function is for the PHASE-SAVING heuristic
In decider() after the variable to be guessed has been selected, we then
need it set it to TRUE of FALSE. Phase-saving says we should set it to our
previous assignment if any.
"""
def get_lit_memoised(self, var: int) -> int:
prev_state = self.prev_assignment[var]
if (prev_state == LiteralState.L_TRUE):
return get_literal(var)
else:
return get_literal(-1 * var)
"""
decide() function selects the next variable to be guessed and the guessed value.
Based on MINISAT decision heuristic. Results in increment of current level.
Score of a varible is the number of clauses in it.
"""
def decide(self) -> SolverState: # MINISAT based decision heuristic
# print("Running decider")
# self.print_curr_assignment()
# print("Activity: ", self.activity)
# Find an unassigned one with maximum score
# Some inputs have unused variables, so we select only those with positive score.
selected_lit = 0
unassigned_var_found = False
while self.score2var:
max_score, var = self.score2var.pop()
self.global_max_score = max(self.global_max_score, max_score)
if self.curr_assignment[var] == LiteralState.L_UNASSIGNED:
unassigned_var_found = True
selected_lit = self.get_lit_memoised(var)
break
if not unassigned_var_found:
return SolverState.S_SATISFIED
# print(selected_lit, selected_var, max_activity_till_now)
assert selected_lit != 0
self.decision_count += 1
self.curr_level += 1
# We need to track this new assignment
if (self.curr_level > self.max_level):
# This branch is separate since we are at a new decision level,
# so push_back is required instead of update
self.max_level = self.curr_level
self.assignments_upto_level.append(len(self.assigned_till_now))
self.conflicts_upto_level.append(self.learnt_clauses_count)
else:
self.assignments_upto_level[self.curr_level] = len(
self.assigned_till_now)
self.conflicts_upto_level[
self.curr_level] = self.learnt_clauses_count
# Now we assign the literal as TRUE, and since put the (FALSE) opposite literal to bcp stack
self.assert_nonunary_literal(selected_lit)
self.bcp_stack.append(get_opposite_literal(selected_lit))
return SolverState.S_UNRESOLVED
"""
analyse_conflict() takes a conflicting clause and returns the level to backtrack to, and a learned clause
We use the nearest UIP (Unique Implication Point) finding method as highlighted in Kroening's book.
"""
def analyze_conflict(self, conflicting_clause: Clause) -> (int, int):
# print("Running analyse_conflict")
curr_literals = [lit for lit in conflicting_clause.literals]
learned_clause = Clause([])
backtrack_level = 0 # to be returned by this function
to_resolve_count = 0
watch_lit = 0 # a watcher for the new learned literal
marked = [False] * (self.var_count + 1)
trail_index = len(self.assigned_till_now) - 1
resolve_lit = 0
resolve_var = 0
iter = 0
"""
This loop outputs the learned clause, it works as follows:
Invariant 1: curr_literals is the clause to be fused into learned_clause
Invariant 2: learned caluse contains exactly one variable assigned at current level (UIP)
All other literals are assigned before.
"""
while (iter == 0 or to_resolve_count > 0):
iter += 1
for lit in curr_literals:
var = get_variable(lit)
if marked[var]:
continue
marked[var] = True
if (self.assignment_level[var] == self.curr_level):
to_resolve_count += 1
else:
learned_clause.insert_literal(lit)
if (self.assignment_level[var] > backtrack_level):
# watch_lit: 2nd highest assigment level, first is UIP
backtrack_level = self.assignment_level[var]
watch_lit = len(learned_clause.literals) - 1
# Find a variable to be resolved by traversing the recently assigned literals first
while (trail_index >= 0):
resolve_lit = self.assigned_till_now[trail_index]
resolve_var = get_variable(resolve_lit)
trail_index -= 1
if marked[resolve_var]:
break
marked[resolve_var] = False
to_resolve_count -= 1
if not to_resolve_count:
# Just one literal remaining with current level assignment, we are done
continue
antecedent_id = self.antecedent[resolve_var]
curr_literals = [
lit for lit in self.clauses[antecedent_id].literals
if lit != resolve_lit
]
# The learned clause becomes an unit clause after backtracking
# This is because every other literal in the learned clause was assigned before
# the backtrack level
# resolve_lit is an UIP
self.learnt_clauses_count += 1
opposite_resolv_lit = get_opposite_literal(resolve_lit)
learned_clause.insert_literal(opposite_resolv_lit)
self.increment_value /= CONSTANTS.VAR_DECAY_RATE
if learned_clause.is_unary:
# Not that we are inserting to bcp_stack without asserting UIP
# Asserting will be done immediately after backtrack (see backtrack())
self.bcp_stack.append(resolve_lit)
self.unary_clauses.append(learned_clause)
else:
self.bcp_stack.append(resolve_lit)
self.insert_clause(learned_clause, watch_lit,
len(learned_clause.literals) - 1)
# for lit in learned_clause.literals:
# var = get_variable(lit)
# print("({}, {})".format(var, self.assignment_level[var]))
return backtrack_level, opposite_resolv_lit
# RESTART heuristic, reset all assignments except ground level and start afresh
# Note that learned claused are not deleted only we start assignments from the beginning
def reset_state(self):
# print("Restart")
"""
The threshold system works as follows eg. (chainsaw graph)
1. We have a range [1, 10]. Threshold increases after every restart till it crosses the ub
2. At that point the threshold is rest and the range is also increased to let it go even higher
"""
self.restart_count += 1
self.restart_threshold = int(self.restart_threshold *
CONSTANTS.THRESHOLD_MULTIPLIER)
if (self.restart_threshold > self.restart_upper_bound):
self.restart_threshold = CONSTANTS.RESTART_LOWER_BOUND
self.restart_upper_bound = int(self.restart_upper_bound *
CONSTANTS.THRESHOLD_MULTIPLIER)
# Resets are similar to backtrack() function below, except that it resets evrything to ground level
for var in range(1, self.var_count + 1):
if (self.assignment_level[var] > 0):
self.assign_variable(var, LiteralState.L_UNASSIGNED)
# self.curr_assignment[var] = LiteralState.L_UNASSIGNED
self.bump_var_score(var)
self.bcp_stack.clear()
self.assigned_till_now.clear()
self.assignments_upto_level = [0]
self.conflicts_upto_level = [0]
self.curr_level = 0
self.max_level = 0
# Function to backtrack based on the output of analyse_conflict()
def backtrack(self, k: int, uip_lit):
# print("Running backtrack")
# Invoke restart heuristic if too many clauses have been learnt after backtrack target level
if k > 0 and (self.learnt_clauses_count - self.conflicts_upto_level[k]
> self.restart_threshold):
self.reset_state()
return
# Iterate over the variables assigned at level >= k + 1 and unassign them
for index in range(self.assignments_upto_level[k + 1],
len(self.assigned_till_now)):
var = get_variable(self.assigned_till_now[index])
if (self.assignment_level[var] > k):
self.assign_variable(var, LiteralState.L_UNASSIGNED)
# self.curr_assignment[var] = LiteralState.L_UNASSIGNED
self.bump_var_score(var)
# analyse_function() returns an asserting clause with the UIP just ready for assignment
# This helps to immediately put the learnt clause into practice
self.assigned_till_now = self.assigned_till_now[:self.
assignments_upto_level[
k + 1]]
self.curr_level = k
if k == 0:
# We had learnt a unary clause
self.assert_unary_literal(uip_lit)
else:
self.assert_nonunary_literal(uip_lit)
self.antecedent[get_variable(uip_lit)] = len(self.clauses) - 1
# Function to verify output assignment if any
def verify_assignment(self):
non_true_clauses = []
all_clauses = self.clauses + self.unary_clauses
# Every clause including learnt and unary must have atleast one TRUE literal
for clause in all_clauses:
true_literal_found = False
for lit in clause.literals:
if self.curr_literal_assignment[lit] == LiteralState.L_TRUE:
# if (self.get_literal_status(lit) == LiteralState.L_TRUE):
true_literal_found = True
break
if not true_literal_found:
non_true_clauses.append(clause)
if not non_true_clauses:
print("AC, All clauses evaluate to true under given assignment")
else:
print("WA, {} unsatisfied clauses found".format(
len(non_true_clauses)))
"""
Function implementing the standard CDCL framework:
1. [Outer Loop] Run BCP and decide() alternately, bcp first because of unary clauses
2. [INNER LOOP] Run till BCP gives UNRESOLVED result on which point guesswork must be done.
If bcp encounters conflict a analyse, backtrack pair is done
"""
def run_cdcl(self) -> SolverState:
result: SolverState
while (True):
while (True):
result, conflicting_clause_id = self.bcp()
# print("BCP result was {}".format(result))
if (result == SolverState.S_UNSATISFIED):
return result
if (result == SolverState.S_CONFLICT):
assert conflicting_clause_id != -1
backtrack_level, uip_lit = self.analyze_conflict(
self.clauses[conflicting_clause_id])
# print("Analyze result was k = {}, uip = {}".format(backtrack_level, uip_lit))
self.backtrack(backtrack_level, uip_lit)
else:
break
result = self.decide()
# print("Decide result was {}".format(result))
if (result == SolverState.S_UNSATISFIED
or result == SolverState.S_SATISFIED):
return result
# Wrapper function to print the result of the CDCL framework
def solve(self):
# print("Solving")
result: SolverState = self.run_cdcl()
if (result == SolverState.S_SATISFIED):
print("SATISFIABLE")
self.verify_assignment()
with open("assignment.txt", 'w') as assignment_file:
for var, state in enumerate(self.curr_assignment):
if (var == 0):
assignment_file.write("State: ")
continue
assignment_file.write("{} ".format(
-1 * var if state == LiteralState.L_FALSE else var))
else:
print("UNSATISFIABLE")
def print_statistics(self, solve_time):
print("## Statistics: ")
print("# Restarts: ", self.restart_count)
print("# Learned clauses: ", self.learnt_clauses_count)
print("# Decisions: ", self.decision_count)
print("# Implications: ", self.assignments_count - self.decision_count)
print("# Max score: ", self.global_max_score)
print("# Time (s): ", solve_time)
