def get_orthogonal(q: defaultdict):
priority = min(q.keys())
ch = q[priority].popleft()
if len(q[priority]) == 0:
q.pop(priority)
return priority, ch
def switch_color(tiles: defaultdict, coord: Tuple[int, int]):
if tiles[coord]:
tiles.pop(coord)
else:
tiles[coord] = True
return tiles
def prune(self, count_dict: defaultdict):
"""
This function returns filtered itemSets based on the min_support
Support = Count(itemSet) / total_items
"""
for itemSet in list(count_dict):
support = count_dict[itemSet] / self.total_items
if support < self.min_support:
count_dict.pop(itemSet)
return count_dict
class JobNoticeSender:
def __init__(self, interval=86400, cacheFile='jobNotice.cache'):
self.cacheFile = cacheFile #disk file to persist self.last_jobNotice
self.last_jobNotice = MyTool.readFile(cacheFile) #{jid:ts}
if not self.last_jobNotice:
self.last_jobNotice = DDict(int)
self.interval_secs = interval #interval to send repeated notice for the same job
def sendNotice (self, ts, jobs):
curr_jids = set(jobs.keys())
last_jids = set(self.last_jobNotice.keys())
dup_jids = curr_jids.intersection(last_jids)
rmv_jids = last_jids.difference(dup_jids)
new_jids = curr_jids.difference(dup_jids)
#remove the job notice that is not repeated this time, if job's utilization is fluctrated around border-line->more than expected dup notices
for jid in rmv_jids:
self.last_jobNotice.pop(jid)
#send notice for new appeared job
for jid in new_jids:
self.sendJobNotice (ts, jobs[jid])
self.last_jobNotice[jid] = ts
#send notice for dup job if long interval passed
dup_jids_send = list(filter(lambda x: ts - self.last_jobNotice[x] > self.interval_secs, dup_jids))
#print('{}: send notice for {} conditionly ({})'.format(ts, dup_jids_send, dup_jids))
for jid in dup_jids_send:
self.sendJobNotice(ts, jobs[jid])
self.last_jobNotice[jid] = ts
MyTool.writeFile (self.cacheFile, self.last_jobNotice)
def sendJobNotice (self, ts, job):
user =MyTool.getUserStruct(int(job['user_id']))
groups =MyTool.getUserGroups(user.pw_name)
if 'scc' in groups:
return
userName = user.pw_name
userFullName = user.pw_gecos.split(',')[0]
addr ='http://*****:*****@flatironinstitute.org'.format(userName)]
to_list=RECIPIENTS
msg = EmailMessage()
msg.set_content(content)
msg['Subject'] = 'Long runnig job with low utilization at slurm cluster -- Job {} by {}'.format(job['job_id'], userName)
msg['From'] = 'yliu'
msg['To'] = ', '.join(to_list)
with smtplib.SMTP('smtp-relay.gmail.com') as s:
s.send_message(msg)
def step(
bots: defaultdict, bins: defaultdict, all_move: List[List[str]]
) -> Tuple[defaultdict, defaultdict]:
for bot, v in bots.items():
if len(v) == 2:
break
else:
raise ValueError("not bot with 2 chuips!")
bots.pop(bot)
low = min(v)
high = max(v)
next_move = all_move[bot]
for target_type, target_id, value in zip(
["low_type", "high_type"], ["low_target", "high_target"], [low, high]
):
if next_move[target_type] == "bot": # type: ignore
bots[next_move[target_id]].append(value) # type: ignore
elif next_move[target_type] == "output": # type: ignore
bins[next_move[target_id]].append(value) # type: ignore
return bots, bins
class KVS(object):
'''Get/Put/View implements a client-server key value store. If no
value is associated with a given key, clients will block on get or
view until a value is available. Multiple values may be associated
with any given key.
This is, by design, a very simple, lightweight service that only
depends on standard Python modules.
'''
def __init__(self, getIndex=0, viewIndex=-1):
self.getIndex, self.viewIndex = getIndex, viewIndex #TODO: Add sanity checks?
self.key2mon = DD(lambda:DD(set)) # Maps a normal key to keys that monitor it.
self.monkeys = set() # List of monitor keys.
# store and waiters are mutually exclusive, and could be kept in the same place
self.store = DD(list)
self.waiters = DD(list)
self.opCounts = {b'get': 0, b'put': 0, b'view': 0, b'wait': 0}
self.ac, self.rc = 0, 0
def _doMonkeys(self, op, k):
# Don't monitor operations on monitor keys.
if k in self.monkeys: return
#DEBUGOFF logger.debug('doMonkeys: %s %s %s', op, k, repr(self.key2mon[True][op] | self.key2mon[k][op]))
for p in (True, k):
for mk in self.key2mon[p][op]:
self.put(mk, (b'ASTR', repr((op, k))))
def dump(self):
'''Utility function that returns a snapshot of the KV store.'''
def vrep(v):
t = v[0].tobytes()
# Omit or truncate some values, in which cases add the original length as a third value
if v == b'JSON' or t == b'HTML': return (t, v[1].tobytes())
if t != b'ASTR': return (t, None, len(v[1]))
if v[1][:6].tobytes().lower() == '<html>': return (t, v[1].tobytes()) # for backwards compatibility only
if len(v[1]) > 50: return (t, v[1][:24].tobytes() + '...' + v[1][-23:].tobytes(), len(v[1]))
return (t, v[1].tobytes())
return PDS(([self.opCounts[b'get'], self.opCounts[b'put'], self.opCounts[b'view'], self.opCounts[b'wait'], self.ac, self.rc], [(k, len(v)) for k, v in self.waiters.items() if v], [(k, len(vv), vrep(vv[-1])) for k, vv in self.store.items() if vv]))
def wait(self, waiter):
'''Atomically (remove and) return a value associated with key k. If
none, block.'''
#DEBUGOFF logger.debug('wait: %s, %s', repr(waiter.key), repr(waiter.op))
self._doMonkeys(waiter.op, waiter.key)
vv = self.store.get(waiter.key)
if vv:
if waiter.delete:
v = vv.pop(self.getIndex)
if not vv: self.store.pop(waiter.key)
else:
v = vv[self.viewIndex]
self.opCounts[waiter.op] += 1
#DEBUGOFF logger.debug('_gv (%s): %s => %s (%d)', waiter.op, waiter.key, repr(v[0]), len(v[1]))
waiter.handler(v)
else:
self.waiters[waiter.key].append(waiter)
self.opCounts[b'wait'] += 1
self._doMonkeys(b'wait', waiter.key)
#DEBUGOFF logger.debug('(%s) %s acquiring', repr(waiter), repr(s))
self.ac += 1
def cancel_wait(self, waiter):
ww = self.waiters.get(waiter.key)
if ww:
try:
ww.remove(waiter)
except ValueError:
pass
if not ww: self.waiters.pop(waiter.key)
def monkey(self, mkey, v):
'''Make Mkey a monitor key. Value encodes what events to monitor and
for which key:
Key:Events
Whenever a listed event occurs for "Key", a put will be done
to "Mkey" with the value "<event> <key>". If 'Key' is empty,
the events listed will be monitored for all keys. 'Events' is
some subset of 'g', 'p', 'v' and 'w' (get, put, view and
wait). Monitoring of any event *not* listed is turned off for
the specified key.
'''
#DEBUGOFF logger.debug('monkey: %s %s', mkey, v)
if b':' not in v: return #TODO: Add some sort of error handling?
self.monkeys.add(mkey)
k, events = v.rsplit(b':', 1)
if not k: k = True
for e, op in [(b'g', b'get'), (b'p', b'put'), (b'v', b'view'), (b'w', b'wait')]:
if e in events:
self.key2mon[k][op].add(mkey)
else:
try: self.key2mon[k][op].remove(mkey)
except KeyError: pass
#DEBUGOFF logger.debug('monkey: %s', repr(self.key2mon))
def put(self, k, v):
'''Add value v to those associated with the key k.'''
#DEBUGOFF logger.debug('put: %s, %s', repr(k), repr(v))
self.opCounts[b'put'] += 1
ww = self.waiters.get(k) # No waiters is probably most common, so optimize for
# that. ww will be None if no waiters have been
# registered for key k.
consumed = False
if ww:
while ww:
waiter = ww.pop(0)
#DEBUGOFF logger.debug('%s releasing', repr(waiter))
self.rc += 1
self.opCounts[waiter.op] += 1
waiter.handler(v)
if waiter.delete:
consumed = True
break
if not ww: self.waiters.pop(k)
if not consumed: self.store[k].append(v)
self._doMonkeys(b'put', k)
def removeStopWords(Frequencies: defaultdict):
for i in stop_words:
Frequencies.pop(i, None)
