def __init__(self, clock, serverKey, factory):
self._clock = clock
self._serverKey = serverKey
self._factory = factory
self._received = IntervalSet()
self._weAcked = IntervalSet()
self._sent = IntervalSet()
self._theyAcked = IntervalSet()
self._sentMessages = set()
self._previousID = 0
self._fragment = []
self._congestion = Chicago()
self._sentMessageAt = {}
self._delayedCalls = {}
self._messageQueue = []
self._enqueuedMessages = set()
self._deferred = defer.Deferred()
self._nonce = 0
self._theirLastNonce = 0
self._counter = 1
self._ourStreamEnd = None
self._theirStreamEnd = None
self._reads = self._writes = None
self._done = False
self._outstandingMessages = 0
self._onDone = []
def merge_timelines(working=[], busy=[]):
return [to_time(x) for x in IntervalSet([
to_interval(beg, end) for beg, end in working
]) - IntervalSet([
to_interval(beg, end)
for beg, end in busy
])]
def slow(ips):
from interval import Interval, IntervalSet
r1 = IntervalSet([Interval(0, 4294967295)])
r2 = IntervalSet([Interval(ip[0], ip[1]) for ip in ips])
diff = r1 - r2
total = 0
for i in diff:
start = i.lower_bound + 1
total += i.upper_bound - start
print('Part 2: %d' % total)
def init(self):
self.arch = Architecture['EVM']
self.platform = Architecture['EVM'].standalone_platform
self.add_entry_point(0)
file_size = len(self.raw)
# Find swarm hashes and make them data
bytes = self.raw.read(0, file_size)
# code is everything that isn't a swarm hash
code = IntervalSet([Interval(0, file_size)])
swarm_hashes = self.find_swarm_hashes(bytes)
for start, sz in swarm_hashes:
self.add_auto_segment(
start - len(EVM_HEADER), sz, start, sz,
SegmentFlag.SegmentContainsData
| SegmentFlag.SegmentDenyExecute | SegmentFlag.SegmentReadable
| SegmentFlag.SegmentDenyWrite)
code -= IntervalSet([Interval(start, start + sz)])
print "Code Segments: {}".format(code)
for interval in code:
self.add_auto_segment(
interval.lower_bound, interval.upper_bound - len(EVM_HEADER),
len(EVM_HEADER), interval.upper_bound,
(SegmentFlag.SegmentReadable | SegmentFlag.SegmentExecutable))
self.define_auto_symbol(
Symbol(SymbolType.FunctionSymbol, 0, '_dispatcher'))
invalidJumpCallbackNotification = InvalidJumpCallback()
self.register_notification(invalidJumpCallbackNotification)
dispatcherCallbackNotification = DispatcherCallback()
self.register_notification(dispatcherCallbackNotification)
dynamicJumpCallbackNotification = DynamicJumpCallback()
self.register_notification(dynamicJumpCallbackNotification)
# disable linear sweep
self.store_metadata("ephemeral",
{"binaryninja.analysis.autorunLinearSweep": False})
return True
def __init__(self, page):
self.page = page
self.matcher = SequenceMatcher()
self.wordtuples = [word for word in page.get_words()]
self.words = [word.text for word in self.wordtuples]
self.rawwords = [word for word in page.get_words_raw()]
# XXX MM DEBUG
# self.words = [word.text for word in page.get_words_raw()]
# self.words = [word.text for word in page.get_words()]
# print self.words
# for i in range(4):
# if i >= len(self.words):
# break
# # if i < len(self.words):
# # self.words.pop(i)
# if self.words[i] == 'contents':
# self.words[i] = ''
# if self.words[i] == 'page':
# self.words[i] = ''
self.wordhash = {}
for word in self.words:
self.wordhash[word] = True
self.matcher.set_seq2(self.words)
# l('\n'.join('%s:%s' % (i, w) for i, w in enumerate(self.words)))
# l('content len: %s' % len(self.words))
self.lookaside = {}
self.pageno_cands = self.find_pageno_cands(filtered=True)
self.pageno_cands_unfiltered = self.find_pageno_cands(filtered=False)
self.matches = IntervalSet()
self.matchinfo = {}
self.score = 0
self.adj_score = 0 # 'adjusted' score; as to boost last page in toc
self.set_base_score()
def loseConnection(self, success=True):
d = defer.Deferred()
d.addCallback(self._doneWritingAcked)
streamEnd = self._ourStreamEnd = self._sent.upper_bound(
) if self._sent else 0
resolution = self._ourResolution = 'success' if success else 'failure'
interval = IntervalSet([halfOpen(streamEnd, streamEnd + 1)])
self._enqueue(
1, QueuedResolution(interval, streamEnd, [d], [], [], resolution))
self._writes = 'closing'
return d
def _parseMessage(self, now, message):
message = parseMessage(message)
sentAt = self._sentMessageAt.pop(message.previousID, None)
if sentAt is not None:
self._congestion.processDelta(now, now - sentAt)
if message.id:
self._clock.callLater(0, self._sendAMessage, ack=message.id)
self._theyAcked.update(message.ranges)
for qd in list(self._sentMessages):
if qd.interval & self._theyAcked:
qd.interval.difference_update(self._theyAcked)
if not qd.interval:
for d in qd.deferreds:
d.callback(now)
self._cancel(qd)
self._outstandingMessages -= 1
self._sentMessages.remove(qd)
if message.resolution and self._theirStreamEnd is None:
self._theirStreamEnd = message.dataPos
self._theirResolution = message.resolution
self._received.add(halfOpen(message.dataPos, message.dataPos + 1))
self._reads = 'closing'
self._checkTheirResolution()
return
elif not message.data:
return
i = halfOpen(message.dataPos, message.dataPos + len(message.data))
new = IntervalSet([i]) - self._received
if not new:
return
self._received.add(i)
newData = message.data[new.lower_bound() -
i.lower_bound:new.upper_bound() -
i.upper_bound or None]
bisect.insort(self._fragment, (i.lower_bound, newData))
if len(self._received) > 1 or self._received.lower_bound() != 0:
return
newData = ''.join([d for _, d in self._fragment])
self._protocol.dataReceived(newData)
self._fragment = []
self._checkTheirResolution()
def spacerDistData(tranFN, outFN):
'''chr strand tranStart tranEnd'''
chrom_length = bioLibCG.returnChromLengthDict('hg19')
chrom_strand_iSet = {}
for chrom in chrom_length:
for strand in ('+', '-'):
chrom_strand_iSet.setdefault(chrom,
{}).setdefault(strand, IntervalSet())
print 'making intervals'
f = open(tranFN, 'r')
for line in f:
ls = line.strip().split('\t')
tranStart, tranEnd = int(ls[3]), int(ls[4])
strand = ls[2]
chrom = ls[1]
chrom_strand_iSet[chrom][strand].add(Interval(tranStart, tranEnd))
f.close()
spacerData = []
print 'creating spacer data'
for chrom in chrom_strand_iSet:
for strand in chrom_strand_iSet[chrom]:
iSet = chrom_strand_iSet[chrom][strand]
for i, interv in enumerate(iSet):
if interv == iSet[-1]: break
nextInterv = iSet[i + 1]
seperation = nextInterv.lower_bound - interv.upper_bound
spacerData.append(seperation)
f = open(outFN, 'w')
outLines = [str(x) + '\n' for x in spacerData]
f.writelines(outLines)
f.close()
def collapseUTRs(utrFile, outFN):
chrom_strand_iSet = {}
f = open(utrFile, 'r')
for line in f:
ls = line.strip().split('\t')
chrom, strand, start, end = bioLibCG.tccSplit(ls[0])
chrom_strand_iSet.setdefault(chrom, {}).setdefault(strand, IntervalSet())
chrom_strand_iSet[chrom][strand].add(Interval(start, end))
f.close()
fOut = open(outFN, 'w')
for chrom in chrom_strand_iSet:
for strand in chrom_strand_iSet[chrom]:
iSet = chrom_strand_iSet[chrom][strand]
for interv in iSet:
try:
newTcc = bioLibCG.makeTcc(chrom, strand, interv.lower_bound, interv.upper_bound)
fOut.write('%s\n' % newTcc)
except AttributeError:
print interv, 'was not added'
fOut.close()
def write(self, data):
if not data:
return defer.succeed(None)
elif self._writes in ('closing', 'closed'):
return defer.fail(
e.CurveCPConnectionDone(
'attempted a write after closing writes'))
d = defer.Deferred()
qds = []
lowerBound = self._sent.upper_bound() if self._sent else 0
while data:
ds = []
queueableData = data[:1024]
dataRange = halfOpen(lowerBound, lowerBound + len(queueableData))
qd = QueuedData(IntervalSet([dataRange]), lowerBound, ds, [], [],
queueableData)
self._sent.add(dataRange)
lowerBound += len(queueableData)
data = data[1024:]
qds.append(qd)
ds.append(d)
self._enqueue(1, *qds)
return d
header= True
continue
strand= line[strandIdx]
cdsStart= int(line[cdsStartIdx])
cdsEnd= int(line[cdsEndIdx])
# if cdsStart == cdsEnd:
# "No coding sequence in this transcripts, skip it"
# continue
txStart= int(line[txStartIdx])
txEnd= int(line[txEndIdx])
exonStarts= line[exStartsIdx].strip(',').split(',')
exonEnds= line[exEndsIdx].strip(',').split(',')
exon_intervals= []
for s,e in zip(exonStarts, exonEnds):
exon_intervals.append(Interval(int(s), int(e)))
exon_set= IntervalSet(exon_intervals) ## EXAMPLE= exon_set= IntervalSet([Interval(1, 50), Interval(100, 200), Interval(800, 1300)])
txStart_cdsStart= IntervalSet([Interval(txStart, cdsStart)]) ## EXAMPLE= IntervalSet([Interval(1, 1000, lower_closed=True, upper_closed=True)])
txEnd_cdsEnd= IntervalSet([Interval(cdsEnd, txEnd)])
if strand == '+':
utr5= txStart_cdsStart - (txStart_cdsStart - exon_set)
utr3= txEnd_cdsEnd - (txEnd_cdsEnd - exon_set)
if strand == '-':
utr3= txStart_cdsStart - (txStart_cdsStart - exon_set)
utr5= txEnd_cdsEnd - (txEnd_cdsEnd - exon_set)
for x in utr5:
if type(x) == int:
"Skip transcripts which don't have UTR"
continue
name5utr= '_'.join([line[chromIdx], str(x.lower_bound), str(x.upper_bound), line[nameIdx], '5utr'])
utr_list.append(([line[chromIdx], x.lower_bound, x.upper_bound, name5utr, line[nameIdx], line[strandIdx], line[name2Idx]]))
def score_short_answer(gold_label_list, pred_label):
"""Scores a short answer as correct or not.
1) First decide if there is a gold short answer with SHORT_NO_NULL_THRESHOLD.
2) The prediction will get a match if:
a. There is a gold short answer.
b. The prediction span *set* match exactly with *one* of the non-null gold
short answer span *set*.
Args:
gold_label_list: A list of NQLabel.
pred_label: A single NQLabel.
Returns:
gold_has_answer, pred_has_answer, f1, score
"""
# There is a gold short answer if gold_label_list not empty and non null
# answers is over the threshold (sum over annotators).
gold_has_answer = util.gold_has_short_answer(gold_label_list)
# There is a pred long answer if pred_label is not empty and short answer
# set is not empty.
pred_has_answer = pred_label and (
(not util.is_null_span_list(pred_label.short_answer_span_list))
or pred_label.yes_no_answer != 'none')
f1 = 0
p = 0
r = 0
score = pred_label.short_score
# Both sides have short answers, which contains yes/no questions.
if gold_has_answer and pred_has_answer:
if pred_label.yes_no_answer != 'none': # System thinks its y/n questions.
for gold_label in gold_label_list:
if pred_label.yes_no_answer == gold_label.yes_no_answer:
f1 = 1
p = 1
r = 1
break
else:
for gold_label in gold_label_list:
# if util.span_set_equal(gold_label.short_answer_span_list,
# pred_label.short_answer_span_list):
# is_correct = True
# break
gold_set = IntervalSet([Interval(span.start_token_idx, span.end_token_idx) \
for span in gold_label.short_answer_span_list])
pred_set = IntervalSet([Interval(span.start_token_idx, span.end_token_idx) \
for span in pred_label.short_answer_span_list])
overlap_set = gold_set & pred_set
def calc_len(interval_set):
sum = 0
for span in interval_set:
sum += span.upper_bound - span.lower_bound + 1
return sum
precision = safe_divide(calc_len(overlap_set),
calc_len(pred_set))
recall = safe_divide(calc_len(overlap_set), calc_len(gold_set))
if safe_divide(2 * precision * recall,
precision + recall) > f1:
f1 = safe_divide(2 * precision * recall,
precision + recall)
p = precision
r = recall
elif not gold_has_answer and not pred_has_answer:
f1 = 1
p = 1
r = 1
return gold_has_answer, pred_has_answer, f1, p, r, score
def add_match(self, page, match):
# l('ADDING ' + str(match))
info = RangeMatch(self, page, match)
# l(info)
pageno = page.info['number']
pagenoval = rnum_to_int(pageno)
if pagenoval == 0 and len(pageno) > 0:
pagenoval = int(pageno)
matchint = Interval.between(match.b, match.b + match.size)
overlaps = [m for m in self.matches if m & matchint]
# if nearnos matches either, mark flag and amp score
if pageno:
nearnos = self.find_nearnos(match)
# l("GREPME near is [%s] pagenoval %s" % (nearnos, pagenoval))
# for no in nearnos[1], nearnos[0]:
if nearnos is None: # XXX SHOULDN"T BE NEEDED!!!!!!!!!!!!
nearnos = []
for no in nearnos[1], nearnos[0]:
# for no in nearnos:
if no is not None:
# l(no.val)
if no.val == pagenoval:
info.notes += 'nearno: %s' % pageno
# l("GOODMATCH tc %s, %s %s" % (self.page.index, pageno, self.score))
self.score += 1
info.nearno = no.word_index
break
if no.val > pagenoval - 10 and match.a < 10:
self.score += .01
break
# cases: no overlap
if len(overlaps) == 0:
self.matchinfo[matchint] = info
self.matches = self.matches + IntervalSet([matchint])
else:
start = match.b
end = match.b + match.size
for i in overlaps:
oinfo = self.matchinfo[i]
ostart = oinfo.match.b
oend = oinfo.match.b + oinfo.match.size
scootback = 0
if ostart < start:
scootback = start - ostart
start = ostart
if oend > end:
end = oend
info.match = Match(info.match.a - scootback, start,
end - start)
if oinfo.nearno != -1:
# assert(info.nearno == -1)
info.nearno = oinfo.nearno
# info.score += oinfo.score
# info.pageno = oinfo.pageno
# info.notes = info.notes + ' ' + info.notes
# for opageno in oinfo.pagenos:
# opagecount = oinfo.pagenos[opageno]
# if opageno in info.pagenos:
# info.pagenos[opageno] += opagecount
# else:
# info.pagenos[opageno] = opagecount
self.matches += IntervalSet([matchint])
(new_i, ) = [m for m in self.matches if m & matchint]
self.matchinfo[new_i] = info
from interval import Interval, IntervalSet
prices = [
8.8, 39.0, 49.0, 59.0, 69.0, 79.0, 86.0, 88.0, 89.0, 95.0, 97.0, 98.0,
99.0, 101.0, 105.0, 109.0, 115.0, 119.0, 125.0, 126.0, 129.0, 131.0, 139.0,
145.0, 149.0, 150.0, 159.0, 165.0, 169.0, 174.5, 175.0, 179.0, 186.0,
188.0, 189.0, 194.0, 195.0, 196.0, 198.0, 199.0, 199.5, 209.0, 215.0,
215.0, 219.0, 219.0, 224.0, 224.5, 225.0, 228.0, 229.0, 233.0, 239.0,
240.0, 245.0, 249.0, 249.5, 254.0, 255.0, 259.0, 268.0, 269.0, 274.0,
275.0, 276.0, 278.0, 279.0, 285.0, 289.0, 295.0, 298.0, 299.0, 299.5,
305.0, 309.0, 310.0, 314.0, 319.0, 324.0, 329.0, 339.0, 349.0, 349.5,
355.0, 358.0, 359.0, 367.0, 369.0, 374.0, 378.0, 379.0, 384.0, 389.0,
395.0, 398.0, 399.0, 402.0, 409.0, 418.0, 419.0, 424.0, 429.0, 435.0,
439.0, 449.0, 449.5, 459.0, 469.0, 479.0, 485.0, 489.0, 495.0, 498.0,
499.0, 509.0, 519.0, 529.0, 538.0, 539.0, 548.0, 549.0, 559.0, 579.0,
589.0, 598.0, 599.0, 609.0, 619.0, 629.0, 649.0, 659.0, 669.0, 679.0,
689.0, 698.0, 699.0, 719.0, 729.0, 749.0, 759.0, 769.0, 779.0, 789.0,
798.0, 799.0, 809.0, 829.0, 839.0, 849.0, 879.0, 899.0, 949.0, 959.0,
969.0, 989.0, 998.0, 999.0, 999.0, 1019.0, 1049.0, 1079.0, 1098.0, 1099.0,
1099.0, 1109.0, 1119.0, 1129.0, 1139.0, 1149.0, 1199.0, 1259.0, 1299.0,
1319.0, 1349.0, 1379.0, 1399.0, 1439.0, 1499.0, 1567.02, 1598.0, 1599.0,
1619.0, 1639.0, 1679.0, 1697.0, 1699.0, 1729.0, 1759.0, 1763.02, 1799.0,
1861.02, 1899.0, 1998.0, 1999.0, 2039.0, 2099.0, 2149.0, 2198.0, 2199.0,
2219.0, 2220.0, 2299.0, 2399.0, 2446.0, 2599.0, 2749.0, 2999.0, 5999.0,
9999.0
]
lst = [Interval(round(_ * 0.8, 2), round(_ * 1.2, 2)) for _ in prices]
print lst
print[(_.lower_bound, _.upper_bound) for _ in IntervalSet(lst)]
def __init__(self, low, high):
"""
Initialize a range item with lower bound and higher bound.
"""
self.items = IntervalSet([Interval(low, high)])
from interval import Interval, IntervalSet r1 = IntervalSet([Interval(1, 1000), Interval(1100, 1200)]) r2 = IntervalSet([Interval(30, 50), Interval(60, 200), Interval(1150, 1300)]) r3 = IntervalSet([Interval(1000, 3000)]) r4 = IntervalSet([Interval(1000, 3000)]) r5 = IntervalSet([Interval(30000, 12000)]) # print (r3 - r4), (r4 - r3), r3 & r4 # print len(IntervalSet.empty()) if r3 & r4 == r4: print 'yes' # print r3 & r4 # if (r3 - r4).is_empty(): # print "true" # print (r3 - r4).empty()
def chercher_balise(source,
type="",
cond_attr={},
cond_style={},
sous_balises=[],
inclure_balises=False,
interval_set=False):
"""
Arguments:
type: type de balise (e.g. div, h1 ...)
cond_attr: condition sur les attributs de la balise (e.g. class="c1")
cond_style: condition sur le style css (e.g. font-style: italic)
sous_balises: liste de types de sous-balises à inclure dans le contenu, ["all"] pour inclure tous les types de sous-balises
inclure_balises: inclure ou non les balises d'ouverture/de fermeture dans le contenu
Sortie:
liste de (début, fin) correspondant aux indices délimitant le contenu de la balise recherchée
"""
result = []
exclude = []
if type != "":
starts = list(findall("<" + type + " ", source)) + list(
findall("<" + type + ">", source))
else:
starts = list(findall("<", source))
starts.sort()
for start in starts:
if source[start + 1:start + 2] == "/":
continue
if inclure_balises:
debut_contenu = start
else:
debut_contenu = start + source[start:].find(">") + 1
if type == "":
type_ = source[start + 1:start + 1 + minfind(
source[start + 1:], [" ", ">", "/"]
)] #Dans le cas où type="", permet de récupérer le type de balise
else:
type_ = type
fin_contenu = start + source[start + 1:].find("</" + type_) + 1
if inclure_balises:
fin_contenu += source[fin_contenu:].find(">") + 1
contenu = source[debut_contenu:fin_contenu]
attributs_ = source[start + 1 + len(type_):start +
source[start:].find(">")]
attributs_ = attributs_.split('" ')
attributs = {}
for a in attributs_:
key = a[:a.find("=")].replace(" ", "")
val = a[a.find("=") + 2:].replace('"', '')
if key in attributs.keys():
attributs[key] += val
else:
attributs[key] = val
style = {}
if 'style' in attributs.keys():
style_ = attributs['style']
style_ = style_.split(';')
for s in style_:
if s.replace(" ", "") != "":
key = s[:s.find(":")].replace(" ", "")
val = s[s.find(":") + 1:].replace(" ", "")
style[key] = val
cond = True
for k, v in cond_attr.iteritems():
if k not in attributs.keys() or attributs[k] != v:
cond = False
for k, v in cond_style.iteritems():
if k not in style.keys() or style[k] != v:
cond = False
if cond:
result.append((debut_contenu, fin_contenu))
if not inclure_balises:
for (debut_sous,
fin_sous) in chercher_balise(contenu,
inclure_balises=True):
type_sous = contenu[debut_sous + 1:debut_sous + 1 +
minfind(contenu[debut_sous +
1:], [" ", ">", "/"])]
if type_sous not in sous_balises and "all" not in sous_balises:
exclude.append((debut_contenu + debut_sous,
debut_contenu + fin_sous))
r1 = IntervalSet([Interval(d, f) for (d, f) in result])
r2 = IntervalSet([Interval(d, f) for (d, f) in exclude])
R = r1 - r2
if interval_set:
return R
if inclure_balises:
return result
else:
return [(x.lower_bound, x.upper_bound) for x in R.intervals]
def showMessage(tag, message, sentAt=None):
print tag, message.id, message.previousID, IntervalSet(message.ranges),
print halfOpen(message.dataPos, message.dataPos + len(message.data)),
print len(message.data), sentAt and len(sentAt)
# coding: utf-8
# __author__: ""
from interval import Interval, IntervalSet
volume1 = Interval.between("A", "Foe")
volume2 = Interval.between("Fog", "McAfee")
volume3 = Interval.between("McDonalds", "Space")
volume4 = Interval.between("Spade", "Zygote")
encyclopedia = IntervalSet([volume1, volume2, volume3, volume4])
mySet = IntervalSet([volume1, volume3, volume4])
"Meteor" in encyclopedia
"Goose" in encyclopedia
"Goose" in mySet
volume2 in (encyclopedia ^ mySet)
print [(_.lower_bound, _.upper_bound) for _ in IntervalSet(encyclopedia)]
OfficeHours = IntervalSet.between("08:00", "17:00")
myLunch = IntervalSet.between("11:30", "12:30")
myHours = IntervalSet.between("08:30", "19:30") - myLunch
myHours.issubset(OfficeHours)
"12:00" in myHours
"15:30" in myHours
