def merge_timelines(working=[], busy=[]):
return [to_time(x) for x in IntervalSet([
to_interval(beg, end) for beg, end in working
]) - IntervalSet([
to_interval(beg, end)
for beg, end in busy
])]
def __init__(self, clock, serverKey, factory):
self._clock = clock
self._serverKey = serverKey
self._factory = factory
self._received = IntervalSet()
self._weAcked = IntervalSet()
self._sent = IntervalSet()
self._theyAcked = IntervalSet()
self._sentMessages = set()
self._previousID = 0
self._fragment = []
self._congestion = Chicago()
self._sentMessageAt = {}
self._delayedCalls = {}
self._messageQueue = []
self._enqueuedMessages = set()
self._deferred = defer.Deferred()
self._nonce = 0
self._theirLastNonce = 0
self._counter = 1
self._ourStreamEnd = None
self._theirStreamEnd = None
self._reads = self._writes = None
self._done = False
self._outstandingMessages = 0
self._onDone = []
def event_to_minutes(event):
minutes = IntervalSet.between(0, 0)
d = {"MO": 0, "TU": 1, "WE": 2, "TH": 3, "FR": 4}
start, stop = event["DTSTART"].dt, event["DTEND"].dt
for day in event["RRULE"]["BYDAY"]:
b = d[day] * 1440 + start.hour * 60 + start.minute
e = d[day] * 1440 + stop.hour * 60 + stop.minute
minutes = minutes + IntervalSet.between(b, e)
return minutes
def slow(ips):
from interval import Interval, IntervalSet
r1 = IntervalSet([Interval(0, 4294967295)])
r2 = IntervalSet([Interval(ip[0], ip[1]) for ip in ips])
diff = r1 - r2
total = 0
for i in diff:
start = i.lower_bound + 1
total += i.upper_bound - start
print('Part 2: %d' % total)
def calc_break_time(bts_level,t0,t1):
'''
根据告警发生时间 t0 和 告警结束时间 t1
计算出在三个考核时段内告警的时长，单位：minutes
'''
# t0 = df_block.loc[0,'告警发生时间']
# t1 = df_block.loc[0,'告警清除时间']
date_list = list(rrule(DAILY, dtstart=parse(t0), until=parse(t1)))
day_list = [x.strftime('%Y-%m-%d') for x in date_list]
# 创建断站的时间段
start_time = datetime.strptime(t0, '%Y-%m-%d %H:%M:%S')
end_time = datetime.strptime(t1, '%Y-%m-%d %H:%M:%S')
break_interval = IntervalSet.between(start_time, end_time, closed=True)
break_time_6to8 = 0
break_time_8to22 = 0
break_time_22to24 = 0
for day in day_list:
# 创建三个考核时间段，分别是6-8点，8-22点，22-24点，
interval_6to8 = IntervalSet.between(
datetime.strptime(
day + ' ' + '06:00:00',
'%Y-%m-%d %H:%M:%S'),
datetime.strptime(
day + ' ' + '08:00:00',
'%Y-%m-%d %H:%M:%S'), closed=True)
interval_8to22 = IntervalSet.between(
datetime.strptime(
day + ' ' + '08:00:00',
'%Y-%m-%d %H:%M:%S'),
datetime.strptime(
day + ' ' + '22:00:00',
'%Y-%m-%d %H:%M:%S'), closed=True)
interval_22to24 = IntervalSet.between(
datetime.strptime(
day + ' ' + '22:00:00',
'%Y-%m-%d %H:%M:%S'),
datetime.strptime(
day + ' ' + '23:59:59',
'%Y-%m-%d %H:%M:%S'), closed=True)
break_interval_6to8 = break_interval & interval_6to8
break_interval_8to22 = break_interval & interval_8to22
break_interval_22to24 = break_interval & interval_22to24
break_time_6to8 += calc_time(break_interval_6to8)
break_time_8to22 += calc_time(break_interval_8to22)
break_time_22to24 += calc_time(break_interval_22to24)
if bts_level == 'A' or bts_level == 'B':
return (break_time_6to8,break_time_8to22 * 1.2,break_time_22to24)
else:
return (break_time_6to8,break_time_8to22,break_time_22to24)
def __init__(self, clock, serverKey, factory):
self._clock = clock
self._serverKey = serverKey
self._factory = factory
self._received = IntervalSet()
self._weAcked = IntervalSet()
self._sent = IntervalSet()
self._theyAcked = IntervalSet()
self._sentMessages = set()
self._previousID = 0
self._fragment = []
self._congestion = Chicago()
self._sentMessageAt = {}
self._delayedCalls = {}
self._messageQueue = []
self._enqueuedMessages = set()
self._deferred = defer.Deferred()
self._nonce = 0
self._theirLastNonce = 0
self._counter = 1
self._ourStreamEnd = None
self._theirStreamEnd = None
self._reads = self._writes = None
self._done = False
self._outstandingMessages = 0
self._onDone = []
def init(self):
self.arch = Architecture['EVM']
self.platform = Architecture['EVM'].standalone_platform
self.add_entry_point(0)
file_size = len(self.raw)
# Find swarm hashes and make them data
bytes = self.raw.read(0, file_size)
# code is everything that isn't a swarm hash
code = IntervalSet([Interval(0, file_size)])
swarm_hashes = self.find_swarm_hashes(bytes)
for start, sz in swarm_hashes:
self.add_auto_segment(
start - len(EVM_HEADER), sz, start, sz,
SegmentFlag.SegmentContainsData
| SegmentFlag.SegmentDenyExecute | SegmentFlag.SegmentReadable
| SegmentFlag.SegmentDenyWrite)
code -= IntervalSet([Interval(start, start + sz)])
print "Code Segments: {}".format(code)
for interval in code:
self.add_auto_segment(
interval.lower_bound, interval.upper_bound - len(EVM_HEADER),
len(EVM_HEADER), interval.upper_bound,
(SegmentFlag.SegmentReadable | SegmentFlag.SegmentExecutable))
self.define_auto_symbol(
Symbol(SymbolType.FunctionSymbol, 0, '_dispatcher'))
invalidJumpCallbackNotification = InvalidJumpCallback()
self.register_notification(invalidJumpCallbackNotification)
dispatcherCallbackNotification = DispatcherCallback()
self.register_notification(dispatcherCallbackNotification)
dynamicJumpCallbackNotification = DynamicJumpCallback()
self.register_notification(dynamicJumpCallbackNotification)
# disable linear sweep
self.store_metadata("ephemeral",
{"binaryninja.analysis.autorunLinearSweep": False})
return True
def __init__(self, page):
self.page = page
self.matcher = SequenceMatcher()
self.wordtuples = [word for word in page.get_words()]
self.words = [word.text for word in self.wordtuples]
self.rawwords = [word for word in page.get_words_raw()]
# XXX MM DEBUG
# self.words = [word.text for word in page.get_words_raw()]
# self.words = [word.text for word in page.get_words()]
# print self.words
# for i in range(4):
# if i >= len(self.words):
# break
# # if i < len(self.words):
# # self.words.pop(i)
# if self.words[i] == 'contents':
# self.words[i] = ''
# if self.words[i] == 'page':
# self.words[i] = ''
self.wordhash = {}
for word in self.words:
self.wordhash[word] = True
self.matcher.set_seq2(self.words)
# l('\n'.join('%s:%s' % (i, w) for i, w in enumerate(self.words)))
# l('content len: %s' % len(self.words))
self.lookaside = {}
self.pageno_cands = self.find_pageno_cands(filtered=True)
self.pageno_cands_unfiltered = self.find_pageno_cands(filtered=False)
self.matches = IntervalSet()
self.matchinfo = {}
self.score = 0
self.adj_score = 0 # 'adjusted' score; as to boost last page in toc
self.set_base_score()
def _parseMessage(self, now, message):
message = parseMessage(message)
sentAt = self._sentMessageAt.pop(message.previousID, None)
if sentAt is not None:
self._congestion.processDelta(now, now - sentAt)
if message.id:
self._clock.callLater(0, self._sendAMessage, ack=message.id)
self._theyAcked.update(message.ranges)
for qd in list(self._sentMessages):
if qd.interval & self._theyAcked:
qd.interval.difference_update(self._theyAcked)
if not qd.interval:
for d in qd.deferreds:
d.callback(now)
self._cancel(qd)
self._outstandingMessages -= 1
self._sentMessages.remove(qd)
if message.resolution and self._theirStreamEnd is None:
self._theirStreamEnd = message.dataPos
self._theirResolution = message.resolution
self._received.add(halfOpen(message.dataPos, message.dataPos + 1))
self._reads = 'closing'
self._checkTheirResolution()
return
elif not message.data:
return
i = halfOpen(message.dataPos, message.dataPos + len(message.data))
new = IntervalSet([i]) - self._received
if not new:
return
self._received.add(i)
newData = message.data[new.lower_bound() -
i.lower_bound:new.upper_bound() -
i.upper_bound or None]
bisect.insort(self._fragment, (i.lower_bound, newData))
if len(self._received) > 1 or self._received.lower_bound() != 0:
return
newData = ''.join([d for _, d in self._fragment])
self._protocol.dataReceived(newData)
self._fragment = []
self._checkTheirResolution()
def get_working_time(onduty_time, offduty_time, work_date):
"""
提供上下班打卡时间，计算工作时长
onduty_time 上班打卡时间， 时间戳毫秒
offduty_time 下班打卡时间， 时间戳毫秒
work_date 工作日 '2018-10-10'
@author： lanxiong
@time：2018/11/15
print get_working_time(1538355625000, 1538388501000, '2018-10-01')
"""
from interval import IntervalSet
#TODO;从参数表获取
onduty_time = onduty_time / 1000
offduty_time = offduty_time / 1000
work_date = convert_timestamp_to_date(work_date, date_format="%Y-%m-%d")
am_onduty_time = "00:00"
am_offduty_time = "12:00"
pm_onduty_time = "13:30"
pm_offduty_time = "23:59"
am_onduty_time_stamp = convert_date_to_timestamp(work_date + ' ' + am_onduty_time, date_format="%Y-%m-%d %H:%M")
am_offduty_time_stamp = convert_date_to_timestamp(work_date + ' ' + am_offduty_time, date_format="%Y-%m-%d %H:%M")
pm_onduty_time_stamp = convert_date_to_timestamp(work_date + ' ' + pm_onduty_time, date_format="%Y-%m-%d %H:%M")
pm_offduty_time_stamp = convert_date_to_timestamp(work_date + ' ' + pm_offduty_time, date_format="%Y-%m-%d %H:%M")
am_work = IntervalSet.between(am_onduty_time_stamp, am_offduty_time_stamp)
lunch_time = IntervalSet.between(am_offduty_time_stamp, pm_onduty_time_stamp)
pm_work = IntervalSet.between(pm_onduty_time_stamp, pm_offduty_time_stamp)
if offduty_time in am_work:
return offduty_time - onduty_time
elif offduty_time in lunch_time:
return am_offduty_time_stamp - onduty_time
elif offduty_time in pm_work:
if onduty_time in am_work:
return offduty_time - onduty_time - 1.5*60*60
elif onduty_time in lunch_time:
return offduty_time - pm_onduty_time_stamp
elif onduty_time in pm_work:
return offduty_time - onduty_time
else:
# 干到第二天了
return offduty_time - onduty_time
def _parseMessage(self, now, message):
message = parseMessage(message)
sentAt = self._sentMessageAt.pop(message.previousID, None)
if sentAt is not None:
self._congestion.processDelta(now, now - sentAt)
if message.id:
self._clock.callLater(0, self._sendAMessage, ack=message.id)
self._theyAcked.update(message.ranges)
for qd in list(self._sentMessages):
if qd.interval & self._theyAcked:
qd.interval.difference_update(self._theyAcked)
if not qd.interval:
for d in qd.deferreds:
d.callback(now)
self._cancel(qd)
self._outstandingMessages -= 1
self._sentMessages.remove(qd)
if message.resolution and self._theirStreamEnd is None:
self._theirStreamEnd = message.dataPos
self._theirResolution = message.resolution
self._received.add(halfOpen(message.dataPos, message.dataPos + 1))
self._reads = 'closing'
self._checkTheirResolution()
return
elif not message.data:
return
i = halfOpen(message.dataPos, message.dataPos + len(message.data))
new = IntervalSet([i]) - self._received
if not new:
return
self._received.add(i)
newData = message.data[new.lower_bound() - i.lower_bound:new.upper_bound() - i.upper_bound or None]
bisect.insort(self._fragment, (i.lower_bound, newData))
if len(self._received) > 1 or self._received.lower_bound() != 0:
return
newData = ''.join([d for _, d in self._fragment])
self._protocol.dataReceived(newData)
self._fragment = []
self._checkTheirResolution()
def loseConnection(self, success=True):
d = defer.Deferred()
d.addCallback(self._doneWritingAcked)
streamEnd = self._ourStreamEnd = self._sent.upper_bound(
) if self._sent else 0
resolution = self._ourResolution = 'success' if success else 'failure'
interval = IntervalSet([halfOpen(streamEnd, streamEnd + 1)])
self._enqueue(
1, QueuedResolution(interval, streamEnd, [d], [], [], resolution))
self._writes = 'closing'
return d
def calc_time(interval):
'''
根据告警的interval计算出时长，单位：minutes
'''
if interval != IntervalSet.empty():
low = interval.lower_bound()
upper = interval.upper_bound()
break_time = upper - low
break_minutes = round(break_time.seconds/60,2)
else:
break_minutes = 0
return break_minutes
def read_gff(infileName):
pCDS = IntervalSet()
mCDS = IntervalSet()
infile = open(infileName, 'r')
for line in infile:
if line[0] == "#":
continue
if line[0] == ">":
break
line = line.strip()
entries = line.split()
strand = entries[6]
feature = entries[2]
start = int(entries[3])
stop = int(entries[4])
if strand == "+":
if feature == "CDS":
pCDS.add(Interval(start,stop))
elif strand == "-":
if feature == "CDS":
mCDS.add(Interval(start,stop))
gffDict = {}
gffDict['pCDS'] = pCDS
gffDict['mCDS'] = mCDS
infile.close()
return gffDict
def compare_intersection():
cyinterval_set1 = CyIntervalSet(CyInterval(0., 1., upper_closed=False),
CyInterval(1., 3., lower_closed=False))
cyinterval_set2 = CyIntervalSet(CyInterval(.5, 1.5))
orinterval_set1 = OrigIntervalSet([
CyInterval(0., 1., upper_closed=False),
OrigInterval(1., 3., lower_closed=False)
])
orinterval_set2 = OrigIntervalSet([OrigInterval(.5, 1.5)])
n = 100000
t0 = time.time()
for _ in range(n):
cyinterval_set1.intersection(cyinterval_set2)
t1 = time.time()
print('cyinterval took %f seconds for %d iterations' % (t1 - t0, n))
d_cy = t1 - t0
t0 = time.time()
for _ in range(n):
orinterval_set1.intersection(orinterval_set2)
t1 = time.time()
d_orig = t1 - t0
print('interval took %f seconds for %d iterations' % (t1 - t0, n))
print('cyinterval was %f times as fast as interval' % (d_orig / d_cy))
def decouples(self, rhs):
"""
Decouple this item with the other
"""
assert isinstance(rhs, RangeItem)
# print self.items, rhs.items
new_values = [self.items - rhs.items, rhs.items - self.items, self.items & rhs.items]
new_items = []
for value in new_values:
# print value
# print '%s - %s' % str(value.lower_bound()), str(value.upper_bound)
if len(value) != 0:
new_items.append(RangeItem(value.lower_bound(), value.upper_bound()))
else:
empty_range = RangeItem(0, 0)
empty_range.items = IntervalSet.empty()
new_items.append(empty_range)
return new_items
def decouples(self, rhs):
"""
Decouple this item with the other
"""
assert isinstance(rhs, RangeItem)
# print self.items, rhs.items
new_values = [
self.items - rhs.items, rhs.items - self.items,
self.items & rhs.items
]
new_items = []
for value in new_values:
# print value
# print '%s - %s' % str(value.lower_bound()), str(value.upper_bound)
if len(value) != 0:
new_items.append(
RangeItem(value.lower_bound(), value.upper_bound()))
else:
empty_range = RangeItem(0, 0)
empty_range.items = IntervalSet.empty()
new_items.append(empty_range)
return new_items
def spacerDistData(tranFN, outFN):
'''chr strand tranStart tranEnd'''
chrom_length = bioLibCG.returnChromLengthDict('hg19')
chrom_strand_iSet = {}
for chrom in chrom_length:
for strand in ('+', '-'):
chrom_strand_iSet.setdefault(chrom,
{}).setdefault(strand, IntervalSet())
print 'making intervals'
f = open(tranFN, 'r')
for line in f:
ls = line.strip().split('\t')
tranStart, tranEnd = int(ls[3]), int(ls[4])
strand = ls[2]
chrom = ls[1]
chrom_strand_iSet[chrom][strand].add(Interval(tranStart, tranEnd))
f.close()
spacerData = []
print 'creating spacer data'
for chrom in chrom_strand_iSet:
for strand in chrom_strand_iSet[chrom]:
iSet = chrom_strand_iSet[chrom][strand]
for i, interv in enumerate(iSet):
if interv == iSet[-1]: break
nextInterv = iSet[i + 1]
seperation = nextInterv.lower_bound - interv.upper_bound
spacerData.append(seperation)
f = open(outFN, 'w')
outLines = [str(x) + '\n' for x in spacerData]
f.writelines(outLines)
f.close()
def collapseUTRs(utrFile, outFN):
chrom_strand_iSet = {}
f = open(utrFile, 'r')
for line in f:
ls = line.strip().split('\t')
chrom, strand, start, end = bioLibCG.tccSplit(ls[0])
chrom_strand_iSet.setdefault(chrom, {}).setdefault(strand, IntervalSet())
chrom_strand_iSet[chrom][strand].add(Interval(start, end))
f.close()
fOut = open(outFN, 'w')
for chrom in chrom_strand_iSet:
for strand in chrom_strand_iSet[chrom]:
iSet = chrom_strand_iSet[chrom][strand]
for interv in iSet:
try:
newTcc = bioLibCG.makeTcc(chrom, strand, interv.lower_bound, interv.upper_bound)
fOut.write('%s\n' % newTcc)
except AttributeError:
print interv, 'was not added'
fOut.close()
def write(self, data):
if not data:
return defer.succeed(None)
elif self._writes in ('closing', 'closed'):
return defer.fail(
e.CurveCPConnectionDone(
'attempted a write after closing writes'))
d = defer.Deferred()
qds = []
lowerBound = self._sent.upper_bound() if self._sent else 0
while data:
ds = []
queueableData = data[:1024]
dataRange = halfOpen(lowerBound, lowerBound + len(queueableData))
qd = QueuedData(IntervalSet([dataRange]), lowerBound, ds, [], [],
queueableData)
self._sent.add(dataRange)
lowerBound += len(queueableData)
data = data[1024:]
qds.append(qd)
ds.append(d)
self._enqueue(1, *qds)
return d
class _CurveCPBaseTransport(DatagramProtocol):
timeouts = 1, 1, 2, 3, 5, 8, 13
_generateKey = staticmethod(PrivateKey.generate)
_generateKeydir = staticmethod(EphemeralKey)
def __init__(self, clock, serverKey, factory):
self._clock = clock
self._serverKey = serverKey
self._factory = factory
self._received = IntervalSet()
self._weAcked = IntervalSet()
self._sent = IntervalSet()
self._theyAcked = IntervalSet()
self._sentMessages = set()
self._previousID = 0
self._fragment = []
self._congestion = Chicago()
self._sentMessageAt = {}
self._delayedCalls = {}
self._messageQueue = []
self._enqueuedMessages = set()
self._deferred = defer.Deferred()
self._nonce = 0
self._theirLastNonce = 0
self._counter = 1
self._ourStreamEnd = None
self._theirStreamEnd = None
self._reads = self._writes = None
self._done = False
self._outstandingMessages = 0
self._onDone = []
def _now(self):
return self._clock.seconds()
def _timedOutHandshaking(self):
self._deferred.errback(e.HandshakeTimeout())
def _write(self, data):
self.transport.write(data, self._peerHost)
def _retrySendingForHandshake(self, data):
mt = MultiTimeout(
self._clock, self.timeouts, self._timedOutHandshaking, self._write, data)
mt.reset()
return mt
messageMap = {}
def datagramReceived(self, data, host_port):
if self._done:
return
handler = self._messageMap.get(data[:8])
if not handler:
return
meth = getattr(self, '_datagram_' + handler)
meth(data, host_port)
_nonceInfix = ''
def _encryptForNonce(self, which, box, data):
packedNonce = _nonceStruct.pack(self._nonce)
self._nonce += 1
nonce = 'CurveCP-%s-%s%s' % (self._nonceInfix, which, packedNonce)
return packedNonce + box.encrypt(data, nonce).ciphertext
def _verifyNonce(self, nonce):
unpacked, = _nonceStruct.unpack(nonce)
ret = unpacked >= self._theirLastNonce
self._theirLastNonce = unpacked
return ret
def _serializeMessage(self, message):
return ''
def _sendMessage(self, message):
packet = self._serializeMessage(message)
self._write(packet)
if message.id:
self._sentMessageAt[message.id] = self._congestion.lastSentAt = self._now()
self._weAcked.update(message.ranges)
def _parseMessage(self, now, message):
message = parseMessage(message)
sentAt = self._sentMessageAt.pop(message.previousID, None)
if sentAt is not None:
self._congestion.processDelta(now, now - sentAt)
if message.id:
self._clock.callLater(0, self._sendAMessage, ack=message.id)
self._theyAcked.update(message.ranges)
for qd in list(self._sentMessages):
if qd.interval & self._theyAcked:
qd.interval.difference_update(self._theyAcked)
if not qd.interval:
for d in qd.deferreds:
d.callback(now)
self._cancel(qd)
self._outstandingMessages -= 1
self._sentMessages.remove(qd)
if message.resolution and self._theirStreamEnd is None:
self._theirStreamEnd = message.dataPos
self._theirResolution = message.resolution
self._received.add(halfOpen(message.dataPos, message.dataPos + 1))
self._reads = 'closing'
self._checkTheirResolution()
return
elif not message.data:
return
i = halfOpen(message.dataPos, message.dataPos + len(message.data))
new = IntervalSet([i]) - self._received
if not new:
return
self._received.add(i)
newData = message.data[new.lower_bound() - i.lower_bound:new.upper_bound() - i.upper_bound or None]
bisect.insort(self._fragment, (i.lower_bound, newData))
if len(self._received) > 1 or self._received.lower_bound() != 0:
return
newData = ''.join([d for _, d in self._fragment])
self._protocol.dataReceived(newData)
self._fragment = []
self._checkTheirResolution()
def _checkTheirResolution(self):
if self._theirStreamEnd is None:
return
if len(self._received) != 1 or self._received.lower_bound() != 0:
return
self._reads = 'closed'
self._protocol.readConnectionLost()
self._checkBothResolutions()
def notifyFinish(self):
if self._done:
return defer.succeed(None)
d = defer.Deferred()
self._onDone.append(d)
return d
def _checkBothResolutions(self):
if self._reads == self._writes == 'closed' and not self._done:
self._protocol.connectionLost(Failure(e.resolution_map[self._theirResolution]()))
self._cancel('message')
deferreds, self._onDone = self._onDone, None
for d in deferreds:
d.callback(None)
# this used to be done on a callLater, but I can't remember why
self._done = True
def _sendAMessage(self, ack=None):
now = self._now()
nextActionIn = None
message = Message(
id=self._counter,
previousID=0,
ranges=list(self._received)[:6],
resolution=None,
dataPos=0,
data='',
)
if ack is not None:
message = message._replace(id=0, previousID=ack)
elif self._messageQueue:
_, _, qd = heapq.heappop(self._messageQueue)
self._enqueuedMessages.remove(qd)
message = qd.fillInMessage(message)
self._counter += 1
if qd.sentAt:
self._congestion.timedOut(now)
self._sentMessageAt.pop(qd.messageIDs[-1], None)
elif self._congestion.window is not None and self._outstandingMessages > self._congestion.window:
self._enqueue(1, qd)
return
else:
self._outstandingMessages += 1
qd.sentAt.append(now)
qd.messageIDs.append(message.id)
self._sentMessages.add(qd)
self._reschedule(qd)
else:
return 60
self._sendMessage(message)
return nextActionIn
def _reschedule(self, what, nextActionIn=None):
now = self._now()
if nextActionIn is None:
if what == 'message':
nextActionIn = self._congestion.nextMessageIn(now)
else:
nextActionIn = self._congestion.nextTimeoutIn(now, what)
delayedCall = self._delayedCalls.get(what)
if delayedCall is not None and delayedCall.active():
delayedCall.reset(nextActionIn)
else:
self._delayedCalls[what] = self._clock.callLater(
nextActionIn, self._scheduledAction, what)
def _cancel(self, what):
delayedCall = self._delayedCalls.pop(what, None)
if delayedCall is not None and delayedCall.active():
delayedCall.cancel()
def _scheduledAction(self, what):
nextActionIn = None
if what == 'message':
nextActionIn = self._sendAMessage()
self._reschedule(what, nextActionIn=nextActionIn)
else:
self._sentMessages.remove(what)
if what.interval:
self._enqueue(0, what)
def _enqueue(self, priority, *data):
self._reschedule('message')
for datum in data:
if datum not in self._enqueuedMessages and datum.interval:
heapq.heappush(self._messageQueue, (priority, datum.lowerBound, datum))
self._enqueuedMessages.add(datum)
def write(self, data):
if not data:
return defer.succeed(None)
elif self._writes in ('closing', 'closed'):
return defer.fail(e.CurveCPConnectionDone(
'attempted a write after closing writes'))
d = defer.Deferred()
qds = []
lowerBound = self._sent.upper_bound() if self._sent else 0
while data:
ds = []
queueableData = data[:1024]
dataRange = halfOpen(lowerBound, lowerBound + len(queueableData))
qd = QueuedData(
IntervalSet([dataRange]), lowerBound, ds, [], [], queueableData)
self._sent.add(dataRange)
lowerBound += len(queueableData)
data = data[1024:]
qds.append(qd)
ds.append(d)
self._enqueue(1, *qds)
return d
def _peerEstablished(self):
self._protocol = self._factory.buildProtocol(self.getPeer())
self._protocol.makeConnection(self)
self._deferred.callback(self._protocol)
self._reads = 'open'
self._writes = 'open'
def _doneWritingAcked(self, when):
self._writes = 'closed'
self._checkBothResolutions()
return when
def loseConnection(self, success=True):
d = defer.Deferred()
d.addCallback(self._doneWritingAcked)
streamEnd = self._ourStreamEnd = self._sent.upper_bound() if self._sent else 0
resolution = self._ourResolution = 'success' if success else 'failure'
interval = IntervalSet([halfOpen(streamEnd, streamEnd + 1)])
self._enqueue(1, QueuedResolution(interval, streamEnd, [d], [], [], resolution))
self._writes = 'closing'
return d
def add_match(self, page, match):
# l('ADDING ' + str(match))
info = RangeMatch(self, page, match)
# l(info)
pageno = page.info['number']
pagenoval = rnum_to_int(pageno)
if pagenoval == 0 and len(pageno) > 0:
pagenoval = int(pageno)
matchint = Interval.between(match.b, match.b + match.size)
overlaps = [m for m in self.matches if m & matchint]
# if nearnos matches either, mark flag and amp score
if pageno:
nearnos = self.find_nearnos(match)
# l("GREPME near is [%s] pagenoval %s" % (nearnos, pagenoval))
# for no in nearnos[1], nearnos[0]:
if nearnos is None: # XXX SHOULDN"T BE NEEDED!!!!!!!!!!!!
nearnos = []
for no in nearnos[1], nearnos[0]:
# for no in nearnos:
if no is not None:
# l(no.val)
if no.val == pagenoval:
info.notes += 'nearno: %s' % pageno
# l("GOODMATCH tc %s, %s %s" % (self.page.index, pageno, self.score))
self.score += 1
info.nearno = no.word_index
break
if no.val > pagenoval - 10 and match.a < 10:
self.score += .01
break
# cases: no overlap
if len(overlaps) == 0:
self.matchinfo[matchint] = info
self.matches = self.matches + IntervalSet([matchint])
else:
start = match.b
end = match.b + match.size
for i in overlaps:
oinfo = self.matchinfo[i]
ostart = oinfo.match.b
oend = oinfo.match.b + oinfo.match.size
scootback = 0
if ostart < start:
scootback = start - ostart
start = ostart
if oend > end:
end = oend
info.match = Match(info.match.a - scootback, start,
end - start)
if oinfo.nearno != -1:
# assert(info.nearno == -1)
info.nearno = oinfo.nearno
# info.score += oinfo.score
# info.pageno = oinfo.pageno
# info.notes = info.notes + ' ' + info.notes
# for opageno in oinfo.pagenos:
# opagecount = oinfo.pagenos[opageno]
# if opageno in info.pagenos:
# info.pagenos[opageno] += opagecount
# else:
# info.pagenos[opageno] = opagecount
self.matches += IntervalSet([matchint])
(new_i, ) = [m for m in self.matches if m & matchint]
self.matchinfo[new_i] = info
def __init__(self, debug=False):
self.unsimplified_demands = []
self.interval_set = IntervalSet.get_top()
self.debug = debug
from interval import Interval, IntervalSet
prices = [
8.8, 39.0, 49.0, 59.0, 69.0, 79.0, 86.0, 88.0, 89.0, 95.0, 97.0, 98.0,
99.0, 101.0, 105.0, 109.0, 115.0, 119.0, 125.0, 126.0, 129.0, 131.0, 139.0,
145.0, 149.0, 150.0, 159.0, 165.0, 169.0, 174.5, 175.0, 179.0, 186.0,
188.0, 189.0, 194.0, 195.0, 196.0, 198.0, 199.0, 199.5, 209.0, 215.0,
215.0, 219.0, 219.0, 224.0, 224.5, 225.0, 228.0, 229.0, 233.0, 239.0,
240.0, 245.0, 249.0, 249.5, 254.0, 255.0, 259.0, 268.0, 269.0, 274.0,
275.0, 276.0, 278.0, 279.0, 285.0, 289.0, 295.0, 298.0, 299.0, 299.5,
305.0, 309.0, 310.0, 314.0, 319.0, 324.0, 329.0, 339.0, 349.0, 349.5,
355.0, 358.0, 359.0, 367.0, 369.0, 374.0, 378.0, 379.0, 384.0, 389.0,
395.0, 398.0, 399.0, 402.0, 409.0, 418.0, 419.0, 424.0, 429.0, 435.0,
439.0, 449.0, 449.5, 459.0, 469.0, 479.0, 485.0, 489.0, 495.0, 498.0,
499.0, 509.0, 519.0, 529.0, 538.0, 539.0, 548.0, 549.0, 559.0, 579.0,
589.0, 598.0, 599.0, 609.0, 619.0, 629.0, 649.0, 659.0, 669.0, 679.0,
689.0, 698.0, 699.0, 719.0, 729.0, 749.0, 759.0, 769.0, 779.0, 789.0,
798.0, 799.0, 809.0, 829.0, 839.0, 849.0, 879.0, 899.0, 949.0, 959.0,
969.0, 989.0, 998.0, 999.0, 999.0, 1019.0, 1049.0, 1079.0, 1098.0, 1099.0,
1099.0, 1109.0, 1119.0, 1129.0, 1139.0, 1149.0, 1199.0, 1259.0, 1299.0,
1319.0, 1349.0, 1379.0, 1399.0, 1439.0, 1499.0, 1567.02, 1598.0, 1599.0,
1619.0, 1639.0, 1679.0, 1697.0, 1699.0, 1729.0, 1759.0, 1763.02, 1799.0,
1861.02, 1899.0, 1998.0, 1999.0, 2039.0, 2099.0, 2149.0, 2198.0, 2199.0,
2219.0, 2220.0, 2299.0, 2399.0, 2446.0, 2599.0, 2749.0, 2999.0, 5999.0,
9999.0
]
lst = [Interval(round(_ * 0.8, 2), round(_ * 1.2, 2)) for _ in prices]
print lst
print[(_.lower_bound, _.upper_bound) for _ in IntervalSet(lst)]
class RangeItem:
items = IntervalSet.empty()
def __init__(self, low, high):
"""
Initialize a range item with lower bound and higher bound.
"""
self.items = IntervalSet([Interval(low, high)])
def __eq__(self, rhs):
"""
Redefine == operator
"""
assert isinstance(rhs, RangeItem)
return self.items == rhs.items
def empty(self):
if len(self.items) == 0:
return True
return False
def dump(self, type='ip'):
if self.empty():
print 'Empty RangeItem'
else:
if type == 'int':
print 'RangeItem %s - %s' % (self.items.lower_bound(),
self.items.upper_bound())
elif type == 'ip':
print 'RangeItem %s - %s' % (long2ip(self.items.lower_bound()),
long2ip(self.items.upper_bound()))
def produce(self):
return RangeItem(self.items.lower_bound(), self.items.upper_bound())
def overlaps(self, rhs):
"""
If this item is overlapped with the other
"""
if len(self.items & rhs.items) != 0:
return True
return False
def includes(self, rhs):
"""
If this item includes the other
"""
if self.items & rhs.items == rhs.items:
return True
return False
def decouples(self, rhs):
"""
Decouple this item with the other
"""
assert isinstance(rhs, RangeItem)
# print self.items, rhs.items
new_values = [
self.items - rhs.items, rhs.items - self.items,
self.items & rhs.items
]
new_items = []
for value in new_values:
# print value
# print '%s - %s' % str(value.lower_bound()), str(value.upper_bound)
if len(value) != 0:
new_items.append(
RangeItem(value.lower_bound(), value.upper_bound()))
else:
empty_range = RangeItem(0, 0)
empty_range.items = IntervalSet.empty()
new_items.append(empty_range)
return new_items
def __init__(self, low, high):
"""
Initialize a range item with lower bound and higher bound.
"""
self.items = IntervalSet([Interval(low, high)])
if __name__ == "__main__":
insns = get_insns(file(sys.argv[1]))
trace = list(enumerate(TraceEntry(t) for t in insns))
tbs = make_tbs(trace)
tbdict = make_tbdict(tbs)
cfg = make_slice_cfg(tbs)
memops = filter(lambda x: is_memop(x[1]), trace)
ranges = [(i,) + getrange(insn) for i, insn in memops]
d = defaultdict(list)
for i, st, ed, m in ranges:
d[(st, ed)].append(m)
d = dict(d)
# Things written to before they were read
writes = [k for k in d if d[k][0] == "w"]
# Coalesce overlapping writes
iset = IntervalSet()
for w in writes:
iset.add(Interval(*w))
iset = FrozenIntervalSet(iset)
tbmap = dict((tb.range(), tb) for tb in tbs)
trace_ranges = sorted(tbmap.keys())
embedshell = IPython.Shell.IPShellEmbed(argv=[])
embedshell()
def call_on(call_user, subject):
convert_subject = tool.ip_convert(subject)
call_subject = convert_subject.split(':')[2].strip().replace('.', '点')
url = 'http://apiserver:8765/phoneWarn/callUp/v1/gen'
payload = {
'tos': call_user['1']["phone"],
'name': call_user['1']["name"],
'content': call_subject
}
result = requests.post(url, data=payload)
alert_data = {
"alert_data": subject.split(':')[2].strip(),
"alert_name": call_user[num]['name']
}
log_info = {}
log_info["info"] = alert_data
log_info["result"] = json.loads(result.text.strip())
logger.info(json.dumps(log_info))
if __name__ == '__main__':
subject = sys.argv[1]
start_time = IntervalSet.between("21:00", "23:59")
end_time = IntervalSet.between("00:00", "10:30")
now_time = datetime.now().strftime('%H:%M')
alert_time = datetime.now().strftime("%Y%m%d")
if now_time in start_time or now_time in end_time:
call_on(call_user, subject)
else:
logger.info('非电话报警时间，报警取消..')
def chercher_balise(source,
type="",
cond_attr={},
cond_style={},
sous_balises=[],
inclure_balises=False,
interval_set=False):
"""
Arguments:
type: type de balise (e.g. div, h1 ...)
cond_attr: condition sur les attributs de la balise (e.g. class="c1")
cond_style: condition sur le style css (e.g. font-style: italic)
sous_balises: liste de types de sous-balises à inclure dans le contenu, ["all"] pour inclure tous les types de sous-balises
inclure_balises: inclure ou non les balises d'ouverture/de fermeture dans le contenu
Sortie:
liste de (début, fin) correspondant aux indices délimitant le contenu de la balise recherchée
"""
result = []
exclude = []
if type != "":
starts = list(findall("<" + type + " ", source)) + list(
findall("<" + type + ">", source))
else:
starts = list(findall("<", source))
starts.sort()
for start in starts:
if source[start + 1:start + 2] == "/":
continue
if inclure_balises:
debut_contenu = start
else:
debut_contenu = start + source[start:].find(">") + 1
if type == "":
type_ = source[start + 1:start + 1 + minfind(
source[start + 1:], [" ", ">", "/"]
)] #Dans le cas où type="", permet de récupérer le type de balise
else:
type_ = type
fin_contenu = start + source[start + 1:].find("</" + type_) + 1
if inclure_balises:
fin_contenu += source[fin_contenu:].find(">") + 1
contenu = source[debut_contenu:fin_contenu]
attributs_ = source[start + 1 + len(type_):start +
source[start:].find(">")]
attributs_ = attributs_.split('" ')
attributs = {}
for a in attributs_:
key = a[:a.find("=")].replace(" ", "")
val = a[a.find("=") + 2:].replace('"', '')
if key in attributs.keys():
attributs[key] += val
else:
attributs[key] = val
style = {}
if 'style' in attributs.keys():
style_ = attributs['style']
style_ = style_.split(';')
for s in style_:
if s.replace(" ", "") != "":
key = s[:s.find(":")].replace(" ", "")
val = s[s.find(":") + 1:].replace(" ", "")
style[key] = val
cond = True
for k, v in cond_attr.iteritems():
if k not in attributs.keys() or attributs[k] != v:
cond = False
for k, v in cond_style.iteritems():
if k not in style.keys() or style[k] != v:
cond = False
if cond:
result.append((debut_contenu, fin_contenu))
if not inclure_balises:
for (debut_sous,
fin_sous) in chercher_balise(contenu,
inclure_balises=True):
type_sous = contenu[debut_sous + 1:debut_sous + 1 +
minfind(contenu[debut_sous +
1:], [" ", ">", "/"])]
if type_sous not in sous_balises and "all" not in sous_balises:
exclude.append((debut_contenu + debut_sous,
debut_contenu + fin_sous))
r1 = IntervalSet([Interval(d, f) for (d, f) in result])
r2 = IntervalSet([Interval(d, f) for (d, f) in exclude])
R = r1 - r2
if interval_set:
return R
if inclure_balises:
return result
else:
return [(x.lower_bound, x.upper_bound) for x in R.intervals]
class _CurveCPBaseTransport(DatagramProtocol):
timeouts = 1, 1, 2, 3, 5, 8, 13
_generateKey = staticmethod(PrivateKey.generate)
_generateKeydir = staticmethod(EphemeralKey)
def __init__(self, clock, serverKey, factory):
self._clock = clock
self._serverKey = serverKey
self._factory = factory
self._received = IntervalSet()
self._weAcked = IntervalSet()
self._sent = IntervalSet()
self._theyAcked = IntervalSet()
self._sentMessages = set()
self._previousID = 0
self._fragment = []
self._congestion = Chicago()
self._sentMessageAt = {}
self._delayedCalls = {}
self._messageQueue = []
self._enqueuedMessages = set()
self._deferred = defer.Deferred()
self._nonce = 0
self._theirLastNonce = 0
self._counter = 1
self._ourStreamEnd = None
self._theirStreamEnd = None
self._reads = self._writes = None
self._done = False
self._outstandingMessages = 0
self._onDone = []
def _now(self):
return self._clock.seconds()
def _timedOutHandshaking(self):
self._deferred.errback(e.HandshakeTimeout())
def _write(self, data):
self.transport.write(data, self._peerHost)
def _retrySendingForHandshake(self, data):
mt = MultiTimeout(self._clock, self.timeouts,
self._timedOutHandshaking, self._write, data)
mt.reset()
return mt
messageMap = {}
def datagramReceived(self, data, host_port):
if self._done:
return
handler = self._messageMap.get(data[:8])
if not handler:
return
meth = getattr(self, '_datagram_' + handler)
meth(data, host_port)
_nonceInfix = ''
def _encryptForNonce(self, which, box, data):
packedNonce = _nonceStruct.pack(self._nonce)
self._nonce += 1
nonce = 'CurveCP-%s-%s%s' % (self._nonceInfix, which, packedNonce)
return packedNonce + box.encrypt(data, nonce).ciphertext
def _verifyNonce(self, nonce):
unpacked, = _nonceStruct.unpack(nonce)
ret = unpacked >= self._theirLastNonce
self._theirLastNonce = unpacked
return ret
def _serializeMessage(self, message):
return ''
def _sendMessage(self, message):
packet = self._serializeMessage(message)
self._write(packet)
if message.id:
self._sentMessageAt[
message.id] = self._congestion.lastSentAt = self._now()
self._weAcked.update(message.ranges)
def _parseMessage(self, now, message):
message = parseMessage(message)
sentAt = self._sentMessageAt.pop(message.previousID, None)
if sentAt is not None:
self._congestion.processDelta(now, now - sentAt)
if message.id:
self._clock.callLater(0, self._sendAMessage, ack=message.id)
self._theyAcked.update(message.ranges)
for qd in list(self._sentMessages):
if qd.interval & self._theyAcked:
qd.interval.difference_update(self._theyAcked)
if not qd.interval:
for d in qd.deferreds:
d.callback(now)
self._cancel(qd)
self._outstandingMessages -= 1
self._sentMessages.remove(qd)
if message.resolution and self._theirStreamEnd is None:
self._theirStreamEnd = message.dataPos
self._theirResolution = message.resolution
self._received.add(halfOpen(message.dataPos, message.dataPos + 1))
self._reads = 'closing'
self._checkTheirResolution()
return
elif not message.data:
return
i = halfOpen(message.dataPos, message.dataPos + len(message.data))
new = IntervalSet([i]) - self._received
if not new:
return
self._received.add(i)
newData = message.data[new.lower_bound() -
i.lower_bound:new.upper_bound() -
i.upper_bound or None]
bisect.insort(self._fragment, (i.lower_bound, newData))
if len(self._received) > 1 or self._received.lower_bound() != 0:
return
newData = ''.join([d for _, d in self._fragment])
self._protocol.dataReceived(newData)
self._fragment = []
self._checkTheirResolution()
def _checkTheirResolution(self):
if self._theirStreamEnd is None:
return
if len(self._received) != 1 or self._received.lower_bound() != 0:
return
self._reads = 'closed'
self._protocol.readConnectionLost()
self._checkBothResolutions()
def notifyFinish(self):
if self._done:
return defer.succeed(None)
d = defer.Deferred()
self._onDone.append(d)
return d
def _checkBothResolutions(self):
if self._reads == self._writes == 'closed' and not self._done:
self._protocol.connectionLost(
Failure(e.resolution_map[self._theirResolution]()))
self._cancel('message')
deferreds, self._onDone = self._onDone, None
for d in deferreds:
d.callback(None)
# this used to be done on a callLater, but I can't remember why
self._done = True
def _sendAMessage(self, ack=None):
now = self._now()
nextActionIn = None
message = Message(
id=self._counter,
previousID=0,
ranges=list(self._received)[:6],
resolution=None,
dataPos=0,
data='',
)
if ack is not None:
message = message._replace(id=0, previousID=ack)
elif self._messageQueue:
_, _, qd = heapq.heappop(self._messageQueue)
self._enqueuedMessages.remove(qd)
message = qd.fillInMessage(message)
self._counter += 1
if qd.sentAt:
self._congestion.timedOut(now)
self._sentMessageAt.pop(qd.messageIDs[-1], None)
elif self._congestion.window is not None and self._outstandingMessages > self._congestion.window:
self._enqueue(1, qd)
return
else:
self._outstandingMessages += 1
qd.sentAt.append(now)
qd.messageIDs.append(message.id)
self._sentMessages.add(qd)
self._reschedule(qd)
else:
return 60
self._sendMessage(message)
return nextActionIn
def _reschedule(self, what, nextActionIn=None):
now = self._now()
if nextActionIn is None:
if what == 'message':
nextActionIn = self._congestion.nextMessageIn(now)
else:
nextActionIn = self._congestion.nextTimeoutIn(now, what)
delayedCall = self._delayedCalls.get(what)
if delayedCall is not None and delayedCall.active():
delayedCall.reset(nextActionIn)
else:
self._delayedCalls[what] = self._clock.callLater(
nextActionIn, self._scheduledAction, what)
def _cancel(self, what):
delayedCall = self._delayedCalls.pop(what, None)
if delayedCall is not None and delayedCall.active():
delayedCall.cancel()
def _scheduledAction(self, what):
nextActionIn = None
if what == 'message':
nextActionIn = self._sendAMessage()
self._reschedule(what, nextActionIn=nextActionIn)
else:
self._sentMessages.remove(what)
if what.interval:
self._enqueue(0, what)
def _enqueue(self, priority, *data):
self._reschedule('message')
for datum in data:
if datum not in self._enqueuedMessages and datum.interval:
heapq.heappush(self._messageQueue,
(priority, datum.lowerBound, datum))
self._enqueuedMessages.add(datum)
def write(self, data):
if not data:
return defer.succeed(None)
elif self._writes in ('closing', 'closed'):
return defer.fail(
e.CurveCPConnectionDone(
'attempted a write after closing writes'))
d = defer.Deferred()
qds = []
lowerBound = self._sent.upper_bound() if self._sent else 0
while data:
ds = []
queueableData = data[:1024]
dataRange = halfOpen(lowerBound, lowerBound + len(queueableData))
qd = QueuedData(IntervalSet([dataRange]), lowerBound, ds, [], [],
queueableData)
self._sent.add(dataRange)
lowerBound += len(queueableData)
data = data[1024:]
qds.append(qd)
ds.append(d)
self._enqueue(1, *qds)
return d
def _peerEstablished(self):
self._protocol = self._factory.buildProtocol(self.getPeer())
self._protocol.makeConnection(self)
self._deferred.callback(self._protocol)
self._reads = 'open'
self._writes = 'open'
def _doneWritingAcked(self, when):
self._writes = 'closed'
self._checkBothResolutions()
return when
def loseConnection(self, success=True):
d = defer.Deferred()
d.addCallback(self._doneWritingAcked)
streamEnd = self._ourStreamEnd = self._sent.upper_bound(
) if self._sent else 0
resolution = self._ourResolution = 'success' if success else 'failure'
interval = IntervalSet([halfOpen(streamEnd, streamEnd + 1)])
self._enqueue(
1, QueuedResolution(interval, streamEnd, [d], [], [], resolution))
self._writes = 'closing'
return d
if __name__ == "__main__":
urls = []
while True:
url = raw_input("Enter a Ninja Courses URL here (enter '.' to stop): ")
if url == '.':
break
url = coerce_to_ics(url)
if url:
urls.append(url)
else:
print "ERROR: Could not extract 's' and 't' from the provided URL"
print "Make sure your URL is in the form http://ninjacourses.com/schedule/view/#######/?t=######## or http://ninjacourses.com/schedule/export/schedule.ics?s=#######&t=########"
calendars = [Calendar.from_ical(requests.get(url).text) for url in urls]
events = sum((calendar.walk() for calendar in calendars), [])
events = [event for event in events if isinstance(event, Event) and "RRULE" in event]
events = [event for event in events if event["RRULE"]["FREQ"] == ["WEEKLY"]]
minutes = IntervalSet.between(0, 7200)
for event in events[::-1]:
minutes = minutes - event_to_minutes(event)
dow = "Monday Tuesday Wednesday Thursday Friday".split()
for interval in minutes:
b, e = interval.lower_bound, interval.upper_bound
b = datetime(2013, 10, 7, 0, 0) + timedelta(minutes=b)
e = datetime(2013, 10, 7, 0, 0) + timedelta(minutes=e)
print "You are all free from %s to %s!" % \
(b.strftime("%a %I:%M %p"), e.strftime("%a %I:%M %p"))
header= True
continue
strand= line[strandIdx]
cdsStart= int(line[cdsStartIdx])
cdsEnd= int(line[cdsEndIdx])
# if cdsStart == cdsEnd:
# "No coding sequence in this transcripts, skip it"
# continue
txStart= int(line[txStartIdx])
txEnd= int(line[txEndIdx])
exonStarts= line[exStartsIdx].strip(',').split(',')
exonEnds= line[exEndsIdx].strip(',').split(',')
exon_intervals= []
for s,e in zip(exonStarts, exonEnds):
exon_intervals.append(Interval(int(s), int(e)))
exon_set= IntervalSet(exon_intervals) ## EXAMPLE= exon_set= IntervalSet([Interval(1, 50), Interval(100, 200), Interval(800, 1300)])
txStart_cdsStart= IntervalSet([Interval(txStart, cdsStart)]) ## EXAMPLE= IntervalSet([Interval(1, 1000, lower_closed=True, upper_closed=True)])
txEnd_cdsEnd= IntervalSet([Interval(cdsEnd, txEnd)])
if strand == '+':
utr5= txStart_cdsStart - (txStart_cdsStart - exon_set)
utr3= txEnd_cdsEnd - (txEnd_cdsEnd - exon_set)
if strand == '-':
utr3= txStart_cdsStart - (txStart_cdsStart - exon_set)
utr5= txEnd_cdsEnd - (txEnd_cdsEnd - exon_set)
for x in utr5:
if type(x) == int:
"Skip transcripts which don't have UTR"
continue
name5utr= '_'.join([line[chromIdx], str(x.lower_bound), str(x.upper_bound), line[nameIdx], '5utr'])
utr_list.append(([line[chromIdx], x.lower_bound, x.upper_bound, name5utr, line[nameIdx], line[strandIdx], line[name2Idx]]))
# coding: utf-8
# __author__: ""
from interval import Interval, IntervalSet
volume1 = Interval.between("A", "Foe")
volume2 = Interval.between("Fog", "McAfee")
volume3 = Interval.between("McDonalds", "Space")
volume4 = Interval.between("Spade", "Zygote")
encyclopedia = IntervalSet([volume1, volume2, volume3, volume4])
mySet = IntervalSet([volume1, volume3, volume4])
"Meteor" in encyclopedia
"Goose" in encyclopedia
"Goose" in mySet
volume2 in (encyclopedia ^ mySet)
print [(_.lower_bound, _.upper_bound) for _ in IntervalSet(encyclopedia)]
OfficeHours = IntervalSet.between("08:00", "17:00")
myLunch = IntervalSet.between("11:30", "12:30")
myHours = IntervalSet.between("08:30", "19:30") - myLunch
myHours.issubset(OfficeHours)
"12:00" in myHours
"15:30" in myHours
def score_short_answer(gold_label_list, pred_label):
"""Scores a short answer as correct or not.
1) First decide if there is a gold short answer with SHORT_NO_NULL_THRESHOLD.
2) The prediction will get a match if:
a. There is a gold short answer.
b. The prediction span *set* match exactly with *one* of the non-null gold
short answer span *set*.
Args:
gold_label_list: A list of NQLabel.
pred_label: A single NQLabel.
Returns:
gold_has_answer, pred_has_answer, f1, score
"""
# There is a gold short answer if gold_label_list not empty and non null
# answers is over the threshold (sum over annotators).
gold_has_answer = util.gold_has_short_answer(gold_label_list)
# There is a pred long answer if pred_label is not empty and short answer
# set is not empty.
pred_has_answer = pred_label and (
(not util.is_null_span_list(pred_label.short_answer_span_list))
or pred_label.yes_no_answer != 'none')
f1 = 0
p = 0
r = 0
score = pred_label.short_score
# Both sides have short answers, which contains yes/no questions.
if gold_has_answer and pred_has_answer:
if pred_label.yes_no_answer != 'none': # System thinks its y/n questions.
for gold_label in gold_label_list:
if pred_label.yes_no_answer == gold_label.yes_no_answer:
f1 = 1
p = 1
r = 1
break
else:
for gold_label in gold_label_list:
# if util.span_set_equal(gold_label.short_answer_span_list,
# pred_label.short_answer_span_list):
# is_correct = True
# break
gold_set = IntervalSet([Interval(span.start_token_idx, span.end_token_idx) \
for span in gold_label.short_answer_span_list])
pred_set = IntervalSet([Interval(span.start_token_idx, span.end_token_idx) \
for span in pred_label.short_answer_span_list])
overlap_set = gold_set & pred_set
def calc_len(interval_set):
sum = 0
for span in interval_set:
sum += span.upper_bound - span.lower_bound + 1
return sum
precision = safe_divide(calc_len(overlap_set),
calc_len(pred_set))
recall = safe_divide(calc_len(overlap_set), calc_len(gold_set))
if safe_divide(2 * precision * recall,
precision + recall) > f1:
f1 = safe_divide(2 * precision * recall,
precision + recall)
p = precision
r = recall
elif not gold_has_answer and not pred_has_answer:
f1 = 1
p = 1
r = 1
return gold_has_answer, pred_has_answer, f1, p, r, score
from interval import Interval, IntervalSet r1 = IntervalSet([Interval(1, 1000), Interval(1100, 1200)]) r2 = IntervalSet([Interval(30, 50), Interval(60, 200), Interval(1150, 1300)]) r3 = IntervalSet([Interval(1000, 3000)]) r4 = IntervalSet([Interval(1000, 3000)]) r5 = IntervalSet([Interval(30000, 12000)]) # print (r3 - r4), (r4 - r3), r3 & r4 # print len(IntervalSet.empty()) if r3 & r4 == r4: print 'yes' # print r3 & r4 # if (r3 - r4).is_empty(): # print "true" # print (r3 - r4).empty()
def showMessage(tag, message, sentAt=None):
print tag, message.id, message.previousID, IntervalSet(message.ranges),
print halfOpen(message.dataPos, message.dataPos + len(message.data)),
print len(message.data), sentAt and len(sentAt)
def __init__(self, low, high): """ Initialize a range item with lower bound and higher bound. """ self.items = IntervalSet([Interval(low, high)])
class RangeItem:
items = IntervalSet.empty()
def __init__(self, low, high):
"""
Initialize a range item with lower bound and higher bound.
"""
self.items = IntervalSet([Interval(low, high)])
def __eq__(self, rhs):
"""
Redefine == operator
"""
assert isinstance(rhs, RangeItem)
return self.items == rhs.items
def empty(self):
if len(self.items) == 0:
return True
return False
def dump(self, type='ip'):
if self.empty():
print 'Empty RangeItem'
else:
if type == 'int':
print 'RangeItem %s - %s' % (self.items.lower_bound(), self.items.upper_bound())
elif type == 'ip':
print 'RangeItem %s - %s' % (long2ip(self.items.lower_bound()), long2ip(self.items.upper_bound()))
def produce(self):
return RangeItem(self.items.lower_bound(), self.items.upper_bound())
def overlaps(self, rhs):
"""
If this item is overlapped with the other
"""
if len(self.items & rhs.items) != 0:
return True
return False
def includes(self, rhs):
"""
If this item includes the other
"""
if self.items & rhs.items == rhs.items:
return True
return False
def decouples(self, rhs):
"""
Decouple this item with the other
"""
assert isinstance(rhs, RangeItem)
# print self.items, rhs.items
new_values = [self.items - rhs.items, rhs.items - self.items, self.items & rhs.items]
new_items = []
for value in new_values:
# print value
# print '%s - %s' % str(value.lower_bound()), str(value.upper_bound)
if len(value) != 0:
new_items.append(RangeItem(value.lower_bound(), value.upper_bound()))
else:
empty_range = RangeItem(0, 0)
empty_range.items = IntervalSet.empty()
new_items.append(empty_range)
return new_items
def get_bottom(debug=False):
res = StrenghenedFormula(debug)
res.interval_set = IntervalSet.get_bottom()
return res
