def _add_transaction(self, tr, state):
"""Add the transaction to the transaction storage, thread-safely.
@type tr: AbstractTransaction
@type state: AbstractTransaction.State
"""
logger.debug('__add %r', tr)
with self.__transactions_lock:
assert tr.uuid not in self.__transactions_by_uuid, \
(tr, self.__transactions_by_uuid)
assert tr.uuid not in self.__states_by_uuid, \
(tr, self.__states_by_uuid)
# 1. By UUID
(self.__transactions_by_uuid[tr.uuid],
self.__states_by_uuid[tr.uuid]) = (tr,
state)
# 2. By end time
assert self.__transactions_by_end_time == \
sorted(self.__transactions_by_end_time), \
repr(self.__transactions_by_end_time)
bisect.insort(self.__transactions_by_end_time,
(tr.max_end_time(), tr))
# 3. By class
self.__states_by_class_name[tr.msg_class.name][tr.uuid] = state
self.__assert_invariant()
def doActivate(self, w): # when activating a timer which has already passed, # simply abort the timer. don't run trough all the stages. if w.shouldSkip(): w.state = PowerTimerEntry.StateEnded else: # when active returns true, this means "accepted". # otherwise, the current state is kept. # the timer entry itself will fix up the delay then. if w.activate(): w.state += 1 try: self.timer_list.remove(w) except: print '[PowerManager]: Remove list failed' # did this timer reached the last state? if w.state < PowerTimerEntry.StateEnded: # no, sort it into active list insort(self.timer_list, w) else: # yes. Process repeated, and re-add. if w.repeated: w.processRepeated() w.state = PowerTimerEntry.StateWaiting self.addTimerEntry(w) else: # Remove old timers as set in config self.cleanupDaily(config.recording.keep_timers.value) insort(self.processed_timers, w) self.stateChanged(w)
def add(self, node, parent, priority=0):
"""Adds the specified node with the specified parent.
If the dep is a soft-dep and the node already has a hard
relationship to the parent, the relationship is left as hard."""
if node not in self.nodes:
self.nodes[node] = ({}, {}, node)
self.order.append(node)
if not parent:
return
if parent not in self.nodes:
self.nodes[parent] = ({}, {}, parent)
self.order.append(parent)
priorities = self.nodes[node][1].get(parent)
if priorities is None:
priorities = []
self.nodes[node][1][parent] = priorities
self.nodes[parent][0][node] = priorities
if not priorities or priorities[-1] is not priority:
bisect.insort(priorities, priority)
def makeShapeDifferentTo(self, paramValue):
#make new shape to compare with
[newShape, newParams] = self.shapeModeler.makeRandomShapeFromTriangular(self.params, self.paramsToVary,
self.bounds, [
paramValue]) #USE ONLY FIRST PARAM FOR SELF-LEARNING ALGORITHM ATM
newParamValue = newParams[self.paramsToVary[0] - 1, 0] #USE ONLY FIRST PARAM FOR SELF-LEARNING ALGORITHM ATM
#ensure it is significantly different
numAttempts = 1
while (abs(newParamValue - paramValue) < self.minParamDiff and numAttempts < maxNumAttempts):
[newShape, newParams] = self.shapeModeler.makeRandomShapeFromTriangular(self.params, self.paramsToVary,
self.bounds, [
paramValue]) #USE ONLY FIRST PARAM FOR SELF-LEARNING ALGORITHM ATM
newParamValue = newParams[
self.paramsToVary[0] - 1, 0] #USE ONLY FIRST PARAM FOR SELF-LEARNING ALGORITHM ATM
numAttempts += 1
if (numAttempts >= maxNumAttempts): #couldn't find a 'different' shape in range
print('Oh no!') #this should be prevented by the convergence test below
#store it as an attempt
if (self.doGroupwiseComparison):
bisect.insort(self.params_sorted, newParamValue)
self.shapeToParamsMapping.append(newParams)
return newShape, newParamValue
def doActivate(self, w): self.timer_list.remove(w) # when activating a timer which has already passed, # simply abort the timer. don't run trough all the stages. if w.shouldSkip(): w.state = TimerEntry.StateEnded else: # when active returns true, this means "accepted". # otherwise, the current state is kept. # the timer entry itself will fix up the delay then. if w.activate(): w.state += 1 # did this timer reached the last state? if w.state < TimerEntry.StateEnded: # no, sort it into active list insort(self.timer_list, w) else: # yes. Process repeated, and re-add. if w.repeated: w.processRepeated() w.state = TimerEntry.StateWaiting self.addTimerEntry(w) else: insort(self.processed_timers, w) self.stateChanged(w)
def update(self,records,**kw):
"""Update one record of a list of records
with new keys and values and update indices"""
# ignore unknown fields
kw = dict([(k,v) for (k,v) in kw.items() if k in self.fields])
if isinstance(records,dict):
records = [ records ]
# update indices
for indx in set(self.indices.keys()) & set (kw.keys()):
for record in records:
if record[indx] == kw[indx]:
continue
_id = record["__id__"]
# remove id for the old value
old_pos = bisect.bisect(self.indices[indx][record[indx]],_id)-1
del self.indices[indx][record[indx]][old_pos]
if not self.indices[indx][record[indx]]:
del self.indices[indx][record[indx]]
# insert new value
bisect.insort(self.indices[indx].setdefault(kw[indx],[]),_id)
for record in records:
# update record values
record.update(kw)
# increment version number
record["__version__"] += 1
def _free(self, block):
# free location and try to merge with neighbours
(arena, start, stop) = block
try:
prev_block = self._stop_to_block[(arena, start)]
except KeyError:
pass
else:
start, _ = self._absorb(prev_block)
try:
next_block = self._start_to_block[(arena, stop)]
except KeyError:
pass
else:
_, stop = self._absorb(next_block)
block = (arena, start, stop)
length = stop - start
try:
self._len_to_seq[length].append(block)
except KeyError:
self._len_to_seq[length] = [block]
bisect.insort(self._lengths, length)
self._start_to_block[(arena, start)] = block
self._stop_to_block[(arena, stop)] = block
def create_index(self,*fields):
"""Create an index on the specified field names
An index on a field is a mapping between the values taken by the field
and the sorted list of the ids of the records whose field is equal to
this value
For each indexed field, an attribute of self is created, an instance
of the class Index (see above). Its name it the field name, with the
prefix _ to avoid name conflicts
"""
reset = False
for f in fields:
if not f in self.fields:
raise NameError("%s is not a field name %s" %(f,self.fields))
# initialize the indices
if self.mode == "open" and f in self.indices:
continue
reset = True
self.indices[f] = {}
for _id,record in self.records.items():
# use bisect to quickly insert the id in the list
bisect.insort(self.indices[f].setdefault(record[f],[]),
_id)
# create a new attribute of self, used to find the records
# by this index
setattr(self,'_'+f,Index(self,f))
if reset:
self.commit()
def insert(self,*args,**kw):
"""Insert a record in the database
Parameters can be positional or keyword arguments. If positional
they must be in the same order as in the create() method
If some of the fields are missing the value is set to None
Returns the record identifier
"""
if args:
kw = dict([(f,arg) for f,arg in zip(self.fields,args)])
# initialize all fields to None
record = dict([(f,None) for f in self.fields])
# raise exception if unknown field
for key in kw:
if not key in self.fields:
raise NameError("Invalid field name : %s" %key)
# set keys and values
for (k,v) in kw.items():
record[k]=v
# add the key __id__ : record identifier
record['__id__'] = self.next_id
# add the key __version__ : version number
record['__version__'] = 0
# create an entry in the dictionary self.records, indexed by __id__
self.records[self.next_id] = record
# update index
for ix in self.indices.keys():
bisect.insort(self.indices[ix].setdefault(record[ix],[]),
self.next_id)
# increment the next __id__
self.next_id += 1
return record['__id__']
def add_global_handler(self, event, handler, priority=0):
"""Adds a global handler function for a specific event type.
Arguments:
event -- Event type (a string). Check the values of the
numeric_events dictionary in irclib.py for possible event
types.
handler -- Callback function.
priority -- A number (the lower number, the higher priority).
The handler function is called whenever the specified event is
triggered in any of the connections. See documentation for
the Event class.
The handler functions are called in priority order (lowest
number is highest priority). If a handler function returns
\"NO MORE\", no more handlers will be called.
"""
if not event in self.handlers:
self.handlers[event] = []
bisect.insort(self.handlers[event], ((priority, handler)))
def _run_timeouts(self):
now = time.time()
while len(self._scheduled_timeouts) > 0 and self._scheduled_timeouts[0][0] < now:
t, t_id = self._scheduled_timeouts.pop(0)
info = self._timeouts.get(t_id, None)
if info is None:
continue
try:
ret = info['func'](*info['args'])
# Only reschedule if returning True
if not ret:
self.timeout_remove(t_id)
continue
# Reschedule
now2 = time.time()
delta = (now2 - t) % info['delay']
t_new = now2 + info['delay'] - delta
logger.debug('Rescheduling timeout %d for %s', t_id, t_new - now2)
bisect.insort(self._scheduled_timeouts, (t_new, t_id))
except Exception, e:
logger.error('Timeout call %d failed: %s', t_id, str(e))
self.timeout_remove(t_id)
def suppressFire_callback(channel,flag=0):
# print GPIO.input(channel), channel
# print GPIO.input(24), 24
# print 'callback'
if not flag:
print 'UVTron just told me about a fire!', GPIO.input(channel)
x,y = nan, nan
while isnan(x) or isnan(y):
print 'no fire in frame yet'
# Need to do some sort of check to filter out random spikes here
ti = time.time()
while GPIO.input(channel):
print 'signal went high for some reason'
if time.time() - ti >= 0.01:
print "Something may be wrong with the UVTron signal"
return
# FireImage = abs(average(ImQueue[-1],-1) - average(ImQueue[0],-1))
print 'grabbing an image'
FireImage = average(ImQueue[0],-1)
x,y = findFire(FireImage)
# print x,y
# fo = '-'.join(map(str, datetime.now().timetuple()[:6]))
# imwrite('fire'+fo+'.bmp',FireImage)
xdivtmp, ydivtmp = xdivs[:], ydivs[:]
bisect.insort(xdivtmp,x) # Insert the fire coordinates into the protection grid
bisect.insort(ydivtmp,y)
xzone = xdivtmp.index(x) - 1 # Find the grid coordinates
yzone = ydivtmp.index(y) - 1
# print 'fire seen in %d,%d' % (xzone,yzone)
del xdivtmp, ydivtmp
print 'putting out fire'
firePorts((xzone,yzone))
print 'Fire at (%.2f,%.2f) in zone %d,%d\nFiring ports %d & %d' % ((x,y,xzone,yzone,) + fireDict[(xzone,yzone)])
def register_middleware(level, middleware, quiet=False):
"""
Register a middleware for HTTP requests.
In addition to standard WSGI entries,
the ``environ`` passed to middlewares will include:
* ``rdfrest.resource``: the requested resource; may be None
* ``rdfrest.requested.uri``: the URI (as an ``rdflib.URIRef``)
requested by the client, without its extension (see below)
* ``rdfrest.requested.extension``: the requested extension; may be ``""``
:param level: a level governing the order of execution of pre-processors;
predefined levels are AUTHENTICATION, AUTHORIZATION
:param middleware: a function accepting a WSGI application,
and producing a WSGI application wrapping the former
"""
if middleware in ( i[1] for i in _MIDDLEWARE_REGISTRY ):
if not quiet:
raise ValueError("middleware already registered")
else:
return
insort(_MIDDLEWARE_REGISTRY, (level, middleware))
global _MIDDLEWARE_STACK_VERSION
_MIDDLEWARE_STACK_VERSION += 1
def add(self, time, event, absolute_time=False):
"""
Adds an event to the schedule.
Events are inserted so that the schedule is always chronologically
sorted.
Parameters
----------
time : float
The time at which the event takes place
event : dict
The properties of the event
absolute_time : bool, optional
Specifies whether the time is expressed as absolute time or as
interval from the latest event of the schedule
"""
# When a new event is inserted, make sure it is inserted in
# chronological order
if absolute_time and time < 0:
raise ValueError('Time must be a positive value')
if absolute_time:
if time < self.attrib['t_end']:
bisect.insort(self.event, (time, event)) # maintain list sorted
return
self.attrib['t_end'] = time
else:
self.attrib['t_end'] += time
time = self.attrib['t_end']
self.event.append((time, event))
def get_service(self, bundle, reference):
"""
Retrieves the service corresponding to the given reference
:param bundle: The bundle requiring the service
:param reference: A service reference
:return: The requested service
:raise BundleException: The service could not be found
"""
with self.__svc_lock:
# Be sure to have the instance
try:
service = self.__svc_registry[reference]
# Indicate the dependency
imports = self.__bundle_imports.setdefault(bundle, [])
bisect.insort(imports, reference)
reference.used_by(bundle)
return service
except KeyError:
# Not found
raise BundleException("Service not found (reference: {0})"
.format(reference))
def init_allocations():
global machine_allocation_dict
global allocations_list
global node_manager
global usages_list
global machine_usage_dict
logger.info("init allocations:")
machines = node_manager.get_allnodes()
for machine in machines:
allocation = AllocationOfMachine()
allocation.machineid = machine
allocation.resources = 2
allocation.reliable_resources_allocation_summary = 0
allocation.reliable_allocations = []
allocation.restricted_allocations = []
machine_allocation_dict[machine] = allocation
bisect.insort(allocations_list,allocation)
usage_of_machine = {}
usage_of_machine['machineid']=machine
usage_of_machine['cpu_utilization']=0.1
usages_list.append(usage_of_machine)
machine_usage_dict[machine] = 0.1
def doActivate(self, w): # when activating a timer which has already passed, # simply abort the timer. don't run trough all the stages. if w.shouldSkip(): w.state = RecordTimerEntry.StateEnded else: # when active returns true, this means "accepted". # otherwise, the current state is kept. # the timer entry itself will fix up the delay then. if w.activate(): w.state += 1 self.timer_list.remove(w) # did this timer reached the last state? if w.state < RecordTimerEntry.StateEnded: # no, sort it into active list insort(self.timer_list, w) else: # yes. Process repeated, and re-add. if w.repeated: w.processRepeated() w.state = RecordTimerEntry.StateWaiting w.first_try_prepare = True self.addTimerEntry(w) else: # correct wrong running timers self.checkWrongRunningTimers() # check for disabled timers, if time as passed set to completed self.cleanupDisabled() # Remove old timers as set in config self.cleanupDaily(config.recording.keep_timers.value) insort(self.processed_timers, w) self.stateChanged(w)
def _add_free_block(self, block):
# make block available and try to merge with its neighbours in the arena
(arena, start, stop) = block
try:
prev_block = self._stop_to_block[(arena, start)]
except KeyError:
pass
else:
start, _ = self._absorb(prev_block)
try:
next_block = self._start_to_block[(arena, stop)]
except KeyError:
pass
else:
_, stop = self._absorb(next_block)
block = (arena, start, stop)
length = stop - start
try:
self._len_to_seq[length].append(block)
except KeyError:
self._len_to_seq[length] = [block]
bisect.insort(self._lengths, length)
self._start_to_block[(arena, start)] = block
self._stop_to_block[(arena, stop)] = block
def mergeFiles():
SOURCE = "E:/Data/Stats/source/IntraDay/Nifty50_2017-10/"
DEST = SOURCE + "OUTPUT/"
if not os.path.exists(DEST):
os.makedirs(DEST)
ScripMap = {}
csvs = (file for file in os.listdir(SOURCE)
if os.path.isfile(os.path.join(SOURCE, file)))
for csv in csvs:
split = csv.split('_')
scrip = split[0]
if( split[1] == 'F1' or split[1] == 'F2'):
scrip += '_' + split[1]
if( scrip not in ScripMap ):
ScripMap[scrip] = [os.path.join(SOURCE, csv)]
else:
a = ScripMap[scrip]
bisect.insort( a, os.path.join(SOURCE, csv) )
for scrip in ScripMap :
print( scrip )
outputFile = open(DEST + scrip + '.txt', 'w')
inputFiles = ScripMap[scrip]
for file in inputFiles :
fileO = open(file,'r')
outputFile.write( fileO.read() )
fileO.close()
outputFile.close()
def addEvidence(self,data,label): """ Adds to the set of training data. If the anchor lists were sorted before the call to this method, the new data will be inserted into the anchor lists using 'bisect.insort' data - a numpy array, either a single point (1D) or a set of points (2D) label - the label for data. A single value, or a list of values in the same order as the points in data. """ if len(data.shape)==1: new_idx = len(self.training_data) self.training_data.append(data) self.data_classes[new_idx] = label if self.is_sorted: for a in self.anchors: dist = self.distfun(data,self.training_data[a]) bisect.insort(self.data_by_anchors[a],(dist,new_idx)) elif len(data.shape)>1: for i in range(len(data)): thing = data[i] new_idx = len(self.training_data) self.training_data.append(thing) self.data_classes[new_idx] = label[i] if self.is_sorted: for a in self.anchors: dist = self.distfun(thing,self.training_data[a]) bisect.insort(self.data_by_anchors[a],(dist,new_idx))
def registerPlugin(self, plugin):
"""Registers a plugin."""
plugin_types = {
'presence': IPresencePlugin,
'chat': IChatPlugin,
'population': IPopulationPlugin,
'base_plugin': BaseInterface,
}
# Everything is, at least, a base plugin.
valid_types = set(['baseplugin'])
# Loop through the types of plugins and check for implentation
# of each.
claimed_compliances = list(implementedBy(type(plugin)))
# Can we use this as a map instead?
for t, interface in plugin_types.iteritems():
if interface in claimed_compliances:
try:
verifyObject(interface, plugin)
except DoesNotImplement:
log.error('Plugin %s claims to be a %s, but is not!',
plugin.name, t)
else:
# If the above succeeded, then `plugin` implements
# `interface`.
insort(self.plugins[t], plugin)
valid_types.add(t)
plugin.setup(self)
log.info('registered plugin %s as %s', plugin.name, valid_types)
def _add_contexts(self, all_messages):
"""Add source lines to the messages.
This method has the desired side effect that messages that only differ in line
number but that do have the same source line, will be considered identical.
Parameters
----------
all_messages : Set([]) of Message instances
All errors encountered in the current branch.
"""
# 1) Collect all messages in a dictionary, where filenames are keys and values
# are sorted lists of messages. Only messages with a filename and a line number
# must be included.
mdict = {}
for message in all_messages:
if message.filename is not None and message.lineno is not None:
l = mdict.setdefault(message.filename, [])
bisect.insort(l, message)
# 2) Loop over all files and collect some source context for each message
for filename, file_messages in mdict.iteritems():
with open(filename) as source_file:
lines = source_file.readlines()
for message in file_messages:
all_messages.discard(message)
# The context starts three lines before the line and ends three lines
# after.
context = "".join(lines[max(0, message.lineno - 3) : min(len(lines), message.lineno + 4)])
all_messages.add(message.add_context(context))
def cycle(self, interval, function, *args, **kwargs):
"""
Schedule a cycle.
A cycle is a deferred that is continuously rescheduled. It will
be run at regular intervals.
Returns a callable which can be used to cancel the cycle.
========= ============
Argument Description
========= ============
interval The interval, in seconds, at which the cycle should be run.
function The callable to be executed when the cycle is run.
args The positional arguments to be passed to the callable.
kwargs The keyword arguments to be passed to the callable.
========= ============
"""
if interval <= 0:
raise ValueError("Interval must be greater than 0 seconds.")
cycle = functools.partial(function, *args, **kwargs)
timer = _Timer(self, cycle, True, interval, self.time + interval)
bisect.insort(self._deferreds, timer)
return timer
def __get_rsync_backups (self):
# get rsync backup dir
self.rsyncsmf = rsyncsmf.RsyncSMF("%s:rsync" %(plugin.PLUGINBASEFMRI))
rsyncBaseDir = self.rsyncsmf.get_target_dir()
sysName,nodeName,rel,ver,arch = os.uname()
self.rsyncDir = os.path.join(rsyncBaseDir,
rsyncsmf.RSYNCDIRPREFIX,
nodeName)
if not os.path.exists(self.rsyncDir):
return
rootBackupDirs = []
for root, dirs, files in os.walk(self.rsyncDir):
if '.time-slider' in dirs:
dirs.remove('.time-slider')
backupDir = os.path.join(root, rsyncsmf.RSYNCDIRSUFFIX)
if os.path.exists(backupDir):
insort(rootBackupDirs, os.path.abspath(backupDir))
for dirName in rootBackupDirs:
os.chdir(dirName)
for d in os.listdir(dirName):
if os.path.isdir(d) and not os.path.islink(d):
s1 = dirName.split ("%s/" % self.rsyncDir, 1)
s2 = s1[1].split ("/%s" % rsyncsmf.RSYNCDIRSUFFIX, 1)
fs = s2[0]
rb = RsyncBackup ("%s/%s" %(dirName, d),
self.rsyncDir,
fs,
d,
os.stat(d).st_mtime)
self.rsynced_backups.append (rb)
def defer(self, delay, function, *args, **kwargs):
"""
Schedule a deferred.
A deferred is a function (or other callable) that is executed
after a certain amount of time has passed.
Returns a callable which can be used to cancel the deferred.
========= =====================================================
Argument Description
========= =====================================================
delay The delay, in seconds, after which the deferred
should be run.
function The callable to be executed when the deferred is run.
args The positional arguments to be passed to the
callable.
kwargs The keyword arguments to be passed to the callable.
========= =====================================================
"""
if delay <= 0:
raise ValueError("Delay must be greater than 0 seconds.")
deferred = functools.partial(function, *args, **kwargs)
timer = _Timer(self, deferred, False, delay, self.time + delay)
bisect.insort(self._deferreds, timer)
return timer
def find_shortest_path(graph, start, end, parent, visited, distance):
import bisect
current = start
candidates = list()
while current and current != end:
neighbours = graph[current]
for neighbour, neighbour_dist in neighbours:
if visited.has_key(neighbour):
continue
if distance[current] + neighbour_dist < distance[neighbour]:
if ((distance[neighbour], neighbour) in candidates):
candidates.remove((distance[neighbour], neighbour))
distance[neighbour] = distance[current] + neighbour_dist
bisect.insort(candidates, (distance[neighbour], neighbour))
parent[neighbour] = current
visited[current] = True
current = None
if len(candidates) > 0:
_, current = candidates.pop(0)
def scan_commands(self):
self._commands = []
if self.mod == None:
return
dic = self.mod.__dict__
if '__root__' in dic:
root = dic['__root__']
else:
root = []
for k in dic:
if k.startswith('_') or k in root:
continue
item = dic[k]
if type(item) == types.FunctionType:
bisect.insort(self._commands, method.Method(item, k, self))
elif type(item) == types.ModuleType and utils.is_commander_module(item):
mod = Module(k, item, self)
bisect.insort(self._commands, mod)
# Insert root functions into this module
for r in mod.roots():
bisect.insert(self._commands, r)
def update(self,record,**kw):
"""Update the record with new keys and values and update indices"""
# validate all the keys and values before updating
for k,v in kw.iteritems():
self._validate(k,v)
# update record values
for k,v in kw.iteritems():
record[k] = v
# if there is no index to update, stop here
if not self.indices.keys():
return
# update index
# get previous version of the record (same __id__)
_id = record['__id__']
try:
old_rec = self.records[_id]
except KeyError:
raise KeyError,"No record with __id__ %s" %_id
# change indices
for indx in self.indices.keys():
if old_rec[indx] == record[indx]:
continue
# remove id for the old value
old_pos = bisect.bisect(self.indices[indx][old_rec[indx]],_id)-1
del self.indices[indx][old_rec[indx]][old_pos]
# insert new value
bisect.insort(self.indices[indx].setdefault(record[indx],[]),_id)
def add(self, child, z=0, name=None ):
"""Adds a child to the container
:Parameters:
`child` : CocosNode
object to be added
`z` : float
the z index of self
`name` : str
Name of the child
:rtype: `CocosNode` instance
:return: self
"""
# child must be a subclass of supported_classes
#if not isinstance( child, self.supported_classes ):
# raise TypeError("%s is not instance of: %s" % (type(child), self.supported_classes) )
if name:
if name in self.children_names:
raise Exception("Name already exists: %s" % name )
self.children_names[ name ] = child
child.parent = self
elem = z, child
bisect.insort( self.children, elem )
if self.is_running:
child.on_enter()
return self
def add_many(self, edges):
"""Add multiple edges."""
for src,dest in edges:
if dest not in self._outgoing[src]:
insort(self._outgoing[src], dest)
insort(self._incoming[dest], src)
def add(self, aCard: Card) -> None:
bisect.insort(self.cards, aCard)
def add_row(self, member):
if not self.have_member(member):
bisect.insort(self.items, member)
self.refresh_collisions()
self.modelReset.emit()
def _initializeLocationCache(self):
"""
CGD uses Faldo ontology for locations, it's a bit complicated.
This function sets up an in memory cache of all locations, which
can be queried via:
locationMap[build][chromosome][begin][end] = location["_id"]
"""
# cache of locations
self._locationMap = {}
locationMap = self._locationMap
triples = self._rdfGraph.triples
Ref = rdflib.URIRef
associations = []
for subj, _, _ in triples((None, RDF.type, Ref(ASSOCIATION))):
associations.append(subj.toPython())
locationIds = []
for association in associations:
for _, _, obj in triples((Ref(association),
Ref(HAS_SUBJECT), None)):
locationIds.append(obj.toPython())
locations = []
for _id in locationIds:
location = {}
location["_id"] = _id
for subj, predicate, obj in triples((Ref(location["_id"]),
None, None)):
if not predicate.toPython() in location:
location[predicate.toPython()] = []
bisect.insort(location[predicate.toPython()], obj.toPython())
if FALDO_LOCATION in location:
locations.append(location)
for location in locations:
for _id in location[FALDO_LOCATION]:
# lookup faldo region, ensure positions are sorted
faldoLocation = {}
faldoLocation["_id"] = _id
for subj, predicate, obj in triples((Ref(faldoLocation["_id"]),
None, None)):
if not predicate.toPython() in faldoLocation:
faldoLocation[predicate.toPython()] = []
bisect.insort(faldoLocation[predicate.toPython()],
obj.toPython())
faldoBegins = []
for _id in faldoLocation[FALDO_BEGIN]:
faldoBegin = {}
faldoBegin["_id"] = _id
for subj, predicate, obj in triples(
(Ref(faldoBegin["_id"]),
None, None)):
faldoBegin[predicate.toPython()] = obj.toPython()
faldoBegins.append(faldoBegin)
faldoReferences = []
for _id in faldoLocation[FALDO_BEGIN]:
faldoReference = {}
faldoReference["_id"] = faldoBegin[FALDO_REFERENCE]
for subj, predicate, obj in triples(
(Ref(faldoReference["_id"]),
None, None)):
faldoReference[predicate.toPython()] = obj.toPython()
faldoReferences.append(faldoReference)
faldoEnds = []
for _id in faldoLocation[FALDO_END]:
faldoEnd = {}
faldoEnd["_id"] = _id
for subj, predicate, obj in triples((Ref(faldoEnd["_id"]),
None, None)):
faldoEnd[predicate.toPython()] = obj.toPython()
faldoEnds.append(faldoEnd)
for idx, faldoReference in enumerate(faldoReferences):
if MEMBER_OF in faldoReference:
build = faldoReference[MEMBER_OF].split('/')[-1]
chromosome = faldoReference[LABEL].split(' ')[0]
begin = faldoBegins[idx][FALDO_POSITION]
end = faldoEnds[idx][FALDO_POSITION]
if build not in locationMap:
locationMap[build] = {}
if chromosome not in locationMap[build]:
locationMap[build][chromosome] = {}
if begin not in locationMap[build][chromosome]:
locationMap[build][chromosome][begin] = {}
if end not in locationMap[build][chromosome][begin]:
locationMap[build][chromosome][begin][end] = {}
locationMap[build][chromosome][begin][end] = \
location["_id"]
locationMap[location["_id"]] = {
"build": build,
"chromosome": chromosome,
"begin": begin,
"end": end,
}
def append(self, item):
bisect.insort(self.A, (self.f(item), item))
output_file.write(CMTE_ID + '|' + ZIP_CODE + '|' +
transaction_year + '|' +
str(int(round(TRANSACTION_AMT))) + '|' +
str(int(round(TRANSACTION_AMT))) + '|' + '1\n')
else:
'''
donation_array = stores all donations from repeat donors with a given tuple = (CMTE_ID, ZIP_CODE, transaction_year)
total_amount = sum of the donation array
'''
donation_array, total_amount = repeat_donors[(CMTE_ID, ZIP_CODE,
transaction_year)]
total_amount += TRANSACTION_AMT #total amount of money donated is increased by the next donation
bisect.insort(
donation_array, TRANSACTION_AMT
) #a new value is inserted and the array is sorted again.
repeat_donors[(CMTE_ID, ZIP_CODE,
transaction_year)] = [donation_array, total_amount]
percentile = donation_array[int(
math.ceil(PERCENTILE_number * len(donation_array))) -
1] #percentile value is calculated
output_file.write(CMTE_ID + '|' + ZIP_CODE + '|' +
transaction_year + '|' +
str(int(round(percentile))) + '|' +
str(int(round(total_amount))) + '|' +
str(len(donation_array)) + '\n')
input_file.close()
output_file.close()
""" heaps : running mean
author : Woongwon Lee
problem : https://www.hackerrank.com/challenges/ctci-find-the-running-median/problem
output : After each new integer is added to the list, print the list's updated
median on a new line as a single double-precision number scaled to 1 decimal
place (i.e., 12.3 format).
"""
import bisect
n = int(input().strip())
list_i = []
a_i = 0
for a_i in range(n):
a_t = int(input().strip())
bisect.insort(list_i, a_t)
if len(list_i) % 2 == 0:
mid_left = len(list_i) // 2 - 1
mid_right = len(list_i) // 2
print('{:0.1f}'.format((list_i[mid_left] + list_i[mid_right]) / 2))
else:
mid = len(list_i) // 2
print('{:0.1f}'.format(list_i[mid]))
lib_dict0[name_dict0.get(name, name)].append(lib_dict[name])
p_mz_rt = []
for name, dat in all_dat0.items():
if name.startswith('ISF of'):
pmz, rt = [float(x) for x in re.findall("\d+\.\d+", name)[:2]]
else:
pmz = dat[0][2]
rt = statistics.median(rt for _, _, _, rt, _, _, _, _ in dat)
pos0 = bisect_left(p_mz_rt, (pmz - .01, ))
pos1 = bisect_left(p_mz_rt, (pmz + .01, ), lo=pos0)
for x in p_mz_rt[pos0:pos1]:
if abs(rt - x[1]) < RT_shift:
break
else:
bisect.insort(p_mz_rt, (pmz, rt))
mz_rt_name = collections.defaultdict(list)
for name, dat in all_dat0.items():
if name.startswith('ISF of'):
pmz, rt = [float(x) for x in re.findall("\d+\.\d+", name)[:2]]
else:
pmz = dat[0][2]
rt = statistics.median(rt for _, _, _, rt, _, _, _, _ in dat)
pos0 = bisect_left(p_mz_rt, (pmz - .01, ))
pos1 = bisect_left(p_mz_rt, (pmz + .01, ), lo=pos0)
for n, x in enumerate(p_mz_rt[pos0:pos1], pos0):
if abs(rt - x[1]) < RT_shift:
mz_rt_name[n].append(name)
break
else:
def add(self, album):
bisect.insort(self.list, album)
list(reversed(weekdays)) # sort a list in place (modifies but does not return the list) simpsons.sort() simpsons.sort(reverse=True) # sort in reverse simpsons.sort(key=len) # sort by a key # return a sorted list (does not modify the original list) sorted(simpsons) sorted(simpsons, reverse=True) sorted(simpsons, key=len) # insert into an already sorted list, and keep it sorted num = [10, 20, 40, 50] from bisect import insort insort(num, 30) # create a second reference to the same list same_num = num same_num[0] = 0 # modifies both 'num' and 'same_num' # copy a list (two ways) new_num = num[:] new_num = list(num) # examine objects num is same_num # returns True (checks whether they are the same object) num is new_num # returns False num == same_num # returns True (checks whether they have the same contents) num == new_num # returns True
def schedule_slot(self, time_slot):
if self.check_availability(time_slot):
raise ScheduleError('DERP! We messed up the scheduling!')
bisect.insort(self.time_slots, time_slot)
def run(self):
socket = QtNetwork.QTcpSocket()
if not socket.setSocketDescriptor(self.socketId):
self.error.emit(socket.error())
return
while socket.state() == QtNetwork.QAbstractSocket.ConnectedState:
nextBlockSize = 0
stream = QtCore.QDataStream(socket)
stream.setVersion(QtCore.QDataStream.Qt_4_2)
if (socket.waitForReadyRead()
and socket.bytesAvailable() >= SIZEOF_UINT16):
nextBlockSize = stream.readUInt16()
else:
self.sendError(socket, "Cannot read client request")
return
if socket.bytesAvailable() < nextBlockSize:
if (not socket.waitForReadyRead(60000)
or socket.bytesAvailable() < nextBlockSize):
self.sendError(socket, "Cannot read client data")
return
action = stream.readQString()
date = QtCore.QDate()
if action in ("BOOK", "UNBOOK"):
room = stream.readQString()
stream >> date
try:
Thread.lock.lockForRead()
bookings = Bookings.get(date.toPython())
finally:
Thread.lock.unlock()
uroom = room
if action == "BOOK":
newlist = False
try:
Thread.lock.lockForRead()
if bookings is None:
newlist = True
finally:
Thread.lock.unlock()
if newlist:
try:
Thread.lock.lockForWrite()
bookings = Bookings[date.toPython()]
finally:
Thread.lock.unlock()
error = None
insert = False
try:
Thread.lock.lockForRead()
if len(bookings) < MAX_BOOKINGS_PER_DAY:
if uroom in bookings:
error = "Cannot accept duplicate booking"
else:
insert = True
else:
error = "{} is fully booked".format(
date.toString(QtCore.Qt.ISODate))
finally:
Thread.lock.unlock()
if insert:
try:
Thread.lock.lockForWrite()
bisect.insort(bookings, uroom)
finally:
Thread.lock.unlock()
self.sendReply(socket, action, room, date)
else:
self.sendError(socket, error)
elif action == "UNBOOK":
error = None
remove = False
try:
Thread.lock.lockForRead()
if bookings is None or uroom not in bookings:
error = "Cannot unbook nonexistent booking"
else:
remove = True
finally:
Thread.lock.unlock()
if remove:
try:
Thread.lock.lockForWrite()
bookings.remove(uroom)
finally:
Thread.lock.unlock()
self.sendReply(socket, action, room, date)
else:
self.sendError(socket, error)
else:
self.sendError(socket, "Unrecognized request")
socket.waitForDisconnected()
try:
Thread.lock.lockForRead()
printBookings()
finally:
Thread.lock.unlock()
def add_result(sm):
if -sm not in res:
insort(res, -sm)
if len(res) > 3:
res.pop()
import bisect
import random
# Sorting is expensive, so once you have a sorted sequence, it's good to keep it that way. That is why bisect.insort was created.
# insort(seq, item) inserts item into seq so as to keep seq in ascending order.
SIZE = 7
random.seed(1729)
my_list = []
for i in range(SIZE):
new_item = random.randrange(SIZE * 2)
bisect.insort(my_list, new_item)
print('%2d ->' % new_item, my_list)
def _push_entry(backend, entry):
insort(backend, entry)
print("")
from array import array
a = array('H', [4000, 10, 700, 22222])
print(sum(a))
print(a[1:3])
from collections import deque
d = deque(["task1", "task2", "task3"])
d.append("task4")
print("Handling", d.popleft())
import bisect
scores = [(100, 'perl'), (200, 'tcl'), (400, 'lua'), (500, 'python')]
bisect.insort(scores, (300, 'ruby'))
print(scores)
from heapq import heapify, heappop, heappush
data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0]
heapify(data) # rearrange the list into heap order
heappush(data, -5) # add a new entry
print([heappop(data) for i in range(3)]) # fetch the three smallest entries
print('')
from decimal import *
print(round(.70 * 1.05, 2))
print(round(Decimal('0.70') * Decimal('1.05'), 2))
getcontext().prec = 36
def step(self, policy_action=None):
assert not self.done, 'Trying to step on an exhausted environment!'
if self.normalise_action and policy_action is not None:
np.multiply(policy_action, self.data_stats['a_std'],
policy_action) # multiply by the std
np.add(policy_action, self.data_stats['a_mean'],
policy_action) # add the mean
df = self.df
now = df['Frame ID'] == self.frame
vehicles = set(
df[now]['Vehicle ID']) - self.vehicles_history - self._black_list[
self._t_slot]
if vehicles:
now_and_on = df['Frame ID'] >= self.frame
for vehicle_id in vehicles:
this_vehicle = df['Vehicle ID'] == vehicle_id
car_df = df[this_vehicle & now_and_on]
if len(car_df) < self.smoothing_window + 1: continue
f = self.font[20] if self.display else None
car = self.EnvCar(car_df,
self.offset,
self.look_ahead,
self.screen_size[0],
f,
self.smoothing_window,
dt=self.delta_t)
self.vehicles.append(car)
if self.controlled_car and \
not self.controlled_car['locked'] and \
self.frame >= self.controlled_car['frame'] and \
(self.controlled_car['v_id'] is None or vehicle_id == self.controlled_car['v_id']):
self.controlled_car['locked'] = car
car.is_controlled = True
car.buffer_size = self.nb_states
car.lanes = self.lanes
car.look_ahead = self.look_ahead
# print(f'Controlling car {car.id}')
# self.dump_folder = f'{self._t_slot}_{car.id}'
# print(f'Creating folder {self.dump_folder}')
# system(f'mkdir -p screen-dumps/{self.dump_folder}')
if self.store_sim_video:
self.ghost = self.EnvCar(car_df,
self.offset,
self.look_ahead,
self.screen_size[0],
f,
self.smoothing_window,
dt=self.delta_t)
self.vehicles_history |= vehicles # union set operation
self.lane_occupancy = [[] for _ in range(7)]
if self.show_frame_count:
print(f'\r[t={self.frame}]', end='')
for v in self.vehicles[:]:
if v.off_screen:
# print(f'vehicle {v.id} [off screen]')
if self.state_image and self.store:
file_name = os.path.join(self.data_dir, self.DUMP_NAME,
os.path.basename(self._t_slot))
print(f'[dumping {v} in {file_name}]')
v.dump_state_image(file_name, 'tensor')
self.vehicles.remove(v)
else:
# Insort it in my vehicle list
lane_idx = v.current_lane
assert v.current_lane < self.nb_lanes, f'{v} is in lane {v.current_lane} at frame {self.frame}'
bisect.insort(self.lane_occupancy[lane_idx], v)
if self.state_image or self.controlled_car and self.controlled_car[
'locked']:
# How much to look far ahead
look_ahead = MAX_SPEED * 1000 / 3600 * self.SCALE
look_sideways = 2 * self.LANE_W
self.render(mode='machine',
width_height=(2 * look_ahead, 2 * look_sideways),
scale=0.25)
for v in self.vehicles:
# Generate symbolic state
lane_idx = v.current_lane
left_vehicles = self._get_neighbours(lane_idx, -1, v) \
if 0 < lane_idx < 6 or lane_idx == 6 and v.front[0] > 18 * LANE_W else None
mid_vehicles = self._get_neighbours(lane_idx, 0, v)
right_vehicles = self._get_neighbours(lane_idx, + 1, v) \
if lane_idx < 5 or lane_idx == 5 and v.front[0] > 18 * LANE_W else None
state = left_vehicles, mid_vehicles, right_vehicles
# Sample an action based on the current state
action = v.policy() if not v.is_autonomous else policy_action
# Perform such action
v.step(action)
# Store state and action pair
if (self.store or v.is_controlled) and v.valid:
v.store('state', state)
v.store('action', action)
if v.is_controlled and v.valid:
v.count_collisions(state)
if v.collisions_per_frame > 0: self.collision = True
# # Create set of off track vehicles
# if v._colour[0] > 128: # one lane away
# if v.id not in self.off_track:
# print(f'Adding {v} to off_track set and saving it to disk')
# self.off_track.add(v.id)
# with open('off_track.pkl', 'wb') as f:
# pickle.dump(self.off_track, f)
# # Point out accidents (as in tracking bugs) in original trajectories
# if self.frame == self.accident['frame']:
# if v.id in self.accident['cars']:
# v.collisions_per_frame = 1
# self.collision = True
# if self.frame == self.accident['frame']:
# print('Colliding vehicles:', self.accident['cars'])
# self.accident = self.get_next_accident()
# Keep the ghost updated
if self.store_sim_video:
if self.ghost and self.ghost.off_screen: self.ghost = None
if self.ghost: self.ghost.step(self.ghost.policy())
self.frame += int(self.delta_t * 10)
# Run out of frames?
self.done = self.frame >= self.max_frame or self.user_is_done
if self.controlled_car and self.controlled_car['locked']:
return_ = self.controlled_car['locked'].get_last(
n=self.nb_states,
done=self.done,
norm_state=self.normalise_state and self.data_stats,
return_reward=self.return_reward,
gamma=self.gamma,
)
if return_: return return_
# return observation, reward, done, info
return None, None, self.done, None
next_is_unicode = False
for entry in line.split('"'):
if next_is_glyph_name:
glyph_name = entry.replace('-', '_')
next_is_glyph_name = False
if next_is_unicode:
code = entry.replace(';', '').replace('&', '').replace(
'#', '').replace('x', '')
next_is_unicode = False
if 'glyph-name=' in entry:
next_is_glyph_name = True
if 'unicode=' in entry:
next_is_unicode = True
if code != '' and glyph_name != '':
t = (glyph_name, code, svg)
bisect.insort(glyphs, t)
in_file.close()
# read fontawesomehash.cpp and write updated version to fontawesomehash.cpp_
cpp_file = os.path.join(project_path, 'src/lib/fontawesomehash.cpp')
cpp_file_tmp = cpp_file + '_'
in_file = open(cpp_file, 'r')
out_file = open(cpp_file_tmp, 'w')
in_auto = False
glyph_names_added = []
fa_var_prefix = 'fa_'
print('Upgrading %s...' % cpp_file, end='')
for line in in_file.readlines():
if '// START AUTO-GENERATED' in line:
in_auto = True
def write(self, entry, filename):
self.out = StringIO()
super(SortedCDXWriter, self).write(entry, filename)
line = self.out.getvalue()
if line:
insort(self.sortlist, line)
def schedule(self, ticks, callable):
if ticks > 0:
bisect.insort(mae.event_q, Event(mae.tick + ticks, callable))
else:
callable()
def _provides_inject(self, pkg):
index = self._provides_index
for atom in pkg.provides:
# Use bisect.insort for ordered match results.
bisect.insort(index[atom], pkg)
def insertChild(self, child):
child.parent = self
bisect.insort(self.children, (child.orderKey(), child))
def test_start_alignment():
dst_dir = '../aligned'
tsc_dir_name = 'transcriptions'
img_dir_name = 'images'
map_dir_name = 'mapping'
fnms = set(f.split('.')[0] for f in os.listdir(IMG_DIR))
fnms = sorted(fnms - set(os.listdir(dst_dir)))
print("aligned so far: ", os.listdir(dst_dir))
#
# this is the output data structure
# {
# filename: {
# row_row_index: {
# bbx: 'transcription'
# ...
# }
# }
# }
#
#
for fnm in fnms:
print('\n###### {} ######'.format(fnm))
stop = input("STOP alignment? [y/n] ")
if stop == "y":
break
else:
# output data structure
bboxes2transcript = rec_defaultdict()
images_path = os.path.join(dst_dir, fnm, img_dir_name)
transcriptions_path = os.path.join(dst_dir, fnm, tsc_dir_name)
mapping_path = os.path.join(dst_dir, fnm, map_dir_name)
# load page image
page_img_og = cv2.imread(os.path.join(IMG_DIR, fnm + IMG_EXT),
cv2.IMREAD_GRAYSCALE)
# cut capital letters at margin
page_img, left_margin = remove_left_margin(page_img_og)
# segment page image into lines
img_lines = page_to_lines(page_img)
# load GT transcription
with open(os.path.join(TSC_DIR,
fnm.split('_')[0] + TSC_EXT),
'r') as tsc_file:
tsc_lines = tsc_file.readlines()
if len(img_lines) != len(tsc_lines):
raise AlignmentException(
"Line mismatch: {} lines segmented, but transcription has {} lines"
.format(len(img_lines), len(tsc_lines)))
#
# alignment begins
#
# for each line: transcription of line L, image of line L
transcr2img = zip(tsc_lines[:], img_lines[:])
# testing specific lines
try:
lines_to_test = list(
map(int,
input("choose lines to test ").strip().split(',')))
print(lines_to_test)
except ValueError as v:
print(v)
lines_to_test = []
for row_ind, (tsc_line, (img_line, top_y)) in enumerate(transcr2img):
if row_ind in lines_to_test or not lines_to_test:
#
# original: no dilatation or erosion
#
# spaces, center = spaces_in_line(img_line)
(spaces, (first,
last)), center = spaces_in_line_simple(img_line)
num_spaces = len(tsc_line.split()) - 1
print('\n', " • ".join(tsc_line.split()), '\n')
other_spaces = min((len(spaces) - num_spaces) // 2, 4)
biggest_spaces = group_nearest_spaces(
sorted(spaces, key=lambda e: -e[1] + e[0]))[:num_spaces +
other_spaces]
image_spaces = highlight_spaces_line(img_line, biggest_spaces,
"red")
#
# manipulated: dilatation and erosion
#
(spaces_m,
(first_m,
last_m)), center_m = spaces_in_line_manipulated(img_line,
discount=10)
other_spaces_m = min((len(spaces_m) - num_spaces) // 2, 3)
biggest_spaces_m = group_nearest_spaces(
sorted(spaces_m,
key=lambda e: -e[1] + e[0]))[:num_spaces +
other_spaces_m]
# image_spaces_m = highlight_spaces_line(img_line, biggest_spaces_m, "green")
spaces_m_flatten = [
el for rng in intervals_to_ranges(biggest_spaces_m)
for el in rng
]
spaces_orig_flatten = [
el for rng in intervals_to_ranges(biggest_spaces)
for el in rng
]
# start of the line
whitest_pnt_start = min(
[(np.argmax(np.count_nonzero(img_line[:, 0:first] == 0,
axis=0),
axis=0), 0),
(np.argmax(np.count_nonzero(img_line[:, 0:first_m] == 0,
axis=0),
axis=0), 1)],
key=lambda e: e[0])
start_text = (0, whitest_pnt_start[0])
# end of the line
end_left_pt = min(last_m, last)
white_count_rev = np.count_nonzero(img_line[:, end_left_pt:],
axis=0)[::-1]
whitest_pnt_end = len(white_count_rev) - 1 - np.argmax(
white_count_rev) + end_left_pt
end_text = (whitest_pnt_end, img_line.shape[1])
####
# show_image(img_line[:, 0:first], name=str(whitest_pnt_start[1] == 0)[0] + ' ' + str(row_ind))
# show_image(img_line[:, 0:first_m], name=str(whitest_pnt_start[1] == 1)[0] + ' manip ' + str(row_ind))
# show_image(img_line[:, 0:max(start_text[1], 1)])
# show_image(img_line[:, end_text[0]:])
#
# intersection
#
spaces_intersection = intersect1d(spaces_m_flatten,
spaces_orig_flatten)
spaces_intersect_intervals = [
(sp[0], sp[-1])
for sp in group_consecutive_values(spaces_intersection,
threshold=1)
if sp[-1] - sp[0] > 5
]
# spaces_intersect_intervals = group_nearest_spaces(sorted(spaces_intersect_intervals))
spaces_intersect_intervals_sorted = sorted(
spaces_intersect_intervals,
key=lambda e: -log(e[1] - e[0]) - count_white_ratio(
image=img_line, interval_x=e))
spaces_intersect_intervals_sorted = remove_trailing_spaces(
spaces_intersect_intervals_sorted,
from_x=start_text[1] + 25,
to_x=end_text[0] - 25)
othersp = min(
(len(spaces_intersect_intervals_sorted) - num_spaces) // 2,
3)
spaces_intersect_intervals = sorted(
spaces_intersect_intervals_sorted[:num_spaces + othersp])
#
# difference
# considering the interval as a hole not just set of consecutive pixels, so if part of an interval
# is in difference then it is discarded
#
spaces_difference = setxor1d(spaces_m_flatten,
spaces_orig_flatten)
spaces_diff_intervals = [
(sp[0], sp[-1]) for row_ind, sp in enumerate(
group_consecutive_values(spaces_difference,
threshold=1))
if sp[0] - 1 not in spaces_orig_flatten and sp[0] -
1 not in spaces_m_flatten and sp[-1] +
1 not in spaces_orig_flatten and sp[-1] +
1 not in spaces_m_flatten
]
spaces_diff_intervals.extend(
spaces_intersect_intervals_sorted[num_spaces + othersp:])
spaces_diff_intervals = sorted(spaces_diff_intervals)
#
# transcriptions
#
# the biggest ones + starting and ending
spaces_list = sorted([start_text] +
spaces_intersect_intervals + [end_text])
# each word of this line found in the transcription
words = tsc_line.split()
words_widths_estimate = []
for ix, w in enumerate(words):
previous_w = "" if ix == 0 else words[ix - 1]
words_widths_estimate.append(
estimate_word_width(w, previous_word=previous_w))
#
# checking that intersection picked the right spaces
#
c = 0
while True:
word_diffs = []
has_anomalous_width = False
c += 1
# print(len(spaces_list)-1 == len(words))
for si in range(1, len(spaces_list)):
try:
left_word_estimate = words_widths_estimate[si - 1]
except IndexError:
# last word reached
pass
left_word_width = spaces_list[si][0] - spaces_list[
si - 1][1]
left_word_diff = (left_word_estimate -
left_word_width) / left_word_estimate
word_diffs.append(left_word_diff)
if abs(left_word_diff) >= 0.5:
has_anomalous_width = True
# print(si, "/", num_spaces, " •• ", left_word_diff, words[si-1])
# assert not has_anomalous_width
if not has_anomalous_width:
break
# each space in space_list is associated with the word at its RIGHT (except the last one)
words_widths_diffs = sorted(
range(len(word_diffs)),
key=lambda row_ind_w: word_diffs[row_ind_w])
widest = words_widths_diffs[0]
narrowest = words_widths_diffs[-1]
# any space to insert?
eval_insertion = word_diffs[widest]
best_candidate_insertion = None
if eval_insertion < -0.5:
sp_left = spaces_list[
widest] # spaces_list[widest - 1]
sp_right = spaces_list[widest +
1] # spaces_list[widest]
# between sp_left/right
candidate_spaces = [
(s_start, s_end)
for s_start, s_end in spaces_diff_intervals
if sp_left[1] <= s_start and s_end <= sp_right[0]
]
if candidate_spaces:
# choosing the space that minimizes the difference between calculated and expected width
for cand in candidate_spaces:
diff_cand = abs((cand[0] - sp_left[1]) -
words_widths_estimate[widest -
1])
diff_no_cand = abs(
(sp_right[0] - sp_left[1]) -
words_widths_estimate[widest - 1])
if diff_cand < diff_no_cand:
best_candidate_insertion = cand
# any space to remove?
eval_removal = word_diffs[narrowest]
eval_space_row_index = narrowest + 1
if eval_removal > 0.5 and eval_space_row_index + 1 < len(
spaces_list):
# narrowest/cand is between sp_left/right
sp_left = spaces_list[eval_space_row_index - 1]
sp_right = spaces_list[eval_space_row_index + 1]
cand = spaces_list[eval_space_row_index]
try:
left_word_estimate = words_widths_estimate[
eval_space_row_index]
except IndexError:
pass
# if minimizes the error (difference)
if eval_removal > abs(
(sp_right[0] - sp_left[1] -
left_word_estimate)) / left_word_estimate:
spaces_list.remove(cand)
spaces_intersect_intervals.remove(cand)
if best_candidate_insertion:
insort(spaces_intersect_intervals,
best_candidate_insertion)
insort(spaces_list, best_candidate_insertion)
spaces_diff_intervals.remove(best_candidate_insertion)
# ensure no infinite loop
if c > 5:
break
has_anomalous_width = False
# END WHILE
spaces_intersect_intervals = sorted(spaces_intersect_intervals)
spaces_diff_intervals = sorted(spaces_diff_intervals)
#
# Second processing step evaluates weather a space in diff_intervals should be taken or not
#
take_spaces = spaces_intersect_intervals
missings = num_spaces - len(take_spaces)
if len(spaces_intersect_intervals) != num_spaces:
# row_indexes of the nearest spaces in space_intersect to the ones in spaces_diff
# if new spaces must be taken, these are placed in nearest_spaces
nearest_spaces = [
argmin([
abs(center_space(its) - center_space(d))
for its in sorted(spaces_intersect_intervals)
]) for d in spaces_diff_intervals
]
candidate_spaces = []
for candidate, near in zip(spaces_diff_intervals,
nearest_spaces):
left, right = None, None
# candidate = tuple(map(np.int, candidate))
near_space = spaces_intersect_intervals[near]
# candidate space is at the left of his nearest space (already taken)
if candidate[1] <= near_space[0]:
right = near_space[0]
if near > 0:
left = spaces_intersect_intervals[near - 1][1]
else:
left = start_text[1]
# width of the next word if candidate space is taken
# candidate_next_word_width = near_space[0] - candidate[1]
estimated_width = words_widths_estimate[
near] # +1?
# print("word ", words[near])
else: # candidate space is at the right of his nearest space (already taken)
left = near_space[1]
try:
right = spaces_intersect_intervals[near + 1][0]
except IndexError:
right = end_text[0]
assert right > left
candidate_next_word_width = candidate[
0] - near_space[1]
try:
estimated_width = words_widths_estimate[near +
1]
except IndexError:
print("## no near +1 ", near + 1)
pass # keeps the last assignment
# try:
# estimated_next_word_width = words_widths_estimate[near + 2]
# except IndexError:
# estimated_next_word_width = 0
candidate_word_width = candidate[0] - left
# current_word_width = right - left
difference = abs(candidate_word_width -
estimated_width) / estimated_width
# print("estimated_next ", estimated_next_word_width)
candidate_area = (candidate[1] -
candidate[0]) * img_line.shape[0] / 2
white = count_white_in_interval(img_line,
interval_x=candidate)
candidate_spaces.append(
(difference, white / candidate_area))
# END FOR
candidate_spaces = sorted(
[(idx, (csp, white))
for idx, (csp, white) in enumerate(candidate_spaces)
if csp <= 0.5 and white >= 1.8],
key=lambda e: e[1][0])[:missings]
take_spaces.extend([
spaces_diff_intervals[c_row_index]
for c_row_index, _ in candidate_spaces
])
image_spaces_taken = highlight_spaces_line(
img_line, take_spaces, "magenta")
take_spaces = sorted(take_spaces)
# SOME INFO
print(
row_ind,
") words: {}, spaces found/expected: {}/{}\n".format(
len(words), len(take_spaces), num_spaces))
# row_indexes of the spaces selected by the user
selected_row_indexes = show_image_ask_spaces(
image_spaces_taken, tot_spaces=len(take_spaces))
image_selected_spaces = highlight_spaces_line(
img_line, [take_spaces[i] for i in selected_row_indexes],
"blue")
# draw missing spaces if any
has_missing_spaces = input(
"draw spaces? (empty = NO else int): ")
selected_spaces = []
if has_missing_spaces:
# valid input
while not isinstance(eval(has_missing_spaces), int):
has_missing_spaces = input(
"insert number of spaces to insert: ")
has_drawn = False
# ask until the number of input spaces is correct
while not has_drawn:
print("draw and press 'q' to exit\n")
try:
num_missings = eval(
has_missing_spaces) # to be inserted
draw_missing_spaces(image_selected_spaces)
drawn = get_drawn_spaces() # drawn spaces
assert len(drawn) == num_missings
for d_lft, d_rt in drawn:
# inserted spaces overlaps start_text or end_text?
if start_text[0] < d_lft <= start_text[1]:
print("### inserted starting space")
start_text = (start_text[0],
np.argmax(np.count_nonzero(
img_line[:, :d_lft],
axis=0),
axis=0))
elif end_text[0] <= d_rt < end_text[1]:
print("### inserted ending space ",
end_text)
white_count_rev = np.count_nonzero(
img_line[:, d_rt:], axis=0)[::-1]
left_end = len(
white_count_rev) - 1 - np.argmax(
white_count_rev, axis=0)
end_text = (left_end, end_text[1])
"""
# the correct cutting columns is likely to be at the end of the line (and of the
# selected sub-image)
guard = min(end_text[1] - d_rt, 10)
# argmax take the "first" max, so start scanning from the right
white_count_rev = np.count_nonzero(img_line[:, d_rt + guard:], axis=0)[::-1]
left_end = len(white_count_rev) - 1 - np.argmax(white_count_rev, axis=0)\
+ d_rt + guard
end_text = (left_end, end_text[1])
"""
selected_spaces = sorted(
[start_text] +
[take_spaces[i]
for i in selected_row_indexes] + drawn +
[end_text])
# exit
has_drawn = True
except AssertionError:
print(
"\nERROR: number of drawn spaces != number of spaces requested, {} {}"
.format(len(drawn), num_missings))
cv2.destroyAllWindows()
else:
selected_spaces = sorted(
[start_text] +
[take_spaces[i]
for i in selected_row_indexes] + [end_text])
image_select_draw_spaces = highlight_spaces_line(
img_line, selected_spaces, "red")
show_image(image_select_draw_spaces, name="selected + drawn")
####
# show_image(img_line[:, start_text[0]:start_text[1]], name="space before text")
for sp in range(1, len(selected_spaces)):
space_left = selected_spaces[sp - 1]
space_right = selected_spaces[sp]
# whitest column left
try:
left = np.argmax(
np.count_nonzero(
img_line[:, space_left[0]:space_left[1]],
axis=0))
except ValueError:
print(space_left[0], space_left[1],
" attempt to get argmin of an empty sequence")
left = 0
left += space_left[0]
# whitest column right
"""
white_count_rev = np.count_nonzero(img_line[:, d_rt + guard:], axis=0)[::-1]
left_end = len(white_count_rev) - 1 - np.argmax(white_count_rev, axis=0)\
+ d_rt + guard
end_text = (left_end, end_text[1])
"""
try:
if sp == len(selected_spaces) - 1: # last space?
print("###### cutting the last space")
white_count_rev = np.count_nonzero(
img_line[:, space_right[0]:], axis=0)[::-1]
right = len(white_count_rev) - 1 - np.argmax(
white_count_rev)
else:
right = np.argmax(
np.count_nonzero(
img_line[:, space_right[0]:space_right[1]],
axis=0))
except ValueError: # attempt to get argmin of an empty sequence
print(space_right[0], space_right[1],
" attempt to get argmax of an empty sequence")
right = space_right[1]
right += space_right[0]
# left (start bbx x), top_y (start bbx y), width, height
bbx_name = "{}_{}_{}_{}".format(left + left_margin, top_y,
right - left,
img_line.shape[0])
try:
bboxes2transcript[row_ind][bbx_name] = words[sp - 1]
show_image(img_line[:, left:right], name=words[sp - 1])
# print(words[sp - 1], '\n')
except IndexError:
print("words and index = ", sp - 1, " len(words) ",
len(words))
print(
"\n ••••••••••••••••••••••••••••••••••••••••••••••••••••••\n\n"
)
# Like bisect, insort takes optional lo, hi arguments to limit the search to a sub-sequence.
# There is also an insort_left variation that uses bisect_left to find insertion points.
import bisect
import random
SIZE = 7
random.seed(1729)
my_list = []
for i in range(SIZE):
new_item = random.randrange(SIZE * 2)
bisect.insort(my_list, new_item) # Insert x in a in sorted order.
# This is equivalent to a.insert(bisect.bisect_left(a, x, lo, hi), x) assuming that
# a is already sorted. Keep in mind that the O(log n) search is dominated by the slow O(n) insertion step.
print('%2d ->' % new_item, my_list)
def addNum(self, num: int) -> None:
insort(self.nums, num)
def generate_assignee_workloads(self, input_paths, output_paths):
assigned_issues_timelines = {}
unassigned_issues_timelines = {}
workload_timelines = {}
worklogs = self.generate_assignee_worklogs(input_paths, output_paths)
for assignee, worklog in worklogs.items():
assigned_dates, issues_assigned_on, assigned_issues_timeline = (
self.extract_assigned_issues(assignee, worklog))
timeline_entry = {
"assigned_dates": assigned_dates,
"issues_assigned_on": issues_assigned_on,
"assigned_issues_timeline": assigned_issues_timeline
} # noqa
assigned_issues_timelines[assignee] = timeline_entry
unassigned_dates, issues_unassigned_on, unassigned_issues_timeline = ( # noqa
self.extract_unassigned_issues(assignee, worklog))
timeline_entry = {
"unassigned_dates": unassigned_dates,
"issues_unassigned_on": issues_unassigned_on,
"unassigned_issues_timeline": unassigned_issues_timeline
} # noqa
unassigned_issues_timelines[assignee] = timeline_entry
workload_timeline = {}
workload_dates = []
for date in assigned_dates:
if workload_timeline.get(date) is None:
bisect.insort(workload_dates, date)
workload_timeline[date] = -1
for date in unassigned_dates:
if workload_timeline.get(date) is None:
bisect.insort(workload_dates, date)
workload_timeline[date] = -1
assigned = 0
unassigned = 0
for date in workload_dates:
if assigned_issues_timeline.get(date):
assigned = assigned_issues_timeline[date]
if unassigned_issues_timeline.get(date):
unassigned = unassigned_issues_timeline[date]
workload_timeline[date] = assigned - unassigned
timeline_entry = {
"workload_dates": workload_dates,
"workload_timeline": workload_timeline
}
workload_timelines[assignee] = timeline_entry
output_path = os.path.join(output_paths["cross_issue"],
"workload_timelines.pickle")
with open(output_path, 'wb') as fp:
pickle.dump(workload_timelines, fp)
output_path = os.path.join(output_paths["cross_issue"],
"assigned_issues_timelines.json")
self.save_dict_as_json(output_path, assigned_issues_timelines)
output_path = os.path.join(output_paths["cross_issue"],
"unassigned_issues_timelines.json")
self.save_dict_as_json(output_path, unassigned_issues_timelines)
output_path = os.path.join(output_paths["cross_issue"],
"workloads_timelines.json")
self.save_dict_as_json(output_path, workload_timelines)
return workload_timelines
# -*- coding: utf-8 -*-
# author:xls
"""
bisect模块实现二分查找和插入算法.
"""
import bisect
x1= 200
x2= 500
list1 = [1, 3, 6, 24, 55, 78, 454, 555, 1234, 6900]
list2 = [1, 3, 6, 24, 55, 78, 454, 555, 1234, 6900]
ret1 = bisect.bisect(list1, x1) #获取插入位置
ret2 = bisect.insort(list2, x2) #返回插入结果
print("返回值：", ret1)
print("list1 = ", list1)
print("返回值：", ret2)
print("list1 = ", list2)
import bisect list = [10, 20, 30] bisect.insort(list, 25) bisect.insort(list, 15) print list
def generate_reporter_reputations(self, input_paths, output_paths):
open_issues_timelines = {}
close_issues_timelines = {}
reputation_timelines = {}
worklogs = self.generate_reporter_worklogs(input_paths, output_paths)
for reporter, worklog in worklogs.items():
open_dates, issues_opened_on, open_issues_timeline = (
self.extract_opened_issues(reporter, worklog))
timeline_entry = {
"open_dates": open_dates,
"issues_opened_on": issues_opened_on,
"open_issues_timeline": open_issues_timeline
}
open_issues_timelines[reporter] = timeline_entry
close_dates, issues_closed_on, close_issues_timeline = (
self.extract_closed_issues(reporter, worklog))
timeline_entry = {
"close_dates": close_dates,
"issues_closed_on": issues_closed_on,
"close_issues_timeline": close_issues_timeline
}
close_issues_timelines[reporter] = timeline_entry
reputation_timeline = {}
reputation_dates = []
for date in open_dates:
if reputation_timeline.get(date) is None:
bisect.insort(reputation_dates, date)
reputation_timeline[date] = -1
for date in close_dates:
if reputation_timeline.get(date) is None:
bisect.insort(reputation_dates, date)
reputation_timeline[date] = -1
opened = 0
fixed = 0
for date in reputation_dates:
if open_issues_timeline.get(date):
opened = open_issues_timeline[date]
if close_issues_timeline.get(date):
fixed = close_issues_timeline[date]
reputation_timeline[date] = (fixed / (opened + 1))
timeline_entry = {
"reputation_dates": reputation_dates,
"reputation_timeline": reputation_timeline
}
reputation_timelines[reporter] = timeline_entry
output_path = os.path.join(output_paths["cross_issue"],
"reputation_timelines.pickle")
with open(output_path, 'wb') as fp:
pickle.dump(reputation_timelines, fp)
output_path = os.path.join(output_paths["cross_issue"],
"open_issues_timelines.json")
self.save_dict_as_json(output_path, open_issues_timelines)
output_path = os.path.join(output_paths["cross_issue"],
"close_issues_timelines.json")
self.save_dict_as_json(output_path, close_issues_timelines)
output_path = os.path.join(output_paths["cross_issue"],
"reputation_timelines.json")
self.save_dict_as_json(output_path, reputation_timelines)
return reputation_timelines
from bisect import insort
n = int(input())
k = int(input())
arr = []
for _ in range(n):
#Insort is for when we insert new element we kept the array in the sorting order
value = int(input())
insort(arr, value)
minfairnes = 10**10
for i in range(n - k + 1):
currentfairnes = arr[i + k - 1] - arr[i]
if (currentfairnes < minfairnes):
minfairnes = currentfairnes
print(minfairnes)
def get_new_balancing_breakpoints(instance, active_hx, inactive_hx, weaken):
new_th_breakpoints = {}
new_thx_breakpoints = {}
new_tc_breakpoints = {}
new_tcx_breakpoints = {}
active_th_indices = list(set((i, k) for i, _, k in active_hx[stream_hx]))
active_tc_indices = list(set((j, k+1) for _, j, k in active_hx[stream_hx]))
for index in active_th_indices:
breakpoint = instance.th[index].value
if breakpoint not in th_breakpoints[index]:
bisect.insort(th_breakpoints[index], breakpoint)
new_th_breakpoints[index] = breakpoint
for index in active_hx[stream_hx]:
breakpoint = instance.thx[index].value
if breakpoint not in thx_breakpoints[index]:
bisect.insort(thx_breakpoints[index], breakpoint)
new_thx_breakpoints[index] = breakpoint
for index in active_tc_indices:
breakpoint = instance.tc[index].value
if breakpoint not in tc_breakpoints[index]:
bisect.insort(tc_breakpoints[index], breakpoint)
new_tc_breakpoints[index] = breakpoint
for index in active_hx[stream_hx]:
breakpoint = instance.tcx[index].value
if breakpoint not in tcx_breakpoints[index]:
bisect.insort(tcx_breakpoints[index], breakpoint)
new_tcx_breakpoints[index] = breakpoint
if not weaken:
for index in inactive_hx[stream_hx]:
if instance.bh_out[index].value > 0.000001:
breakpoint = instance.thx[index].value
if breakpoint not in thx_breakpoints[index]:
bisect.insort(thx_breakpoints[index], breakpoint)
new_thx_breakpoints[index] = breakpoint
if instance.bc_out[index].value > 0.000001:
breakpoint = instance.tcx[index].value
if breakpoint not in tcx_breakpoints[index]:
bisect.insort(tcx_breakpoints[index], breakpoint)
new_tcx_breakpoints[index] = breakpoint
new_balancing_breakpoints = {}
if not len(new_th_breakpoints) == 0:
new_balancing_breakpoints['th'] = new_th_breakpoints
if not len(new_thx_breakpoints) == 0:
new_balancing_breakpoints['thx'] = new_thx_breakpoints
if not len(new_tc_breakpoints) == 0:
new_balancing_breakpoints['tc'] = new_tc_breakpoints
if not len(new_tcx_breakpoints) == 0:
new_balancing_breakpoints['tcx'] = new_tcx_breakpoints
return new_balancing_breakpoints
