def scaffolding(self):
"""Call Grouper.py to prepare the partition tables."""
g = Grouper(self.names['master_table'])
g.main()
return
def run(self):
if len(self.handles) is 0:
raise RuntimeError('未找到任务处理')
for handle in self.handles:
# print(handle)
if len(handle) is 0:
raise RuntimeError('未找到操作片段')
results = {}
for segment in handle:
key = list(segment.keys())[0]
value = list(segment.values())[0]
if key == '(SELECT)':
selecter = Selecter(self.databases, results, value)
selecter.run()
print(results)
elif key == '(JOIN)':
joiner = Joiner(results, value)
joiner.run()
print(results)
elif key == '(UNION)':
unioner = Unioner(results, value)
unioner.run()
print(results)
elif key == '(ADD)':
adder = Adder(results, value)
minuser.run()
print(results)
elif key == '(MINUS)':
minuser = Minuser(results, value)
minuser.run()
print(results)
elif key == '(GROUP)':
grouper = Grouper(results, value)
grouper.run()
print(results)
elif key == '(输出)':
outputer = Outputer(results, value)
outputer.run()
print(results)
elif key == '(邮件)':
emailer = Emailer(self.email, value[0], results)
emailer.run()
def jobMap(grouperNum, chunksQueue, listSaveStateNameGrouper, listListLastCallNum):
if self.silent<1:
print 'Starting mapper worker ' + str(grouperNum)
while True:
# Get new chunck to process
chunk = chunksQueue.get()
# Work
if self.silent<1:
print 'Worker ' + str(grouperNum) + ' mapping chunk ' + str(chunk)
MapIterator = MapChunkIterator(mapChunksNameGenerator(chunk)) # Iterator to iterate through the chunck
theContext = MapContext(groupChunksNameGenerator(chunk),MapIterator)
self.Mapper.map(theContext)
if self.silent<1:
print 'Worker ' + str(grouperNum) + ' grouping locally chunck ' + str(chunk)
idx = listListLastCallNum[grouperNum]+1
theGrouper = Grouper(grouperNum,idx,idx-1,directory);
listSaveStateNameGrouper[grouperNum] = theGrouper.group(theContext)
listListLastCallNum[grouperNum] = idx ;
# "Close" chunk
chunksQueue.task_done()
def _group(self):
result = tk.Variable()
def callback(*args):
master_activity = result.get()[0]
for sub_activity in result.get()[1]:
self._translator[sub_activity] = master_activity
result.trace("w", callback=callback)
sub_activities = set(a for a in self._translator.keys())
master_activities = set(a for a in self._translator.values())
Grouper(self._activities,
result,
omit_master=sub_activities,
omit_sub=master_activities)
reader = UnicodeReader(open(input))
pts = []
for row in reader:
try:
pt = LatLng(float(row[0]),float(row[1]))
pts.append(pt)
except AssertionError:
print row
#except IndexError:
# print row
pts.sort(LatLngCompare)
grouper = Grouper()
for (i,pt) in enumerate(pts):
grouper.append(pt)
optimizer = GroupSwapOptimizer(grouper)
optimizer.process()
#Output generation
dom = Kml()
group_style = xml.dom.minidom.parseString("""
<Style id="group">
<IconStyle>
<color>ff00ff00</color>
clusters = []
# Cluster loading
entities = Cluster.all().fetch(100)
while entities:
for entry in entities:
clusters.append(entry)
entities = Cluster.all().filter('__key__ >', entities[-1].key()).fetch(100)
print "%d of clusters loaded" % len(clusters)
# Start processing
clusterer = Grouper()
for c in clusters: # Initializer clusters
g = LatLngGroup()
g.set_data(c)
for key in c.members:
pt = LatLng()
s = stops[key.id_or_name()]
pt.lat = s.latlng.lat
pt.lng = s.latlng.lon
pt.set_data(s)
g.append(pt)
if c.station != None:
g.set_fixed_centroid(LatLng(c.station.latlng.lat ,c.station.latlng.lon ) )
def execute(self):
def mapChunksNameGenerator(idx):
pathName = 'mc/'
if not os.path.exists(pathName):
os.makedirs(pathName)
return pathName + 'mc_'+str(idx) # map chunk
def groupChunksNameGenerator(idx):
pathName = 'gc/'
if not os.path.exists(pathName):
os.makedirs(pathName)
return pathName + 'gc_'+str(idx) # grouper chunk
##################
# Create chuncks #
##################
if self.silent<1:
print 'Creating chuncks ...'
cf = ChuncksFactory(self.listOfFiles)
cf.divideIntoChunks(mapChunksNameGenerator)
if self.silent<1:
print str(cf.nChunks) + ' chunks created.'
###################
# Prepare grouper #
###################
totalNumberOfGrouper = min(self.nThreads,cf.nChunks) ; # One grouper = 1 core, basically
if self.silent<1:
print 'Actual number of threads to be used : {}'.format(totalNumberOfGrouper)
listGrouperNum = [0 for i in range(totalNumberOfGrouper)];
listLastCallNum = [-1 for i in range(totalNumberOfGrouper)];
saveStateNameGrouper = [0 for i in range(totalNumberOfGrouper)];
listGrouperNum = range(0,totalNumberOfGrouper);
directory = 'mgl/' # mapper and grouper log
if not os.path.exists(directory):
os.makedirs(directory)
##################################
# Map and local group (parallel) #
##################################
if self.silent<1:
print '------------------------------'
print 'Mapping and local grouping ...'
print '------------------------------'
chunkLeft = cf.nChunks ;
def jobMap(grouperNum, chunksQueue, listSaveStateNameGrouper, listListLastCallNum):
if self.silent<1:
print 'Starting mapper worker ' + str(grouperNum)
while True:
# Get new chunck to process
chunk = chunksQueue.get()
# Work
if self.silent<1:
print 'Worker ' + str(grouperNum) + ' mapping chunk ' + str(chunk)
MapIterator = MapChunkIterator(mapChunksNameGenerator(chunk)) # Iterator to iterate through the chunck
theContext = MapContext(groupChunksNameGenerator(chunk),MapIterator)
self.Mapper.map(theContext)
if self.silent<1:
print 'Worker ' + str(grouperNum) + ' grouping locally chunck ' + str(chunk)
idx = listListLastCallNum[grouperNum]+1
theGrouper = Grouper(grouperNum,idx,idx-1,directory);
listSaveStateNameGrouper[grouperNum] = theGrouper.group(theContext)
listListLastCallNum[grouperNum] = idx ;
# "Close" chunk
chunksQueue.task_done()
# Create workers
chunksQueue = Queue()
for i in range(totalNumberOfGrouper):
worker = Thread(target=jobMap, args=(i, chunksQueue, saveStateNameGrouper, listLastCallNum))
worker.setDaemon(True)
worker.start()
# Feed workers
for chunk in range(chunkLeft):
chunksQueue.put(chunk)
# Wait for map completion
if self.silent<1:
print '\nMain thread waiting ...'
chunksQueue.join()
if self.silent<1:
print 'All workers have finished.'
print 'Mapping and local grouping done. {} chuncks grouped by {} threads.'.format(cf.nChunks, totalNumberOfGrouper)
################
# Global group #
################
if self.silent<1:
print '------------------'
print('Global grouping...')
print '------------------'
listOfDirectory = []
globalGrouperDirectory = 'mgl/' # mapper and grouper log
for i in range(0,totalNumberOfGrouper):
listOfDirectory.append(globalGrouperDirectory)
globalDict = Grouper.globalGrouper(saveStateNameGrouper,listGrouperNum,listLastCallNum,listOfDirectory,globalGrouperDirectory)
if self.silent<1:
print('Global grouping done.')
############
# Reducing #
############
def jobReduce(reducerNum, jobsQueue, outputDictionary):
if self.silent<1:
print 'Starting reducer worker ' + str(reducerNum)
while True:
# Get new (key,globalNodeFileName) to process
(key, globalNodeFileName) = jobsQueue.get()
# Work
reduceIterator = ReduceFromGroupIterator(globalNodeFileName)
theReduceContext = ReduceContext(key,reduceIterator)
outputDictionary[key] = self.Reducer.reduce(theReduceContext)
# "Close" job
jobsQueue.task_done()
# Create workers
jobsQueue = Queue()
outputDict = dict()
for i in range(totalNumberOfGrouper):
worker = Thread(target=jobReduce, args=(i, jobsQueue, outputDict))
worker.setDaemon(True)
worker.start()
# Feed workers
for key, globalNodeFileName in globalDict.iteritems():
jobsQueue.put((key, globalNodeFileName))
# Wait for reduce completion
if self.silent<1:
print '\nMain thread waiting ...'
jobsQueue.join()
if self.silent<1:
print 'All workers have finished.'
print 'Reducing done by {} threads.'.format(totalNumberOfGrouper)
# Serial version
# if self.silent<1:
# print '------------'
# print('Reducing ...')
# print '------------'
# outputDict = dict()
# for key, globalNodeFileName in globalDict.iteritems():
# reduceIterator = ReduceFromGroupIterator(globalNodeFileName)
# theReduceContext = ReduceContext(key,reduceIterator)
# outputDict[key] = self.Reducer.reduce(theReduceContext)
# if self.silent<1:
# print('Reducing done.')
##########
# OUTPUT #
##########
if self.silent == -1:
print '\n------------------------------\nOutput\n------------------------------\n'
for key in outputDict :
print str(key) + ' - ' + str(outputDict[key])
return outputDict;
