def main(argv): try: opts, args = getopt.getopt(argv, "hc:d:", ["help", "city=", "direcotry="]) except getopt.GetoptError as err: # print help information and exit: print str(err) # will print something like "option -a not recognized" usage() sys.exit(2) city, dic = 'beijing', '/home/taojiang/datasets/pois/beijing' for opt, arg in opts: if opt == '-h': usage() sys.exit() elif opt in ("-c", "--city"): city = arg elif opt in ("-d", "--direcotry"): dic = arg + '/' + city cityins = POICollection(city, getCityLocs(city), 'POI', dic) # 建立 POI 列表 pois = cityins.readFiletoJson("%s.fPOI.geojson" % city) # 向 POI 完善附加信息并存入文件 cityins.writeFile(cityins.appendAttr(pois), "%s.aPOI.geojson" % city)
def main(argv): try: opts, args = getopt.getopt( argv, "hc:d:s:t:", ["help", "city=", "direcotry=", "split=", "tasks="]) except getopt.GetoptError as err: # print help information and exit: print str(err) # will print something like "option -a not recognized" usage() sys.exit(2) city, dic, tasks, split = 'beijing', '/home/taojiang/tools', 22, 0.001 for opt, arg in opts: if opt == '-h': usage() sys.exit() elif opt in ("-c", "--city"): city = arg elif opt in ("-d", "--direcotry"): dic = arg elif opt in ("-s", "--split"): split = float(arg) elif opt in ("-t", "--tasks"): tasks = int(arg) ufeature = UserFeature(city, split, getCityLocs(city)) # 20 threads to split the whole task ulist = ufeature.getUserList(tasks) # Enum user records and assign them with region grid UID # ufeature.sepUserTaskes(ulist, split) # Filter out region grid UID list and aggreate user feature vector flist, tobj = ufeature.getGridList() ufeature.sepFeatureTaskes(flist, tobj, ulist)
def augment(self, inputfile, outputfile, CITY, FILENUM=1000): reslist = ['' for i in range(FILENUM)] # print "Begin read file at %s" % time.time() with open(inputfile, 'rb') as stream: # print "Finished read file at %s" % time.time() for line in stream: linelist = line.strip('\n').split(',') index = int(linelist[0]) % FILENUM # formatAdmin(linelist[5].strip()) # linelist[5] reslist[index] += linelist[0] + ',' + formatTime( linelist[1]) + ',' + formatAdmin( linelist[5]) + ',' + formatGridID( getCityLocs(CITY), [linelist[3], linelist[2]]) + '\n' stream.close() gc.collect() # print "Begin write file at %s" % time.time() for i in range(FILENUM): tLock.acquire() with open('%s/res-%05d' % (outputfile, i), 'ab') as res: res.write(reslist[i]) res.close() # 释放锁 tLock.release()
def main(argv): try: opts, args = getopt.getopt(argv, "hc:r:s:", ["help", "city=", "radius=", "split="]) except getopt.GetoptError as err: # print help information and exit: print str(err) # will print something like "option -a not recognized" usage() sys.exit(2) city, r, split, IP = 'beijing', 10, 0.003, 'localhost' # another IP in ISCAS should be '192.168.1.42' for opt, arg in opts: if opt == '-h': usage() sys.exit() elif opt in ("-c", "--city"): city = arg elif opt in ("-r", "--radius"): r = arg elif opt in ("-s", "--split"): split = float(arg) start_time = time.time() citylocs = getCityLocs(city) cityGrid = CityGrid(city, citylocs, r, IP) # midLat, midLng = round((citylocs['south'] +citylocs['north']) / 2.0, 3), round((citylocs['west'] + citylocs['east']) / 2.0, 3) # sepcitylocs = sep4Citylocs(citylocs, midLat, midLng, split) # ppservers = () # job_server = pp.Server(ppservers=ppservers) # print "The number of working kernel threads that can be used is %s." % job_server.get_ncpus() # jobs, index = [], 0 # for sublocs in sepcitylocs: # jobs.append( (index, job_server.submit(cityGrid.gridGeneration, (split, sublocs, ), (connectMongo, getCityLocs, gaussian2D, sep4Citylocs,), ('os', 'math', 'sys', 'getopt', 'logging', 'time', 'numpy', 'scipy', 'pymongo', 'MySQLdb', 'geopy.distance', 'CommonFunc' ))) ) # index += 1 # for index, job in jobs: # job() # job_server.print_stats() # cityGrid.gridGeneration(split, citylocs) logging.info("City grid construction consumption: %s s" % str(time.time() - start_time))
def augment(self, inputfile, outputfile, CITY): reslist = ['' for i in range(self.resfilenum)] resnumber = 0 with open(inputfile, 'rb') as stream: for line in stream: resnumber += 1 linelist = line.strip('\n').split(',') index = int(linelist[0]) % self.resfilenum reslist[index] += linelist[0] + ',' + formatTime( linelist[1]) + ',' + formatAdmin( linelist[4]) + ',' + formatGridID( getCityLocs(CITY), [linelist[3], linelist[2]]) + '\n' stream.close() global pLock # localFileStream = [] with pLock: print "CountVal %d at time %s" % ( self.listCount.value, str(time.time() - self.STARTTIME)) self.listCount.value += resnumber if self.listCount.value > self.MAXRECORDS: print "PROCESS ID-%d has one write operation at %s." % ( self.INDEX, str(time.time() - self.STARTTIME)) for x in xrange(0, self.resfilenum): with open('%s/res-%05d' % (outputfile, x), 'ab') as res: res.write(self.strData[x].value + reslist[x]) res.close() # localFileStream.append( self.strData[x].value + reslist[x] ) self.strData[x].value = '' # 计数器重置为 0 self.listCount.value = 0 gc.collect() else: for x in xrange(0, self.resfilenum): self.strData[x].value += reslist[x]
def calGridAoiDis(city): # 初始化网格,构建 POI 类别档案 locs = getCityLocs(city) grids = [] split = 0.05 centerincrement = 0.025 # round(split / 2.0, 4) latnum = int((locs['north'] - locs['south']) / split + 1) lngnum = int((locs['east'] - locs['west']) / split + 1) conn, db = connectMongo('tdnormal') POIs = db['pois_%s' % city] # 构建栅栏数组 disobjs = [] with open( os.path.join( '/home/taojiang/git/living-modes-visual-comparison/conf/data', getAbbName(city) + '.json')) as f: stream = json.load(f) features = stream['features'] for each in features: disobjs.append({ 'name': each['properties']['name'], 'geo': shape(each['geometry']) }) f.close() # 遍历查询网格周围 POI 并更新 for latind in xrange(0, latnum): for lngind in xrange(0, lngnum): # 前11元素均为分类别统计数量,最后一个元素为POI总量 tmpGrid = [0 for x in xrange(0, 15)] vaildGrid = False lat = round(locs['south'] + latind * split, 3) lng = round(locs['west'] + lngind * split, 3) lnginc = round(lng + split, 3) latinc = round(lat + split, 3) lngcen = round(lng + centerincrement, 4) latcen = round(lat + centerincrement, 4) # 一个正方形 geojson 对象,代表当前方块对应的地理边界 coordsarr = [[lng, lat], [lnginc, lat], [lnginc, latinc], [lng, latinc], [lng, lat]] featurelistarray = [0] * 11 typevalid = False # query all the POIs less than maxQRadius nearPOIList = list( POIs.find( { "properties.center": { '$near': { '$geometry': { 'type': "Point", 'coordinates': [lngcen, latcen] }, '$minDistance': 0, '$maxDistance': 2500 * (2**0.5) } } }, {'properties': 1})) # construct vector with POIs types info poilen = len(nearPOIList) if poilen != 0: vaildGrid = True # POI list is not null for each in nearPOIList: curPInd = each['properties']['ftype'] - 1 tmpGrid[curPInd] += 1 tmpGrid[11] += 1 if vaildGrid: tmpGrid[12] = lngcen tmpGrid[13] = latcen point = Point(lngcen, latcen) index = getDisIndex(point, disobjs) tmpGrid[14] = index if index == -1: print 'Invalid Grid Found.' grids.append(tmpGrid) print "%s City with valid grids %s" % (city, str(len(grids))) return grids