def main(argv):
try:
opts, args = getopt.getopt(argv, "hc:d:",
["help", "city=", "direcotry="])
except getopt.GetoptError as err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
usage()
sys.exit(2)
city, dic = 'beijing', '/home/taojiang/datasets/pois/beijing'
for opt, arg in opts:
if opt == '-h':
usage()
sys.exit()
elif opt in ("-c", "--city"):
city = arg
elif opt in ("-d", "--direcotry"):
dic = arg + '/' + city
cityins = POICollection(city, getCityLocs(city), 'POI', dic)
# 建立 POI 列表
pois = cityins.readFiletoJson("%s.fPOI.geojson" % city)
# 向 POI 完善附加信息并存入文件
cityins.writeFile(cityins.appendAttr(pois), "%s.aPOI.geojson" % city)
def main(argv):
try:
opts, args = getopt.getopt(
argv, "hc:d:s:t:",
["help", "city=", "direcotry=", "split=", "tasks="])
except getopt.GetoptError as err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
usage()
sys.exit(2)
city, dic, tasks, split = 'beijing', '/home/taojiang/tools', 22, 0.001
for opt, arg in opts:
if opt == '-h':
usage()
sys.exit()
elif opt in ("-c", "--city"):
city = arg
elif opt in ("-d", "--direcotry"):
dic = arg
elif opt in ("-s", "--split"):
split = float(arg)
elif opt in ("-t", "--tasks"):
tasks = int(arg)
ufeature = UserFeature(city, split, getCityLocs(city))
# 20 threads to split the whole task
ulist = ufeature.getUserList(tasks)
# Enum user records and assign them with region grid UID
# ufeature.sepUserTaskes(ulist, split)
# Filter out region grid UID list and aggreate user feature vector
flist, tobj = ufeature.getGridList()
ufeature.sepFeatureTaskes(flist, tobj, ulist)
def augment(self, inputfile, outputfile, CITY, FILENUM=1000):
reslist = ['' for i in range(FILENUM)]
# print "Begin read file at %s" % time.time()
with open(inputfile, 'rb') as stream:
# print "Finished read file at %s" % time.time()
for line in stream:
linelist = line.strip('\n').split(',')
index = int(linelist[0]) % FILENUM
# formatAdmin(linelist[5].strip())
# linelist[5]
reslist[index] += linelist[0] + ',' + formatTime(
linelist[1]) + ',' + formatAdmin(
linelist[5]) + ',' + formatGridID(
getCityLocs(CITY),
[linelist[3], linelist[2]]) + '\n'
stream.close()
gc.collect()
# print "Begin write file at %s" % time.time()
for i in range(FILENUM):
tLock.acquire()
with open('%s/res-%05d' % (outputfile, i), 'ab') as res:
res.write(reslist[i])
res.close()
# 释放锁
tLock.release()
def main(argv):
try:
opts, args = getopt.getopt(argv, "hc:r:s:", ["help", "city=", "radius=", "split="])
except getopt.GetoptError as err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
usage()
sys.exit(2)
city, r, split, IP = 'beijing', 10, 0.003, 'localhost' # another IP in ISCAS should be '192.168.1.42'
for opt, arg in opts:
if opt == '-h':
usage()
sys.exit()
elif opt in ("-c", "--city"):
city = arg
elif opt in ("-r", "--radius"):
r = arg
elif opt in ("-s", "--split"):
split = float(arg)
start_time = time.time()
citylocs = getCityLocs(city)
cityGrid = CityGrid(city, citylocs, r, IP)
# midLat, midLng = round((citylocs['south'] +citylocs['north']) / 2.0, 3), round((citylocs['west'] + citylocs['east']) / 2.0, 3)
# sepcitylocs = sep4Citylocs(citylocs, midLat, midLng, split)
# ppservers = ()
# job_server = pp.Server(ppservers=ppservers)
# print "The number of working kernel threads that can be used is %s." % job_server.get_ncpus()
# jobs, index = [], 0
# for sublocs in sepcitylocs:
# jobs.append( (index, job_server.submit(cityGrid.gridGeneration, (split, sublocs, ), (connectMongo, getCityLocs, gaussian2D, sep4Citylocs,), ('os', 'math', 'sys', 'getopt', 'logging', 'time', 'numpy', 'scipy', 'pymongo', 'MySQLdb', 'geopy.distance', 'CommonFunc' ))) )
# index += 1
# for index, job in jobs:
# job()
# job_server.print_stats()
#
cityGrid.gridGeneration(split, citylocs)
logging.info("City grid construction consumption: %s s" % str(time.time() - start_time))
def augment(self, inputfile, outputfile, CITY):
reslist = ['' for i in range(self.resfilenum)]
resnumber = 0
with open(inputfile, 'rb') as stream:
for line in stream:
resnumber += 1
linelist = line.strip('\n').split(',')
index = int(linelist[0]) % self.resfilenum
reslist[index] += linelist[0] + ',' + formatTime(
linelist[1]) + ',' + formatAdmin(
linelist[4]) + ',' + formatGridID(
getCityLocs(CITY),
[linelist[3], linelist[2]]) + '\n'
stream.close()
global pLock
# localFileStream = []
with pLock:
print "CountVal %d at time %s" % (
self.listCount.value, str(time.time() - self.STARTTIME))
self.listCount.value += resnumber
if self.listCount.value > self.MAXRECORDS:
print "PROCESS ID-%d has one write operation at %s." % (
self.INDEX, str(time.time() - self.STARTTIME))
for x in xrange(0, self.resfilenum):
with open('%s/res-%05d' % (outputfile, x), 'ab') as res:
res.write(self.strData[x].value + reslist[x])
res.close()
# localFileStream.append( self.strData[x].value + reslist[x] )
self.strData[x].value = ''
# 计数器重置为 0
self.listCount.value = 0
gc.collect()
else:
for x in xrange(0, self.resfilenum):
self.strData[x].value += reslist[x]
def calGridAoiDis(city):
# 初始化网格,构建 POI 类别档案
locs = getCityLocs(city)
grids = []
split = 0.05
centerincrement = 0.025 # round(split / 2.0, 4)
latnum = int((locs['north'] - locs['south']) / split + 1)
lngnum = int((locs['east'] - locs['west']) / split + 1)
conn, db = connectMongo('tdnormal')
POIs = db['pois_%s' % city]
# 构建栅栏数组
disobjs = []
with open(
os.path.join(
'/home/taojiang/git/living-modes-visual-comparison/conf/data',
getAbbName(city) + '.json')) as f:
stream = json.load(f)
features = stream['features']
for each in features:
disobjs.append({
'name': each['properties']['name'],
'geo': shape(each['geometry'])
})
f.close()
# 遍历查询网格周围 POI 并更新
for latind in xrange(0, latnum):
for lngind in xrange(0, lngnum):
# 前11元素均为分类别统计数量,最后一个元素为POI总量
tmpGrid = [0 for x in xrange(0, 15)]
vaildGrid = False
lat = round(locs['south'] + latind * split, 3)
lng = round(locs['west'] + lngind * split, 3)
lnginc = round(lng + split, 3)
latinc = round(lat + split, 3)
lngcen = round(lng + centerincrement, 4)
latcen = round(lat + centerincrement, 4)
# 一个正方形 geojson 对象，代表当前方块对应的地理边界
coordsarr = [[lng, lat], [lnginc, lat], [lnginc, latinc],
[lng, latinc], [lng, lat]]
featurelistarray = [0] * 11
typevalid = False
# query all the POIs less than maxQRadius
nearPOIList = list(
POIs.find(
{
"properties.center": {
'$near': {
'$geometry': {
'type': "Point",
'coordinates': [lngcen, latcen]
},
'$minDistance': 0,
'$maxDistance': 2500 * (2**0.5)
}
}
}, {'properties': 1}))
# construct vector with POIs types info
poilen = len(nearPOIList)
if poilen != 0:
vaildGrid = True
# POI list is not null
for each in nearPOIList:
curPInd = each['properties']['ftype'] - 1
tmpGrid[curPInd] += 1
tmpGrid[11] += 1
if vaildGrid:
tmpGrid[12] = lngcen
tmpGrid[13] = latcen
point = Point(lngcen, latcen)
index = getDisIndex(point, disobjs)
tmpGrid[14] = index
if index == -1:
print 'Invalid Grid Found.'
grids.append(tmpGrid)
print "%s City with valid grids %s" % (city, str(len(grids)))
return grids