def loadArcData(shapeFileName):
"""
Handler to use PySAL W and dbf readers in place of clusterpy's
Parameters
==========
shapeFileName: string
filename including .shp extension
Returns
=======
layer: clusterpy layer instance
Examples
========
>>> import pysal.contrib.clusterpy as cp
>>> import pysal as ps
>>> shpFile = ps.examples.get_path('columbus.shp')
>>> columbus = cp.loadArcData(shpFile)
>>> columbus.Wrook[0]
[1, 2]
>>> columbus.Wrook[1]
[0, 2, 3]
>>> columbus.fieldNames[0:10]
['ID', 'AREA', 'PERIMETER', 'COLUMBUS_', 'COLUMBUS_I', 'POLYID', 'NEIG', 'HOVAL', 'INC', 'CRIME']
"""
base = shapeFileName.split(".")[0]
return _clusterpy.importArcData(base)
def loadArcData(shapeFileName):
"""
Handler to use PySAL W and dbf readers in place of clusterpy's
Parameters
==========
shapeFileName: string
filename including .shp extension
Returns
=======
layer: clusterpy layer instance
Examples
========
>>> import pysal.contrib.clusterpy as cp
>>> import pysal as ps
>>> shpFile = ps.examples.get_path('columbus.shp')
>>> columbus = cp.loadArcData(shpFile)
>>> columbus.Wrook[0]
[1, 2]
>>> columbus.Wrook[1]
[0, 2, 3]
>>> columbus.fieldNames[0:10]
['ID', 'AREA', 'PERIMETER', 'COLUMBUS_', 'COLUMBUS_I', 'POLYID', 'NEIG', 'HOVAL', 'INC', 'CRIME']
"""
base = shapeFileName.split(".")[0]
return _clusterpy.importArcData(base)
def setUp(self):
self.layer = importArcData(sample_input_path)
""" Simple 10x10 Grid with valid Region WRook configuration:
'**********' area 0-9
'**********' area 1-19
'**********' area 20-29
'*******ooo' area 30-39
'*********o' area 40-49
'oooooooooo' area 50-59
'oooooooooo' area 60-69
'oooooooooo' area 70-79
'oooooooooo' area 80-89
'oooooooooo' area 90-99
"""
self.possible_rook_solution = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 0, 0, 0,
1, 1, 1, 1, 1, 1, 1, 1, 1, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
""" Simple 10x10 Grid with valid Region WQueen configuration:
'**********' area 0-9
'**********' area 1-19
'**********' area 20-29
'*******ooo' area 30-39
'********oo' area 40-49
'oooooooo*o' area 50-59
'oooooooooo' area 60-69
'oooooooooo' area 70-79
'oooooooooo' area 80-89
'oooooooooo' area 90-99
"""
self.possible_queen_solution = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 0, 0, 0,
1, 1, 1, 1, 1, 1, 1, 1, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
def setUp(self):
self.map_instance = clusterpy.importArcData(sample_input_path)
# numThres = [2,5]
# import itertools
# exp =list(itertools.product(numArea, numThres, range(1,3)))
exp = [(3, 2, 1), (3, 2, 2), (3, 5, 1), (3, 5, 2), (4, 2, 1),
(4, 2, 2), (4, 5, 1), (4, 5, 2), (5, 2, 1), (5, 2, 2),
(5, 5, 1), (5, 5, 2), (6, 2, 1), (6, 2, 2), (6, 5, 1),
(6, 5, 2), (7, 2, 1), (7, 2, 2), (7, 5, 1), (7, 5, 2),
(8, 2, 1), (8, 2, 2), (8, 5, 1), (8, 5, 2)]
tup = exp[conseq]
area = tup[0]
thres = tup[1]
form = tup[2] #formulation
lay = clusterpy.importArcData("minpInstances/minp"+str(area*area))
#generate .txt file
if form==1: #Order
f = open(str(conseq)+"-"+str(area)+"-"+str(thres)+"-O"+".txt","w")
f.write(str(area*area)+" "+str(thres)+" O ")
f.close()
# Order formulation
lay.cluster('minpOrder',['SAR1', 'Uniform2'],threshold=thres, conseq=conseq)
minpExp1=lay.fieldNames[-1]
output1=lay.outputCluster[minpExp1]
f = open(str(conseq)+"-"+str(area)+"-"+str(thres)+"-O"+".txt","a")
f.write(str(output1["bestBound"])+" "+str(output1["objectiveFunction"])+" "+str(output1["running time"])+" "+str(output1["p"])+" ")
(0.5, 5, 1), (0.5, 5, 2), (0.5, 5, 3), (0.5, 5, 4), (0.5, 5, 5),
(0.5, 5, 6), (0.5, 5, 7), (0.5, 5, 8), (0.5, 5, 9), (0.5, 5, 10),
(0.5, 7, 1), (0.5, 7, 2), (0.5, 7, 3), (0.5, 7, 4), (0.5, 7, 5), (0.5, 7, 6),
(0.5, 7, 7), (0.5, 7, 8), (0.5, 7, 9), (0.5, 7, 10), (0.9, 3, 1), (0.9, 3, 2),
(0.9, 3, 3), (0.9, 3, 4), (0.9, 3, 5), (0.9, 3, 6), (0.9, 3, 7), (0.9, 3, 8),
(0.9, 3, 9), (0.9, 3, 10), (0.9, 5, 1), (0.9, 5, 2), (0.9, 5, 3), (0.9, 5, 4),
(0.9, 5, 5), (0.9, 5, 6), (0.9, 5, 7), (0.9, 5, 8), (0.9, 5, 9), (0.9, 5, 10),
(0.9, 7, 1), (0.9, 7, 2), (0.9, 7, 3), (0.9, 7, 4), (0.9, 7, 5), (0.9, 7, 6),
(0.9, 7, 7), (0.9, 7, 8), (0.9, 7, 9), (0.9, 7, 10)]
tup = exp[conseq]
rho = tup[0]
p = tup[1]
inst = tup[2]
lay = clusterpy.importArcData("HCinstances/hc25-"+str(rho)+"-"+str(inst))
# #QUITAR
# y=lay.Y
# w=lay.Wrook
# start = tm.time()
# # Number of areas
# n = len(y)
# # Region iterator
# numR = range(p)
# # Area iterator
# numA = range(n)
def setUp(self):
map_instance = importArcData(sample_input_path)
self.Y = map_instance.Y
self.Wrook = map_instance.Wrook
self.Wqueen = map_instance.Wqueen
def cluster_with_clusterpy(new_coor,
new_weight,
ref_list,
nb_data,
list_coor,
out_file,
out_file_2,
hierarchi='False'):
# GeoSOM
data = cp.importArcData("map_test/test_big_grid")
var = data.fieldNames
var_algo = var[0:len(var) - 1]
if hierarchi == 'False':
it = 100
nRow = 22
nCol = 22
else:
it = 300
nRow = 30
nCol = 30
data.cluster('geoSom',
var_algo,
wType='queen',
nRows=nRow,
nCols=nCol,
iters=it)
#data.exportRegions2area("test.csv")
"""
with open('test.csv','rb') as csvfile:
reader = csv.reader(csvfile,delimiter=';')
for row in reader:
id_clust.append(int(row[1]))
"""
cluster_init = data.region2areas
id_clust = []
for row in cluster_init:
id_clust.append(row)
### Create geoson file for the map
nb_clust = nRow * nCol
print nb_clust
print new_weight.shape
# Centroid points
# Polygon preparation
# Preparation of hierarchical clustering
centroid = data.oCentroid
dataCentroid = data.dataCentroid
nb_feat = new_weight.shape[1]
if hierarchi == 'True':
nb_clust = 800
id_clust = hierarchical_clustering(nb_clust, nb_feat, centroid,
cluster_init, dataCentroid)
else:
nb_clust = 380
id_clust = hierarchical_clustering(nb_clust, nb_feat, centroid,
cluster_init, dataCentroid)
feature_list = []
polygon_list = []
idx = 0
# Creation of the food list for each cluster (we take only the term with the highest frequency inside the cluster)
term_list = []
for i in range(nb_clust):
term_list.append([])
cor_idx = 0
for i in new_weight:
idx = 0
for j in i:
if j > 0:
term_list[id_clust[cor_idx]].append(ref_list[idx])
idx += 1
cor_idx += 1
final_term_list = []
for i in range(nb_clust):
c = Counter(term_list[i])
print c
if c.values():
m = max(c.values())
r = [k for k in c if c[k] == m]
else:
r = []
final_term_list.append(r)
print "r: ", r
# Modify cluster numbers so that they are consecutive
new_clust_id = []
list_clust = []
for i in id_clust:
if i not in list_clust:
list_clust.append(i)
print len(list_clust)
for i in id_clust:
new_clust_id.append(list_clust.index(i))
idx = 0
for feat in new_coor:
food_list = []
f_idx = 0
for f in new_weight[idx]:
if f > 0:
food_list.append(ref_list[f_idx])
f_idx = f_idx + 1
feature_list.append({
"type": "Feature",
"geometry": {
"type": "Point",
"coordinates": [feat[0], feat[1]],
},
"properties": {
"category": food_list,
"cluster": new_clust_id[idx]
}
})
polygon_list.append({
"type": "Feature",
"geometry": {
"type": "Polygon",
"coordinates": [[list_coor[idx]]],
},
"properties": {
"category": final_term_list[id_clust[idx]],
"cluster": new_clust_id[idx]
}
})
idx = idx + 1
geojson = {"type": "FeatureCollection", "features": feature_list}
geojson_2 = {"type": "FeatureCollection", "features": polygon_list}
output = open(out_file, 'w')
json.dump(geojson, output)
output_2 = open(out_file_2, 'w')
json.dump(geojson_2, output_2)
print new_coor
print weight
def setUp(self):
self.map_instance = clusterpy.importArcData(sample_input_path)
(7, 11, 0.5, 4, 1), (7, 11, 0.5, 4, 2), (7, 11, 0.5, 5, 1), (7, 11, 0.5, 5, 2),
(7, 11, 0.7, 1, 1), (7, 11, 0.7, 1, 2), (7, 11, 0.7, 2, 1), (7, 11, 0.7, 2, 2),
(7, 11, 0.7, 3, 1), (7, 11, 0.7, 3, 2), (7, 11, 0.7, 4, 1), (7, 11, 0.7, 4, 2),
(7, 11, 0.7, 5, 1), (7, 11, 0.7, 5, 2), (7, 11, 0.9, 1, 1), (7, 11, 0.9, 1, 2),
(7, 11, 0.9, 2, 1), (7, 11, 0.9, 2, 2), (7, 11, 0.9, 3, 1), (7, 11, 0.9, 3, 2),
(7, 11, 0.9, 4, 1), (7, 11, 0.9, 4, 2), (7, 11, 0.9, 5, 1), (7, 11, 0.9, 5, 2)]
tup = exp[conseq]
area = tup[0]
reg = tup[1]
rho = tup[2]
inst = tup[3]
form = tup[4]
lay = clusterpy.importArcData("pregInstances/preg"+str(area*area)+"-"+str(rho)+"-"+str(inst))
#f.write("n p rho instance best-bound best-int time sol best-boundCP best-intCP timeCP solCP\n")
#generate .txt file
if form==1: # without
f = open(str(conseq)+"-"+str(area)+"-"+str(reg)+"-"+str(rho)+"-"+str(inst)+"-E"+".txt","w")
f.write(str(area*area)+" "+str(reg)+" "+str(rho)+" "+str(inst)+" E ")
f.close()
# WITHOUT CUTTING PLANES
lay.cluster('pRegionsExact',['SAR1'],p=reg, rho=rho, inst=inst, conseq=conseq)
pregExp=lay.fieldNames[-1]
output=lay.outputCluster[pregExp]
# starts at the second element of argv since the first one is the script name
# extraparms are extra arguments passed after all option/keywords are assigned
# opts is a list containing the pair "option"/"value"
# print 'Opts:',opts
# print 'Extra parameters:',extraparams
inputsPath = None
valuesOn = None
for o,p in opts:
if o in ['-i','--input']:
inputsPath = p
if o in ['-v','--values']:
valuesOn = p
if (inputsPath is not None):
try:
# Load data
data = clusterpy.importArcData(inputsPath)
# Instantiate json return string
json = "{\"statistics\":["
jsonValues = "\"values\":["
# Loop through columns, compute basic statistics or get histogram, and add to json
for index, field in enumerate(data.fieldNames,start=1):
variable = "{\"name\":\""+field+"\","
variableValues = "{\"name\":\""+field+"\","
print field
values = [i[0] for i in data.getVars(field).values()]
stringVars = False
variableValues+="\"values\":[";
def setUp(self):
map_instance = importArcData(sample_input_path)
self.Y = map_instance.Y
self.Wrook = map_instance.Wrook
self.Wqueen = map_instance.Wqueen
# opts is a list containing the pair "option"/"value"
# print 'Opts:',opts
# print 'Extra parameters:',extraparams
inputsPath = None
operation = None
for o,p in opts:
if o in ['-s','--shape']:
shapePath = p
elif o in ['-c','--csv']:
csvPath = p
print(shapePath)
print(csvPath)
if (shapePath is not None and csvPath is not None):
arc = clusterpy.importArcData(shapePath)
data = clusterpy.importCSV(csvPath)
try:
os.remove(shapePath+"_backup.shp");
os.remove(shapePath+"_backup.shx");
os.remove(shapePath+"_backup.dbf");
except Exception as e:
print(e);
os.rename(shapePath+".shp",shapePath+"_backup.shp")
os.rename(shapePath+".shx",shapePath+"_backup.shx")
os.rename(shapePath+".dbf",shapePath+"_backup.dbf")
try:
arc.addVariable(data[1],data[0])
(0.5, 5, 6), (0.5, 5, 7), (0.5, 5, 8), (0.5, 5, 9), (0.5, 5, 10),
(0.5, 7, 1), (0.5, 7, 2), (0.5, 7, 3), (0.5, 7, 4), (0.5, 7, 5),
(0.5, 7, 6), (0.5, 7, 7), (0.5, 7, 8), (0.5, 7, 9), (0.5, 7, 10),
(0.9, 3, 1), (0.9, 3, 2), (0.9, 3, 3), (0.9, 3, 4), (0.9, 3, 5),
(0.9, 3, 6), (0.9, 3, 7), (0.9, 3, 8), (0.9, 3, 9), (0.9, 3, 10),
(0.9, 5, 1), (0.9, 5, 2), (0.9, 5, 3), (0.9, 5, 4), (0.9, 5, 5),
(0.9, 5, 6), (0.9, 5, 7), (0.9, 5, 8), (0.9, 5, 9), (0.9, 5, 10),
(0.9, 7, 1), (0.9, 7, 2), (0.9, 7, 3), (0.9, 7, 4), (0.9, 7, 5),
(0.9, 7, 6), (0.9, 7, 7), (0.9, 7, 8), (0.9, 7, 9), (0.9, 7, 10)]
tup = exp[conseq]
rho = tup[0]
p = tup[1]
inst = tup[2]
lay = clusterpy.importArcData("HCinstances/hc25-" + str(rho) + "-" + str(inst))
# #QUITAR
# y=lay.Y
# w=lay.Wrook
# start = tm.time()
# # Number of areas
# n = len(y)
# # Region iterator
# numR = range(p)
# # Area iterator
# numA = range(n)
def cluster_with_clusterpy(new_coor, new_weight, ref_list, nb_data, list_coor, out_file, out_file_2):
# GeoSOM
data = cp.importArcData("map_test/test_big_grid")
var = data.fieldNames
var_algo = var[0:len(var)-1]
print len(var)
nRow = 20
nCol = 20
data.cluster('geoSom', var_algo,wType='queen', nRows = nRow, nCols = nCol,iters=50)
data.exportRegions2area("test.csv")
id_clust = []
with open('test.csv','rb') as csvfile:
reader = csv.reader(csvfile,delimiter=';')
for row in reader:
id_clust.append(int(row[1]))
### Create geoson file for the map
nb_cluster = nRow*nCol
print nb_cluster
# Centroid points
# Polygon preparation
feature_list = []
polygon_list = []
idx = 0
for feat in new_coor:
food_list = []
f_idx = 0
for f in new_weight[idx]:
if f > 0:
food_list.append(ref_list[f_idx])
f_idx = f_idx + 1
feature_list.append({
"type": "Feature",
"geometry" : {
"type": "Point",
"coordinates": [feat[0], feat[1]],
},
"properties" : {
"category": food_list,
"cluster": id_clust[idx]
}
})
polygon_list.append({
"type": "Feature",
"geometry" : {
"type": "Polygon",
"coordinates": [[list_coor[idx]]],
},
"properties" : {
"category": food_list,
"cluster": id_clust[idx]
}
})
idx = idx + 1
geojson = {
"type": "FeatureCollection",
"features": feature_list
}
geojson_2 = {
"type": "FeatureCollection",
"features": polygon_list
}
output = open(out_file, 'w')
json.dump(geojson, output)
output_2 = open(out_file_2, 'w')
json.dump(geojson_2, output_2)
import clusterpy
c = clusterpy.importArcData("input/voronoi_40percent")
c.cluster('maxpTabu', ['price_per_', 'capacity'], threshold=150, std=1)
c.exportArcData("output/maxp_capacity_150")
# Change the following paths acording to the location on you computer
start = time.time()
if (inputsPath is not None) & (outputsPath is not None) & (algorithm is not None) & (vars is not None):
fileName = inputsPath
output = outputsPath
logPath = outputsPath[:(outputsPath.rindex("/")+1)]+"LOG.txt"
logcsvPath = outputsPath[:(outputsPath.rindex("/")+1)]+"LOG.csv"
errorPath = outputsPath[:(outputsPath.rindex("/")+1)]+"ERROR.txt"
success = False
end = 0
print vars
try:
if (algorithm == 'arisel')& (pReg is not None):
instance = clusterpy.importArcData(fileName)
instance.cluster('arisel', vars, pReg, wType=wType, std=std, inits=inits, convTabu=convTabu, tabuLength=tabuLength, dissolve=dissolve)
#instance.cluster('arisel', ['SAR1'], regions=pReg, dissolve=1)
#field = instance.fieldNames[-1]
#results.append(instance.outputCluster[field])
instance.save(output)
instance.exportArcData(output)
#instance.dissolveMap()
#instance.results[0].exportArcData(output)
elif (algorithm == 'azp')& (pReg is not None):
instance = clusterpy.importArcData(fileName)
instance.cluster('azp', vars, pReg, wType=wType, std=std, dissolve=dissolve)
#field = instance.fieldNames[-1]
#results.append(instance.outputCluster[field])
instance.save(output)
instance.exportArcData(output)
# numArea = [3,4,5,6,7,8]
# numThres = [2,5]
# import itertools
# exp =list(itertools.product(numArea, numThres, range(1,3)))
exp = [(3, 2, 1), (3, 2, 2), (3, 5, 1), (3, 5, 2), (4, 2, 1), (4, 2, 2),
(4, 5, 1), (4, 5, 2), (5, 2, 1), (5, 2, 2), (5, 5, 1), (5, 5, 2),
(6, 2, 1), (6, 2, 2), (6, 5, 1), (6, 5, 2), (7, 2, 1), (7, 2, 2),
(7, 5, 1), (7, 5, 2), (8, 2, 1), (8, 2, 2), (8, 5, 1), (8, 5, 2)]
tup = exp[conseq]
area = tup[0]
thres = tup[1]
form = tup[2] #formulation
lay = clusterpy.importArcData("minpInstances/minp" + str(area * area))
#generate .txt file
if form == 1: #Order
f = open(
str(conseq) + "-" + str(area) + "-" + str(thres) + "-O" + ".txt", "w")
f.write(str(area * area) + " " + str(thres) + " O ")
f.close()
# Order formulation
lay.cluster('minpOrder', ['SAR1', 'Uniform2'],
threshold=thres,
conseq=conseq)
minpExp1 = lay.fieldNames[-1]
