def test_sids(self):
w = pysal.open(pysal.examples.get_path("sids2.gal")).read()
f = pysal.open(pysal.examples.get_path("sids2.dbf"))
SIDR = np.array(f.by_col("SIDR74"))
mi = moran.Moran(SIDR, w, two_tailed=False)
np.testing.assert_allclose(mi.I, 0.24772519320480135, atol=ATOL, rtol=RTOL)
self.assertAlmostEqual(mi.p_norm, 5.7916539074498452e-05)
def test___init__(self):
f = ps.open(ps.examples.get_path('usjoin.csv'))
pci = np.array([f.by_col[str(y)]
for y in range(1929, 2010)]).transpose()
w = ps.open(ps.examples.get_path("states48.gal")).read()
lm = LISA_Markov(pci, w)
obs = np.array([1, 2, 3, 4])
np.testing.assert_array_almost_equal(obs, lm.classes)
ss = np.array([0.28561505, 0.14190226, 0.40493672, 0.16754598])
np.testing.assert_array_almost_equal(lm.steady_state, ss)
transitions = np.array(
[[1.08700000e+03, 4.40000000e+01, 4.00000000e+00, 3.40000000e+01],
[4.10000000e+01, 4.70000000e+02, 3.60000000e+01, 1.00000000e+00],
[5.00000000e+00, 3.40000000e+01, 1.42200000e+03, 3.90000000e+01],
[3.00000000e+01, 1.00000000e+00, 4.00000000e+01, 5.52000000e+02]])
np.testing.assert_array_almost_equal(lm.transitions, transitions)
p = np.array([[0.92985458, 0.03763901, 0.00342173, 0.02908469],
[0.07481752, 0.85766423, 0.06569343, 0.00182482],
[0.00333333, 0.02266667, 0.948, 0.026],
[0.04815409, 0.00160514, 0.06420546, 0.88603531]])
np.testing.assert_array_almost_equal(lm.p, p)
np.random.seed(10)
lm_random = LISA_Markov(pci, w, permutations=99)
expected = np.array(
[[1.12328098e+03, 1.15377356e+01, 3.47522158e-01, 3.38337644e+01],
[3.50272664e+00, 5.28473882e+02, 1.59178880e+01, 1.05503814e-01],
[1.53878082e-01, 2.32163556e+01, 1.46690710e+03, 9.72266513e+00],
[9.60775143e+00, 9.86856346e-02, 6.23537392e+00, 6.07058189e+02]])
np.testing.assert_allclose(lm_random.expected_t, expected, RTOL)
c = np.array([1058.207904, 0., 9.])
np.testing.assert_allclose(lm_random.chi_2, c, RTOL)
def setUp(self):
f = pysal.open(pysal.examples.get_path("sids2.dbf"))
varnames = ['SIDR74', 'SIDR79', 'NWR74', 'NWR79']
self.names = varnames
vars = [np.array(f.by_col[var]) for var in varnames]
self.vars = vars
self.w = pysal.open(pysal.examples.get_path("sids2.gal")).read()
def __init__(self, shapefile, idvariable=None, attribute=False):
self.points = {}
self.npoints = 0
if idvariable:
ids = get_ids(shapefile, idvariable)
else:
ids = None
pts = ps.open(shapefile)
# Get attributes if requested
if attribute == True:
dbname = os.path.splitext(shapefile)[0] + '.dbf'
db = ps.open(dbname)
else:
db = None
for i, pt in enumerate(pts):
if ids and db:
self.points[ids[i]] = {'coordinates': pt, 'properties': db[i]}
elif ids and not db:
self.points[ids[i]] = {'coordinates': pt, 'properties': None}
elif not ids and db:
self.points[i] = {'coordinates': pt, 'properties': db[i]}
else:
self.points[i] = {'coordinates': pt, 'properties': None}
pts.close()
if db:
db.close()
self.npoints = len(self.points.keys())
def setUp(self):
f = ps.open(ps.examples.get_path('usjoin.csv'))
pci = np.array([f.by_col[str(y)] for y in range(1929, 2010)])
pci = pci.transpose()
self.rpci = pci / (pci.mean(axis=0))
self.discretized = (self.rpci * 100).astype(int) % 4
self.w = ps.open(ps.examples.get_path("states48.gal")).read()
self.w.transform = 'r'
def test___init__(self):
f = libpysal.open(libpysal.examples.get_path('usjoin.csv'))
pci = np.array([f.by_col[str(y)] for y in range(1929, 2010)])
pci = pci.transpose()
rpci = pci / (pci.mean(axis=0))
w = libpysal.open(libpysal.examples.get_path("states48.gal")).read()
w.transform = 'r'
sm = gapi.Spatial_Markov(rpci, w, fixed=True, k=5)
S = np.array(
[[0.43509425, 0.2635327, 0.20363044, 0.06841983, 0.02932278],
[0.13391287, 0.33993305, 0.25153036, 0.23343016, 0.04119356],
[0.12124869, 0.21137444, 0.2635101, 0.29013417, 0.1137326],
[0.0776413, 0.19748806, 0.25352636, 0.22480415, 0.24654013],
[0.01776781, 0.19964349, 0.19009833, 0.25524697, 0.3372434]])
np.testing.assert_array_almost_equal(S, sm.S)
def setUp(self):
f = libpysal.open(libpysal.examples.get_path("mexico.csv"))
vnames = ["pcgdp%d" % dec for dec in range(1940, 2010, 10)]
y = np.transpose(np.array([f.by_col[v] for v in vnames]))
self.y = y[:, 0]
regimes = np.array(f.by_col('hanson98'))
self.w = ps.block_weights(regimes)
def plot_poly_lines(shp_link, savein=None, poly_col='none'):
'''
Quick plotting of shapefiles
...
Arguments
---------
shp_link : str
Path to shapefile
savein : str
Path to png file where to dump the plot. Optional,
defaults to None
poly_col : str
Face color of polygons
'''
fig = plt.figure()
shp = ps.open(shp_link)
patchco = map_poly_shp(shp)
patchco.set_facecolor('none')
patchco.set_edgecolor('0.8')
ax = setup_ax([patchco], [shp.bbox])
fig.add_axes(ax)
if savein:
plt.savefig(savein)
else:
print('calling plt.show()')
plt.show()
return None
def setUp(self):
f = open(libpysal.examples.get_path("spi_download.csv"), 'r')
lines = f.readlines()
f.close()
lines = [line.strip().split(",") for line in lines]
names = [line[2] for line in lines[1:-5]]
data = np.array([list(map(int, line[3:])) for line in lines[1:-5]])
sids = list(range(60))
out = ['"United States 3/"',
'"Alaska 3/"',
'"District of Columbia"',
'"Hawaii 3/"',
'"New England"',
'"Mideast"',
'"Great Lakes"',
'"Plains"',
'"Southeast"',
'"Southwest"',
'"Rocky Mountain"',
'"Far West 3/"']
snames = [name for name in names if name not in out]
sids = [names.index(name) for name in snames]
states = data[sids, :]
us = data[0]
years = np.arange(1969, 2009)
rel = states / (us * 1.)
gal = libpysal.open(libpysal.examples.get_path('states48.gal'))
self.w = gal.read()
self.w.transform = 'r'
self.Y = rel[:, [0, -1]]
def test___init__(self):
f = ps.open(ps.examples.get_path('usjoin.csv'))
pci = np.array([f.by_col[str(y)] for y in range(1929, 2010)])
q5 = np.array([mc.Quantiles(y).yb for y in pci]).transpose()
m = Markov(q5)
res = np.array(
[0.08988764, 0.21468144, 0.21125, 0.20194986, 0.07259074])
np.testing.assert_array_almost_equal(prais(m.p), res)
def test___init__(self):
import numpy as np
f = ps.open(ps.examples.get_path('usjoin.csv'))
pci = np.array([f.by_col[str(y)] for y in range(1929, 2010)])
q5 = np.array([mc.Quantiles(y).yb for y in pci]).transpose()
m = Markov(q5)
np.testing.assert_array_almost_equal(markov_mobility(m.p, measure="P"),
0.19758992000997844)
def test_by_col(self):
from libpysal.io import geotable as pdio
df = pdio.read_files(pysal.examples.get_path('sids2.dbf'))
w = pysal.open(pysal.examples.get_path("sids2.gal")).read()
mi = moran.Moran.by_col(df, ['SIDR74'], w=w, two_tailed=False)
sidr = np.unique(mi.SIDR74_moran.values)
pval = np.unique(mi.SIDR74_p_sim.values)
np.testing.assert_allclose(sidr, 0.24772519320480135, atol=ATOL, rtol=RTOL)
self.assertAlmostEqual(pval, 0.001)
def setUp(self):
db = libpysal.open(libpysal.examples.get_path('columbus.dbf'), 'r')
y = np.array(db.by_col("HOVAL"))
y = np.reshape(y, (49, 1))
self.y = np.round(y).astype(int)
X = []
X.append(db.by_col("INC"))
X.append(db.by_col("CRIME"))
self.X = np.array(X).T
def test___init__(self):
import numpy as np
f = libpysal.open(libpysal.examples.get_path('usjoin.csv'))
pci = np.array([f.by_col[str(y)] for y in range(1929, 2010)])
q5 = np.array([mc.Quantiles(y).yb for y in pci]).transpose()
m = gapi.Markov(q5)
res = np.matrix(
[[0.08988764, 0.21468144, 0.21125, 0.20194986, 0.07259074]])
np.testing.assert_array_almost_equal(gapi.prais(m.p), res)
def test___init__(self):
f = libpysal.open(libpysal.examples.get_path("mexico.csv"))
vnames = ["pcgdp%d" % dec for dec in range(1940, 2010, 10)]
y = np.transpose(np.array([f.by_col[v] for v in vnames]))
regimes = np.array(f.by_col('hanson98'))
np.random.seed(10)
theil_ds = gapi.TheilDSim(y, regimes, 999)
np.testing.assert_almost_equal(
theil_ds.bg_pvalue,
np.array([0.4, 0.344, 0.001, 0.001, 0.034, 0.072, 0.032]))
def setUp(self):
data_path = os.path.join(os.path.dirname(__file__),
'georgia/GData_utm.csv')
data = libpysal.open(data_path)
self.coords = list(zip(data.by_col('X'), data.by_col('Y')))
self.y = np.array(data.by_col('PctBach')).reshape((-1, 1))
rural = np.array(data.by_col('PctRural')).reshape((-1, 1))
pov = np.array(data.by_col('PctPov')).reshape((-1, 1))
black = np.array(data.by_col('PctBlack')).reshape((-1, 1))
fb = np.array(data.by_col('PctFB')).reshape((-1, 1))
self.X = np.hstack([rural, pov, black])
self.mgwr_X = np.hstack([fb, black, rural])
def test___init__(self):
# theil = Theil(y)
f = libpysal.open(libpysal.examples.get_path("mexico.csv"))
vnames = ["pcgdp%d" % dec for dec in range(1940, 2010, 10)]
y = np.transpose(np.array([f.by_col[v] for v in vnames]))
theil_y = gapi.Theil(y)
np.testing.assert_almost_equal(
theil_y.T,
np.array([
0.20894344, 0.15222451, 0.10472941, 0.10194725, 0.09560113,
0.10511256, 0.10660832
]))
def test_chi2(self):
f = ps.open(ps.examples.get_path('usjoin.csv'))
pci = np.array([f.by_col[str(y)] for y in range(1929, 2010)])
pci = pci.transpose()
rpci = pci / (pci.mean(axis=0))
w = ps.open(ps.examples.get_path("states48.gal")).read()
w.transform = 'r'
sm = Spatial_Markov(rpci, w, fixed=True, k=5, m=5)
chi = np.array([[4.05598541e+01, 6.44644317e-04, 1.60000000e+01],
[5.54751974e+01, 2.97033748e-06, 1.60000000e+01],
[1.77528996e+01, 3.38563882e-01, 1.60000000e+01],
[4.00390961e+01, 7.68422046e-04, 1.60000000e+01],
[4.67966803e+01, 7.32512065e-05, 1.60000000e+01]])
obs = np.array(sm.chi2)
np.testing.assert_array_almost_equal(obs, chi)
obs = np.array([[4.61209613e+02, 0.00000000e+00, 4.00000000e+00],
[1.48140694e+02, 0.00000000e+00, 4.00000000e+00],
[6.33129261e+01, 5.83089133e-13, 4.00000000e+00],
[7.22778509e+01, 7.54951657e-15, 4.00000000e+00],
[2.32659201e+02, 0.00000000e+00, 4.00000000e+00]])
np.testing.assert_array_almost_equal(obs, np.array(sm.shtest))
def setUp(self):
sids = pysal.open(pysal.examples.get_path('sids2.dbf'), 'r')
self.w = pysal.open(pysal.examples.get_path('sids2.gal'), 'r').read()
self.b, self.e = np.array(sids[:, 8]), np.array(sids[:, 9])
self.er = [0.453433, 0.000000, 0.775871, 0.973810, 3.133190]
self.eb = [0.0016973, 0.0017054, 0.0017731, 0.0020129, 0.0035349]
self.sr = [0.0009922, 0.0012639, 0.0009740, 0.0007605, 0.0050154]
self.smr = [0.00083622, 0.00109402, 0.00081567, 0.0, 0.0048209]
self.smr_w = [0.00127146, 0.00127146, 0.0008433, 0.0, 0.0049889]
self.smr2 = [0.00091659, 0.00087641, 0.00091073, 0.0, 0.00467633]
self.s_ebr10 = np.array([
4.01485749e-05, 3.62437513e-05, 4.93034844e-05, 5.09387329e-05,
3.72735210e-05, 3.69333797e-05, 5.40245456e-05, 2.99806055e-05,
3.73034109e-05, 3.47270722e-05
]).reshape(-1, 1)
self.stl = pysal.open(pysal.examples.get_path('stl_hom.csv'), 'r')
self.stl_e, self.stl_b = np.array(self.stl[:,
10]), np.array(self.stl[:,
13])
self.stl_w = pysal.open(pysal.examples.get_path('stl.gal'), 'r').read()
if not self.stl_w.id_order_set:
self.stl_w.id_order = list(range(1, len(self.stl) + 1))
if not PANDAS_EXTINCT:
self.df = pysal.open(pysal.examples.get_path('sids2.dbf')).to_df()
self.ename = 'SID74'
self.bname = 'BIR74'
self.stl_df = pysal.open(
pysal.examples.get_path('stl_hom.csv')).to_df()
self.stl_ename = 'HC7984'
self.stl_bname = 'PO7984'
def test___init__(self):
# theil_d = TheilD(y, partition)
f = libpysal.open(libpysal.examples.get_path("mexico.csv"))
vnames = ["pcgdp%d" % dec for dec in range(1940, 2010, 10)]
y = np.transpose(np.array([f.by_col[v] for v in vnames]))
regimes = np.array(f.by_col('hanson98'))
theil_d = gapi.TheilD(y, regimes)
np.testing.assert_almost_equal(
theil_d.bg,
np.array([
0.0345889, 0.02816853, 0.05260921, 0.05931219, 0.03205257,
0.02963731, 0.03635872
]))
def setUp(self):
sids = pysal.open(pysal.examples.get_path('sids2.shp'), 'r')
self.sids = sids
self.d = np.array([i.centroid for i in sids])
self.w = KNN.from_array(self.d, k=5)
if not self.w.id_order_set:
self.w.id_order = self.w.id_order
sids_db = pysal.open(pysal.examples.get_path('sids2.dbf'), 'r')
self.b, self.e = np.array(sids_db[:, 8]), np.array(sids_db[:, 9])
self.sids_hb_rr5 = np.array(
[0.00075586, 0., 0.0008285, 0.0018315, 0.00498891])
self.sids_hb_r2r5 = np.array(
[0.0008285, 0.00084331, 0.00086896, 0.0018315, 0.00498891])
self.sids_hb_r3r5 = np.array(
[0.00091659, 0., 0.00156838, 0.0018315, 0.00498891])
if not PANDAS_EXTINCT:
self.df = sids_db.to_df()
self.ename = 'SID74'
self.bname = 'BIR74'
self.enames = [self.ename, 'SID79']
self.bnames = [self.bname, 'BIR79']
self.sids79r = np.array(
[.000563, .001659, .001879, .002410, .002720])
def test___init__(self):
pi = np.array([0.1, 0.2, 0.2, 0.4, 0.1])
f = ps.open(ps.examples.get_path('usjoin.csv'))
pci = np.array([f.by_col[str(y)] for y in range(1929, 2010)])
q5 = np.array([mc.Quantiles(y).yb for y in pci]).transpose()
m = Markov(q5)
np.testing.assert_array_almost_equal(markov_mobility(m.p, measure="P"),
0.19758992000997844)
np.testing.assert_array_almost_equal(
markov_mobility(m.p, measure="D"),
0.60684854623695594)
np.testing.assert_array_almost_equal(
markov_mobility(m.p, measure="L2"),
0.039782002308159647)
np.testing.assert_array_almost_equal(
markov_mobility(m.p, measure="B1", ini=pi),
0.2277675878319787)
np.testing.assert_array_almost_equal(
markov_mobility(m.p, measure="B2", ini=pi),
0.046366601194789261)
def test___init__(self):
# markov = Markov(class_ids, classes)
f = ps.open(ps.examples.get_path('usjoin.csv'))
pci = np.array([f.by_col[str(y)] for y in range(1929, 2010)])
q5 = np.array([mc.Quantiles(y).yb for y in pci]).transpose()
m = Markov(q5)
expected = np.array([[729., 71., 1., 0., 0.], [72., 567., 80., 3., 0.],
[0., 81., 631., 86.,
2.], [0., 3., 86., 573., 56.],
[0., 0., 1., 57., 741.]])
np.testing.assert_array_equal(m.transitions, expected)
expected = np.array(
[[0.91011236, 0.0886392, 0.00124844, 0., 0.],
[0.09972299, 0.78531856, 0.11080332, 0.00415512, 0.],
[0., 0.10125, 0.78875, 0.1075, 0.0025],
[0., 0.00417827, 0.11977716, 0.79805014, 0.07799443],
[0., 0., 0.00125156, 0.07133917, 0.92740926]])
np.testing.assert_array_almost_equal(m.p, expected)
expected = np.array(
[0.20774716, 0.18725774, 0.20740537, 0.18821787, 0.20937187])
np.testing.assert_array_almost_equal(m.steady_state, expected)
def _extractnetwork(self):
"""
Used internally, to extract a network from a polyline shapefile.
"""
nodecount = 0
shps = ps.open(self.in_shp)
for shp in shps:
vertices = shp.vertices
for i, v in enumerate(vertices[:-1]):
v = self._round_sig(v)
try:
vid = self.nodes[v]
except:
self.nodes[v] = vid = nodecount
nodecount += 1
v2 = self._round_sig(vertices[i + 1])
try:
nvid = self.nodes[v2]
except:
self.nodes[v2] = nvid = nodecount
nodecount += 1
self.adjacencylist[vid].append(nvid)
self.adjacencylist[nvid].append(vid)
# Sort the edges so that mono-directional keys can be stored.
edgenodes = sorted([vid, nvid])
edge = tuple(edgenodes)
self.edges.append(edge)
length = util.compute_length(v, vertices[i + 1])
self.edge_lengths[edge] = length
if self.unique_segs == True:
# Remove duplicate edges and duplicate adjacent nodes.
self.edges = list(set(self.edges))
for k, v in self.adjacencylist.iteritems():
self.adjacencylist[k] = list(set(v))
def setUp(self):
self.w = pysal.open(pysal.examples.get_path("stl.gal")).read()
f = pysal.open(pysal.examples.get_path("stl_hom.txt"))
self.y = np.array(f.by_col['HR8893'])
# In[6]:
m.p
# This means that if any of the 5 pixels was in state 'c', the probability of staying at 'c' or transitioning to any other states ('a', 'b') in the next period is the same (0.333). If a pixel was in state 'b', there is a high possibility that it would take on state 'c' in the next period because $\hat{p}_{2,3}=0.667$.
#
# This simple example illustrates the basic creation of a Markov instance, but the small sample size makes it unrealistic for the more advanced features of this approach. For a larger example, we will look at an application of Markov methods to understanding regional income dynamics in the US. Here we will load in data on per capita incomes observed annually from 1929 to 2010 for the lower 48 US states:
# #### Regional income dynamics in the US
# Firstly, we load in data on per capita incomes observed annually from 1929 to 2009 for the lower 48 US states. We use the example dataset in [**libpysal**](https://github.com/pysal/libpysal) which was downloaded from [US Bureau of Economic Analysis](https://www.bea.gov).
# In[7]:
f = libpysal.open(libpysal.examples.get_path("usjoin.csv"))
pci = np.array([f.by_col[str(y)] for y in range(1929, 2010)])
pci.shape
# The first row of the array is the per capita incomes for the 48 US states for the year 1929:
# In[8]:
pci[0, :]
# In order to apply the classic Markov approach to this series, we first have to discretize the distribution by defining our classes. There are many ways to do this including quantiles classification scheme, equal interval classification scheme, Fisher Jenks classification scheme, etc. For a list of classification methods, please refer to the pysal package [**mapclassify**](https://github.com/pysal/mapclassify).
#
# Here we will use the quintiles for each annual income distribution to define the classes. It should be noted that using quintiles for the pooled income distribution to define the classes will result in a different interpretation of the income dynamics. Quintiles for each annual income distribution (the former) will reveal more of relative income dynamics while those for the pooled income distribution (the latter) will provide insights in absolute dynamics.
# In[9]:
def setUp(self):
np.random.seed(10)
self.w = pysal.open(pysal.examples.get_path("desmith.gal")).read()
f = pysal.open(pysal.examples.get_path("desmith.txt"))
self.y = np.array(f.by_col['z'])
sids = [names.index(name) for name in snames]
states = data[sids, :]
us = data[0]
years = np.arange(1969, 2009)
rel = states / (us * 1.)
Y = rel[:, [0, -1]]
###############################################################################
# Spatial Weights
# ---------------
#
# We will use a simple contiguity structure to define neighbors. The file
# states48.gal encodes the adjacency structure of the 48 states. We read this in
# and row-normalize the weights:
gal = lps.open(lps.examples.get_path('states48.gal'))
w = gal.read()
w.transform = 'r'
##########################################
# Visualization
# ==============
#
# The Rose class creates a circular histogram that can be used to examine the distribution
# of LISA Vectors across segments of the histogram:
r4 = Rose(Y, w, k=4)
##########################################
# LISA Vectors
# ------------
def setUp(self):
self.w = pysal.open(pysal.examples.get_path("sids2.gal")).read()
f = pysal.open(pysal.examples.get_path("sids2.dbf"))
self.e = np.array(f.by_col['SID79'])
self.b = np.array(f.by_col['BIR79'])
def setUp(self):
np.random.seed(10)
self.w = pysal.open(pysal.examples.get_path("sids2.gal")).read()
f = pysal.open(pysal.examples.get_path("sids2.dbf"))
self.x = np.array(f.by_col['SIDR79'])
self.y = np.array(f.by_col['SIDR74'])