def _get_source_folder(source_path_or_url, temporary_folder, file_extensions):
if isdir(source_path_or_url):
return source_path_or_url
if source_path_or_url.startswith(
'http://') or source_path_or_url.startswith('https://'):
source_name = basename(source_path_or_url).strip()
source_path = join(temporary_folder, source_name.lower())
urlretrieve(source_path_or_url, source_path)
else:
source_path = source_path_or_url
source_extension = splitext(source_path)[1]
target_folder = make_unique_folder(temporary_folder)
if source_extension not in file_extensions:
try:
return uncompress(source_path, target_folder)
except BadFormat:
pass
for extension in ARCHIVE_EXTENSIONS:
try:
link_path = link_safely(
make_unique_path(temporary_folder, extension), source_path)
return uncompress(link_path, target_folder)
except BadArchive:
pass
raise GeoTableError(source_path)
def test_include_external_link(self, sandbox, target_folder):
source_folder = sandbox.source_folder
target_path = source_folder + self.extension
with raises(IOError):
compress(source_folder, target_path)
target_folder = uncompress(target_path, target_folder)
assert_archive_contents(target_folder, sandbox)
target_path = compress(source_folder, target_path, [
sandbox.external_folder])
target_folder = uncompress(target_path, target_folder)
assert_archive_contents(target_folder, sandbox)
def test_include_external_link(self, sandbox, target_folder):
source_folder = sandbox.source_folder
target_path = source_folder + self.extension
with raises(IOError):
compress(source_folder, target_path)
target_folder = uncompress(target_path, target_folder)
assert_archive_contents(target_folder, sandbox)
target_path = compress(source_folder, target_path,
[sandbox.external_folder])
target_folder = uncompress(target_path, target_folder)
assert_archive_contents(target_folder, sandbox)
def test_resolve_source_folder_link(self, sandbox, target_folder):
source_folder = sandbox.source_folder_link_path
target_path = source_folder + self.extension
compress(source_folder, target_path, [
sandbox.external_folder])
target_folder = uncompress(target_path, target_folder)
assert_archive_contents(target_folder, sandbox)
def test_load(self, tmpdir):
t = load(join(FOLDER, 'xyz.kmz'))
assert len(t.iloc[0]['geometry_object'].coords[0]) == 3
assert len(t) == 3
t = load(join(FOLDER, 'xyz.kmz'), drop_z=True, bounding_box=(
-83.743038, 42.280826, -83.743037, 42.280825))
assert len(t.iloc[0]['geometry_object'].coords[0]) == 2
assert len(t) == 1
t = GeoTable.load(join(FOLDER, 'shp', 'a.shp'))
assert t['date'].dtype.name == 'datetime64[ns]'
t = GeoTable.load(join(FOLDER, 'csv', 'lat-lon.csv'))
assert t.iloc[0]['geometry_object'].x == -83.7430378
t = GeoTable.load(join(FOLDER, 'csv', 'latitude-longitude.csv'))
assert t.iloc[0]['geometry_object'].x == -91.5305465
t = GeoTable.load(join(FOLDER, 'csv', 'latitude-longitude-wkt.csv'))
assert t.iloc[0]['geometry_object'].x == -71.10973349999999
t = GeoTable.load(join(FOLDER, 'csv', 'longitude-latitude-wkt.csv'))
assert t.iloc[0]['geometry_object'].x == -71.10973349999999
t = GeoTable.load(join(FOLDER, 'csv', 'wkt.csv'), parse_dates=['date'])
assert t['date'].dtype.name == 'datetime64[ns]'
assert t.iloc[0]['geometry_object'].type == 'Point'
t = GeoTable.load(join(FOLDER, 'csv.zip'), parse_dates=['date'])
assert t['date'].dtype.name == 'datetime64[ns]'
x_folder = uncompress(join(FOLDER, 'csv.zip'), tmpdir.join('csv'))
t = GeoTable.load(x_folder)
assert len(t) > 1
x_path_object = tmpdir.join('x.txt')
x_path_object.write('x')
x_path = str(x_path_object)
with raises(GeoTableError):
GeoTable.load(x_path)
t = GeoTable.load(join(FOLDER, 'csv-bad.tar.gz'))
assert len(t) == 1
t = GeoTable.load(join(FOLDER, 'shp-bad.tar.gz'))
assert len(t) == 2
t = GeoTable.load(
'https://data.cityofnewyork.us/api/geospatial/tqmj-j8zm'
'?method=export&format=Original')
assert len(t) == 5
def receive_result_request(endpoint_url, queue_token, parent_folder):
response = requests.get(endpoint_url,
headers={'Authorization': 'Bearer ' + queue_token})
if response.status_code == 200:
pass
elif response.status_code == 204:
raise HTTPNoContent
elif response.status_code == 401:
raise HTTPUnauthorized
else:
raise HTTPBadRequest
archive_path = make_unique_path(parent_folder, '.zip', length=16)
open(archive_path, 'wb').write(response.content)
result_folder = uncompress(archive_path)
return result_folder
def receive_result_request(endpoint_url, queue_token, parent_folder):
response = requests.get(endpoint_url, headers={
'Authorization': 'Bearer ' + queue_token})
if response.status_code == 200:
pass
elif response.status_code == 204:
raise HTTPNoContent
elif response.status_code == 401:
raise HTTPUnauthorized
else:
raise HTTPBadRequest
archive_path = make_unique_path(parent_folder, '.zip', length=16)
open(archive_path, 'wb').write(response.content)
result_folder = uncompress(archive_path)
return result_folder
def load(Class, source_path, source_proj4=None, target_proj4=None, **kw):
with TemporaryStorage() as storage:
try:
source_folder = uncompress(source_path, storage.folder)
except BadFormat:
if source_path.endswith('.csv'):
return Class.from_csv(source_path, source_proj4,
target_proj4, **kw)
else:
return Class.from_gdal(source_path, source_proj4,
target_proj4)
try:
return concatenate_tables(
Class.from_csv(x, source_proj4, target_proj4, **kw)
for x in find_paths(source_folder, '*.csv'))
except (GeoTableError, ValueError):
pass
try:
return concatenate_tables(
Class.from_gdal(x, source_proj4, target_proj4)
for x in find_paths(source_folder, '*.shp'))
except (GeoTableError, ValueError):
pass
return Class()
def test_recognize_bad_archive(self, tmpdir):
target_path = str(tmpdir.join('x' + self.extension))
open(target_path, 'wt').write('123')
with raises(BadArchive):
uncompress(target_path)
def test_resolve_source_folder_link(self, sandbox, target_folder):
source_folder = sandbox.source_folder_link_path
target_path = source_folder + self.extension
compress(source_folder, target_path, [sandbox.external_folder])
target_folder = uncompress(target_path, target_folder)
assert_archive_contents(target_folder, sandbox)
def test_recognize_bad_archive(self, tmpdir):
target_path = str(tmpdir.join('x' + self.extension))
open(target_path, 'wt').write('123')
with raises(BadArchive):
uncompress(target_path)
def test_resolve_source_folder_link(self, sandbox, tmpdir):
source_folder = sandbox.source_folder_link_path
target_path = compress(source_folder, source_folder + self.extension)
target_folder = uncompress(target_path, str(tmpdir))
assert_contents(target_folder, sandbox)
def test_include_external_link(self, sandbox, tmpdir):
source_folder = sandbox.source_folder
target_path = compress(source_folder, source_folder + self.extension)
target_folder = uncompress(target_path, str(tmpdir))
assert_contents(target_folder, sandbox)
target_folder = make_unique_folder('/tmp')
g = geopy.GoogleV3('AIzaSyDNqc0tWzXHx_wIp1w75-XTcCk4BSphB5w').geocode
location = g('Greensboro, NC')
p = Point(location.longitude, location.latitude)
proj4s = open('proj4s.txt').read().splitlines()
target_proj4 = geotable.LONGITUDE_LATITUDE_PROJ4
archive_url = ('https://egriddata.org/sites/default/files/'
'GSO_RNM_GIS_Network.zip')
archive_path = '/tmp/greensboro-synthetic-network.zip'
archive_folder = splitext(archive_path)[0]
if not exists(archive_path):
urlretrieve(archive_url, archive_path)
if not exists(archive_folder):
archive_folder = uncompress(archive_path)
source_folder = join(archive_folder, 'GSO_RNM_GIS_Network', 'Rural')
source_path = join(source_folder, 'Line_N.shp')
t = line_table = geotable.load(source_path)
source_geometry = t.geometries[0]
target_geometry = p
best_index = 0
best_distance = np.inf
for index, proj4 in enumerate(proj4s):
f = get_transform_shapely_geometry(proj4, target_proj4)
distance = p.distance(f(source_geometry))
if distance < best_distance:
best_index = index
best_distance = distance