def test_compress_7zip(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
fold = get_temp_folder(__file__, "temp_compress_7zip")
url = "https://docs.python.org/3/library/ftplib.html"
f = download(url, fold)
out7 = os.path.join(fold, "try.7z")
zip7_files(out7, [f], fLOG=fLOG, temp_folder=fold)
if not os.path.exists(out7):
raise FileNotFoundError(out7)
typbytes = bytes
from py7zlib import COMPRESSION_METHOD_COPY
fLOG("***", COMPRESSION_METHOD_COPY)
res = un7zip_files(out7)
self.assertTrue(isinstance(res, list))
self.assertEqual(len(res), 1)
fLOG(res[0][0])
if not isinstance(res[0][1], (typbytes, str)):
raise TypeError(type(res[0][1]))
self.assertTrue(res[0][0].endswith("ftplib.html"))
fold = get_temp_folder(__file__, "temp_compress_7zip2")
res = un7zip_files(out7, where_to=fold, fLOG=fLOG)
self.assertEqual(len(res), 1)
s = res[0].replace("\\", "/")
if not s.endswith("_unittests/ut_filehelper/temp_compress_7zip2/ftplib.html"):
raise Exception(res[0])
def test_uncompress_7zip_lzma2_cmd(self):
import pylzma
# use github version, not pypi version (2016-11-11)
# this version does not include a fix to read file produced by the
# latest version of 7z
self.assertTrue(pylzma)
fold = get_temp_folder(__file__, "temp_uncompress_7zip_lzma2_cmd")
data = os.path.join(fold, "..", "data", "donnees.7z")
self.assertExists(data)
files = un7zip_files(data, where_to=fold, cmd_line=True)
self.assertEqual(len(files), 1)
self.assertTrue(files[0].endswith('donnees.txt'))
def test_uncompress_7zip(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
import pylzma
self.assertTrue(pylzma)
fold = get_temp_folder(__file__, "temp_uncompress_7zip")
data = os.path.join(fold, "..", "data", "ftplib.7z")
files = un7zip_files(data, where_to=fold, fLOG=fLOG)
self.assertEqual(len(files), 1)
def test_uncompress_7zip(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
import pylzma
self.assertTrue(pylzma)
fold = get_temp_folder(__file__, "temp_uncompress_7zip")
data = os.path.join(fold, "..", "data", "ftplib.7z")
files = un7zip_files(data, where_to=fold, fLOG=fLOG)
self.assertEqual(len(files), 1)
def test_compress_7zip(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
fold = get_temp_folder(__file__, "temp_compress_7zip")
url = "https://docs.python.org/3/library/ftplib.html"
f = download(url, fold)
out7 = os.path.join(fold, "try.7z")
zip7_files(out7, [f], fLOG=fLOG, temp_folder=fold)
if not os.path.exists(out7):
raise FileNotFoundError(out7)
typbytes = bytes
if is_travis_or_appveyor() == "appveyor":
return
from py7zlib import COMPRESSION_METHOD_COPY
fLOG("***", COMPRESSION_METHOD_COPY)
res = un7zip_files(out7)
self.assertTrue(isinstance(res, list))
self.assertEqual(len(res), 1)
fLOG(res[0][0])
if not isinstance(res[0][1], (typbytes, str)):
raise TypeError(type(res[0][1]))
self.assertTrue(res[0][0].endswith("ftplib.html"))
fold = get_temp_folder(__file__, "temp_compress_7zip2")
res = un7zip_files(out7, where_to=fold, fLOG=fLOG)
self.assertEqual(len(res), 1)
s = res[0].replace("\\", "/")
if not s.endswith("_unittests/ut_filehelper/temp_compress_7zip2/ftplib.html"):
raise Exception(res[0])
def unzip_files(self, group):
"""
Unzips files and convert notebooks into :epkg:`HTML`.
@param group group name
@return list of new filess
"""
def fvalid(zip_name, local_name):
if "__pycache__" in zip_name:
return False
if zip_name.endswith(".pyc"):
return False
return True
names = list(self.enumerate_group_files(group))
files = []
for name in names:
if "attachments" not in name:
continue
ext = os.path.splitext(name)[-1]
if ext == ".zip":
folder = os.path.splitext(name)[0] + "_zip"
folder = folder.replace(" ", "_").replace(",", "_")
if not os.path.exists(folder):
self.fLOG(
"[ProjectsRepository.unzip_files] unzip '{0}'".format(
name))
self.fLOG(
"[ProjectsRepository.unzip_files] creating '{0}'".
format(folder))
os.mkdir(folder)
try:
lf = unzip_files(name,
folder,
fLOG=self.fLOG,
fvalid=fvalid,
fail_if_error=False)
except zipfile.BadZipFile as e:
self.fLOG(
"[ProjectsRepository.unzip_files] ERROR: unable to unzip '{0}' because of '{1}']"
.format(name, e))
lf = []
files.extend(lf)
else:
# already done, we do not do it again
pass
elif ext == ".7z":
folder = os.path.splitext(name)[0] + "_7z"
folder = folder.replace(" ", "_").replace(",", "_")
if not os.path.exists(folder):
self.fLOG("[ProjectsRepository.un7zip_files] un7zip '{0}'".
format(name))
self.fLOG(
"[ProjectsRepository.un7zip_files] creating '{0}'".
format(folder))
os.mkdir(folder)
lf = un7zip_files(name,
folder,
fLOG=self.fLOG,
fvalid=fvalid)
files.extend(lf)
else:
# already done, we do not do it again
pass
elif ext == ".rar":
folder = os.path.splitext(name)[0] + "_rar"
folder = folder.replace(" ", "_").replace(",", "_")
if not os.path.exists(folder):
self.fLOG(
"[ProjectsRepository.unrar_files] unrar '{0}'".format(
name))
self.fLOG(
"[ProjectsRepository.unrar_files] creating '{0}'".
format(folder))
os.mkdir(folder)
lf = unrar_files(name,
folder,
fLOG=self.fLOG,
fvalid=fvalid)
files.extend(lf)
else:
# already done, we do not do it again
pass
elif ext == ".gz":
folder = os.path.splitext(name)[0] + "_gz"
folder = folder.replace(" ", "_").replace(",", "_")
if not os.path.exists(folder):
self.fLOG("[ProjectsRepository.ungzip_files] ungzip '{0}'".
format(name))
self.fLOG(
"[ProjectsRepository.ungzip_files] creating '{0}'".
format(folder))
os.mkdir(folder)
unzip = "pkl.gz" not in name
lf = ungzip_files(name,
folder,
fLOG=self.fLOG,
fvalid=fvalid,
unzip=unzip)
files.extend(lf)
else:
# already done, we do not do it again
pass
elif ext == ".tar.gz":
raise Exception("unable to process such a file: " + name)
return files
def unzip_files(self, group):
"""
unzip files and convert notebooks into html
@param group group name
@return list of new filess
"""
def fvalid(zip_name, local_name):
if "__pycache__" in zip_name:
return False
if zip_name.endswith(".pyc"):
return False
return True
names = list(self.enumerate_group_files(group))
files = []
for name in names:
if "attachments" not in name:
continue
ext = os.path.splitext(name)[-1]
if ext == ".zip":
folder = os.path.splitext(name)[0] + "_zip"
folder = folder.replace(" ", "_").replace(",", "_")
if not os.path.exists(folder):
self.fLOG(
"ProjectsRepository.unzip_files [unzip {0}]".format(
name))
self.fLOG(
"ProjectsRepository.unzip_files [creating {0}]".format(
folder))
os.mkdir(folder)
lf = unzip_files(name,
folder,
fLOG=self.fLOG,
fvalid=fvalid)
files.extend(lf)
else:
# already done, we do not do it again
pass
elif ext == ".7z":
folder = os.path.splitext(name)[0] + "_7z"
folder = folder.replace(" ", "_").replace(",", "_")
if not os.path.exists(folder):
self.fLOG(
"ProjectsRepository.un7zip_files [un7zip {0}]".format(
name))
self.fLOG("ProjectsRepository.un7zip_files [creating {0}]".
format(folder))
os.mkdir(folder)
lf = un7zip_files(name,
folder,
fLOG=self.fLOG,
fvalid=fvalid)
files.extend(lf)
else:
# already done, we do not do it again
pass
elif ext == ".gz":
folder = os.path.splitext(name)[0] + "_gz"
folder = folder.replace(" ", "_").replace(",", "_")
if not os.path.exists(folder):
self.fLOG(
"ProjectsRepository.ungzip_files [ungzip {0}]".format(
name))
self.fLOG("ProjectsRepository.ungzip_files [creating {0}]".
format(folder))
os.mkdir(folder)
lf = ungzip_files(name,
folder,
fLOG=self.fLOG,
fvalid=fvalid)
files.extend(lf)
else:
# already done, we do not do it again
pass
elif ext == ".tar.gz":
raise Exception("unable to process such a file: " + name)
return files
def unzip_files(self, group):
"""
Unzips files and convert notebooks into :epkg:`HTML`.
@param group group name
@return list of new filess
"""
def fvalid(zip_name, local_name):
if "__pycache__" in zip_name:
return False
if zip_name.endswith(".pyc"):
return False
return True
names = list(self.enumerate_group_files(group))
files = []
for name in names:
if "attachments" not in name:
continue
ext = os.path.splitext(name)[-1]
if ext == ".zip":
folder = os.path.splitext(name)[0] + "_zip"
folder = folder.replace(" ", "_").replace(",", "_")
if not os.path.exists(folder):
self.fLOG(
"[ProjectsRepository.unzip_files] unzip '{0}'".format(name))
self.fLOG(
"[ProjectsRepository.unzip_files] creating '{0}'".format(folder))
os.mkdir(folder)
try:
lf = unzip_files(
name, folder, fLOG=self.fLOG, fvalid=fvalid, fail_if_error=False)
except zipfile.BadZipFile as e:
self.fLOG(
"[ProjectsRepository.unzip_files] ERROR: unable to unzip '{0}' because of '{1}']".format(name, e))
lf = []
files.extend(lf)
else:
# already done, we do not do it again
pass
elif ext == ".7z":
folder = os.path.splitext(name)[0] + "_7z"
folder = folder.replace(" ", "_").replace(",", "_")
if not os.path.exists(folder):
self.fLOG(
"[ProjectsRepository.un7zip_files] un7zip '{0}'".format(name))
self.fLOG(
"[ProjectsRepository.un7zip_files] creating '{0}'".format(folder))
os.mkdir(folder)
lf = un7zip_files(
name, folder, fLOG=self.fLOG, fvalid=fvalid)
files.extend(lf)
else:
# already done, we do not do it again
pass
elif ext == ".rar":
folder = os.path.splitext(name)[0] + "_rar"
folder = folder.replace(" ", "_").replace(",", "_")
if not os.path.exists(folder):
self.fLOG(
"[ProjectsRepository.unrar_files] unrar '{0}'".format(name))
self.fLOG(
"[ProjectsRepository.unrar_files] creating '{0}'".format(folder))
os.mkdir(folder)
lf = unrar_files(
name, folder, fLOG=self.fLOG, fvalid=fvalid)
files.extend(lf)
else:
# already done, we do not do it again
pass
elif ext == ".gz":
folder = os.path.splitext(name)[0] + "_gz"
folder = folder.replace(" ", "_").replace(",", "_")
if not os.path.exists(folder):
self.fLOG(
"[ProjectsRepository.ungzip_files] ungzip '{0}'".format(name))
self.fLOG(
"[ProjectsRepository.ungzip_files] creating '{0}'".format(folder))
os.mkdir(folder)
unzip = "pkl.gz" not in name
lf = ungzip_files(
name, folder, fLOG=self.fLOG, fvalid=fvalid, unzip=unzip)
files.extend(lf)
else:
# already done, we do not do it again
pass
elif ext == ".tar.gz":
raise Exception("unable to process such a file: " + name)
return files
def load_french_departements(cache=None):
"""
Returns the definition of French departments.
@param cache cache folder
@return French departments as a :epkg:`GeoDataFrame`
.. exref::
:title: Loads French departments polygons
Simple example to retrieve French departements.
.. runpython::
:showcode:
from pyensae.datasource import load_french_departements
df = load_french_departements()
print(df.head(2).T)
"""
if cache is None:
cache = '.'
# delayed import
import shapefile
import geopandas
from shapely.geometry import Polygon
from shapely.ops import unary_union
name = "GEOFLA_2-1_DEPARTEMENT_SHP_LAMB93_FXX_2015-12-01.7z"
try:
download_data(
name,
whereTo=cache,
website=
"https://wxs-telechargement.ign.fr/oikr5jryiph0iwhw36053ptm/telechargement/inspire/"
"GEOFLA_THEME-DEPARTEMENTS_2015_2$GEOFLA_2-1_DEPARTEMENT_SHP_LAMB93_FXX_2015-12-01/file/"
)
except Exception as e:
# au cas le site n'est pas accessible
download_data(name, website="xd", whereTo=cache)
full_name = os.path.join(cache, name)
un7zip_files(full_name, where_to=os.path.join(cache, "shapefiles"))
departements = os.path.join(
cache,
'shapefiles/GEOFLA_2-1_DEPARTEMENT_SHP_LAMB93_FXX_2015-12-01/GEOFLA/1_DONNEES_LIVRAISON_2015/'
'GEOFLA_2-1_SHP_LAMB93_FR-ED152/DEPARTEMENT/DEPARTEMENT.shp')
r = shapefile.Reader(departements)
shapes = r.shapes()
records = r.records()
polys = []
datas = []
for i, (record, shape) in enumerate(zip(records, shapes)):
# coordinates in Lambert 93
geo_points = [lambert932WGPS(x, y) for x, y in shape.points]
rec = {}
for k in [
'CODE_DEPT', 'CODE_REG', 'CODE_CHF', 'ID_GEOFLA', 'NOM_CHF',
'NOM_DEPT', 'NOM_REG', 'X_CENTROID', 'X_CHF_LIEU',
'Y_CENTROID', 'Y_CHF_LIEU'
]:
rec[k] = getattr(record, k, None)
if len(shape.parts) == 1:
# only one polygon
poly = Polygon(geo_points)
else:
# merge them
ind = list(shape.parts) + [len(shape.points)]
pols = [
Polygon(geo_points[ind[i]:ind[i + 1]])
for i in range(0, len(shape.parts))
]
try:
poly = unary_union(pols)
except Exception as e:
poly = Polygon(geo_points)
polys.append(poly)
datas.append(rec)
geom = geopandas.GeoDataFrame(geometry=polys)
data = pandas.DataFrame(datas)
return pandas.concat([geom, data], axis=1)
def unzip_files(self, group):
"""
unzip files and convert notebooks into html
@param group group name
@return list of new filess
"""
def fvalid(zip_name, local_name):
if "__pycache__" in zip_name:
return False
if zip_name.endswith(".pyc"):
return False
return True
names = list(self.enumerate_group_files(group))
files = []
for name in names:
if "attachments" not in name:
continue
ext = os.path.splitext(name)[-1]
if ext == ".zip":
folder = os.path.splitext(name)[0] + "_zip"
folder = folder.replace(" ", "_").replace(",", "_")
if not os.path.exists(folder):
self.fLOG(
"ProjectsRepository.unzip_files [unzip {0}]".format(name))
self.fLOG(
"ProjectsRepository.unzip_files [creating {0}]".format(folder))
os.mkdir(folder)
lf = unzip_files(
name, folder, fLOG=self.fLOG, fvalid=fvalid)
files.extend(lf)
else:
# already done, we do not do it again
pass
elif ext == ".7z":
folder = os.path.splitext(name)[0] + "_7z"
folder = folder.replace(" ", "_").replace(",", "_")
if not os.path.exists(folder):
self.fLOG(
"ProjectsRepository.un7zip_files [un7zip {0}]".format(name))
self.fLOG(
"ProjectsRepository.un7zip_files [creating {0}]".format(folder))
os.mkdir(folder)
lf = un7zip_files(
name, folder, fLOG=self.fLOG, fvalid=fvalid)
files.extend(lf)
else:
# already done, we do not do it again
pass
elif ext == ".gz":
folder = os.path.splitext(name)[0] + "_gz"
folder = folder.replace(" ", "_").replace(",", "_")
if not os.path.exists(folder):
self.fLOG(
"ProjectsRepository.ungzip_files [ungzip {0}]".format(name))
self.fLOG(
"ProjectsRepository.ungzip_files [creating {0}]".format(folder))
os.mkdir(folder)
lf = ungzip_files(
name, folder, fLOG=self.fLOG, fvalid=fvalid)
files.extend(lf)
else:
# already done, we do not do it again
pass
elif ext == ".tar.gz":
raise Exception("unable to process such a file: " + name)
return files
