def test_compress_7zip(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") fold = get_temp_folder(__file__, "temp_compress_7zip") url = "https://docs.python.org/3/library/ftplib.html" f = download(url, fold) out7 = os.path.join(fold, "try.7z") zip7_files(out7, [f], fLOG=fLOG, temp_folder=fold) if not os.path.exists(out7): raise FileNotFoundError(out7) typbytes = bytes from py7zlib import COMPRESSION_METHOD_COPY fLOG("***", COMPRESSION_METHOD_COPY) res = un7zip_files(out7) self.assertTrue(isinstance(res, list)) self.assertEqual(len(res), 1) fLOG(res[0][0]) if not isinstance(res[0][1], (typbytes, str)): raise TypeError(type(res[0][1])) self.assertTrue(res[0][0].endswith("ftplib.html")) fold = get_temp_folder(__file__, "temp_compress_7zip2") res = un7zip_files(out7, where_to=fold, fLOG=fLOG) self.assertEqual(len(res), 1) s = res[0].replace("\\", "/") if not s.endswith("_unittests/ut_filehelper/temp_compress_7zip2/ftplib.html"): raise Exception(res[0])
def test_uncompress_7zip_lzma2_cmd(self): import pylzma # use github version, not pypi version (2016-11-11) # this version does not include a fix to read file produced by the # latest version of 7z self.assertTrue(pylzma) fold = get_temp_folder(__file__, "temp_uncompress_7zip_lzma2_cmd") data = os.path.join(fold, "..", "data", "donnees.7z") self.assertExists(data) files = un7zip_files(data, where_to=fold, cmd_line=True) self.assertEqual(len(files), 1) self.assertTrue(files[0].endswith('donnees.txt'))
def test_uncompress_7zip(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") import pylzma self.assertTrue(pylzma) fold = get_temp_folder(__file__, "temp_uncompress_7zip") data = os.path.join(fold, "..", "data", "ftplib.7z") files = un7zip_files(data, where_to=fold, fLOG=fLOG) self.assertEqual(len(files), 1)
def test_uncompress_7zip(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") import pylzma self.assertTrue(pylzma) fold = get_temp_folder(__file__, "temp_uncompress_7zip") data = os.path.join(fold, "..", "data", "ftplib.7z") files = un7zip_files(data, where_to=fold, fLOG=fLOG) self.assertEqual(len(files), 1)
def test_compress_7zip(self): fLOG( __file__, self._testMethodName, OutputPrint=__name__ == "__main__") fold = get_temp_folder(__file__, "temp_compress_7zip") url = "https://docs.python.org/3/library/ftplib.html" f = download(url, fold) out7 = os.path.join(fold, "try.7z") zip7_files(out7, [f], fLOG=fLOG, temp_folder=fold) if not os.path.exists(out7): raise FileNotFoundError(out7) typbytes = bytes if is_travis_or_appveyor() == "appveyor": return from py7zlib import COMPRESSION_METHOD_COPY fLOG("***", COMPRESSION_METHOD_COPY) res = un7zip_files(out7) self.assertTrue(isinstance(res, list)) self.assertEqual(len(res), 1) fLOG(res[0][0]) if not isinstance(res[0][1], (typbytes, str)): raise TypeError(type(res[0][1])) self.assertTrue(res[0][0].endswith("ftplib.html")) fold = get_temp_folder(__file__, "temp_compress_7zip2") res = un7zip_files(out7, where_to=fold, fLOG=fLOG) self.assertEqual(len(res), 1) s = res[0].replace("\\", "/") if not s.endswith("_unittests/ut_filehelper/temp_compress_7zip2/ftplib.html"): raise Exception(res[0])
def unzip_files(self, group): """ Unzips files and convert notebooks into :epkg:`HTML`. @param group group name @return list of new filess """ def fvalid(zip_name, local_name): if "__pycache__" in zip_name: return False if zip_name.endswith(".pyc"): return False return True names = list(self.enumerate_group_files(group)) files = [] for name in names: if "attachments" not in name: continue ext = os.path.splitext(name)[-1] if ext == ".zip": folder = os.path.splitext(name)[0] + "_zip" folder = folder.replace(" ", "_").replace(",", "_") if not os.path.exists(folder): self.fLOG( "[ProjectsRepository.unzip_files] unzip '{0}'".format( name)) self.fLOG( "[ProjectsRepository.unzip_files] creating '{0}'". format(folder)) os.mkdir(folder) try: lf = unzip_files(name, folder, fLOG=self.fLOG, fvalid=fvalid, fail_if_error=False) except zipfile.BadZipFile as e: self.fLOG( "[ProjectsRepository.unzip_files] ERROR: unable to unzip '{0}' because of '{1}']" .format(name, e)) lf = [] files.extend(lf) else: # already done, we do not do it again pass elif ext == ".7z": folder = os.path.splitext(name)[0] + "_7z" folder = folder.replace(" ", "_").replace(",", "_") if not os.path.exists(folder): self.fLOG("[ProjectsRepository.un7zip_files] un7zip '{0}'". format(name)) self.fLOG( "[ProjectsRepository.un7zip_files] creating '{0}'". format(folder)) os.mkdir(folder) lf = un7zip_files(name, folder, fLOG=self.fLOG, fvalid=fvalid) files.extend(lf) else: # already done, we do not do it again pass elif ext == ".rar": folder = os.path.splitext(name)[0] + "_rar" folder = folder.replace(" ", "_").replace(",", "_") if not os.path.exists(folder): self.fLOG( "[ProjectsRepository.unrar_files] unrar '{0}'".format( name)) self.fLOG( "[ProjectsRepository.unrar_files] creating '{0}'". format(folder)) os.mkdir(folder) lf = unrar_files(name, folder, fLOG=self.fLOG, fvalid=fvalid) files.extend(lf) else: # already done, we do not do it again pass elif ext == ".gz": folder = os.path.splitext(name)[0] + "_gz" folder = folder.replace(" ", "_").replace(",", "_") if not os.path.exists(folder): self.fLOG("[ProjectsRepository.ungzip_files] ungzip '{0}'". format(name)) self.fLOG( "[ProjectsRepository.ungzip_files] creating '{0}'". format(folder)) os.mkdir(folder) unzip = "pkl.gz" not in name lf = ungzip_files(name, folder, fLOG=self.fLOG, fvalid=fvalid, unzip=unzip) files.extend(lf) else: # already done, we do not do it again pass elif ext == ".tar.gz": raise Exception("unable to process such a file: " + name) return files
def unzip_files(self, group): """ unzip files and convert notebooks into html @param group group name @return list of new filess """ def fvalid(zip_name, local_name): if "__pycache__" in zip_name: return False if zip_name.endswith(".pyc"): return False return True names = list(self.enumerate_group_files(group)) files = [] for name in names: if "attachments" not in name: continue ext = os.path.splitext(name)[-1] if ext == ".zip": folder = os.path.splitext(name)[0] + "_zip" folder = folder.replace(" ", "_").replace(",", "_") if not os.path.exists(folder): self.fLOG( "ProjectsRepository.unzip_files [unzip {0}]".format( name)) self.fLOG( "ProjectsRepository.unzip_files [creating {0}]".format( folder)) os.mkdir(folder) lf = unzip_files(name, folder, fLOG=self.fLOG, fvalid=fvalid) files.extend(lf) else: # already done, we do not do it again pass elif ext == ".7z": folder = os.path.splitext(name)[0] + "_7z" folder = folder.replace(" ", "_").replace(",", "_") if not os.path.exists(folder): self.fLOG( "ProjectsRepository.un7zip_files [un7zip {0}]".format( name)) self.fLOG("ProjectsRepository.un7zip_files [creating {0}]". format(folder)) os.mkdir(folder) lf = un7zip_files(name, folder, fLOG=self.fLOG, fvalid=fvalid) files.extend(lf) else: # already done, we do not do it again pass elif ext == ".gz": folder = os.path.splitext(name)[0] + "_gz" folder = folder.replace(" ", "_").replace(",", "_") if not os.path.exists(folder): self.fLOG( "ProjectsRepository.ungzip_files [ungzip {0}]".format( name)) self.fLOG("ProjectsRepository.ungzip_files [creating {0}]". format(folder)) os.mkdir(folder) lf = ungzip_files(name, folder, fLOG=self.fLOG, fvalid=fvalid) files.extend(lf) else: # already done, we do not do it again pass elif ext == ".tar.gz": raise Exception("unable to process such a file: " + name) return files
def unzip_files(self, group): """ Unzips files and convert notebooks into :epkg:`HTML`. @param group group name @return list of new filess """ def fvalid(zip_name, local_name): if "__pycache__" in zip_name: return False if zip_name.endswith(".pyc"): return False return True names = list(self.enumerate_group_files(group)) files = [] for name in names: if "attachments" not in name: continue ext = os.path.splitext(name)[-1] if ext == ".zip": folder = os.path.splitext(name)[0] + "_zip" folder = folder.replace(" ", "_").replace(",", "_") if not os.path.exists(folder): self.fLOG( "[ProjectsRepository.unzip_files] unzip '{0}'".format(name)) self.fLOG( "[ProjectsRepository.unzip_files] creating '{0}'".format(folder)) os.mkdir(folder) try: lf = unzip_files( name, folder, fLOG=self.fLOG, fvalid=fvalid, fail_if_error=False) except zipfile.BadZipFile as e: self.fLOG( "[ProjectsRepository.unzip_files] ERROR: unable to unzip '{0}' because of '{1}']".format(name, e)) lf = [] files.extend(lf) else: # already done, we do not do it again pass elif ext == ".7z": folder = os.path.splitext(name)[0] + "_7z" folder = folder.replace(" ", "_").replace(",", "_") if not os.path.exists(folder): self.fLOG( "[ProjectsRepository.un7zip_files] un7zip '{0}'".format(name)) self.fLOG( "[ProjectsRepository.un7zip_files] creating '{0}'".format(folder)) os.mkdir(folder) lf = un7zip_files( name, folder, fLOG=self.fLOG, fvalid=fvalid) files.extend(lf) else: # already done, we do not do it again pass elif ext == ".rar": folder = os.path.splitext(name)[0] + "_rar" folder = folder.replace(" ", "_").replace(",", "_") if not os.path.exists(folder): self.fLOG( "[ProjectsRepository.unrar_files] unrar '{0}'".format(name)) self.fLOG( "[ProjectsRepository.unrar_files] creating '{0}'".format(folder)) os.mkdir(folder) lf = unrar_files( name, folder, fLOG=self.fLOG, fvalid=fvalid) files.extend(lf) else: # already done, we do not do it again pass elif ext == ".gz": folder = os.path.splitext(name)[0] + "_gz" folder = folder.replace(" ", "_").replace(",", "_") if not os.path.exists(folder): self.fLOG( "[ProjectsRepository.ungzip_files] ungzip '{0}'".format(name)) self.fLOG( "[ProjectsRepository.ungzip_files] creating '{0}'".format(folder)) os.mkdir(folder) unzip = "pkl.gz" not in name lf = ungzip_files( name, folder, fLOG=self.fLOG, fvalid=fvalid, unzip=unzip) files.extend(lf) else: # already done, we do not do it again pass elif ext == ".tar.gz": raise Exception("unable to process such a file: " + name) return files
def load_french_departements(cache=None): """ Returns the definition of French departments. @param cache cache folder @return French departments as a :epkg:`GeoDataFrame` .. exref:: :title: Loads French departments polygons Simple example to retrieve French departements. .. runpython:: :showcode: from pyensae.datasource import load_french_departements df = load_french_departements() print(df.head(2).T) """ if cache is None: cache = '.' # delayed import import shapefile import geopandas from shapely.geometry import Polygon from shapely.ops import unary_union name = "GEOFLA_2-1_DEPARTEMENT_SHP_LAMB93_FXX_2015-12-01.7z" try: download_data( name, whereTo=cache, website= "https://wxs-telechargement.ign.fr/oikr5jryiph0iwhw36053ptm/telechargement/inspire/" "GEOFLA_THEME-DEPARTEMENTS_2015_2$GEOFLA_2-1_DEPARTEMENT_SHP_LAMB93_FXX_2015-12-01/file/" ) except Exception as e: # au cas le site n'est pas accessible download_data(name, website="xd", whereTo=cache) full_name = os.path.join(cache, name) un7zip_files(full_name, where_to=os.path.join(cache, "shapefiles")) departements = os.path.join( cache, 'shapefiles/GEOFLA_2-1_DEPARTEMENT_SHP_LAMB93_FXX_2015-12-01/GEOFLA/1_DONNEES_LIVRAISON_2015/' 'GEOFLA_2-1_SHP_LAMB93_FR-ED152/DEPARTEMENT/DEPARTEMENT.shp') r = shapefile.Reader(departements) shapes = r.shapes() records = r.records() polys = [] datas = [] for i, (record, shape) in enumerate(zip(records, shapes)): # coordinates in Lambert 93 geo_points = [lambert932WGPS(x, y) for x, y in shape.points] rec = {} for k in [ 'CODE_DEPT', 'CODE_REG', 'CODE_CHF', 'ID_GEOFLA', 'NOM_CHF', 'NOM_DEPT', 'NOM_REG', 'X_CENTROID', 'X_CHF_LIEU', 'Y_CENTROID', 'Y_CHF_LIEU' ]: rec[k] = getattr(record, k, None) if len(shape.parts) == 1: # only one polygon poly = Polygon(geo_points) else: # merge them ind = list(shape.parts) + [len(shape.points)] pols = [ Polygon(geo_points[ind[i]:ind[i + 1]]) for i in range(0, len(shape.parts)) ] try: poly = unary_union(pols) except Exception as e: poly = Polygon(geo_points) polys.append(poly) datas.append(rec) geom = geopandas.GeoDataFrame(geometry=polys) data = pandas.DataFrame(datas) return pandas.concat([geom, data], axis=1)
def unzip_files(self, group): """ unzip files and convert notebooks into html @param group group name @return list of new filess """ def fvalid(zip_name, local_name): if "__pycache__" in zip_name: return False if zip_name.endswith(".pyc"): return False return True names = list(self.enumerate_group_files(group)) files = [] for name in names: if "attachments" not in name: continue ext = os.path.splitext(name)[-1] if ext == ".zip": folder = os.path.splitext(name)[0] + "_zip" folder = folder.replace(" ", "_").replace(",", "_") if not os.path.exists(folder): self.fLOG( "ProjectsRepository.unzip_files [unzip {0}]".format(name)) self.fLOG( "ProjectsRepository.unzip_files [creating {0}]".format(folder)) os.mkdir(folder) lf = unzip_files( name, folder, fLOG=self.fLOG, fvalid=fvalid) files.extend(lf) else: # already done, we do not do it again pass elif ext == ".7z": folder = os.path.splitext(name)[0] + "_7z" folder = folder.replace(" ", "_").replace(",", "_") if not os.path.exists(folder): self.fLOG( "ProjectsRepository.un7zip_files [un7zip {0}]".format(name)) self.fLOG( "ProjectsRepository.un7zip_files [creating {0}]".format(folder)) os.mkdir(folder) lf = un7zip_files( name, folder, fLOG=self.fLOG, fvalid=fvalid) files.extend(lf) else: # already done, we do not do it again pass elif ext == ".gz": folder = os.path.splitext(name)[0] + "_gz" folder = folder.replace(" ", "_").replace(",", "_") if not os.path.exists(folder): self.fLOG( "ProjectsRepository.ungzip_files [ungzip {0}]".format(name)) self.fLOG( "ProjectsRepository.ungzip_files [creating {0}]".format(folder)) os.mkdir(folder) lf = ungzip_files( name, folder, fLOG=self.fLOG, fvalid=fvalid) files.extend(lf) else: # already done, we do not do it again pass elif ext == ".tar.gz": raise Exception("unable to process such a file: " + name) return files