def open(path):
if isinstance(path, bytes):
return core.open_jay(path)
if not isinstance(path, str):
raise TTypeError("Parameter `path` should be a string")
path = os.path.expanduser(path)
if not os.path.exists(path):
msg = "Path %s does not exist" % path
if not path.startswith("/"):
msg += " (current directory = %s)" % os.getcwd()
raise ValueError(msg)
if not os.path.isdir(path):
return core.open_jay(path)
nff_version = None
nrows = 0
metafile = os.path.join(path, "_meta.nff")
with _builtin_open(metafile, encoding="utf-8") as inp:
info = []
for line in inp:
if line.startswith("#"):
info.append(line[1:].strip())
else:
break
if not (info and info[0].startswith("NFF")):
raise ValueError("File _meta.nff has invalid format")
if info[0] == "NFF1":
nff_version = 1
elif info[0] == "NFF1+":
nff_version = 1.5
elif info[0] == "NFF2":
nff_version = 2
if nff_version:
assert len(info) == 2
mm = re.match("nrows\s*=\s*(\d+)", info[1])
if mm:
nrows = int(mm.group(1))
else:
raise ValueError("nrows info not found in line %r" % info[1])
else:
raise ValueError("Unknown NFF format: %s" % info[0])
coltypes = [dt.stype.str32] * 4
if nff_version > 1:
coltypes += [None] * 2
f0 = dt.fread(metafile, sep=",", columns=coltypes)
f1 = f0(select=["filename", "stype"])
colnames = f0[:, "colname"].to_list()[0]
df = core.datatable_load(f1.internal, nrows, path, nff_version < 2,
colnames)
assert df.nrows == nrows, "Wrong number of rows read: %d" % df.nrows
return df
def open(path):
if isinstance(path, bytes):
return core.open_jay(path)
if not isinstance(path, str):
raise TTypeError("Parameter `path` should be a string")
path = os.path.expanduser(path)
if not os.path.exists(path):
msg = "Path %s does not exist" % path
if not path.startswith("/"):
msg += " (current directory = %s)" % os.getcwd()
raise TValueError(msg)
if os.path.isdir(path):
raise TValueError("Path %s is a directory" % path)
return core.open_jay(path)
def _resolve_archive(self, filename, subpath=None):
ext = os.path.splitext(filename)[1]
if subpath and subpath[0] == "/":
subpath = subpath[1:]
if ext == ".zip":
import zipfile
zf = zipfile.ZipFile(filename)
# MacOS is found guilty of adding extra files into the Zip archives
# it creates. The files are hidden, and in the directory __MACOSX/.
# We remove those files from the list, since they are not real user
# files, and have an unknown binary format.
zff = [name for name in zf.namelist()
if not(name.startswith("__MACOSX/") or name.endswith("/"))]
if subpath:
if subpath in zff:
zff = [subpath]
else:
raise TValueError("File `%s` does not exist in archive "
"`%s`" % (subpath, filename))
if len(zff) > 1:
warnings.warn("Zip file %s contains multiple compressed "
"files: %r. Only the first of them will be used."
% (filename, zff), category=FreadWarning)
if len(zff) == 0:
raise TValueError("Zip file %s is empty" % filename)
if self._verbose:
self._logger.debug("Extracting %s to temporary directory %s"
% (filename, self.tempdir))
self._tempfiles.append(zf.extract(zff[0], path=self.tempdir))
self._file = self._tempfiles[-1]
elif ext == ".gz":
import gzip
zf = gzip.GzipFile(filename, mode="rb")
if self._verbose:
self._logger.debug("Extracting %s into memory" % filename)
self._text = zf.read()
if self._verbose:
self._logger.debug("Extracted: size = %d" % len(self._text))
elif ext == ".bz2":
import bz2
with bz2.open(filename, mode="rb") as zf:
if self._verbose:
self._logger.debug("Extracting %s into memory" % filename)
self._text = zf.read()
if self._verbose:
self._logger.debug("Extracted: size = %d" % len(self._text))
elif ext == ".xz":
import lzma
with lzma.open(filename, mode="rb") as zf:
if self._verbose:
self._logger.debug("Extracting %s into memory" % filename)
self._text = zf.read()
if self._verbose:
self._logger.debug("Extracted: size = %d" % len(self._text))
elif ext == ".xlsx" or ext == ".xls":
self._result = read_xls_workbook(filename, subpath)
elif ext == ".jay":
self._result = core.open_jay(filename)
else:
self._file = filename
def _resolve_archive(filename, subpath, tempfiles):
logger = tempfiles._logger
ext = os.path.splitext(filename)[1]
if subpath and subpath[0] in ["/", "\\"]:
subpath = subpath[1:]
out_file = None
out_text = None
out_result = None
# TODO: file extarction should be lazy
if ext == ".zip":
import zipfile
with zipfile.ZipFile(filename) as zf:
# MacOS is found guilty of adding extra files into the Zip archives
# it creates. The files are hidden, and in the directory __MACOSX/.
# We remove those files from the list, since they are not real user
# files, and have an unknown binary format.
zff = [name for name in zf.namelist()
if not(name.startswith("__MACOSX/") or name.endswith("/"))]
if subpath:
if subpath in zff:
filename = os.path.join(filename, subpath)
zff = [subpath]
else:
raise IOError("File `%s` does not exist in archive `%s`"
% (subpath, filename))
extracted_files = []
for zf_file in zff:
if logger:
logger.debug("Extracting %s/%s to temporary directory %s"
% (filename, zf_file, tempfiles.tempdir))
newfile = zf.extract(zf_file, path=tempfiles.tempdir)
srcname = os.path.join(filename, zf_file)
tempfiles.add(newfile)
extracted_files.append(((srcname, newfile, None, None), None))
if len(extracted_files) == 1:
out_file = extracted_files[0][0][1]
else:
return (None, None, None, None), extracted_files
elif filename.endswith(".tar.gz") or filename.endswith(".tgz"):
import tarfile
zf = tarfile.open(filename, mode="r:gz")
zff = [entry.name for entry in zf.getmembers() if entry.isfile()]
if subpath:
if subpath in zff:
filename = os.path.join(filename, subpath)
zff = [subpath]
else:
raise IOError("File `%s` does not exist in archive `%s`"
% (subpath, filename))
extracted_files = []
for entryname in zff:
if logger:
logger.debug("Extracting %s/%s to temporary directory %s"
% (filename, entryname, tempfiles.tempdir))
newfile = tempfiles.create_temp_file()
with zf.extractfile(entryname) as inp, open(newfile, "wb") as out:
out.write(inp.read())
srcname = os.path.join(filename, entryname)
extracted_files.append(((srcname, newfile, None, None), None))
if len(extracted_files) == 1:
out_file = extracted_files[0][0][1]
else:
return (None, None, None, None), extracted_files
elif ext == ".gz":
import gzip
zf = gzip.GzipFile(filename, mode="rb")
if logger:
logger.debug("Extracting %s into memory" % filename)
out_text = zf.read()
if logger:
logger.debug("Extracted: size = %d" % len(out_text))
elif ext == ".bz2":
import bz2
with bz2.open(filename, mode="rb") as zf:
if logger:
logger.debug("Extracting %s into memory" % filename)
out_text = zf.read()
if logger:
logger.debug("Extracted: size = %d" % len(out_text))
elif ext == ".xz":
import lzma
with lzma.open(filename, mode="rb") as zf:
if logger:
logger.debug("Extracting %s into memory" % filename)
out_text = zf.read()
if logger:
logger.debug("Extracted: size = %d" % len(out_text))
elif ext == ".xlsx" or ext == ".xls":
out_result = read_xls_workbook(filename, subpath)
if subpath:
filename = os.path.join(filename, subpath)
elif ext == ".jay":
out_result = core.open_jay(filename)
else:
out_file = filename
# src, file, fileno, text, result
return (filename, out_file, None, out_text), out_result