def save(self, dest, format="nff", _strategy="auto"):
"""
Save Frame in binary NFF/Jay format.
:param dest: destination where the Frame should be saved.
:param format: either "nff" or "jay"
:param _strategy: one of "mmap", "write" or "auto"
"""
if _strategy not in ("auto", "write", "mmap"):
raise TValueError("Invalid parameter _strategy: only 'write' / 'mmap' "
"/ 'auto' are allowed")
if format not in ("nff", "jay"):
raise TValueError("Invalid parameter `format`: only 'nff' or 'jay' "
"are supported")
dest = os.path.expanduser(dest)
if os.path.exists(dest):
pass
elif format == "nff":
os.makedirs(dest)
if format == "jay":
self.internal.save_jay(dest, self.names, _strategy)
return
self.materialize()
mins = self.min().topython()
maxs = self.max().topython()
metafile = os.path.join(dest, "_meta.nff")
with _builtin_open(metafile, "w", encoding="utf-8") as out:
out.write("# NFF2\n")
out.write("# nrows = %d\n" % self.nrows)
out.write('filename,stype,meta,colname,min,max\n')
l = len(str(self.ncols))
for i in range(self.ncols):
filename = "c%0*d" % (l, i + 1)
colname = self.names[i].replace('"', '""')
_col = self.internal.column(i)
stype = _col.stype
meta = _col.meta
if stype == dt.stype.obj64:
dtwarn("Column %r of type obj64 was not saved" % self.names[i])
continue
if meta is None:
meta = ""
smin = _stringify(mins[i][0])
smax = _stringify(maxs[i][0])
out.write('%s,%s,%s,"%s",%s,%s\n'
% (filename, stype.code, meta, colname, smin, smax))
filename = os.path.join(dest, filename)
_col.save_to_disk(filename, _strategy)
def _dedup_names(names) -> Tuple[Tuple[str, ...], Dict[str, int]]:
if not names:
return tuple(), dict()
inames = {}
tnames = []
dupnames = []
min_c = options.frame.names_auto_index
prefix = options.frame.names_auto_prefix
fill_default_names = False
for i, name in enumerate(names):
if not name:
fill_default_names = True
tnames.append(None) # Placeholder, filled in below
continue
if not isinstance(name, str):
raise TTypeError("Invalid `names` list: element %d is not a "
"string" % i)
if name[:len(prefix)] == prefix and name[len(prefix):].isdigit():
min_c = max(min_c, int(name[len(prefix):]) + 1)
else:
name = re.sub(_dedup_names_re0, ".", name)
if name in inames:
mm = re.match(_dedup_names_re1, name)
if mm:
base = mm.group(1)
count = int(mm.group(2)) + 1
else:
base = name + "."
count = 1
newname = name
while newname in inames:
newname = "%s%d" % (base, count)
count += 1
dupnames.append(name)
else:
newname = name
inames[newname] = i
tnames.append(newname)
if fill_default_names:
for i, name in enumerate(names):
if not name:
newname = prefix + str(min_c)
tnames[i] = newname
inames[newname] = i
min_c += 1
if dupnames:
dtwarn("Duplicate column names found: %r. They were assigned "
"unique names." % dupnames)
assert len(inames) == len(tnames) == len(names)
return (tuple(tnames), inames)
def _dedup_names(names) -> Tuple[Tuple[str, ...], Dict[str, int]]:
inames = {}
tnames = []
dupnames = []
min_c = 0
fill_default_names = False
for i, name in enumerate(names):
if not name:
fill_default_names = True
tnames.append(None) # Placeholder, filled in below
continue
if re.match(_dedup_names_re1, name):
min_c = max(min_c, int(name[1:]) + 1)
else:
name = re.sub(_dedup_names_re0, ".", name)
if name in inames:
mm = re.match(_dedup_names_re2, name)
if mm:
base = mm.group(1)
count = int(mm.group(2)) + 1
else:
base = name + "."
count = 1
newname = name
while newname in inames:
newname = "%s%d" % (base, count)
count += 1
dupnames.append(name)
else:
newname = name
inames[newname] = i
tnames.append(newname)
if fill_default_names:
for i, name in enumerate(names):
if not name:
newname = "C%d" % min_c
tnames[i] = newname
inames[newname] = i
min_c += 1
if dupnames:
dtwarn("Duplicate column names found: %r. They were assigned "
"unique names." % dupnames)
assert len(inames) == len(tnames) == len(names)
return (tuple(tnames), inames)
def __init__(self, src=None, names=None, stypes=None, **kwargs):
if "stype" in kwargs:
stypes = [kwargs.pop("stype")]
if kwargs:
if src is None:
src = kwargs
else:
dtwarn("Unknown options %r to Frame()" % kwargs)
Frame._id_counter_ += 1
self._id = Frame._id_counter_ # type: int
self._ncols = 0 # type: int
self._nrows = 0 # type: int
self._ltypes = None # type: Tuple[ltype]
self._stypes = None # type: Tuple[stype]
self._names = None # type: Tuple[str]
# Mapping of column names to their indices
self._inames = None # type: Dict[str, int]
self._dt = None # type: core.DataTable
self._fill_from_source(src, names=names, stypes=stypes)
def __init__(self, src=None, names=None, stypes=None, **kwargs):
if "colnames" in kwargs and names is None:
names = kwargs.pop("colnames")
dtwarn("Parameter `colnames` in Frame constructor is "
"deprecated. Use `names` instead.")
if "stype" in kwargs:
stypes = [kwargs.pop("stype")]
if kwargs:
dtwarn("Unknown options %r to Frame()" % kwargs)
Frame._id_counter_ += 1
self._id = Frame._id_counter_ # type: int
self._ncols = 0 # type: int
self._nrows = 0 # type: int
self._ltypes = None # type: Tuple[ltype]
self._stypes = None # type: Tuple[stype]
self._names = None # type: Tuple[str]
# Mapping of column names to their indices
self._inames = None # type: Dict[str, int]
self._dt = None # type: core.DataTable
self._fill_from_source(src, names=names, stypes=stypes)
def save(self, dest, _strategy="auto"):
"""
Save Frame in binary NFF format.
:param dest: destination where the Frame should be saved.
:param _strategy: one of "mmap", "write" or "auto"
"""
if _strategy not in ("auto", "write", "mmap"):
raise TValueError("Invalid parameter _strategy: only 'write' / 'mmap' "
"/ 'auto' are allowed")
dest = os.path.expanduser(dest)
if os.path.exists(dest):
# raise ValueError("Path %s already exists" % dest)
pass
else:
os.makedirs(dest)
if self.internal.isview:
# Materialize before saving
self._dt = self.internal.materialize()
metafile = os.path.join(dest, "_meta.nff")
with _builtin_open(metafile, "w", encoding="utf-8") as out:
out.write("# NFF1\n")
out.write("# nrows = %d\n" % self.nrows)
out.write('filename,stype,meta,colname\n')
l = len(str(self.ncols))
for i in range(self.ncols):
filename = "c%0*d" % (l, i + 1)
colname = self.names[i].replace('"', '""')
_col = self.internal.column(i)
stype = _col.stype
meta = _col.meta
if stype == dt.stype.obj64:
dtwarn("Column %r of type obj64 was not saved" % self.names[i])
continue
if meta is None:
meta = ""
out.write('%s,%s,%s,"%s"\n' % (filename, stype.code, meta, colname))
filename = os.path.join(dest, filename)
_col.save_to_disk(filename, _strategy)
def save_nff(self, dest, _strategy="auto"):
"""
Save Frame in binary NFF/Jay format.
:param dest: destination where the Frame should be saved.
:param _strategy: one of "mmap", "write" or "auto"
"""
if _strategy not in ("auto", "write", "mmap"):
raise TValueError("Invalid parameter _strategy: only 'write' / 'mmap' "
"/ 'auto' are allowed")
dest = os.path.expanduser(dest)
if not os.path.exists(dest):
os.makedirs(dest)
self.materialize()
mins = self.min().to_list()
maxs = self.max().to_list()
metafile = os.path.join(dest, "_meta.nff")
with _builtin_open(metafile, "w", encoding="utf-8") as out:
out.write("# NFF2\n")
out.write("# nrows = %d\n" % self.nrows)
out.write('filename,stype,meta,colname,min,max\n')
l = len(str(self.ncols))
for i in range(self.ncols):
filename = "c%0*d" % (l, i + 1)
colname = self.names[i].replace('"', '""')
stype = self.stypes[i]
if stype == dt.stype.obj64:
dtwarn("Column %r of type obj64 was not saved" % self.names[i])
continue
smin = _stringify(mins[i][0])
smax = _stringify(maxs[i][0])
out.write('%s,%s,,"%s",%s,%s\n' %
(filename, stype.code, colname, smin, smax))
filename = os.path.join(dest, filename)
core._column_save_to_disk(self, i, filename, _strategy)
def __init__(self,
anysource=None,
*,
file=None,
text=None,
url=None,
cmd=None,
columns=None,
sep=None,
max_nrows=None,
header=None,
na_strings=None,
verbose=False,
fill=False,
encoding=None,
dec=".",
skip_to_string=None,
skip_to_line=None,
save_to=None,
nthreads=None,
logger=None,
skip_blank_lines=True,
strip_whitespace=True,
quotechar='"',
**args):
self._src = None # type: str
self._file = None # type: str
self._files = None # type: List[str]
self._fileno = None # type: int
self._tempfiles = [] # type: List[str]
self._tempdir = None # type: str
self._tempdir_own = False # type: bool
self._text = None # type: Union[str, bytes]
self._sep = None # type: str
self._dec = None # type: str
self._maxnrows = None # type: int
self._header = None # type: bool
self._nastrings = [] # type: List[str]
self._verbose = False # type: bool
self._fill = False # type: bool
self._encoding = encoding # type: str
self._quotechar = None # type: str
self._skip_to_line = None
self._skip_blank_lines = True
self._skip_to_string = None
self._strip_whitespace = True
self._columns = None
self._save_to = save_to
self._nthreads = nthreads
self._logger = None
self._colnames = None
self._bar_ends = None
self._bar_symbols = None
self._result = None
if na_strings is None:
na_strings = ["NA"]
if "_tempdir" in args:
self.tempdir = args.pop("_tempdir")
self.verbose = verbose
self.logger = logger
if verbose:
self.logger.debug("[1] Prepare for reading")
self._resolve_source(anysource, file, text, cmd, url)
self.columns = columns
self.sep = sep
self.dec = dec
self.max_nrows = max_nrows
self.header = header
self.na_strings = na_strings
self.fill = fill
self.skip_to_string = skip_to_string
self.skip_to_line = skip_to_line
self.skip_blank_lines = skip_blank_lines
self.strip_whitespace = strip_whitespace
self.quotechar = quotechar
if "separator" in args:
self.sep = args.pop("separator")
if "show_progress" in args:
dtwarn("Parameter `show_progress` is ignored")
args.pop("show_progress")
if "progress_fn" in args:
dtwarn("Parameter `progress_fn` is ignored")
args.pop("progress_fn")
if args:
raise TTypeError("Unknown argument(s) %r in FReader(...)" %
list(args.keys()))