def _ensure_out_file(out_path, inp_path, force, out_frmt):
"""
:param str out_path:
If `None`, same folder as `inp_path` assumed.
"""
basename = osp.basename(inp_path)
if not osp.exists(out_path):
out_file = out_path
folders = osp.dirname(out_path)
if not osp.isdir(folders):
if force:
log.info('Creating intermediate folders: %r...', folders)
os.makedirs(folders)
else:
raise CmdException("Intermediate folders %r do not exist! \n"
"Tip: specify --force to create them." %
out_path)
elif osp.isfile(out_path):
out_file = out_path
elif osp.isdir(out_path):
out_file = osp.join(out_path, out_frmt % osp.splitext(basename))
else:
assert False, 'Unexpected file-type: %r' % out_path
assert out_file, (out_path, inp_path, force, out_frmt)
out_file = osp.abspath(osp.expanduser(osp.expandvars(out_file)))
if osp.isfile(out_file):
if force:
log.info('Overwritting datasync-file: %r...', out_file)
else:
raise CmdException("Output file exists! \n"
"\n To overwrite add '-f' option!")
return out_file
def _cmd_template(opts):
dst_fpaths = opts.get('<excel-file-path>', None)
is_gui = opts.get('--gui', False)
if is_gui and not dst_fpaths:
import easygui as eu
fpath = eu.filesavebox(msg='Create INPUT-TEMPLATE file as:',
title='%s-v%s' % (proj_name, proj_ver),
default='template.xlsx')
if not fpath:
raise CmdException('User abort creating INPUT-TEMPLATE file.')
dst_fpaths = [fpath]
elif not dst_fpaths:
raise CmdException('Missing destination filepath for INPUT-TEMPLATE!')
if opts['--cycle'] is not None:
profile = _re_template.match(opts['--cycle'])
if profile is None:
raise CmdException('Cycle %s not allowed' % opts['--cycle'])
df = _get_theoretical(profile.groupdict())
def overwrite_ref(fpath):
book = load_workbook(fpath)
writer = pd.ExcelWriter(fpath, engine='openpyxl')
writer.book = book
writer.sheets = dict((ws.title, ws) for ws in book.worksheets)
df.to_excel(writer, "ref", index=False, startrow=2, header=False)
writer.save()
else:
def overwrite_ref(fpath):
pass
template = _get_input_template_fpath()
force = opts['--force']
for fpath in dst_fpaths:
if not fpath.endswith('.xlsx'):
fpath = '%s.xlsx' % fpath
if osp.exists(fpath) and not force and not is_gui:
raise CmdException("Writing file '%s' skipped, already exists! "
"Use '-f' to overwrite it." % fpath)
if osp.isdir(fpath):
raise CmdException(
"Expecting a file-name instead of directory '%s'!" % fpath)
log.info("Creating INPUT-TEMPLATE file '%s'...", fpath)
with open(fpath, 'wb') as fd:
shutil.copyfileobj(template, fd, 16 * 1024)
overwrite_ref(fpath)
def _cmd_template(opts):
dst_fpaths = opts.get('<excel-file-path>', None)
if not dst_fpaths:
raise CmdException('Missing destination filepath for INPUT-TEMPLATE!')
if opts['--cycle'] is not None:
profile = _re_template.match(opts['--cycle'])
if profile is None:
raise CmdException('Cycle %s not allowed' % opts['--cycle'])
df = _get_theoretical(profile.groupdict())
def overwrite_ref(fpath):
book = openpyxl.load_workbook(fpath)
writer = pd.ExcelWriter(fpath, engine='openpyxl')
writer.book = book
writer.sheets = dict((ws.title, ws) for ws in book.worksheets)
df.to_excel(writer, "ref", index=False, startrow=2, header=False)
writer.save()
else:
def overwrite_ref(fpath):
pass
template = _get_input_template_fpath()
force = opts['--force']
for fpath in dst_fpaths:
if not fpath.endswith('.xlsx'):
fpath = '%s.xlsx' % fpath
if osp.exists(fpath) and not force:
raise CmdException("File '%s', already exists! "
"Use '-f' to overwrite it." % fpath)
if not osp.splitext(fpath)[-1]:
raise CmdException(
"Expecting a file-name instead of directory '%s'!" % fpath)
dir_name = osp.dirname(fpath)
if dir_name and not osp.isdir(dir_name):
if force:
os.makedirs(dir_name)
else:
raise CmdException("Directory '%s' does not exists! "
"Use '-f' to create it." % dir_name)
log.info("Creating INPUT-TEMPLATE file '%s'...", fpath)
shutil.copy(template, fpath)
overwrite_ref(fpath)
def bool_env(env_var, default):
"""
- false: 0, f, false, n, no, off
- true: 1, t, true, y, yes, on
- default: missing, or empty string
Any other value raise CmdException.
.. Attention::
On *Windows* it's impossible to assign the empty-string to a variable!
"""
v = os.environ.get(env_var)
if not v:
return default
false = '0 f false n no off'
true = '1 t true y yes on'
v = v.lower()
for flag, values in zip((0, 1), (false, true)):
if v in values.split():
return flag
raise CmdException("Invalid value '%s' for env-var[%s]!"
"\n Should be one of (%s %s)." %
(v, env_var, false, true))
def consume_next_xlref(self, xlref, lasso):
i = len(self.tables)
try:
return self._consume_next_xlref(xlref, lasso)
except CmdException as ex:
raise CmdException('Cannot read sync-sheet(%i: %s) due to: %s' %
(i, xlref, ex.args[0]))
except Exception as ex:
log.error('Failed reading sync-sheet(%i: %s) due to: %s', i, xlref,
ex)
raise
def _consume_next_xlref(self, xlref, lasso):
"""
:param str xlref:
an xlref that may not contain hash(`#`); in that case,
it is taken as *file-part* or as *fragment-part* depending
on the existence of prev lasso's `url_file`.
:param Lasso lasso:
reuses `url_file` & `sheet` if missing from xlref
"""
xlref = _guess_xlref_without_hash(xlref,
bias_on_fragment=bool(
lasso.url_file))
lasso = xleash.lasso(xlref,
sheets_factory=self._sheets_factory,
url_file=lasso.url_file,
sheet=lasso.sheet,
return_lasso=True)
values = lasso.values
if values: # Skip blank sheets.
# TODO: Convert column monkeybiz into pure-pandas using xleash.
str_row_indices = [
i for i, r in enumerate(values) if any(
isinstance(v, str) for v in r)
]
req_labels = IndexedSet(self.required_labels)
for k in str_row_indices:
if set(values[k]) >= req_labels:
break
else:
raise CmdException(
"Columns %r not found in table of sheet(%r) in book(%r)!" %
(self.required_labels, lasso.sheet._sheet.name,
lasso.sheet.book_fname))
ix = values[k]
i = max(str_row_indices, default=0) + 1
h = pd.DataFrame(values[:i], columns=ix)
self.headers.append((sheet_name(lasso), k, h))
values = pd.DataFrame(values[i:], columns=ix)
values.dropna(how='all', inplace=True)
values.dropna(axis=1, how='any', inplace=True)
if values.empty:
log.warning("Empty table of sheet(%r) in book (%r)!" %
(lasso.sheet._sheet.name, lasso.sheet.book_fname))
else:
self.tables.append(values)
return lasso
def _get_interp_method(interpolation_method):
methods = _interpolation_methods()
try:
kw = _re_interpolation_method.match(interpolation_method)
if kw:
kw = {k: v for k, v in kw.groupdict().items() if v is not None}
if 'order' in kw:
kw['order'] = int(kw['order'])
kind = kw.pop('kind').lower()
if kind == 'spline':
kw['kind'] = kw.pop('order')
return fnt.partial(methods[kind], **kw)
else:
raise KeyError
except KeyError:
msg = '`%s` is not implemented as re-sampling method!\n' \
'Please choose one of: \n %s'
msg %= interpolation_method, ', '.join(sorted(methods))
raise CmdException(msg)
def _guess_xlref_without_hash(xlref, bias_on_fragment):
if not xlref:
raise CmdException("An xlref cannot be empty-string!")
if '#' not in xlref:
xlref = ('#%s!' if bias_on_fragment else '%s#:') % xlref
return xlref