def _procfpath(p):
p = pndlu.convpath(p)
if osp.isdir(p):
p = osp.join(p, default_config_fname())
else:
p = osp.splitext(p)[0]
return p
def build_cmd_line(self, **convpath_kwds):
"Build cli-options for `project append` preserving pair-order."
import string
import itertools as itt
ok_file_chars = set(string.ascii_letters + string.digits + '-')
def quote(s):
"best-effort that works also on Windows (no single-quote)"
if set(s) - ok_file_chars:
s = '"%s"' % s
return s
def append_opt(l, opt, fpath):
if fpath:
l.append(opt)
l.append(quote(convpath(fpath, **convpath_kwds)))
args = []
for inp, out in itt.zip_longest(self.inp, self.out, fillvalue=()):
append_opt(args, '--inp', inp)
append_opt(args, '--out', out)
args.extend(quote(convpath(f, **convpath_kwds)) for f in self.other)
return args
def _logger_configed(self, log):
"""The ``log.name`` becomes the (expanded) filename to write into."""
from logging import Formatter
wfpath = log.name
if wfpath.startswith('+'):
append = True
wfpath = wfpath[1:]
else:
append = False
wfpath = convpath(wfpath, True)
if append:
from logging.handlers import RotatingFileHandler
handler = RotatingFileHandler(wfpath, 'a',
**self.rotating_handler_cstor_kwds)
formatter = Formatter(self.report_log_format,
self.report_date_format)
handler.setFormatter(formatter)
else:
from logging import FileHandler
handler = FileHandler(wfpath, 'w', encoding='utf-8')
handler.setLevel(0)
log.addHandler(handler)
log.setLevel(0)
log.propagate = False
return log
def check_files_exist(self, name):
from .utils import joinstuff
badfiles = self.find_nonfiles()
if badfiles:
raise CmdException("%s: cannot find %i file(s): %s" %
(name, len(badfiles),
joinstuff((convpath(f, abs_path=True)
for f in badfiles), '', '\n %s')))
def yield_files(self, *fpaths):
"""
:return:
a 2 tuple `(fpath, file_text)`
"""
import io
import os
from boltons.setutils import IndexedSet as iset
fpaths = iset(fpaths) or ['-']
for fpath in fpaths:
if fpath == '-':
msg = "Reading STDIN."
if getattr(sys.stdin, 'isatty', lambda: False)():
msg += ("..paste text, then [Ctrl+%s] to exit!" %
'Z' if sys.platform == 'win32' else 'D')
self.log.info(msg)
text = sys.stdin.read()
yield "<STDIN: %i-chars>" % len(text), text
else:
fpath = convpath(fpath, abs_path=False)
if osp.exists(fpath):
afpath = convpath(fpath, abs_path=True)
if osp.exists(afpath):
fpath = afpath
else:
self.log.error(
"File to read '%s' not found!"
"\n CWD: %s", fpath, os.curdir)
continue
try:
with io.open(fpath, 'rt') as fin:
text = fin.read()
yield fpath, text
except Exception as ex:
self.log.error(
"Reading file-path '%s' failed due to: %r",
fpath,
ex,
exc_info=self.verbose) # WARN: from `cmdlets.Spec`
continue
def convpath(fpath, abs_path=None, exp_user=True, exp_vars=True):
"""
Override `abs_path` functioning..
:param abs_path:
3-state, None: expand if it exists
Useful to preserve POSIX fpaths under Windows, e.g. ``/dev/null``.
"""
from pandalone.utils import convpath
if abs_path is None:
fpath = convpath(fpath, False, exp_user, exp_vars)
if osp.exists(fpath):
afpath = convpath(fpath, True, False, False)
if osp.exists(afpath):
fpath = afpath
else:
fpath = convpath(fpath, abs_path, exp_user, exp_vars)
return fpath
def write_default_config(self, config_file=None):
if not config_file:
config_file = default_config_fpath()
else:
config_file = pndlu.convpath(config_file)
if osp.isdir(config_file):
config_file = osp.join(config_file, default_config_fname())
config_file = pndlu.ensure_file_ext(config_file, '.py')
op = 'Over-writting' if osp.isfile(config_file) else 'Writting'
self.log.info('%s config-file %r...', op, config_file)
pndlu.ensure_dir_exists(os.path.dirname(config_file), 0o700)
config_text = self.generate_config_file()
with io.open(config_file, mode='wt') as fp:
fp.write(config_text)
class GenConfigCmd(Cmd):
"""
Store config defaults into specified path(s); '{confpath}' assumed if none specified.
- If a path resolves to a folder, the filename '{appname}_config.py' is appended.
- It OVERWRITES any pre-existing configuration file(s)!
SYNTAX
co2dice gen-config [<config-path-1>] ...
"""
## Class-docstring CANNOT contain string-interpolations!
description = trt.Unicode(
__doc__.format(confpath=pndlu.convpath('~/.%s_config.py' % APPNAME),
appname=APPNAME))
examples = trt.Unicode("""
Generate a config-file at your home folder:
co2dice gen-config ~/my_conf
Re-use this custom config-file:
co2dice --config-files=~/my_conf ...
""")
def run(self, *args):
from co2mpas.sampling import project, report
## INFO: Add all conf-classes here
pp = project.ProjectCmd
self.classes = [
pp,
pp.CurrentCmd,
pp.ListCmd,
pp.AddCmd,
pp.OpenCmd,
pp.ExamineCmd,
pp.BackupCmd,
report.ReportCmd,
GenConfigCmd,
baseapp.Spec,
project.ProjectsDB,
report.Report,
MainCmd,
]
args = args or [None]
for fpath in args:
self.write_default_config(fpath)
class GpgSpec(trtc.SingletonConfigurable, baseapp.Spec):
"""Provider of GnuPG high-level methods."""
exec_path = trt.Unicode(None,
allow_none=True,
help="""
The path to GnuPG executable; if None, the first one in PATH variable is used: '{gpgexec}'.
""".format(gpgexec=pndlu.convpath(pndlu.which('gpg')))).tag(
config=True)
home = trt.Unicode(None,
allow_none=True,
help="""
The default home directory containing the keys; if None given and nor env-var GNUPGHOME exist,
the executable decides (e.g. `~/.gpg` on POSIX, `%APPDATA%\Roaming\GnuPG` on Windows).
""").tag(config=True)
def yield_from_iofiles(self, iofiles: PFiles):
for fpath in iofiles.inp:
yield self.extract_input_params(pndlu.convpath(fpath))
for fpath in iofiles.out:
yield from self.extract_output_tables(pndlu.convpath(fpath))
def default_config_dir():
"""The folder of to user's config-file."""
return pndlu.convpath('~/.%s' % APPNAME)
def append_opt(l, opt, fpath):
if fpath:
l.append(opt)
l.append(quote(convpath(fpath, **convpath_kwds)))
def _make_report_tuples_from_iofiles(self,
iofiles: PFiles,
expected_vfid=None):
"""
Parses input/output files and yields their *unique* vehicle-family-id and any dice-reports.
:param expected_vfid:
raise
:return:
A generator that begins by yielding the following 3-tuple
for each input/output file::
('inp' | 'out', <abs-fpath>, <report-df>)
- `<report>` is series/data-frame, because that's extracted from excel.
- For *input* files, the ``<report>`` has this index:
``['vehicle_family_id': <expectec_vfid>}``;
- For *output* files, the ``<report>`` is a pandas data-frame.
- For *other* files, the ``<report>`` is None.
:raise:
CmdException if *vehicle_family_id* not matching among each other,
and `expected_vfid` when provided, unless --force.
"""
import pandalone.utils as pndlu
import pandalone.pandata as pndat
def check_vfid_missmatch(fpath, file_vfid):
nonlocal expected_vfid
if expected_vfid is None:
expected_vfid = file_vfid
elif expected_vfid != file_vfid:
return (
"mismatch `vehicle_family_id` between this file(%s) and the rest: "
"'%s' != expected('%s')'" %
(fpath, file_vfid, expected_vfid))
rtuples = []
input_report = [] # a list of dicts {file: ..., input_data: raw_data}
for fpath in iofiles.inp:
try:
fpath = pndlu.convpath(fpath)
file_vfid, inp_data = self._parse_input_xlsx(fpath)
msg = check_vfid_missmatch(fpath, file_vfid)
if msg:
msg = "File('%s') %s!" % (fpath, msg)
if self.force:
self.log.warning(msg)
else:
raise CmdException(msg)
rtuples.append((fpath, 'inp',
OrderedDict([
('vehicle_family_id', file_vfid),
('timestamp', int(osp.getmtime(fpath))),
])))
if inp_data:
input_report.append({
'file': fpath,
'input_data': inp_data
})
except CmdException:
raise
except pndat.RefResolutionError as ex:
raise CmdException(
"Cannot parse '%s' as INPUT-file due to: %s" % (fpath, ex))
except Exception as ex:
raise CmdException(
"Cannot parse '%s' as INPUT-file due to: %s: %s" %
(fpath, type(ex).__name__, ex)) from ex
for fpath in iofiles.out:
try:
fpath = pndlu.convpath(fpath)
file_vfid, dice_report = self._extract_dice_report_from_output(
fpath)
msg1 = self._check_is_ta(fpath, dice_report)
msg2 = check_vfid_missmatch(fpath, file_vfid)
msg3 = self._check_deviations_are_valid(fpath, dice_report)
msgs = [m for m in [msg1, msg2, msg3] if m]
if any(msgs):
msg = ';\n also '.join(msgs)
msg = "File('%s') %s!" % (fpath, msg)
if self.force:
self.log.warning(msg)
else:
raise CmdException(msg)
rtuples.append((fpath, 'out', dice_report))
except CmdException:
raise
except Exception as ex:
raise CmdException(
"Cannot parse '%s' as OUTPUT-file due to: %s: %s" %
(fpath, type(ex).__name__, ex)) from ex
for fpath in iofiles.other:
fpath = pndlu.convpath(fpath)
rtuples.append((fpath, 'other', None))
if input_report:
self.log.info("Attaching input-report...")
rtuples.append(
('inputs.yaml', 'cipher', self._encrypt_data(input_report)))
return rtuples