def fixup(item):
if isinstance(item, dict):
d = {k: fixup(v) for k, v in iteritems(item)}
if len(d) == 1 and first_value(d) is None and first_value(item) is not None:
return {}
return d
if isinstance(item, (list, tuple, set,)):
l = [fixup(v) for v in item]
if l == [None] and list(item) != [None]:
l = []
return type(item)(l)
if isinstance(item, type):
return None
assert isinstance(item, (bytes, unicode, int, float, long, bool, OptionEnum, NoneType, datetime.datetime, datetime.date, datetime.time, datetime.timedelta)), type(item)
return item
def _merge_auto_single(self, it, ix):
# find a non-empty one, so we can look at the data in it
data = next(it)
if isinstance(data, num_types):
# special case for when you have something like (count, dict)
return sum(it, data)
if isinstance(data, list):
for part in it:
data.extend(part)
return data
while not data:
try:
data = next(it)
except StopIteration:
# All were empty, return last one
return data
depth = 0
to_check = data
while hasattr(to_check, "values"):
if not to_check:
raise self._exc("Empty value at depth %d (index %d)" % (
depth,
ix,
))
to_check = first_value(to_check)
depth += 1
if hasattr(to_check, "update"): # like a set
depth += 1
if not depth:
raise self._exc("Top level has no .values (index %d)" % (ix, ))
def upd(aggregate, part, level):
if level == depth:
aggregate.update(part)
else:
for k, v in iteritems(part):
if k in aggregate:
upd(aggregate[k], v, level + 1)
else:
aggregate[k] = v
for part in it:
upd(data, part, 1)
return data
"blutti",
),
# big value - will change if it roundtrips through (any type of) float, semibig to find 32bit issues, and a float.
"number":
(1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,
13578058080989382, 1 / 3),
"json": ({
"a": [1, 2, {
"b": {}
}]
}, None, "bl\xe4"),
}
value_cnt = {len(v) for v in data.values()}
assert len(value_cnt) == 1, "All tuples in data must have the same length."
value_cnt = first_value(value_cnt)
not_none_capable = {
"bits64",
"bits32",
}
def sort_data_for_slice(sliceno):
# numeric types use only (modified) v[0], other types cycle through their values.
# int64 goes down one every other line,
# all other numeric columns go up sliceno + 1 every line.
# json starts as 0 (pretending to be a numeric type).
def add(offset):
res = []
for k, v in sorted(data.items()):
def convert(default_v, v):
if isinstance(default_v, RequiredOption):
if v is None and not default_v.none_ok:
raise OptionException(
'Option %s on method %s requires a non-None value (%r)'
% (
k,
method,
default_v.value,
))
default_v = default_v.value
if default_v is None or v is None:
if default_v is OptionString:
raise OptionException(
'Option %s on method %s requires a non-empty string value'
% (
k,
method,
))
if hasattr(default_v,
'_valid') and v not in default_v._valid:
raise OptionException(
'Option %s on method %s requires a value in %s' % (
k,
method,
default_v._valid,
))
if isinstance(default_v, OptionDefault):
v = default_v.default
return v
if isinstance(default_v, OptionDefault):
default_v = default_v.value
if isinstance(default_v, dict) and isinstance(v, dict):
if default_v:
sample_v = first_value(default_v)
for chk_v in itervalues(default_v):
assert isinstance(chk_v, type(sample_v))
return {
k: convert(sample_v, v)
for k, v in iteritems(v)
}
else:
return v
if isinstance(default_v, (
list,
set,
tuple,
)) and isinstance(v, str_types + (
list,
set,
tuple,
)):
if isinstance(v, str_types):
v = (e.strip() for e in v.split(','))
if default_v:
sample_v = first_value(default_v)
for chk_v in default_v:
assert isinstance(chk_v, type(sample_v))
v = (convert(sample_v, e) for e in v)
return type(default_v)(v)
if isinstance(default_v, (
OptionEnum,
OptionEnumValue,
)):
if not (v or None) in default_v._valid:
ok = False
for cand_prefix in default_v._prefixes:
if v.startswith(cand_prefix):
ok = True
break
if not ok:
raise OptionException(
'%r not a permitted value for option %s on method %s (%s)'
% (v, k, method, default_v._valid))
return v or None
if isinstance(default_v, str_types + num_types) and isinstance(
v, str_types + num_types):
if default_v is OptionString:
v = str(v)
if not v:
raise OptionException(
'Option %s on method %s requires a non-empty string value'
% (
k,
method,
))
return v
if isinstance(default_v, unicode) and isinstance(v, bytes):
return v.decode('utf-8')
return type(default_v)(v)
if (isinstance(default_v, type)
and isinstance(v, typefuzz(default_v))) or isinstance(
v, typefuzz(type(default_v))):
return v
if isinstance(default_v, bool) and isinstance(v, (str, int)):
lv = str(v).lower()
if lv in (
'true',
'1',
't',
'yes',
'on',
):
return True
if lv in (
'false',
'0',
'f',
'no',
'off',
'',
):
return False
if isinstance(default_v, _date_types):
default_v = type(default_v)
if default_v in _date_types:
try:
return typing_conv[default_v.__name__](v)
except Exception:
raise OptionException(
'Failed to convert option %s %r to %s on method %s'
% (
k,
v,
default_v,
method,
))
if isinstance(v, str_types) and not v:
return type(default_v)()
if isinstance(default_v, type): # JobWithFile or similar
default_v = default_v()
if isinstance(default_v, JobWithFile):
defaults = type(default_v).__new__.__defaults__
if not isinstance(v, (
list,
tuple,
)) or len(v) > len(defaults):
raise OptionException(
'Option %s (%r) on method %s is not %s compatible'
% (k, v, method, type(default_v)))
v = tuple(v) + defaults[len(
v):] # so all of default_v gets convert()ed.
v = [convert(dv, vv) for dv, vv in zip(default_v, v)]
return type(default_v)(*v)
raise OptionException(
'Failed to convert option %s of %s to %s on method %s' % (
k,
type(v),
type(default_v),
method,
))
