def fixup(item):
if isinstance(item, dict):
d = {k: fixup(v) for k, v in iteritems(item)}
if len(d) == 1 and first_value(d) is None and first_value(item) is not None:
return {}
return d
if isinstance(item, (list, tuple, set,)):
l = [fixup(v) for v in item]
if l == [None] and list(item) != [None]:
l = []
return type(item)(l)
if isinstance(item, (type, OptionEnum)):
return None
assert isinstance(item, (bytes, unicode, int, float, long, bool, OptionEnum, NoneType, datetime.datetime, datetime.date, datetime.time, datetime.timedelta)), type(item)
return item
def _merge_auto_single(self, it, ix):
# find a non-empty one, so we can look at the data in it
data = next(it)
if isinstance(data, num_types):
# special case for when you have something like (count, dict)
return sum(it, data)
if isinstance(data, list):
for part in it:
data.extend(part)
return data
while not data:
try:
data = next(it)
except StopIteration:
# All were empty, return last one
return data
depth = 0
to_check = data
while hasattr(to_check, "values"):
if not to_check:
raise self._exc("Empty value at depth %d (index %d)" % (
depth,
ix,
))
to_check = first_value(to_check)
depth += 1
if hasattr(to_check, "update"): # like a set
depth += 1
if not depth:
raise self._exc("Top level has no .values (index %d)" % (ix, ))
def upd(aggregate, part, level):
if level == depth:
aggregate.update(part)
else:
for k, v in iteritems(part):
if k in aggregate:
upd(aggregate[k], v, level + 1)
else:
aggregate[k] = v
for part in it:
upd(data, part, 1)
return data
def convert(default_v, v):
if isinstance(default_v, RequiredOption):
if v is None and not default_v.none_ok:
raise OptionException('Option %s on method %s requires a non-None value (%r)' % (k, method, default_v.value,))
default_v = default_v.value
if default_v is None or v is None:
if isinstance(default_v, _OptionString):
raise OptionException('Option %s on method %s requires a non-empty string value' % (k, method,))
if hasattr(default_v, '_valid') and v not in default_v._valid:
raise OptionException('Option %s on method %s requires a value in %s' % (k, method, default_v._valid,))
if isinstance(default_v, OptionDefault):
v = default_v.default
return v
if isinstance(default_v, OptionDefault):
default_v = default_v.value
if isinstance(default_v, dict) and isinstance(v, dict):
if default_v:
sample_v = first_value(default_v)
for chk_v in itervalues(default_v):
assert isinstance(chk_v, type(sample_v))
return {k: convert(sample_v, v) for k, v in iteritems(v)}
else:
return v
if isinstance(default_v, (list, set, tuple,)) and isinstance(v, str_types + (list, set, tuple,)):
if isinstance(v, str_types):
v = (e.strip() for e in v.split(','))
if default_v:
sample_v = first_value(default_v)
for chk_v in default_v:
assert isinstance(chk_v, type(sample_v))
v = (convert(sample_v, e) for e in v)
return type(default_v)(v)
if isinstance(default_v, (OptionEnum, OptionEnumValue,)):
if not (v or None) in default_v._valid:
ok = False
for cand_prefix in default_v._prefixes:
if v.startswith(cand_prefix):
ok = True
break
if not ok:
raise OptionException('%r not a permitted value for option %s on method %s (%s)' % (v, k, method, default_v._valid))
return v or None
if isinstance(default_v, str_types + num_types) and isinstance(v, str_types + num_types):
if isinstance(default_v, _OptionString):
v = str(v)
if not v:
raise OptionException('Option %s on method %s requires a non-empty string value' % (k, method,))
return v
if isinstance(default_v, unicode) and isinstance(v, bytes):
return v.decode('utf-8')
return type(default_v)(v)
if (isinstance(default_v, type) and isinstance(v, typefuzz(default_v))) or isinstance(v, typefuzz(type(default_v))):
return v
if isinstance(default_v, bool) and isinstance(v, (str, int)):
lv = str(v).lower()
if lv in ('true', '1', 't', 'yes', 'on',):
return True
if lv in ('false', '0', 'f', 'no', 'off', '',):
return False
if isinstance(default_v, _date_types):
default_v = type(default_v)
if default_v in _date_types:
try:
return typing_conv[default_v.__name__](v)
except Exception:
raise OptionException('Failed to convert option %s %r to %s on method %s' % (k, v, default_v, method,))
if isinstance(v, str_types) and not v:
return type(default_v)()
if isinstance(default_v, JobWithFile) or default_v is JobWithFile:
defaults = ('', '', False, None,)
if default_v is JobWithFile:
default_v = defaults
if not isinstance(v, (list, tuple,)) or not (2 <= len(v) <= 4):
raise OptionException('Option %s (%r) on method %s is not %s compatible' % (k, v, method, type(default_v)))
v = tuple(v) + defaults[len(v):] # so all of default_v gets convert()ed.
v = [convert(dv, vv) for dv, vv in zip(default_v, v)]
return JobWithFile(*v)
raise OptionException('Failed to convert option %s of %s to %s on method %s' % (k, type(v), type(default_v), method,))
"blutti",
),
# big value - will change if it roundtrips through (any type of) float, semibig to find 32bit issues, and a float.
"number":
(1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,
13578058080989382, 1 / 3),
"json": ({
"a": [1, 2, {
"b": {}
}]
}, None, "bl\xe4"),
}
value_cnt = {len(v) for v in data.values()}
assert len(value_cnt) == 1, "All tuples in data must have the same length."
value_cnt = first_value(value_cnt)
not_none_capable = {
"bits64",
"bits32",
}
def sort_data_for_slice(sliceno):
# numeric types use only (modified) v[0], other types cycle through their values.
# int64 goes down one every other line,
# all other numeric columns go up sliceno + 1 every line.
# json starts as 0 (pretending to be a numeric type).
def add(offset):
res = []
for k, v in sorted(data.items()):