def analysis(sliceno, slices, prepare_res, job):
lst = prepare_res[sliceno::slices]
msg = ProgressMsg(lst)
with status('extracting') as update:
with ZipFile(join(job.input_directory, options.filename), 'r') as z:
for tmpfn, zfn, dsn in lst:
update(msg.step('extracting'))
with z.open(zfn) as rfh:
with job.open(tmpfn, 'wb', temp=True) as wfh:
copyfileobj(rfh, wfh)
def synthesis(job, slices):
if not options.sliced:
filename = '%d.gz' if options.filename.lower().endswith(
'.gz') else '%d.csv'
with job.open(options.filename, "wb") as outfh:
for sliceno in range(slices):
if exists(filename % sliceno):
with status("Assembling %s (%d/%d)" %
(options.filename, sliceno, slices)):
with open(filename % sliceno, "rb") as infh:
copyfileobj(infh, outfh)
unlink(filename % sliceno)
def analysis(sliceno, prepare_res, slices, job):
trend, bg, cases, zone2poly, ts2zone2cnts, map_size = prepare_res
ts2zone2cnts = [v[sliceno::slices] for v in ts2zone2cnts]
fonts = {
90: ImageFont.truetype('DejaVuSerif', 90),
}
gray64 = (64, 64, 64)
gray192 = (192, 192, 192)
x_start = (SPACING + map_size[0]) * 2 + SPACING
def text(y_pos, msg, size=90, fill=gray192):
if size not in fonts:
fonts[size] = ImageFont.truetype('DejaVuSans', size)
pos = ((BOX_WIDTH - draw.textsize(msg, font=fonts[size])[0]) // 2 +
x_start, y_pos)
draw.text(pos,
msg,
font=fonts[size],
fill=fill,
stroke_fill=gray64,
stroke_width=1)
ix = sliceno
with status('rendering') as update:
for (ts, am), (ts_l, left), (ts_r,
right) in zip(trend[sliceno::slices],
*ts2zone2cnts):
assert ts == ts_r # ts_l is ~1y earlier, not verified
im1 = render_one(zone2poly, left, map_size)
im2 = render_one(zone2poly, right, map_size)
im = bg.copy() # Image.new('RGB', (1920,1080))
im.paste(im1, (20, 160), mask=im1)
im.paste(im2, (20 + 667 + 20, 160), mask=im2)
draw = Draw(im)
date = ts
text(610 + 80,
"%2d%%" % (100 - round(100 * am), ),
fill=(255, 128, 64))
text(410 + 70, date.strftime("%H:%M"))
text(160 + 80, date.strftime("%a"), size=60)
text(160 + 160, date.strftime("%B %d"), size=50)
text(870 + 50,
"%d" % (cases.get(date.date(), 1), ),
fill=(100, 160, 255))
filename = 'frames_inorder/frame_%05d.jpg' % (ix, )
im.save(filename)
job.register_file(filename)
ix += slices
update('Rendered %d/%d frames.' %
(ix // slices, len(ts2zone2cnts[0])))
def synthesis(prepare_res):
opts = DotDict((k, v) for k, v in options.items() if k in a_csvimport.options)
lst = prepare_res
previous = datasets.previous
msg = ProgressMsg(lst)
with status('importing') as update:
for fn, info, dsn in lst:
update(msg.step('importing'))
opts.filename = fn
show_fn = '%s:%s' % (options.filename, info.filename,)
ds = build('csvimport', options=opts, previous=previous, caption='Import of ' + show_fn).dataset()
previous = ds.link_to_here(dsn, filename=show_fn)
if options.chaining == 'off':
previous = datasets.previous
if (len(lst) == 1 or options.chaining != 'off') and dsn != 'default':
ds.link_to_here('default', filename=show_fn)
def synthesis(job, slices):
if not options.sliced:
def msg(sliceno):
return "Assembling %s (%d/%d)" % (
options.filename,
sliceno + 1,
slices,
)
with status(msg(0)) as update:
with job.open(options.filename, "wb") as outfh:
for sliceno in range(slices):
filename = str(sliceno)
if exists(filename):
update(msg(sliceno))
with open(filename, "rb") as infh:
copyfileobj(infh, outfh)
unlink(filename)
def synthesis(job):
dw = job.datasetwriter()
todo = {
'ascii',
'bits32',
'bits64',
'bool',
'bytes',
'complex32',
'complex64',
'date',
'datetime',
'float32',
'float64',
'int32',
'int64',
'json',
'number',
'pickle',
'time',
'unicode',
}
check = {n for n in _convfuncs if not n.startswith('parsed:')}
assert todo == check, 'Missing/extra column types: %r %r' % (
check - todo,
todo - check,
)
no_none = {'bits32', 'bits64', 'json'}
for name in sorted(todo):
if PY2 and name == 'pickle':
# pickle columns are not supported on python 2.
t = 'ascii'
else:
t = name
dw.add(name, t, none_support=name not in no_none)
write = dw.get_split_write()
write(
'a',
0xffffffff,
0xfedcba9876543210,
True,
b'hello',
42,
1e100 + 0.00000000000000001j,
date(2020, 6, 23),
datetime(2020, 6, 23, 12, 13, 14),
1.0,
float('-inf'),
-10,
-20,
{'json': True},
0xfedcba9876543210beef,
'...' if PY2 else 1 + 2j,
time(12, 13, 14),
'bl\xe5',
)
d = {}
d['recursion'] = d
write(
'b',
0,
0,
False,
b'bye',
2 - 3j,
-7,
date(1868, 1, 3),
datetime(1868, 1, 3, 13, 14, 5),
float('inf'),
float('nan'),
0,
0,
[False, None],
42.18,
'...' if PY2 else d,
time(13, 14, 5),
'bl\xe4',
)
write(
None,
72,
64,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
)
ds = dw.finish()
sep = '\x1e'
for sep, q, none_as, last_line in (
('\x1e', '', None, (
'None',
'72',
'64',
'None',
'None',
'None',
'None',
'None',
'None',
'None',
'None',
'None',
'None',
'null',
'None',
'None',
'None',
'None',
)),
('\x1e', 'a', '', (
'',
'72',
'64',
'',
'',
'',
'',
'',
'',
'',
'',
'',
'',
'',
'',
'',
'',
'',
)),
('\x00', '0', None, (
'None',
'72',
'64',
'None',
'None',
'None',
'None',
'None',
'None',
'None',
'None',
'None',
'None',
'null',
'None',
'None',
'None',
'None',
)),
(':', '"', '"', (
'"',
'72',
'64',
'"',
'"',
'"',
'"',
'"',
'"',
'"',
'"',
'"',
'"',
'"',
'"',
'"',
'"',
'"',
)),
(':', '"', {
'time': 'never',
'float32': '"0"'
}, (
'None',
'72',
'64',
'None',
'None',
'None',
'None',
'None',
'None',
'"0"',
'None',
'None',
'None',
'null',
'None',
'None',
'never',
'None',
)),
):
with status("Checking with sep=%r, q=%r, none_as=%r" % (
sep,
q,
none_as,
)):
exp = subjobs.build('csvexport',
filename='test.csv',
separator=sep,
source=ds,
quote_fields=q,
none_as=none_as,
lazy_quotes=False)
with exp.open('test.csv', 'r', encoding='utf-8') as fh:
def expect(*a):
want = sep.join(q + v.replace(q, q + q) + q
for v in a) + '\n'
got = next(fh)
assert want == got, 'wanted %r, got %r from %s (export of %s)' % (
want,
got,
exp,
ds,
)
expect(*sorted(todo))
expect(
'a',
'4294967295',
'18364758544493064720',
'True',
'hello',
'(42+0j)',
'(1e+100+1e-17j)',
'2020-06-23',
'2020-06-23 12:13:14',
'1.0',
'-inf',
'-10',
'-20',
'{"json": true}',
'1203552815971897489538799',
'...' if PY2 else '(1+2j)',
'12:13:14',
'bl\xe5',
)
expect(
'b',
'0',
'0',
'False',
'bye',
'(2-3j)',
'(-7+0j)',
'1868-01-03',
'1868-01-03 13:14:05',
'inf',
'nan',
'0',
'0',
'[false, null]',
'42.18',
'...' if PY2 else "{'recursion': {...}}",
'13:14:05',
'bl\xe4',
)
expect(*last_line)
def prepare(job):
chain = datasets.source.chain(stop_ds={jobs.previous: 'source'},
length=options.length)
columns = defaultdict(set)
none_support = defaultdict(bool)
hashlabel = set()
seen_all = set(chain[0].columns)
for ds in chain:
if options.column not in ds.columns:
raise Exception('%r does not have column %r' % (
ds,
options.column,
))
hashlabel.add(ds.hashlabel)
seen_all &= set(ds.columns)
for name, col in ds.columns.items():
columns[name].add(col.type)
none_support[name] |= col.none_support
seen_all.discard(options.column)
if not seen_all:
raise Exception('Chain has no common columns (except %r)' %
(options.column, ))
columns = {k: columns[k] for k in seen_all}
if len(hashlabel) == 1:
hashlabel = hashlabel.pop()
if hashlabel == options.column:
hashlabel = None
else:
hashlabel = None
for name, types in columns.items():
# this relies on internal knowledge that copy_mode is compatible for these types.
if 'unicode' in types:
types.discard('ascii')
if 'number' in types:
types.discard('bits32')
types.discard('bits64')
types.discard('int32')
types.discard('int64')
types.discard('float32')
types.discard('float64')
if 'bits64' in types:
types.discard('bits32')
if 'complex64' in types:
types.discard('complex32')
if 'float64' in types:
types.discard('float32')
if 'int64' in types:
types.discard('bits32')
types.discard('int32')
if len(types) > 1 and not (
types -
{'bits32', 'bits64', 'int32', 'int64', 'float32', 'float64'}):
types = {'number'}
if len(types) > 1:
raise Exception("Column %r has incompatible types: %r" % (
name,
types,
))
columns[name] = (
types.pop(),
none_support[name],
)
collect = subjobs.build(
'dataset_fanout_collect',
source=datasets.source,
previous=jobs.previous,
column=options.column,
length=options.length,
)
values = collect.load()
if jobs.previous:
previous = {ds.name: ds for ds in jobs.previous.datasets}
values.update(previous)
else:
previous = {}
with status('Creating %d datasets' % (len(values), )):
writers = {
name: job.datasetwriter(
name=name,
columns=columns,
hashlabel=hashlabel,
previous=previous.get(name),
copy_mode=True,
)
for name in values
}
return writers, sorted(columns), chain