def parse_csv(csv, out, bench, bench_out):
"""Inspect a CSV file to figure out about the dialect, comment and header lines and the overall structure."""
click.echo("Input CSV {}".format(csv))
table = anycsv.reader(csv)
if out:
if out.endswith(".gz"):
fout = gzip.open(out, 'wt', newline='')
else:
fout = open(out, "wt")
else:
fout=sys.stdout
import csv
writer = csv.writer(fout)
for row in table:
writer.writerow(row)
if out:
fout.close()
if bench:
click.echo("TIMING")
click.echo(Timer.printStats())
if bench_out:
Timer.to_csv(bench_out)
def showcase_pattern(local=None,
provider=None,
method=None,
size=10,
seed="ascd",
pattern1=True,
pattern2=False,
verbose=False):
for l in list_locals():
if not local or l == local:
for pm in list_providers_methods(local=l,
max_unqiue=size,
only_max=True,
seed=seed):
p = "{}.{}".format(pm[0], pm[1])
key = "{}-{}".format(l, p)
try:
header, data = create_data_provider_list(
providers=[["{}".format(p), 1]],
size=size,
local=l,
seed=seed)
data = data[header[0]]
data = [str(d) for d in data]
#res=None
#if pattern1:
# with Timer(key="{}.{}-m1".format(key, size)) as t1:
# res=pattern(data)
print("\n### {} - {}".format(key, pm[2]))
if verbose:
print_columns(data, 6)
#else:
# print(" #=< {}".format(data))
#print(" #=> {}".format(res))
#print(" TIMING: {}".format(Timer.printStats(keys=["{}.{}-m1".format(key, size)], header=False)))
except Exception as e:
print(" #- {} - {}".format(key, pm[2]))
print(" #=> {} {}".format(type(e).__name__, e))
print(Timer.printStats(keys=['pattern1']))
print(Timer.printStats())
def _pattern1(key, values):
with Timer(key="p1-{}".format(key)) as t:
pat = value_pattern_detection.pattern(values, pf=pf)
print("---- value_pattern_detection.pattern(values, pf=pf) ----")
print(" ", pat)
print(" CALL: pattern_to_string(pat, collapse_level=2)")
pat_str = pattern_to_string(pat, collapse_level=2)
print(" P1>> {}".format(pat_str))
print("-----------")
def inspect_csv(csv, bench,bench_out):
"""Inspect a CSV file to figure out about the dialect, comment and header lines and the overall structure."""
click.echo("Input CSV {}".format(csv))
reader = anycsv.reader(csv)
for i,row in enumerate(reader):
pass
click.echo("{:-^80}".format(" Table Info "))
click.echo(" input: {}".format(reader.csv))
click.echo(" encoding: {}".format(reader.encoding))
click.echo(" md5: {}".format(reader.digest))
click.echo(" dialect:")
for k,v in reader.dialect._asdict().items():
click.echo(" {}: {}".format(k,v))
if bench:
click.echo("TIMING")
click.echo(Timer.printStats())
if bench_out:
Timer.to_csv(bench_out)
def demo(size, provider, method, local, p1, p2, groups):
"""Showcase pattern generator based on mimesis data provider"""
click.echo(
"datagenerator(local={}, size={}, provider={}, method={})".format(
local, size, provider, method))
gen = datagenerator(local=local,
size=size,
provider=provider,
method=method)
for key, values in gen:
print("\n-- {}".format(key))
print_columns(values, columns=None, max_rows=4, indent=1)
if p1:
_pattern1(key, values)
if p2:
_pattern2(key, values, groups)
print(Timer.printStats(key=key))
print("Overall timming stats")
print(Timer.printStats())
def pattern3(key, values):
with Timer(key=key) as t:
pat = value_pattern_detection.pattern(values, pf=pf)
print(t.printStats(key=key))
print(pat)
pat_str = pattern_to_string(pat, collapse_level=0)
print("PAT>> {} {}".format(pat_str, pat))
pat_str = pattern_to_string(pat, collapse_level=1, with_gap=False)
print("PATL1>> {} {}".format(pat_str, pat))
pat_str = pattern_to_string(pat, collapse_level=2)
print("PATL2>> {} {}".format(pat_str, pat))
hl = higher_level(pat)
hl_str = pattern_to_string(hl, collapse_level=2)
print("HL>> {} {}".format(hl_str, hl))
def demo(size, provider, method, local):
"""Showcase pattern generator based on mimesis data provider"""
click.echo("datagenerator(local={}, size={}, provider={}, method={})".format(local, size, provider, method))
gen = datagenerator(local=local, size=size, provider=provider, method=method)
for key, values in gen:
print("\n-- {}".format(key))
print_columns(values, columns=None, max_rows=4, indent=1)
with Timer(key=key) as t:
pat = value_pattern_detection.pattern(values, pf=pf)
print(" ",pat)
print(" CALL: pattern_to_string(pat, collapse_level=0)")
pat_str = pattern_to_string(pat, collapse_level=0)
print(" >> {}".format(pat_str))
print(" CALL: pattern_to_string(pat, collapse_level=1)")
pat_str = pattern_to_string(pat, collapse_level=1)
print(" >> {}".format(pat_str))
print(" CALL: pattern_to_string(pat, collapse_level=2)")
pat_str = pattern_to_string(pat, collapse_level=2)
print(" >> {}".format(pat_str))
print(" CALL: pattern_to_string( higher_level(pat), collapse_level=2)")
hl = higher_level(pat)
print(" ", hl)
hl_str = pattern_to_string(hl, collapse_level=2)
print(" >> {}".format(hl_str))
print(t.printStats(key=key))
print("Overall timming stats")
print(t.printStats())
('a', ['+10', '-20', '40']),
('a', ['10', '20', '40']),
('a', ['+10', '+20', '+40']),
('a', ['-10', '-20', '-40']),
]
#gen=datetime
max_pattern = 2
#provider, method=None, None
for key, values in gen:
print("\n-- {}".format(key))
print(" V: {}".format(values[:10]))
patterns = translate_all(values,
trans_table=string_to_uc_level2,
trans_func=translate)
print(" P: {}".format(patterns[:10]))
# L1 = l1_aggregate(patterns)
# print(" L1: {}".format(L1))
#
# L2 = l2_aggregate(grouped=L1)
# print(" L2: {}".format(L2))
patter2(values, max_pattern=max_pattern)
pattern3(key, values)
print(Timer.printStats())
def _pattern2(key, values, groups):
with Timer(key="p2-{}".format(key)) as t:
print("---- pattern2(values, max_pattern={}) ----".format(groups))
pm = pattern2(values, max_pattern=groups)
print(" P2>> {}".format(pm.info()))
print("-----------")