def find_examples(dir_):
f = glob.glob(emit(path.join(dir_, "**", "*.json"), 'path'),
recursive=True)
emit(len(f), 'number of examples')
r = [x for x in [load_example(x) for x in f] if not skip_joint_filters(x)]
random.shuffle(r)
return r
def emit_z3c_objects(indent, db, options, table):
emit_table(indent, db, options, table)
emit(indent + ('class %(tn)sObject(MappedClassBase): pass\n'
'%(tab)smapper(%(tn)sObject, %(tn)s)') % {
'tn': table.name,
'tab': indent
})
def main():
os.makedirs(FRAMES_DIR, exist_ok=True)
clear_old_frames()
v = find_examples(VID_DIR)
X, Y = [[], []]
r = []
for x in v:
emit(x, 'vx')
joint_dict = x['joints']
from_ = list(joint_dict['from'].values())
to_ = list(joint_dict['to'].values())
for vid in x['vids']:
emit(vid, 'extracting')
first, last = save_frames(vid)
if not first or not last:
emit(vid, 'WARNING: no first or last frame')
continue
X.append(first)
Y.append(from_)
X.append(last)
Y.append(to_)
emit(len(X), 'X items')
emit(len(Y), 'Y items')
open(DATASET_FILE, 'w').write(json.dumps({'x': X, 'y': Y}))
def skip_joint_filters(example):
if list(example['joints']['from'].keys()) != FILTER_JOINTS_ONLY:
emit(example['joints']['from'].keys(), 'skipping')
# emit(example, 'skipping')
return True
else:
return False
def emit_index(indent, db, options, table):
"""docstring for emit_index"""
if not options.noindex:
indexes = []
if not table.indexes:
# for certain dialects we need to include index support
if hasattr(db.dialect, 'indexloader'):
indexes = db.dialect.indexloader(db).indexes(table)
else:
print >> config.err, 'It seems that this dialect does not support indexes!'
else:
indexes = list(table.indexes)
emit(*[indent + repr(index) for index in indexes])
def emit_index(indent, db, options, table):
"""docstring for emit_index"""
if not options.noindex:
indexes = []
if not table.indexes:
# for certain dialects we need to include index support
if hasattr(db.dialect, 'indexloader'):
indexes = db.dialect.indexloader(db).indexes(table)
else:
print >>config.err, 'It seems that this dialect does not support indexes!'
else:
indexes = list(table.indexes)
emit(*[indent + repr(index) for index in indexes])
def extract_frames(vid):
p = emit(vid.split("/"), 'p')
fn = '_'.join(p[:-1]) + p[-1].replace('.h264', '-%d.png')
out = path.join(FRAMES_DIR, fn)
cmd = f'ffmpeg -i {vid} -s {W}x{H} {out}'
emit(cmd, 'save thumbs')
system(cmd)
r = glob.glob(out.replace('-%d.png', '-*.png'))
r.sort(key=lambda x: int(x.split("-")[-1].replace('.png', '')))
emit(r, 'frames')
if len(r) < MIN_FRAMES:
return None
keepers = {x: r[x] for x in OFS}
for x in r:
if not x in keepers.values():
unlink(x)
return keepers
def save_frames(vid):
p = emit(vid.split("/"), 'p')
fn = '_'.join(p[:-1]) + p[-1].replace('.h264', '-%d.png')
out = path.join(FRAMES_DIR, fn)
cmd = f'ffmpeg -i {vid} -s {W}x{H} {out}'
emit(cmd, 'save thumbs')
os.system(cmd)
r = glob.glob(out.replace('-%d.png', '-*.png'))
r.sort(key=lambda x: int(x.split("-")[-1].replace('.png', '')))
emit(r, 'frames')
if len(r) < 10:
return [None, None]
assert len(r) > 10
first = r[0]
last = r[-1]
for x in r:
if x != first and x != last:
os.unlink(x)
pass
return [first, last]
def main():
start_time = timestamp()
last_loss = '(loss)'
model_fname = MODEL_OUTPUT_FNAME.format_map({**globals(), **locals()})
emit(model_fname, 'output file')
[y_size, Xn, Yn, Xs, Ys] = prep()
X_tr, X_te, Y_tr, Y_te = train_test_split(Xn, Yn, test_size=0.15)
emit((X_tr.shape, X_te.shape), 'tr/te')
net = model(y_size)
o = net.fit(X_tr,
Y_tr,
epochs=EPOCHS,
validation_data=(X_te, Y_te),
batch_size=BATCH)
emit(o, 'model.fit')
last_loss = o.history['loss'][-1]
model_fname = MODEL_OUTPUT_FNAME.format_map({**globals(), **locals()})
o.model.save(model_fname)
def main():
config.configure()
options = config.options
if options.declarative:
config.interactive = None
if options.interactive:
config.interactive = True
config.schema = None
if options.schema:
config.schema = options.schema
config.example=False
if options.example:
config.example=True
factory = ModelFactory(config)
emit(repr(factory))
config.out.close()
config.out = sys.stdout
print >>config.err, "Output written to %s" % options.output
return
import formatter
formatter.monkey_patch_sa()
import sqlalchemy
db, options = config.engine, config.options
metadata = sqlalchemy.MetaData(db)
print >>config.err, 'Starting...'
conn = db.connect()
if options.schema != None:
reflection_schema=options.schema
else:
try:
reflection_schema = db.dialect.get_default_schema_name(conn)
except NotImplementedError:
reflection_schema = None
tablenames = db.dialect.table_names(conn, reflection_schema)
# fixme: don't set up output until we're sure there's work to do!
if options.tables:
subset, missing, unglobbed = util.glob_intersection(tablenames, options.tables)
for identifier in missing:
print >>config.err, 'Table "%s" not found.' % identifier
for glob in unglobbed:
print >>config.err, '"%s" matched no tables.' % glob
if not subset:
print >>config.err, "No tables matched!"
sys.exit(1)
tablenames = subset
if options.exclude:
tablenames = [table for table in tablenames if not re.search(table, options.exclude)]
# some header with imports
if options.generictypes:
dialect = ''
else:
dialect = 'from sqlalchemy.databases.%s import *\n' % db.name
header = options.z3c and constants.HEADER_Z3C or constants.HEADER
emit(header % {'dialect': dialect, 'encoding': options.encoding})
for tname in tablenames:
print >>config.err, "Generating python model for table %s" % (
util.as_sys_str(tname))
table = sqlalchemy.Table(tname, metadata, schema=reflection_schema,
autoload=True)
if options.schema is None:
# we're going to remove the schema from the table so that it
# isn't rendered in the output. If we don't put back the
# correct value, it may cause errors when other tables reference
# this one.
original_schema = table.schema
table.schema = None
else: original_schema = options.schema
INC = '\n\n'
if options.z3c:
INC = INC + 4*' '
emit('%s%s%s%s = %r' % (INC, options.table_prefix, tname, options.table_suffix, table))
if options.z3c:
emit(INC + ('class %(tn)sObject(MappedClassBase): pass\n'
'mapper(%(tn)sObject, %(tn)s)') % {'tn':tname})
table.schema = original_schema
# directly print indices after table def
if not options.noindex:
indexes = []
if not table.indexes:
# for certain dialects we need to include index support
if hasattr(db.dialect, 'indexloader'):
indexes = db.dialect.indexloader(db).indexes(table)
else:
print >>config.err, 'It seems that this dialect does not support indexes!'
else:
indexes = list(table.indexes)
util.emit(*[repr(index) for index in indexes])
if options.z3c:
emit(constants.FOOTER_Z3C)
# print some example
if options.example:
emit('\n' + constants.FOOTER_EXAMPLE % {
'url': unicode(db.url), 'tablename': tablenames[0]})
if options.output:
emit('\n')
config.out.close()
config.out = sys.stdout
print >>config.err, "Output written to %s" % options.output
def emit_table(indent, db, options, table):
"""Emit table representation."""
emit('%s%s%s%s = %r' % (indent, options.table_prefix, table.name, options.table_suffix, table))
emit_index(indent, db, options, table)
def main():
config.configure()
options = config.options
if options.declarative:
config.interactive = None
if options.interactive:
config.interactive = True
config.schema = None
if options.schema:
config.schema = options.schema
config.example=False
if options.example:
config.example=True
factory = ModelFactory(config)
emit(repr(factory))
config.out.close()
config.out = sys.stdout
print >>config.err, "Output written to %s" % options.output
return
import formatter
formatter.monkey_patch_sa()
import sqlalchemy
from sqlalchemy.engine.reflection import Inspector
db, options = config.engine, config.options
metadata = sqlalchemy.MetaData(db)
print >>config.err, 'Starting...'
conn = db.connect()
inspector = Inspector.from_engine(conn)
if options.schema != None:
reflection_schema=options.schema
else:
try:
reflection_schema = inspector.default_schema_name
except NotImplementedError:
reflection_schema = None
tablenames = inspector.get_table_names(reflection_schema)
# fixme: don't set up output until we're sure there's work to do!
if options.tables:
subset, missing, unglobbed = util.glob_intersection(tablenames,
options.tables)
for identifier in missing:
print >>config.err, 'Table "%s" not found.' % identifier
for glob in unglobbed:
print >>config.err, '"%s" matched no tables.' % glob
if not subset:
print >>config.err, "No tables matched!"
sys.exit(1)
tablenames = subset
# some header with imports
if options.generictypes:
dialect = ''
else:
d1 = 'from sqlalchemy.databases.%s import *\n' % db.name
d2 = 'from sqlalchemy.dialects.%s import *\n' % db.name
#Determine with one is correct...
dialect = util.select_imports([d1, d2])
header = options.z3c and constants.HEADER_Z3C or constants.HEADER
emit(header % {'dialect': dialect, 'encoding': options.encoding})
for tname in tablenames:
print >>config.err, "Generating python model for table %s" % (
util.as_sys_str(tname))
table = sqlalchemy.Table(tname, metadata, schema=reflection_schema, autoload=True)
if options.schema is None:
# we're going to remove the schema from the table so that it
# isn't rendered in the output. If we don't put back the
# correct value, it may cause errors when other tables reference
# this one.
original_schema = table.schema
table.schema = None
else:
original_schema = options.schema
indent = ''
INC = '\n\n'
emit(INC)
if options.z3c:
emit_z3c_objects(constants.TAB, db, options, table)
else:
emit_table('', db, options, table)
table.schema = original_schema
if options.z3c:
emit(constants.FOOTER_Z3C)
# print some example
if options.example:
emit('\n' + constants.FOOTER_EXAMPLE % {
'url': db.url, 'tablename': tablenames[0]})
if options.output:
emit('\n')
config.out.close()
config.out = sys.stdout
print("Output written to %s" % options.output, file=config.err)
def emit_z3c_objects(indent, db, options, table):
emit_table(indent, db, options, table)
emit(indent + ('class %(tn)sObject(MappedClassBase): pass\n'
'%(tab)smapper(%(tn)sObject, %(tn)s)') % {'tn':table.name, 'tab':indent})
def main():
config.configure()
options = config.options
if options.declarative:
config.interactive = None
if options.interactive:
config.interactive = True
config.schema = None
if options.schema:
config.schema = options.schema
config.example = False
if options.example:
config.example = True
factory = ModelFactory(config)
emit(repr(factory))
config.out.close()
config.out = sys.stdout
print >> config.err, "Output written to %s" % options.output
return
import formatter
formatter.monkey_patch_sa()
import sqlalchemy
from sqlalchemy.engine.reflection import Inspector
db, options = config.engine, config.options
metadata = sqlalchemy.MetaData(db)
print >> config.err, 'Starting...'
conn = db.connect()
inspector = Inspector.from_engine(conn)
if options.schema != None:
reflection_schema = options.schema
else:
try:
reflection_schema = inspector.default_schema_name
except NotImplementedError:
reflection_schema = None
tablenames = inspector.get_table_names(reflection_schema)
# fixme: don't set up output until we're sure there's work to do!
if options.tables:
subset, missing, unglobbed = util.glob_intersection(
tablenames, options.tables)
for identifier in missing:
print >> config.err, 'Table "%s" not found.' % identifier
for glob in unglobbed:
print >> config.err, '"%s" matched no tables.' % glob
if not subset:
print >> config.err, "No tables matched!"
sys.exit(1)
tablenames = subset
# some header with imports
if options.generictypes:
dialect = ''
else:
dialect = 'from sqlalchemy.databases.%s import *\n' % db.name
header = options.z3c and constants.HEADER_Z3C or constants.HEADER
emit(header % {'dialect': dialect, 'encoding': options.encoding})
for tname in tablenames:
print >> config.err, "Generating python model for table %s" % (
util.as_sys_str(tname))
table = sqlalchemy.Table(tname,
metadata,
schema=reflection_schema,
autoload=True)
if options.schema is None:
# we're going to remove the schema from the table so that it
# isn't rendered in the output. If we don't put back the
# correct value, it may cause errors when other tables reference
# this one.
original_schema = table.schema
table.schema = None
else:
original_schema = options.schema
INC = '\n\n'
if options.z3c:
INC = INC + 4 * ' '
emit('%s%s%s%s = %r' %
(INC, options.table_prefix, tname, options.table_suffix, table))
if options.z3c:
emit(INC + ('class %(tn)sObject(MappedClassBase): pass\n'
'mapper(%(tn)sObject, %(tn)s)') % {'tn': tname})
table.schema = original_schema
# directly print indices after table def
if not options.noindex:
indexes = []
if not table.indexes:
# for certain dialects we need to include index support
if hasattr(db.dialect, 'indexloader'):
indexes = db.dialect.indexloader(db).indexes(table)
else:
print >> config.err, 'It seems that this dialect does not support indexes!'
else:
indexes = list(table.indexes)
util.emit(*[repr(index) for index in indexes])
if options.z3c:
emit(constants.FOOTER_Z3C)
# print some example
if options.example:
emit('\n' + constants.FOOTER_EXAMPLE % {
'url': unicode(db.url),
'tablename': tablenames[0]
})
if options.output:
emit('\n')
config.out.close()
config.out = sys.stdout
print >> config.err, "Output written to %s" % options.output
def emit_table(indent, db, options, table):
"""Emit table representation."""
emit('%s%s%s%s = %r' % (indent, options.table_prefix, table.name,
options.table_suffix, table))
emit_index(indent, db, options, table)
def main():
config.configure()
options = config.options
if options.declarative:
config.interactive = None
if options.interactive:
config.interactive = True
config.schema = None
if options.schema:
config.schema = options.schema
config.example = False
if options.example:
config.example = True
factory = ModelFactory(config)
emit(repr(factory))
config.out.close()
config.out = sys.stdout
print >> config.err, "Output written to %s" % options.output
return
import formatter
formatter.monkey_patch_sa()
import sqlalchemy
from sqlalchemy.engine.reflection import Inspector
db, options = config.engine, config.options
metadata = sqlalchemy.MetaData(db)
print >> config.err, 'Starting...'
conn = db.connect()
inspector = Inspector.from_engine(conn)
if options.schema != None:
reflection_schema = options.schema
else:
try:
reflection_schema = inspector.default_schema_name
except NotImplementedError:
reflection_schema = None
tablenames = inspector.get_table_names(reflection_schema)
# fixme: don't set up output until we're sure there's work to do!
if options.tables:
subset, missing, unglobbed = util.glob_intersection(
tablenames, options.tables)
for identifier in missing:
print >> config.err, 'Table "%s" not found.' % identifier
for glob in unglobbed:
print >> config.err, '"%s" matched no tables.' % glob
if not subset:
print >> config.err, "No tables matched!"
sys.exit(1)
tablenames = subset
# some header with imports
if options.generictypes:
dialect = ''
else:
d1 = 'from sqlalchemy.databases.%s import *\n' % db.name
d2 = 'from sqlalchemy.dialects.%s import *\n' % db.name
#Determine with one is correct...
dialect = util.select_imports([d1, d2])
header = options.z3c and constants.HEADER_Z3C or constants.HEADER
emit(header % {'dialect': dialect, 'encoding': options.encoding})
for tname in tablenames:
print >> config.err, "Generating python model for table %s" % (
util.as_sys_str(tname))
table = sqlalchemy.Table(tname,
metadata,
schema=reflection_schema,
autoload=True)
if options.schema is None:
# we're going to remove the schema from the table so that it
# isn't rendered in the output. If we don't put back the
# correct value, it may cause errors when other tables reference
# this one.
original_schema = table.schema
table.schema = None
else:
original_schema = options.schema
indent = ''
INC = '\n\n'
emit(INC)
if options.z3c:
emit_z3c_objects(constants.TAB, db, options, table)
else:
emit_table('', db, options, table)
table.schema = original_schema
if options.z3c:
emit(constants.FOOTER_Z3C)
# print some example
if options.example:
emit('\n' + constants.FOOTER_EXAMPLE % {
'url': unicode(db.url),
'tablename': tablenames[0]
})
if options.output:
emit('\n')
config.out.close()
config.out = sys.stdout
print >> config.err, "Output written to %s" % options.output
def prep():
dir_ = EXPERIMENTS_DIR
fs = glob.glob(emit(path.join(dir_, "**", "ALL.json"), 'path'),
recursive=True)
init_exp_config = None
init_ex = None
X_Y = []
for all_fn in fs:
print('loading experiment', all_fn)
experiments = json.loads(open(all_fn, 'r').read())
if not init_exp_config:
init_exp_config = experiments["exp_config"]
init_ex = experiments["examples"][0]
else:
if experiments["exp_config"] != init_exp_config:
print('BAD EXP CONFIG')
all_path = os.path.dirname(all_fn)
examples = experiments["examples"]
fnames = pluck(examples, "fnames")
joints = pluck(pluck(examples, "joints"), JOINTS)
[[X_Y.append([os.path.join(all_path, xx), j]) for xx in x]
for x, j in zip(fnames, joints)]
xy2 = numpy.array(X_Y)
_X = list(xy2.take(0, -1))
_Y = list(xy2.take(-1, 1))
n_joints = len(_Y[0])
emit(n_joints, 'n_joints')
ranges = []
for i in range(n_joints):
j_vals = [yy[i] for yy in _Y]
mx = int(numpy.max(j_vals))
mn = int(numpy.min(j_vals))
emit((mn, mx), ('minmax', i))
ranges.append((mn, mx))
open(JOINT_MAX_FNAME, 'w').write(json.dumps(ranges))
emit(JOINT_MAX_FNAME, 'saved joint maxes / scaling values')
_Y = minmax_scale(_Y)
X = []
Y = []
for x, y in zip(_X, _Y):
img = cv2.imread(x)
if img is not None:
x = cv2.resize(img, (W, H)) / 255.0
X.append(x)
Y.append(y)
emit(len(X), 'len X')
emit(len(Y), 'len Y')
assert len(X) == len(Y)
Xn = numpy.array(X)
Yn = numpy.array(Y)
Xs = Xn.shape
Ys = Yn.shape
emit(Xs, 'shape X')
emit(Ys, 'shape X')
assert Xs[0] == len(X)
assert Xs[1] == H
assert Xs[2] == W
assert Xs[3] == CHAN
assert Ys[0] == len(Y)
assert Ys[1] == n_joints
return [maxes, n_joints, Xn, Yn, Xs, Ys]
def prep():
d = json.loads(open(DATASET_FILE).read())
X = []
Y = []
n_joints = len(d['y'][0])
emit(n_joints, 'n_joints')
maxes = []
for i in range(n_joints):
j_vals = [yy[i] for yy in d['y']]
m = numpy.max(j_vals)
emit(m, ('max', i))
maxes.append(m)
open(JOINT_MAX_FNAME, 'w').write(json.dumps(maxes))
emit(JOINT_MAX_FNAME, 'saved joint maxes / scaling values')
for x, y in zip(d['x'], d['y']):
img = cv2.imread(x)
if img is not None:
x = cv2.resize(img, (W, H)) / 255.0
for i, yy in enumerate(y):
y[i] = y[i] / maxes[i]
X.append(x)
Y.append(y)
emit(len(X), 'len X')
emit(len(Y), 'len Y')
assert len(X) == len(Y)
Xn = numpy.array(X)
Yn = numpy.array(Y)
Xs = Xn.shape
Ys = Yn.shape
emit(Xs, 'shape X')
emit(Ys, 'shape X')
assert Xs[0] == len(X)
assert Xs[1] == H
assert Xs[2] == W
assert Xs[3] == CHAN
assert Ys[0] == len(Y)
assert Ys[1] == n_joints
return [maxes, n_joints, Xn, Yn, Xs, Ys]
