コード例 #1
0
def find_examples(dir_):
    f = glob.glob(emit(path.join(dir_, "**", "*.json"), 'path'),
                  recursive=True)
    emit(len(f), 'number of examples')
    r = [x for x in [load_example(x) for x in f] if not skip_joint_filters(x)]
    random.shuffle(r)
    return r
コード例 #2
0
def emit_z3c_objects(indent, db, options, table):
    emit_table(indent, db, options, table)
    emit(indent + ('class %(tn)sObject(MappedClassBase): pass\n'
                   '%(tab)smapper(%(tn)sObject, %(tn)s)') % {
                       'tn': table.name,
                       'tab': indent
                   })
コード例 #3
0
def main():
    os.makedirs(FRAMES_DIR, exist_ok=True)
    clear_old_frames()
    v = find_examples(VID_DIR)
    X, Y = [[], []]
    r = []
    for x in v:
        emit(x, 'vx')
        joint_dict = x['joints']
        from_ = list(joint_dict['from'].values())
        to_ = list(joint_dict['to'].values())

        for vid in x['vids']:
            emit(vid, 'extracting')

            first, last = save_frames(vid)

            if not first or not last:
                emit(vid, 'WARNING: no first or last frame')
                continue

            X.append(first)
            Y.append(from_)
            X.append(last)
            Y.append(to_)

    emit(len(X), 'X items')
    emit(len(Y), 'Y items')
    open(DATASET_FILE, 'w').write(json.dumps({'x': X, 'y': Y}))
コード例 #4
0
def skip_joint_filters(example):
    if list(example['joints']['from'].keys()) != FILTER_JOINTS_ONLY:
        emit(example['joints']['from'].keys(), 'skipping')
        # emit(example, 'skipping')
        return True
    else:
        return False
コード例 #5
0
def emit_index(indent, db, options, table):
    """docstring for emit_index"""
    if not options.noindex:
        indexes = []
        if not table.indexes:
            # for certain dialects we need to include index support
            if hasattr(db.dialect, 'indexloader'):
                indexes = db.dialect.indexloader(db).indexes(table)
            else:
                print >> config.err, 'It seems that this dialect does not support indexes!'
        else:
            indexes = list(table.indexes)
        emit(*[indent + repr(index) for index in indexes])
コード例 #6
0
ファイル: main.py プロジェクト: hknust/sqlautocode
def emit_index(indent, db, options, table):
    """docstring for emit_index"""
    if not options.noindex:
        indexes = []
        if not table.indexes:
            # for certain dialects we need to include index support
            if hasattr(db.dialect, 'indexloader'):
                indexes = db.dialect.indexloader(db).indexes(table)
            else:
                print >>config.err, 'It seems that this dialect does not support indexes!'
        else:
            indexes = list(table.indexes)
        emit(*[indent + repr(index) for index in indexes])
コード例 #7
0
def extract_frames(vid):
    p = emit(vid.split("/"), 'p')
    fn = '_'.join(p[:-1]) + p[-1].replace('.h264', '-%d.png')
    out = path.join(FRAMES_DIR, fn)
    cmd = f'ffmpeg -i {vid} -s {W}x{H} {out}'
    emit(cmd, 'save thumbs')
    system(cmd)
    r = glob.glob(out.replace('-%d.png', '-*.png'))
    r.sort(key=lambda x: int(x.split("-")[-1].replace('.png', '')))
    emit(r, 'frames')
    if len(r) < MIN_FRAMES:
        return None
    keepers = {x: r[x] for x in OFS}
    for x in r:
        if not x in keepers.values():
            unlink(x)
    return keepers
コード例 #8
0
def save_frames(vid):
    p = emit(vid.split("/"), 'p')
    fn = '_'.join(p[:-1]) + p[-1].replace('.h264', '-%d.png')
    out = path.join(FRAMES_DIR, fn)
    cmd = f'ffmpeg -i {vid} -s {W}x{H} {out}'
    emit(cmd, 'save thumbs')
    os.system(cmd)
    r = glob.glob(out.replace('-%d.png', '-*.png'))
    r.sort(key=lambda x: int(x.split("-")[-1].replace('.png', '')))
    emit(r, 'frames')
    if len(r) < 10:
        return [None, None]

    assert len(r) > 10
    first = r[0]
    last = r[-1]
    for x in r:
        if x != first and x != last:
            os.unlink(x)
            pass

    return [first, last]
コード例 #9
0
def main():
    start_time = timestamp()
    last_loss = '(loss)'
    model_fname = MODEL_OUTPUT_FNAME.format_map({**globals(), **locals()})
    emit(model_fname, 'output file')

    [y_size, Xn, Yn, Xs, Ys] = prep()

    X_tr, X_te, Y_tr, Y_te = train_test_split(Xn, Yn, test_size=0.15)

    emit((X_tr.shape, X_te.shape), 'tr/te')

    net = model(y_size)

    o = net.fit(X_tr,
                Y_tr,
                epochs=EPOCHS,
                validation_data=(X_te, Y_te),
                batch_size=BATCH)
    emit(o, 'model.fit')

    last_loss = o.history['loss'][-1]
    model_fname = MODEL_OUTPUT_FNAME.format_map({**globals(), **locals()})
    o.model.save(model_fname)
コード例 #10
0
ファイル: main.py プロジェクト: DarioGT/SqlAutoCode-
def main():
    config.configure()

    options = config.options
    if options.declarative:
        config.interactive = None
        if options.interactive:
            config.interactive = True
        config.schema = None
        if options.schema:
            config.schema = options.schema
        config.example=False
        if options.example:
            config.example=True
        factory = ModelFactory(config)
        emit(repr(factory))

        config.out.close()
        config.out = sys.stdout
        print >>config.err, "Output written to %s" % options.output
        return

    import formatter
    formatter.monkey_patch_sa()
    
    import sqlalchemy
    db, options = config.engine, config.options
    metadata = sqlalchemy.MetaData(db)

    print >>config.err, 'Starting...'
    conn = db.connect()

    if options.schema != None:
        reflection_schema=options.schema
    else:
        try:
            reflection_schema = db.dialect.get_default_schema_name(conn)
        except NotImplementedError:
            reflection_schema = None

    tablenames = db.dialect.table_names(conn, reflection_schema)

    # fixme: don't set up output until we're sure there's work to do!
    if options.tables:
        subset, missing, unglobbed = util.glob_intersection(tablenames, options.tables)
        for identifier in missing:
            print >>config.err, 'Table "%s" not found.' % identifier
        for glob in unglobbed:
            print >>config.err, '"%s" matched no tables.' % glob
        if not subset:
            print >>config.err, "No tables matched!"
            sys.exit(1)

        tablenames = subset

    if options.exclude:
        tablenames = [table for table in tablenames if not re.search(table, options.exclude)]

    # some header with imports
    if options.generictypes:
        dialect = ''
    else:
        dialect = 'from sqlalchemy.databases.%s import *\n' % db.name

    
    header = options.z3c and constants.HEADER_Z3C or constants.HEADER
    emit(header % {'dialect': dialect, 'encoding': options.encoding})

    for tname in tablenames:
        print >>config.err, "Generating python model for table %s" % (
            util.as_sys_str(tname))

        table = sqlalchemy.Table(tname, metadata, schema=reflection_schema,
                                 autoload=True)
        if options.schema is None:
            # we're going to remove the schema from the table so that it
            #  isn't rendered in the output.  If we don't put back the
            #  correct value, it may cause errors when other tables reference
            #  this one.
            original_schema = table.schema
            table.schema = None

        else: original_schema = options.schema

        INC = '\n\n'
        if options.z3c:
            INC = INC + 4*' '

        emit('%s%s%s%s = %r' % (INC, options.table_prefix, tname, options.table_suffix, table))

        if options.z3c:
            emit(INC + ('class %(tn)sObject(MappedClassBase): pass\n'
                             'mapper(%(tn)sObject, %(tn)s)') % {'tn':tname})

        table.schema = original_schema
        
        # directly print indices after table def
        if not options.noindex:
            indexes = []
            if not table.indexes:
                # for certain dialects we need to include index support
                if hasattr(db.dialect, 'indexloader'):
                    indexes = db.dialect.indexloader(db).indexes(table)
                else:
                    print >>config.err, 'It seems that this dialect does not support indexes!'
            else:
                indexes = list(table.indexes)

            util.emit(*[repr(index) for index in indexes])

    if options.z3c:
        emit(constants.FOOTER_Z3C)

    # print some example
    if options.example:
        emit('\n' + constants.FOOTER_EXAMPLE % {
            'url': unicode(db.url), 'tablename': tablenames[0]})

    if options.output:
        emit('\n')
        config.out.close()
        config.out = sys.stdout
        print >>config.err, "Output written to %s" % options.output
コード例 #11
0
ファイル: main.py プロジェクト: hknust/sqlautocode
def emit_table(indent, db, options, table):
    """Emit table representation."""
    emit('%s%s%s%s = %r' % (indent, options.table_prefix, table.name, options.table_suffix, table))
    emit_index(indent, db, options, table)
コード例 #12
0
ファイル: main.py プロジェクト: hknust/sqlautocode
def main():
    config.configure()

    options = config.options
    if options.declarative:
        config.interactive = None
        if options.interactive:
            config.interactive = True
        config.schema = None
        if options.schema:
            config.schema = options.schema
        config.example=False
        if options.example:
            config.example=True
        factory = ModelFactory(config)
        emit(repr(factory))
        config.out.close()
        config.out = sys.stdout
        print >>config.err, "Output written to %s" % options.output
        return

    import formatter
    formatter.monkey_patch_sa()
    
    import sqlalchemy
    from sqlalchemy.engine.reflection import Inspector
    db, options = config.engine, config.options
    metadata = sqlalchemy.MetaData(db)

    print >>config.err, 'Starting...'
    conn = db.connect()
    inspector = Inspector.from_engine(conn)

    if options.schema != None:
        reflection_schema=options.schema
    else:
        try:
            reflection_schema = inspector.default_schema_name
        except NotImplementedError:
            reflection_schema = None

    tablenames = inspector.get_table_names(reflection_schema)

    # fixme: don't set up output until we're sure there's work to do!
    if options.tables:
        subset, missing, unglobbed = util.glob_intersection(tablenames,
                                                            options.tables)
        for identifier in missing:
            print >>config.err, 'Table "%s" not found.' % identifier
        for glob in unglobbed:
            print >>config.err, '"%s" matched no tables.' % glob
        if not subset:
            print >>config.err, "No tables matched!"
            sys.exit(1)

        tablenames = subset

    # some header with imports
    if options.generictypes:
        dialect = ''
    else:
        d1 = 'from sqlalchemy.databases.%s import *\n' % db.name
        d2 = 'from sqlalchemy.dialects.%s import *\n' % db.name
        #Determine with one is correct...
        dialect = util.select_imports([d1, d2])
    
    header = options.z3c and constants.HEADER_Z3C or constants.HEADER
    emit(header % {'dialect': dialect, 'encoding': options.encoding})

    for tname in tablenames:
        print >>config.err, "Generating python model for table %s" % (
            util.as_sys_str(tname))

        table = sqlalchemy.Table(tname, metadata, schema=reflection_schema, autoload=True)
        if options.schema is None:
            # we're going to remove the schema from the table so that it
            #  isn't rendered in the output.  If we don't put back the
            #  correct value, it may cause errors when other tables reference
            #  this one.
            original_schema = table.schema
            table.schema = None
        else:
            original_schema = options.schema

        indent = ''

        INC = '\n\n'
        emit(INC)
        if options.z3c:
            emit_z3c_objects(constants.TAB, db, options, table)
        else:
            emit_table('', db, options, table)

        table.schema = original_schema

    if options.z3c:
        emit(constants.FOOTER_Z3C)

    # print some example
    if options.example:
        emit('\n' + constants.FOOTER_EXAMPLE % {
            'url': db.url, 'tablename': tablenames[0]})

    if options.output:
        emit('\n')
        config.out.close()
        config.out = sys.stdout
        print("Output written to %s" % options.output, file=config.err)
コード例 #13
0
ファイル: main.py プロジェクト: hknust/sqlautocode
def emit_z3c_objects(indent, db, options, table):
    emit_table(indent, db, options, table)
    emit(indent + ('class %(tn)sObject(MappedClassBase): pass\n'
                    '%(tab)smapper(%(tn)sObject, %(tn)s)') % {'tn':table.name, 'tab':indent})
コード例 #14
0
def main():
    config.configure()

    options = config.options
    if options.declarative:
        config.interactive = None
        if options.interactive:
            config.interactive = True
        config.schema = None
        if options.schema:
            config.schema = options.schema
        config.example = False
        if options.example:
            config.example = True
        factory = ModelFactory(config)
        emit(repr(factory))
        config.out.close()
        config.out = sys.stdout
        print >> config.err, "Output written to %s" % options.output
        return

    import formatter
    formatter.monkey_patch_sa()

    import sqlalchemy
    from sqlalchemy.engine.reflection import Inspector
    db, options = config.engine, config.options
    metadata = sqlalchemy.MetaData(db)

    print >> config.err, 'Starting...'
    conn = db.connect()
    inspector = Inspector.from_engine(conn)

    if options.schema != None:
        reflection_schema = options.schema
    else:
        try:
            reflection_schema = inspector.default_schema_name
        except NotImplementedError:
            reflection_schema = None

    tablenames = inspector.get_table_names(reflection_schema)

    # fixme: don't set up output until we're sure there's work to do!
    if options.tables:
        subset, missing, unglobbed = util.glob_intersection(
            tablenames, options.tables)
        for identifier in missing:
            print >> config.err, 'Table "%s" not found.' % identifier
        for glob in unglobbed:
            print >> config.err, '"%s" matched no tables.' % glob
        if not subset:
            print >> config.err, "No tables matched!"
            sys.exit(1)

        tablenames = subset

    # some header with imports
    if options.generictypes:
        dialect = ''
    else:
        dialect = 'from sqlalchemy.databases.%s import *\n' % db.name

    header = options.z3c and constants.HEADER_Z3C or constants.HEADER
    emit(header % {'dialect': dialect, 'encoding': options.encoding})

    for tname in tablenames:
        print >> config.err, "Generating python model for table %s" % (
            util.as_sys_str(tname))

        table = sqlalchemy.Table(tname,
                                 metadata,
                                 schema=reflection_schema,
                                 autoload=True)
        if options.schema is None:
            # we're going to remove the schema from the table so that it
            #  isn't rendered in the output.  If we don't put back the
            #  correct value, it may cause errors when other tables reference
            #  this one.
            original_schema = table.schema
            table.schema = None
        else:
            original_schema = options.schema

        INC = '\n\n'
        if options.z3c:
            INC = INC + 4 * ' '

        emit('%s%s%s%s = %r' %
             (INC, options.table_prefix, tname, options.table_suffix, table))

        if options.z3c:
            emit(INC + ('class %(tn)sObject(MappedClassBase): pass\n'
                        'mapper(%(tn)sObject, %(tn)s)') % {'tn': tname})

        table.schema = original_schema

        # directly print indices after table def
        if not options.noindex:
            indexes = []
            if not table.indexes:
                # for certain dialects we need to include index support
                if hasattr(db.dialect, 'indexloader'):
                    indexes = db.dialect.indexloader(db).indexes(table)
                else:
                    print >> config.err, 'It seems that this dialect does not support indexes!'
            else:
                indexes = list(table.indexes)

            util.emit(*[repr(index) for index in indexes])

    if options.z3c:
        emit(constants.FOOTER_Z3C)

    # print some example
    if options.example:
        emit('\n' + constants.FOOTER_EXAMPLE % {
            'url': unicode(db.url),
            'tablename': tablenames[0]
        })

    if options.output:
        emit('\n')
        config.out.close()
        config.out = sys.stdout
        print >> config.err, "Output written to %s" % options.output
コード例 #15
0
def emit_table(indent, db, options, table):
    """Emit table representation."""
    emit('%s%s%s%s = %r' % (indent, options.table_prefix, table.name,
                            options.table_suffix, table))
    emit_index(indent, db, options, table)
コード例 #16
0
def main():
    config.configure()

    options = config.options
    if options.declarative:
        config.interactive = None
        if options.interactive:
            config.interactive = True
        config.schema = None
        if options.schema:
            config.schema = options.schema
        config.example = False
        if options.example:
            config.example = True
        factory = ModelFactory(config)
        emit(repr(factory))
        config.out.close()
        config.out = sys.stdout
        print >> config.err, "Output written to %s" % options.output
        return

    import formatter
    formatter.monkey_patch_sa()

    import sqlalchemy
    from sqlalchemy.engine.reflection import Inspector
    db, options = config.engine, config.options
    metadata = sqlalchemy.MetaData(db)

    print >> config.err, 'Starting...'
    conn = db.connect()
    inspector = Inspector.from_engine(conn)

    if options.schema != None:
        reflection_schema = options.schema
    else:
        try:
            reflection_schema = inspector.default_schema_name
        except NotImplementedError:
            reflection_schema = None

    tablenames = inspector.get_table_names(reflection_schema)

    # fixme: don't set up output until we're sure there's work to do!
    if options.tables:
        subset, missing, unglobbed = util.glob_intersection(
            tablenames, options.tables)
        for identifier in missing:
            print >> config.err, 'Table "%s" not found.' % identifier
        for glob in unglobbed:
            print >> config.err, '"%s" matched no tables.' % glob
        if not subset:
            print >> config.err, "No tables matched!"
            sys.exit(1)

        tablenames = subset

    # some header with imports
    if options.generictypes:
        dialect = ''
    else:
        d1 = 'from sqlalchemy.databases.%s import *\n' % db.name
        d2 = 'from sqlalchemy.dialects.%s import *\n' % db.name
        #Determine with one is correct...
        dialect = util.select_imports([d1, d2])

    header = options.z3c and constants.HEADER_Z3C or constants.HEADER
    emit(header % {'dialect': dialect, 'encoding': options.encoding})

    for tname in tablenames:
        print >> config.err, "Generating python model for table %s" % (
            util.as_sys_str(tname))

        table = sqlalchemy.Table(tname,
                                 metadata,
                                 schema=reflection_schema,
                                 autoload=True)
        if options.schema is None:
            # we're going to remove the schema from the table so that it
            #  isn't rendered in the output.  If we don't put back the
            #  correct value, it may cause errors when other tables reference
            #  this one.
            original_schema = table.schema
            table.schema = None
        else:
            original_schema = options.schema

        indent = ''

        INC = '\n\n'
        emit(INC)
        if options.z3c:
            emit_z3c_objects(constants.TAB, db, options, table)
        else:
            emit_table('', db, options, table)

        table.schema = original_schema

    if options.z3c:
        emit(constants.FOOTER_Z3C)

    # print some example
    if options.example:
        emit('\n' + constants.FOOTER_EXAMPLE % {
            'url': unicode(db.url),
            'tablename': tablenames[0]
        })

    if options.output:
        emit('\n')
        config.out.close()
        config.out = sys.stdout
        print >> config.err, "Output written to %s" % options.output
コード例 #17
0
def prep():

    dir_ = EXPERIMENTS_DIR

    fs = glob.glob(emit(path.join(dir_, "**", "ALL.json"), 'path'),
                   recursive=True)

    init_exp_config = None
    init_ex = None

    X_Y = []

    for all_fn in fs:
        print('loading experiment', all_fn)
        experiments = json.loads(open(all_fn, 'r').read())
        if not init_exp_config:
            init_exp_config = experiments["exp_config"]
            init_ex = experiments["examples"][0]
        else:
            if experiments["exp_config"] != init_exp_config:
                print('BAD EXP CONFIG')

        all_path = os.path.dirname(all_fn)

        examples = experiments["examples"]
        fnames = pluck(examples, "fnames")
        joints = pluck(pluck(examples, "joints"), JOINTS)

        [[X_Y.append([os.path.join(all_path, xx), j]) for xx in x]
         for x, j in zip(fnames, joints)]

    xy2 = numpy.array(X_Y)
    _X = list(xy2.take(0, -1))
    _Y = list(xy2.take(-1, 1))

    n_joints = len(_Y[0])
    emit(n_joints, 'n_joints')

    ranges = []
    for i in range(n_joints):
        j_vals = [yy[i] for yy in _Y]
        mx = int(numpy.max(j_vals))
        mn = int(numpy.min(j_vals))
        emit((mn, mx), ('minmax', i))
        ranges.append((mn, mx))

    open(JOINT_MAX_FNAME, 'w').write(json.dumps(ranges))
    emit(JOINT_MAX_FNAME, 'saved joint maxes / scaling values')

    _Y = minmax_scale(_Y)

    X = []
    Y = []

    for x, y in zip(_X, _Y):
        img = cv2.imread(x)
        if img is not None:
            x = cv2.resize(img, (W, H)) / 255.0
            X.append(x)
            Y.append(y)

    emit(len(X), 'len X')
    emit(len(Y), 'len Y')

    assert len(X) == len(Y)

    Xn = numpy.array(X)
    Yn = numpy.array(Y)

    Xs = Xn.shape
    Ys = Yn.shape
    emit(Xs, 'shape X')
    emit(Ys, 'shape X')

    assert Xs[0] == len(X)
    assert Xs[1] == H
    assert Xs[2] == W
    assert Xs[3] == CHAN
    assert Ys[0] == len(Y)
    assert Ys[1] == n_joints

    return [maxes, n_joints, Xn, Yn, Xs, Ys]
コード例 #18
0
def prep():
    d = json.loads(open(DATASET_FILE).read())
    X = []
    Y = []

    n_joints = len(d['y'][0])
    emit(n_joints, 'n_joints')

    maxes = []
    for i in range(n_joints):
        j_vals = [yy[i] for yy in d['y']]
        m = numpy.max(j_vals)
        emit(m, ('max', i))
        maxes.append(m)

    open(JOINT_MAX_FNAME, 'w').write(json.dumps(maxes))
    emit(JOINT_MAX_FNAME, 'saved joint maxes / scaling values')

    for x, y in zip(d['x'], d['y']):
        img = cv2.imread(x)
        if img is not None:
            x = cv2.resize(img, (W, H)) / 255.0
            for i, yy in enumerate(y):
                y[i] = y[i] / maxes[i]
            X.append(x)
            Y.append(y)

    emit(len(X), 'len X')
    emit(len(Y), 'len Y')

    assert len(X) == len(Y)

    Xn = numpy.array(X)
    Yn = numpy.array(Y)

    Xs = Xn.shape
    Ys = Yn.shape
    emit(Xs, 'shape X')
    emit(Ys, 'shape X')

    assert Xs[0] == len(X)
    assert Xs[1] == H
    assert Xs[2] == W
    assert Xs[3] == CHAN
    assert Ys[0] == len(Y)
    assert Ys[1] == n_joints

    return [maxes, n_joints, Xn, Yn, Xs, Ys]