Exemple #1
0
def admin():
    user = session['user']

    if user['id'] not in config['admins']:
        return render_template('gitout.html')
    sort = request.args.get('sort', 'age')
    if sort == 'age_asc':
        keys = r.table('keys').order_by(r.asc('creation_time')).run(get_db())
    elif sort == 'age_desc':
        keys = r.table('keys').order_by(r.desc('creation_time')).run(get_db())
    elif sort == 'usage_asc':
        keys = r.table('keys').order_by(r.asc('total_usage')).run(get_db())
    elif sort == 'usage_desc':
        keys = r.table('keys').order_by(r.desc('total_usage')).run(get_db())
    elif sort == 'accept_asc':
        keys = r.table('keys').order_by(r.asc('acceptance_time')).run(get_db())
    elif sort == 'accept_desc':
        keys = r.table('keys').order_by(r.desc('acceptance_time')).run(
            get_db())
    else:
        keys = r.table('keys').order_by(r.asc('creation_time')).run(get_db())
    apps = r.table('applications').order_by('time').run(get_db())
    return render_template('admin.html',
                           name=user['username'],
                           apps=apps,
                           keys=keys,
                           sort=sort)
Exemple #2
0
    def _get_jobs(self, conditions=None):
        jobs = []
        failed_job_ids = []
        if conditions:
            documents = list(
                self.table.filter(lambda x: x['next_run_time'] != None).filter(
                    conditions).order_by(r.asc('next_run_time'), 'id').pluck(
                        'id', 'job_state').run(self.conn))
        else:
            documents = list(
                self.table.order_by(r.asc('next_run_time'),
                                    'id').pluck('id',
                                                'job_state').run(self.conn))
        for document in documents:
            try:
                jobs.append(self._reconstitute_job(document['job_state']))
            except:
                self._logger.exception(
                    'Unable to restore job "%s" -- removing it',
                    document['id'])
                failed_job_ids.append(document['id'])

        # Remove all the jobs we failed to restore
        if failed_job_ids:
            r.expr(failed_job_ids).for_each(
                lambda job_id: self.table.get_all(job_id).delete()).run(
                    self.conn)

        return jobs
Exemple #3
0
 def test_sort_multi_1_asc(self, conn):
     expected = [
         {'id': 'glen', 'age': 26, 'score': 15},
         {'id': 'joe', 'age': 26, 'score': 60},
         {'id': 'bill', 'age': 35, 'score': 78},
         {'id': 'todd', 'age': 52, 'score': 15},
         {'id': 'pale', 'age': 52, 'score': 30}
     ]
     result = r.db('y').table('scores').order_by(r.asc('age'), r.asc('score')).run(conn)
     assertEqual(expected, list(result))
Exemple #4
0
    def bootstrap(self):
        """
        Get transactions from the backlog that may have been assigned to this while it was
        online (not listening to the changefeed)
        """
        # create bigchain instance
        b = Bigchain()

        # create a queue to store initial results
        q_initial = mp.Queue()

        # get initial results
        initial_results = r.table('backlog')\
            .between([b.me, r.minval], [b.me, r.maxval], index='assignee__transaction_timestamp')\
            .order_by(index=r.asc('assignee__transaction_timestamp'))\
            .run(b.conn)

        # add results to the queue
        for result in initial_results:
            q_initial.put(result)

        for i in range(mp.cpu_count()):
            q_initial.put('stop')

        return q_initial
Exemple #5
0
def listener():
    """ Listener for Sawtooth State changes
    """
    try:
        conn = connect_to_db()

        LOGGER.info("Reading queued Sawtooth transactions")
        while True:
            feed = r.table("inbound_queue").order_by(index=r.asc("timestamp")).run(conn)
            count = 0
            for rec in feed:
                process(rec, conn)
                count = count + 1
            if count == 0:
                break
            LOGGER.info("Processed %s records in the inbound queue", count)
        LOGGER.info("Listening for incoming Sawtooth transactions")
        feed = r.table("inbound_queue").changes().run(conn)
        for rec in feed:
            if rec["new_val"] and not rec["old_val"]:  # only insertions
                process(rec["new_val"], conn)

    except Exception as err:  # pylint: disable=broad-except
        LOGGER.exception("Inbound listener %s exception", type(err).__name__)
        LOGGER.exception(err)

    finally:
        try:
            conn.close()
        except UnboundLocalError:
            pass
Exemple #6
0
    def test_valid_block_voting(self, b):
        q_new_block = mp.Queue()

        genesis = b.create_genesis_block()

        # create valid block
        block = b.create_block([])
        # assert block is valid
        assert b.is_valid_block(block)
        b.write_block(block, durability='hard')

        # create queue and voter
        voter = Voter(q_new_block)

        # vote
        voter.start()
        # wait for vote to be written
        time.sleep(1)
        voter.kill()

        # retrive block from bigchain
        blocks = list(r.table('bigchain')
                       .order_by(r.asc((r.row['block']['timestamp'])))
                       .run(b.conn))

        # validate vote
        assert len(blocks[1]['votes']) == 1
        vote = blocks[1]['votes'][0]

        assert vote['vote']['voting_for_block'] == block['id']
        assert vote['vote']['previous_block'] == genesis['id']
        assert vote['vote']['is_block_valid'] is True
        assert vote['vote']['invalid_reason'] is None
        assert vote['node_pubkey'] == b.me
        assert PublicKey(b.me).verify(util.serialize(vote['vote']), vote['signature']) is True
Exemple #7
0
def get_filter(db_name,
               table_name,
               query,
               limit=None,
               order_by=None,
               sort_order='desc'):
    items = []

    if limit:
        if order_by:
            if 'asc' in sort_order:
                query = r.db(db_name).table(table_name).filter(query).order_by(
                    r.asc(order_by)).limit(limit).run(conn)
            else:
                query = r.db(db_name).table(table_name).filter(query).order_by(
                    r.desc(order_by)).limit(limit).run(conn)
        else:
            query = r.db(db_name).table(table_name).filter(query).limit(
                limit).run(conn)
    else:
        query = r.db(db_name).table(table_name).filter(query).run(conn)

    for x in query:
        items.append(x)
    return items
Exemple #8
0
def lookup_book(author, album, narrator):
    result = r.table(args.rethinktable).filter(~r.row.has_fields('_deleted')).filter({
        'mp3_author': author,
        'mp3_album': album,
        'mp3_narrator': narrator
    }).order_by(r.asc('_item')).limit(1).run()

    if result:
        connection = client[args.mongodb_db][args.mongodb_collection]
        mongo_book = connection\
            .find_one({
                '_deleted': {"$exists": False},
                'mp3_author': author,
                'mp3_album': album,
                'mp3_narrator': narrator
            })
        # Make sure we have it in mongodb...
        if not mongo_book:
            mongo_book = result[0].copy()
            mongo_book['_rethinkdb_id'] = mongo_book.pop('id', None)
            print()
            mongo_log(mongo_book['mp3_author'], mongo_book['mp3_album'], mongo_book['mp3_narrator'])
            connection.insert(mongo_book)

        return True, result[0]

    return False, {}
Exemple #9
0
    def bootstrap(self):
        """
        Get transactions from the backlog that may have been assigned to this while it was
        online (not listening to the changefeed)
        """
        # create bigchain instance
        b = Bigchain()

        # create a queue to store initial results
        q_initial = mp.Queue()

        # get initial results
        initial_results = r.table('backlog')\
            .between([b.me, r.minval], [b.me, r.maxval], index='assignee__transaction_timestamp')\
            .order_by(index=r.asc('assignee__transaction_timestamp'))\
            .run(b.conn)

        # add results to the queue
        for result in initial_results:
            q_initial.put(result)

        for i in range(mp.cpu_count()):
            q_initial.put('stop')

        return q_initial
Exemple #10
0
def init():
    """ init leveldb database by conn"""
    logger.info('leveldb init...')
    conn_bigchain = get_conn('bigchain')
    conn_header = get_conn('header')

    logger.info('leveldb init...')
    logger.info('leveldb/header init...')
    logger.info('leveldb/header init host...' +
                str(bigchaindb.config['database']['host']))
    logger.info('leveldb/header init public_key...' +
                str(bigchaindb.config['keypair']['public']))
    logger.info('leveldb/header init private_key...' +
                str(bigchaindb.config['keypair']['private']))

    update(conn_header, 'host', bigchaindb.config['database']['host'])
    update(conn_header, 'public_key', bigchaindb.config['keypair']['public'])
    update(conn_header, 'private_key', bigchaindb.config['keypair']['private'])

    block_num = int(get_withdefault(conn_header, 'block_num', 0))
    genesis_block_id = get_withdefault(conn_header, 'genesis_block_id', '0')
    if block_num == 0:
        genesis_block = r.db('bigchain').table('bigchain').order_by(
            r.asc(r.row['block']['timestamp'])).limit(1).run(
                bigchaindb.Bigchain().conn)[0]
        genesis_block_id = genesis_block['id']
        insert(conn_bigchain, genesis_block_id, genesis_block)
        insert(conn_header, 'genesis_block_id', genesis_block_id)
        insert(conn_header, 'block_num', 1)
        insert(conn_header, 'current_block_id', genesis_block_id)
    logger.info('leveldb/header genesis_block_id...' + str(genesis_block_id))
    logger.info('leveldb init done')
Exemple #11
0
    def test_valid_block_voting(self, b):
        q_new_block = mp.Queue()

        genesis = b.create_genesis_block()

        # create valid block
        block = b.create_block([])
        # assert block is valid
        assert b.is_valid_block(block)
        b.write_block(block, durability='hard')

        # create queue and voter
        voter = Voter(q_new_block)

        # vote
        voter.start()
        # wait for vote to be written
        time.sleep(1)
        voter.kill()

        # retrive block from bigchain
        blocks = list(r.table('bigchain')
                       .order_by(r.asc((r.row['block']['timestamp'])))
                       .run(b.conn))

        # validate vote
        assert len(blocks[1]['votes']) == 1
        vote = blocks[1]['votes'][0]

        assert vote['vote']['voting_for_block'] == block['id']
        assert vote['vote']['previous_block'] == genesis['id']
        assert vote['vote']['is_block_valid'] is True
        assert vote['vote']['invalid_reason'] is None
        assert vote['node_pubkey'] == b.me
        assert crypto.VerifyingKey(b.me).verify(util.serialize(vote['vote']), vote['signature']) is True
Exemple #12
0
    def test_voter_chains_blocks_with_the_previous_ones(self, b):
        b.create_genesis_block()
        block_1 = b.create_block([])
        b.write_block(block_1, durability='hard')
        block_2 = b.create_block([])
        b.write_block(block_2, durability='hard')

        q_new_block = mp.Queue()

        voter = Voter(q_new_block)
        voter.start()
        time.sleep(1)
        voter.kill()

        # retrive blocks from bigchain
        blocks = list(r.table('bigchain')
                       .order_by(r.asc((r.row['block']['timestamp'])))
                       .run(b.conn))

        assert blocks[0]['block_number'] == 0
        assert blocks[1]['block_number'] == 1
        assert blocks[2]['block_number'] == 2

        # we don't vote on the genesis block right now
        # assert blocks[0]['votes'][0]['vote']['voting_for_block'] == genesis['id']
        assert blocks[1]['votes'][0]['vote']['voting_for_block'] == block_1['id']
        assert blocks[2]['votes'][0]['vote']['voting_for_block'] == block_2['id']
Exemple #13
0
    def test_voter_chains_blocks_with_the_previous_ones(self, b):
        b.create_genesis_block()
        block_1 = b.create_block([])
        b.write_block(block_1, durability='hard')
        block_2 = b.create_block([])
        b.write_block(block_2, durability='hard')

        q_new_block = mp.Queue()

        voter = Voter(q_new_block)
        voter.start()
        time.sleep(1)
        voter.kill()


        # retrive blocks from bigchain
        blocks = list(r.table('bigchain')
                       .order_by(r.asc((r.row['block']['timestamp'])))
                       .run(b.conn))

        assert blocks[0]['block_number'] == 0
        assert blocks[1]['block_number'] == 1
        assert blocks[2]['block_number'] == 2

        # we don't vote on the genesis block right now
        # assert blocks[0]['votes'][0]['vote']['voting_for_block'] == genesis['id']
        assert blocks[1]['votes'][0]['vote']['voting_for_block'] == block_1['id']
        assert blocks[2]['votes'][0]['vote']['voting_for_block'] == block_2['id']
Exemple #14
0
def get_posts_facade(limit=10, skip=0, **params):
    """
    Get posts, and return an array where each
    post is the correct kind based on the `kind` field.
    """

    data = Post.table.filter(params).order_by(r.asc("created")).skip(skip).limit(limit).run(database.db_conn)
    return [instance(d) for d in data]
Exemple #15
0
def getordered(table, conn, key, index, order, direction=None):
    '''Fetch all results out of the database by Secondary Key and order them by non-index'''
    if direction is 'desc':
        return r.table(table).get_all(key, index=index).order_by(
            r.desc(order)).coerce_to('array').run(conn)
    else:
        return r.table(table).get_all(key, index=index).order_by(
            r.asc(order)).coerce_to('array').run(conn)
Exemple #16
0
def get_posts_facade(db_conn, limit=10, skip=0, **params):
    """
    Get posts, and return an array where each
    post is the correct kind based on the `kind` field.
    """

    data = (Post.table.filter(params).order_by(
        r.asc('created')).skip(skip).limit(limit).run(db_conn))
    return [instance(d) for d in data]
def start(request):

    conn = r.connect("localhost", 28015, "wlps")

    profile = Profile.objects.get(user=request.user)

    queued = r.table("queue").filter({"user_id": int(request.user.id)}).order_by(r.asc("date_added")).run(conn)
    notifications = (
        r.table("notifications")
        .filter({"user_id": int(request.user.id)})
        .eq_join("episode_id", r.table("episode"))
        .order_by(r.asc("date_added"))
        .run(conn)
    )

    dic = {"queued": queued, "notifications": notifications, "profile": profile}

    return render_to_response("video/1.html", dic, context_instance=RequestContext(request))
Exemple #18
0
def _get_stop_times_by_vehicle(vehicle, stop_start, stop_end, route, direction, time=arrow.now(), window=45, index='route_direction_vehicle_time'):
    lower_key = [route, direction, vehicle, r.epoch_time(time.replace(minutes=-window).timestamp)]
    upper_key = [route, direction, vehicle, r.epoch_time(time.timestamp)]

    query = r.table('vehicle_stop_times') \
                .between(lower_key, upper_key, index=index) \
                .order_by(index=r.asc(index)) \
                .filter((r.row['stop_id'] == stop_start) | (r.row['stop_id'] == stop_end))

    return list(query.run())
Exemple #19
0
 def get_next_run_time(self):
     results = list(
         self.table
         .filter(r.row['next_run_time'] != None)
         .order_by(r.asc('next_run_time'))
         .map(lambda x: x['next_run_time'])
         .limit(1)
         .run(self.conn)
     )
     return utc_timestamp_to_datetime(results[0]) if results else None
Exemple #20
0
def initial():
    """Return old transactions from the backlog."""

    bigchain = Bigchain()

    return bigchain.connection.run(
        r.table('backlog').between(
            [bigchain.me, r.minval], [bigchain.me, r.maxval],
            index='assignee__transaction_timestamp').order_by(
                index=r.asc('assignee__transaction_timestamp')))
Exemple #21
0
def initial():
    """Return old transactions from the backlog."""

    bigchain = Bigchain()

    return bigchain.connection.run(
        r.table('backlog')
        .between([bigchain.me, r.minval],
                 [bigchain.me, r.maxval],
                 index='assignee__transaction_timestamp')
        .order_by(index=r.asc('assignee__transaction_timestamp')))
Exemple #22
0
    def get_next_run_time(self):
        document = list(
            self.table.filter(lambda x: x['next_run_time'] != None).order_by(
                r.asc('next_run_time')).map(
                    lambda x: x['next_run_time']).limit(1).run(self.conn))
        if document:
            document = utc_timestamp_to_datetime(document[0])
        else:
            document = None

        return document
Exemple #23
0
    def apply_query_parameter_sorts(cls, q, query_parameters):
        if len(query_parameters.sort) == 0:
            q = q.order_by("id")
        else:
            for sort in query_parameters.sort:
                if sort["dir"] == "desc":
                    q = q.order_by(rethinkdb.desc(sort["field"]))
                else:
                    q = q.order_by(rethinkdb.asc(sort["field"]))

        return q
Exemple #24
0
def initial():
    """Return old transactions from the backlog."""

    b = Bigchain()

    rs = r.table('backlog')\
          .between([b.me, r.minval],
                   [b.me, r.maxval],
                   index='assignee__transaction_timestamp')\
          .order_by(index=r.asc('assignee__transaction_timestamp'))\
          .run(b.conn)
    return rs
    def orderBy(self, field, direct="desc"):
        """
        Allows for the results to be ordered by a specific field. If given,
        direction can be set with passing an additional argument in the form
        of "asc" or "desc"
        """
        if direct == "desc":
            self._query = self._query.order_by(r.desc(field))
        else:
            self._query = self._query.order_by(r.asc(field))

        return self
Exemple #26
0
def get_unvoted_blocks(connection, node_pubkey):
    unvoted = connection.run(
            r.table('bigchain', read_mode=READ_MODE)
            .filter(lambda block: r.table('votes', read_mode=READ_MODE)
                                   .get_all([block['id'], node_pubkey], index='block_and_voter')
                                   .is_empty())
            .order_by(r.asc(r.row['block']['timestamp'])))

    # FIXME: I (@vrde) don't like this solution. Filtering should be done at a
    #        database level. Solving issue #444 can help untangling the situation
    unvoted_blocks = filter(lambda block: not utils.is_genesis_block(block), unvoted)
    return unvoted_blocks
Exemple #27
0
    def orderBy(self, field, direct="desc"):
        """
        Allows for the results to be ordered by a specific field. If given,
        direction can be set with passing an additional argument in the form
        of "asc" or "desc"
        """
        if direct == "desc":
            self._query = self._query.order_by(r.desc(field))
        else:
            self._query = self._query.order_by(r.asc(field))

        return self
Exemple #28
0
def get_unvoted_blocks(connection, node_pubkey):
    unvoted = connection.run(
            r.table('bigchain', read_mode=READ_MODE)
            .filter(lambda block: r.table('votes', read_mode=READ_MODE)
                                   .get_all([block['id'], node_pubkey], index='block_and_voter')
                                   .is_empty())
            .order_by(r.asc(r.row['block']['timestamp'])))

    # FIXME: I (@vrde) don't like this solution. Filtering should be done at a
    #        database level. Solving issue #444 can help untangling the situation
    unvoted_blocks = filter(lambda block: not utils.is_genesis_block(block), unvoted)
    return unvoted_blocks
Exemple #29
0
    def get_unvoted_blocks(self):
        """Return all the blocks that has not been voted by this node."""

        unvoted = r.table('bigchain')\
            .filter(lambda doc: doc['votes'].contains(lambda vote: vote['node_pubkey'] == self.me).not_())\
            .order_by(r.asc((r.row['block']['timestamp'])))\
            .run(self.conn)

        if unvoted and unvoted[0].get('block_number') == 0:
            unvoted.pop(0)

        return unvoted
Exemple #30
0
    def get_unvoted_blocks(self):
        """Return all the blocks that has not been voted by this node."""

        unvoted = r.table('bigchain')\
            .filter(lambda doc: doc['votes'].contains(lambda vote: vote['node_pubkey'] == self.me).not_())\
            .order_by(r.asc((r.row['block']['timestamp'])))\
            .run(self.conn)

        if unvoted and unvoted[0].get('block_number') == 0:
            unvoted.pop(0)

        return unvoted
def render_asset(asset_id):
    versions = list(r.table('versions').filter({'asset_id': asset_id}).order_by(r.asc('version_w')).run(g.rdb_conn))

    config = []
    for version in versions:
        #print version['version_width']
        print version
        config.append({"imgwidth": int(version['width'])})
    print config
    filename = os.path.join(app.config['UPLOAD_FOLDER'], asset_id + versions[0]['ext'])
    encode(filename, config)
    return render_template('explore.html')
Exemple #32
0
def initial():
    """Return old transactions from the backlog."""

    b = Bigchain()

    rs = r.table('backlog')\
          .between([b.me, r.minval],
                   [b.me, r.maxval],
                   index='assignee__transaction_timestamp')\
          .order_by(index=r.asc('assignee__transaction_timestamp'))\
          .run(b.conn)
    return rs
Exemple #33
0
def list_posts(params, db_conn):
    """
    Get a list of posts in Sagefy.
    """

    skip = params.get('skip') or 0
    limit = params.get('limit') or 10
    params = omit(params, (
        'skip',
        'limit',
    ))
    query = (r.table(post_schema['tablename']).filter(params).order_by(
        r.asc('created')).skip(skip).limit(limit))
    return list(query.run(db_conn))
Exemple #34
0
    def _get_jobs(self, conditions=None):
        jobs = []
        failed_job_ids = []
        if conditions:
            documents = list(
                self.table
                .filter(
                    lambda x:
                    x['next_run_time'] != None
                )
                .filter(conditions)
                .order_by(r.asc('next_run_time'), 'id')
                .pluck('id', 'job_state')
                .run(self.conn)
            )
        else:
            documents = list(
                self.table
                .order_by(r.asc('next_run_time'), 'id')
                .pluck('id', 'job_state')
                .run(self.conn)
            )
        for document in documents:
            try:
                jobs.append(self._reconstitute_job(document['job_state']))
            except:
                self._logger.exception('Unable to restore job "%s" -- removing it', document['id'])
                failed_job_ids.append(document['id'])

        # Remove all the jobs we failed to restore
        if failed_job_ids:
            r.expr(failed_job_ids).for_each(
                lambda job_id:
                self.table.get_all(job_id).delete()
            ).run(self.conn)

        return jobs
    def get_block_votes(self,id,order_by=False):
        """get all the votes for the special block or all blocks

        Args:
            id:block_id
            order_by:
        Return:
            A cursor for the matching votes.
        """
        if order_by:
            return self.bigchain.connection.run(r.table('votes', read_mode=self.read_mode).filter(
                r.row['vote']['voting_for_block'] == id).order_by(index=r.asc("vote_timestamp")))
        else:
            return self.bigchain.connection.run(r.table('votes',read_mode=self.read_mode).filter(
                r.row['vote']['voting_for_block'] == id))
Exemple #36
0
    def get_unvoted_blocks(self):
        """Return all the blocks that has not been voted by this node."""

        unvoted = r.table('bigchain', read_mode=self.read_mode) \
            .filter(lambda block: r.table('votes', read_mode=self.read_mode)
                    .get_all([block['id'], self.me], index='block_and_voter')
                    .is_empty()) \
            .order_by(r.asc(r.row['block']['timestamp'])) \
            .run(self.conn)

        # FIXME: I (@vrde) don't like this solution. Filtering should be done at a
        #        database level. Solving issue #444 can help untangling the situation
        unvoted = filter(lambda block: not util.is_genesis_block(block), unvoted)

        return list(unvoted)
Exemple #37
0
    def get_unvoted_blocks(self):
        """Return all the blocks that has not been voted by this node."""

        unvoted = (
            r.table("bigchain")
            .filter(lambda block: r.table("votes").get_all([block["id"], self.me], index="block_and_voter").is_empty())
            .order_by(r.asc(r.row["block"]["timestamp"]))
            .run(self.conn)
        )

        # FIXME: I (@vrde) don't like this solution. Filtering should be done at a
        #        database level. Solving issue #444 can help untangling the situation
        unvoted = filter(lambda block: not util.is_genesis_block(block), unvoted)

        return list(unvoted)
Exemple #38
0
def fetch_positions(conn, date):
    # Fetch vehicle positions for the date (in local time)
    if date is None:
        date = arrow.now()
    else:
        date = date.replace(days=1)

    date = arrow.now().replace(year=date.year, month=date.month, day=date.day, hour=0, minute=0, second=0, tzinfo='America/Chicago')
    day_before = date.replace(days=-1)
    LOGGER.info('Fetching positions from {} to {}.'.format(day_before.isoformat(), date.isoformat()))

    query = rethinkdb.table('vehicle_position') \
            .between(day_before.datetime, date.datetime, index='timestamp') \
            .order_by(index=rethinkdb.asc('timestamp')) \
            .without('id', 'bearing')

    return list(query.run(conn))
Exemple #39
0
def get_filter(db_name, table_name, query, limit=None, order_by=None, sort_order='desc'):
    items = []

    if limit:
        if order_by:
            if 'asc' in sort_order:
                query = r.db(db_name).table(table_name).filter(query).order_by(r.asc(order_by)).limit(limit).run(conn)
            else:
                query = r.db(db_name).table(table_name).filter(query).order_by(r.desc(order_by)).limit(limit).run(conn)
        else:
            query = r.db(db_name).table(table_name).filter(query).limit(limit).run(conn)
    else:
        query = r.db(db_name).table(table_name).filter(query).run(conn)

    for x in query:
        items.append(x)
    return items
Exemple #40
0
    def get_next_run_time(self):
        document = list(
            self.table
            .filter(
                lambda x:
                x['next_run_time'] != None
            )
            .order_by(r.asc('next_run_time'))
            .map(lambda x: x['next_run_time'])
            .limit(1)
            .run(self.conn)
        )
        if document:
            document = utc_timestamp_to_datetime(document[0])
        else:
            document = None

        return document
Exemple #41
0
def _get_stop_times_by_vehicle(vehicle,
                               stop_start,
                               stop_end,
                               route,
                               direction,
                               time=arrow.now(),
                               window=45,
                               index='route_direction_vehicle_time'):
    lower_key = [
        route, direction, vehicle,
        r.epoch_time(time.replace(minutes=-window).timestamp)
    ]
    upper_key = [route, direction, vehicle, r.epoch_time(time.timestamp)]

    query = r.table('vehicle_stop_times') \
                .between(lower_key, upper_key, index=index) \
                .order_by(index=r.asc(index)) \
                .filter((r.row['stop_id'] == stop_start) | (r.row['stop_id'] == stop_end))

    return list(query.run())
    def test_valid_block_voting_with_create_transaction(self, b):
        q_new_block = mp.Queue()

        genesis = b.create_genesis_block()

        # create a `CREATE` transaction
        test_user_priv, test_user_pub = crypto.generate_key_pair()
        tx = b.create_transaction(b.me, test_user_pub, None, 'CREATE')
        tx_signed = b.sign_transaction(tx, b.me_private)
        assert b.is_valid_transaction(tx_signed)

        # create valid block
        block = b.create_block([tx_signed])
        # assert block is valid
        assert b.is_valid_block(block)
        b.write_block(block, durability='hard')

        # create queue and voter
        voter = Voter(q_new_block)

        # vote
        voter.start()
        # wait for vote to be written
        time.sleep(1)
        voter.kill()

        # retrive block from bigchain
        blocks = list(r.table('bigchain')
                       .order_by(r.asc((r.row['block']['timestamp'])))
                       .run(b.conn))

        # validate vote
        assert len(blocks[1]['votes']) == 1
        vote = blocks[1]['votes'][0]

        assert vote['vote']['voting_for_block'] == block['id']
        assert vote['vote']['previous_block'] == genesis['id']
        assert vote['vote']['is_block_valid'] is True
        assert vote['vote']['invalid_reason'] is None
        assert vote['node_pubkey'] == b.me
        assert crypto.VerifyingKey(b.me).verify(util.serialize(vote['vote']), vote['signature']) is True
Exemple #43
0
    def get_unvoted_blocks(self, node_pubkey):
        """Return all the blocks that have not been voted by the specified node.

        Args:
            node_pubkey (str): base58 encoded public key

        Returns:
            :obj:`list` of :obj:`dict`: a list of unvoted blocks
        """

        unvoted = self.connection.run(
                r.table('bigchain', read_mode=self.read_mode)
                .filter(lambda block: r.table('votes', read_mode=self.read_mode)
                                       .get_all([block['id'], node_pubkey], index='block_and_voter')
                                       .is_empty())
                .order_by(r.asc(r.row['block']['timestamp'])))

        # FIXME: I (@vrde) don't like this solution. Filtering should be done at a
        #        database level. Solving issue #444 can help untangling the situation
        unvoted_blocks = filter(lambda block: not util.is_genesis_block(block), unvoted)
        return unvoted_blocks
Exemple #44
0
    def get_unvoted_blocks(self, node_pubkey):
        """Return all the blocks that have not been voted by the specified node.

        Args:
            node_pubkey (str): base58 encoded public key

        Returns:
            :obj:`list` of :obj:`dict`: a list of unvoted blocks
        """

        unvoted = self.connection.run(
            r.table('bigchain', read_mode=self.read_mode).filter(
                lambda block: r.table('votes', read_mode=self.read_mode).
                get_all([block['id'], node_pubkey], index='block_and_voter').
                is_empty()).order_by(r.asc(r.row['block']['timestamp'])))

        # FIXME: I (@vrde) don't like this solution. Filtering should be done at a
        #        database level. Solving issue #444 can help untangling the situation
        unvoted_blocks = filter(lambda block: not util.is_genesis_block(block),
                                unvoted)
        return unvoted_blocks
Exemple #45
0
    async def list(self,
                   table,
                   key='get',
                   value='all',
                   index='update_date',
                   order='desc',
                   create_index=False,
                   limit=False):
        if create_index:
            await self.index(table, index)

        if order == "desc":
            direction = r.desc(index)
        else:
            direction = r.asc(index)

        if key == 'get' and value == 'all':
            items = await self.all(table)
        else:
            items = await self.filter(table, {key: value}, limit=limit)
        return items
Exemple #46
0
    def test_invalid_block_voting(self, b):
        # create queue and voter
        q_new_block = mp.Queue()
        voter = Voter(q_new_block)

        # create transaction
        transaction = b.create_transaction(b.me, USER_PUBLIC_KEY, None,
                                           'CREATE')
        transaction_signed = b.sign_transaction(transaction, b.me_private)

        genesis = b.create_genesis_block()

        # create invalid block
        block = b.create_block([transaction_signed])
        # change transaction id to make it invalid
        block['block']['transactions'][0]['id'] = 'abc'
        assert not b.is_valid_block(block)
        b.write_block(block, durability='hard')

        # vote
        voter.start()
        time.sleep(1)
        voter.kill()

        # retrive block from bigchain
        blocks = list(
            r.table('bigchain').order_by(r.asc(
                (r.row['block']['timestamp']))).run(b.conn))

        # validate vote
        assert len(blocks[1]['votes']) == 1
        vote = blocks[1]['votes'][0]

        assert vote['vote']['voting_for_block'] == block['id']
        assert vote['vote']['previous_block'] == genesis['id']
        assert vote['vote']['is_block_valid'] == False
        assert vote['vote']['invalid_reason'] == None
        assert vote['node_pubkey'] == b.me
        assert PublicKey(b.me).verify(b.serialize(vote['vote']),
                                      vote['signature']) == True
    def test_invalid_block_voting(self, b):
        # create queue and voter
        q_new_block = mp.Queue()
        voter = Voter(q_new_block)

        # create transaction
        transaction = b.create_transaction(b.me, USER_PUBLIC_KEY, None, 'CREATE')
        transaction_signed = b.sign_transaction(transaction, b.me_private)

        genesis = b.create_genesis_block()

        # create invalid block
        block = b.create_block([transaction_signed])
        # change transaction id to make it invalid
        block['block']['transactions'][0]['id'] = 'abc'
        assert not b.is_valid_block(block)
        b.write_block(block, durability='hard')


        # vote
        voter.start()
        time.sleep(1)
        voter.kill()

        # retrive block from bigchain
        blocks = list(r.table('bigchain')
                       .order_by(r.asc((r.row['block']['timestamp'])))
                       .run(b.conn))

        # validate vote
        assert len(blocks[1]['votes']) == 1
        vote = blocks[1]['votes'][0]

        assert vote['vote']['voting_for_block'] == block['id']
        assert vote['vote']['previous_block'] == genesis['id']
        assert vote['vote']['is_block_valid'] == False
        assert vote['vote']['invalid_reason'] == None
        assert vote['node_pubkey'] == b.me
        assert PublicKey(b.me).verify(b.serialize(vote['vote']), vote['signature']) == True
    def get_txs_count(self,id=None,order_by=False):
        """get all the transactions count in block

        Args:
            id:block_id
            order_by:
        Return:
            block_count: the count of the blocks
            txs_count: the count of total txs count
            block_txs_count_list: {id:block_txs_count}
        """

        table = 'bigchain'
        block_count = 0
        txs_count = 0
        block_txs_count_list = []

        if id:
            block = self.bigchain.connection.run(r.table(table).get(id))
            block_count = 1
            txs_count = len(block['block']['transactions'])

        else:
            if order_by:
                index = "block_timestamp"
                blocks = self.bigchain.connection.run(r.table(table).order_by(index=r.asc(index)))
            else:
                blocks = self.bigchain.connection.run(r.table(table))

            for block in blocks:
                block_count += 1
                block_id = block['id']
                block_txs_count = len(block['block']['transactions'])
                block_txs_count_dict = dict()
                block_txs_count_dict[block_id] = block_txs_count
                block_txs_count_list.append(block_txs_count_dict)
                txs_count += block_txs_count
                del block_txs_count_dict
        return block_count,txs_count,block_txs_count_list
    def apply_query_parameters(self, query_parameters):
        q = self.table
        if query_parameters.filter is not None:
            for filter in query_parameters.filter["filters"]:
                if filter["operator"] == "startswith":
                    start = filter["value"].lower()
                    end = start
                    end = end[:-1] + chr(ord(end[-1]) + 1)
                    q = q.between(start, end, index="lower_name")
                else:
                    q = q.between(filter["value"].lower(), filter["value"].lower(), index="lower_name",
                                  right_bound="closed")
        for sort in query_parameters.sort:
            if sort["dir"] == "desc":
                q = q.order_by(index=r.desc("lower_name"))
            else:
                q = q.order_by(index=r.asc("lower_name"))

        data = self.uow.run_list(q.skip(query_parameters.skip).limit(query_parameters.take))
        total = self.uow.run(q.count())
        query_result = QueryResult(data, total)
        return query_result
Exemple #50
0
    def _build_cursor_obj(self):
        self._cursor_obj = r.table(self._document.Meta.table_name)
        if self._filter:
            self._cursor_obj = self._cursor_obj.filter(self._filter)

        order_by = self._order_by or self._document.Meta.order_by
        if order_by:
            order_by_r = []
            for field in order_by:
                if field.startswith('-'):
                    order_by_r.append(r.desc(field[1:]))
                else:
                    order_by_r.append(r.asc(field))
            self._cursor_obj = self._cursor_obj.order_by(*order_by_r)

        if self._limit:
            self._cursor_obj = self._cursor_obj.limit(self._limit)

        if self._skip:
            self._cursor_obj = self._cursor_obj.skip(self._skip)

        self._iter_index = 0
        self._cursor_iter = iter(self._cursor_obj.run(get_conn()))
Exemple #51
0
    def _build_cursor_obj(self):
        self._cursor_obj = r.table(self._document.Meta.table_name)
        if self._filter:
            self._cursor_obj = self._cursor_obj.filter(self._filter)

        order_by = self._order_by or self._document.Meta.order_by
        if order_by:
            order_by_r = []
            for field in order_by:
                if field.startswith('-'):
                    order_by_r.append(r.desc(field[1:]))
                else:
                    order_by_r.append(r.asc(field))
            self._cursor_obj = self._cursor_obj.order_by(*order_by_r)

        if self._skip:
            self._cursor_obj = self._cursor_obj.skip(self._skip)

        if self._limit:
            self._cursor_obj = self._cursor_obj.limit(self._limit)

        self._iter_index = 0
        self._cursor_iter = iter(self._cursor_obj.run(get_conn()))
Exemple #52
0
    def test_voter_considers_unvoted_blocks_when_single_node(self, b):
        # simulate a voter going donw in a single node environment
        b.create_genesis_block()

        # insert blocks in the database while the voter process is not listening
        # (these blocks won't appear in the changefeed)
        block_1 = b.create_block([])
        b.write_block(block_1, durability='hard')
        block_2 = b.create_block([])
        b.write_block(block_2, durability='hard')

        # voter is back online, we simulate that by creating a queue and a Voter instance
        q_new_block = mp.Queue()
        voter = Voter(q_new_block)

        # create a new block that will appear in the changefeed
        block_3 = b.create_block([])
        b.write_block(block_3, durability='hard')

        # put the last block in the queue
        q_new_block.put(block_3)

        # vote
        voter.start()
        time.sleep(1)
        voter.kill()

        # retrive blocks from bigchain
        blocks = list(
            r.table('bigchain').order_by(r.asc(
                (r.row['block']['timestamp']))).run(b.conn))

        # FIXME: remove genesis block, we don't vote on it (might change in the future)
        blocks.pop(0)

        assert all(block['votes'][0]['node_pubkey'] == b.me
                   for block in blocks)
Exemple #53
0
    def test_voter_considers_unvoted_blocks_when_single_node(self, b):
        # simulate a voter going donw in a single node environment
        b.create_genesis_block()

        # insert blocks in the database while the voter process is not listening
        # (these blocks won't appear in the changefeed)
        block_1 = b.create_block([])
        b.write_block(block_1, durability='hard')
        block_2 = b.create_block([])
        b.write_block(block_2, durability='hard')

        # voter is back online, we simulate that by creating a queue and a Voter instance
        q_new_block = mp.Queue()
        voter = Voter(q_new_block)

        # create a new block that will appear in the changefeed
        block_3 = b.create_block([])
        b.write_block(block_3, durability='hard')

        # put the last block in the queue
        q_new_block.put(block_3)

        # vote
        voter.start()
        time.sleep(1)
        voter.kill()

        # retrive blocks from bigchain
        blocks = list(r.table('bigchain')
                       .order_by(r.asc((r.row['block']['timestamp'])))
                       .run(b.conn))

        # FIXME: remove genesis block, we don't vote on it (might change in the future)
        blocks.pop(0)

        assert all(block['votes'][0]['node_pubkey'] == b.me for block in blocks)
Exemple #54
0
def newsfeed():
    print "TEST:::::::::::"
    feed_list=[]
    connection=r.connect(host='localhost',port=28015)
    feed_db=r.db('taggem').table('post').filter({'user_id':'*****@*****.**'}).order_by(r.asc('date')).run(connection)
    count=r.db('taggem').table('post').filter({'user_id':'*****@*****.**'}).run(connection)
    print feed_db
    for f in feed_db:
        feed_list.append(f)
    #     print "A"
    # feed={'img_url':'http://girltalkhq.com/wp-content/uploads/2013/09/Image-11.jpg', 'link':'http://www.google.com', 'title':'Colors and paints','text':'Learn more about colors and arts'}
    # feed2={'img_url':'http://girltalkhq.com/wp-content/uploads/2013/09/Image-11.jpg', 'link':'http://www.google.com', 'title':'Colors and paints','text':'Learn more about colors and arts'}
  
    # feed_list.append(feed)
    # feed_list.append(feed2)
    feedJSON=jsonify({'feed':feed_list})


    return feedJSON
def send_initial_data(client):
	try:
	    logging.info("send_initial_data::entering")
	    tconn = yield r.connect(DB_HOST,DB_PORT,DB_NAME)
	    #feed =  yield r.table("observations").pluck('sensorid','datetime','type','temp','measure_temp','pressure_sea','measure_pressure').order_by(r.desc('datetime')).limit(200).order_by(r.asc('datetime')).run(tconn)
	    feed =  yield r.table("observations").order_by(r.desc('timestamp')).limit(200).order_by(r.asc('timestamp')).run(tconn) 
            for document in feed:
            	client.write_message(document) 

	    fan = FAN_Adapter(16,'f_0001')
            d_fan = fan.readJSON()
            d_observation = {'fan':[d_fan],
                             'timestamp': str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")),
                             'type':'actors'
                             }

	    tconn.close()
            logging.info("send_initial_data::leaving")
	except:
            exc_type, exc_value, exc_traceback = sys.exc_info()
            logging.info("send_initial_data::exception %s %s %s",exc_type, exc_value, exc_traceback)
            pass
def get_all_layers_for_asset(asset_id):
    layers = list(r.table('layers').filter({'asset_id': asset_id}).order_by(r.asc('time_stamp')).run(g.rdb_conn))
    return json.dumps(layers)
def get_all_versions_for_asset(asset_id):
    versions = list(r.table('versions').filter({'asset_id': asset_id}).order_by(r.asc('width')).order_by(r.asc('height')).run(g.rdb_conn))
    return json.dumps(versions)
Exemple #58
0
    def test_valid_block_voting_with_transfer_transactions(self, b):
        q_new_block = mp.Queue()

        b.create_genesis_block()

        # create a `CREATE` transaction
        test_user_priv, test_user_pub = crypto.generate_key_pair()
        tx = b.create_transaction(b.me, test_user_pub, None, 'CREATE')
        tx_signed = b.sign_transaction(tx, b.me_private)
        assert b.is_valid_transaction(tx_signed)

        # create valid block
        block = b.create_block([tx_signed])
        # assert block is valid
        assert b.is_valid_block(block)
        b.write_block(block, durability='hard')

        # create queue and voter
        voter = Voter(q_new_block)

        # vote
        voter.start()
        # wait for vote to be written
        time.sleep(1)
        voter.kill()

        # retrive block from bigchain
        blocks = list(r.table('bigchain')
                       .order_by(r.asc((r.row['block']['timestamp'])))
                       .run(b.conn))

        # validate vote
        assert len(blocks[1]['votes']) == 1

        # create a `TRANSFER` transaction
        test_user2_priv, test_user2_pub = crypto.generate_key_pair()
        tx2 = b.create_transaction(test_user_pub, test_user2_pub, {'txid': tx['id'], 'cid': 0}, 'TRANSFER')
        tx2_signed = b.sign_transaction(tx2, test_user_priv)
        assert b.is_valid_transaction(tx2_signed)

        # create valid block
        block = b.create_block([tx2_signed])
        # assert block is valid
        assert b.is_valid_block(block)
        b.write_block(block, durability='hard')

        # create queue and voter
        voter = Voter(q_new_block)

        # vote
        voter.start()
        # wait for vote to be written
        time.sleep(1)
        voter.kill()

        # retrive block from bigchain
        blocks = list(r.table('bigchain')
                       .order_by(r.asc((r.row['block']['timestamp'])))
                       .run(b.conn))

        # validate vote
        assert len(blocks[2]['votes']) == 1

        vote = blocks[2]['votes'][0]

        assert vote['vote']['voting_for_block'] == block['id']
        assert vote['vote']['is_block_valid'] is True
        assert vote['vote']['invalid_reason'] is None
        assert vote['node_pubkey'] == b.me
        assert crypto.VerifyingKey(b.me).verify(util.serialize(vote['vote']), vote['signature']) is True