Example #1
0
    def test_query_with_partially_named_multiindex(self, parser, engine):
        skip_if_no_pandas_parser(parser)
        a = np.random.choice(['red', 'green'], size=10)
        b = np.arange(10)
        index = MultiIndex.from_arrays([a, b])
        index.names = [None, 'rating']
        df = DataFrame(np.random.randn(10, 2), index=index)
        res = df.query('rating == 1', parser=parser, engine=engine)
        ind = Series(df.index.get_level_values('rating').values, index=index,
                     name='rating')
        exp = df[ind == 1]
        assert_frame_equal(res, exp)

        res = df.query('rating != 1', parser=parser, engine=engine)
        ind = Series(df.index.get_level_values('rating').values, index=index,
                     name='rating')
        exp = df[ind != 1]
        assert_frame_equal(res, exp)

        res = df.query('ilevel_0 == "red"', parser=parser, engine=engine)
        ind = Series(df.index.get_level_values(0).values, index=index)
        exp = df[ind == "red"]
        assert_frame_equal(res, exp)

        res = df.query('ilevel_0 != "red"', parser=parser, engine=engine)
        ind = Series(df.index.get_level_values(0).values, index=index)
        exp = df[ind != "red"]
        assert_frame_equal(res, exp)
Example #2
0
    def check_str_query_method(self, parser, engine):
        tm.skip_if_no_ne(engine)
        df = DataFrame(randn(10, 1), columns=['b'])
        df['strings'] = Series(list('aabbccddee'))
        expect = df[df.strings == 'a']

        if parser != 'pandas':
            col = 'strings'
            lst = '"a"'

            lhs = [col] * 2 + [lst] * 2
            rhs = lhs[::-1]

            eq, ne = '==', '!='
            ops = 2 * ([eq] + [ne])

            for lhs, op, rhs in zip(lhs, ops, rhs):
                ex = '{lhs} {op} {rhs}'.format(lhs=lhs, op=op, rhs=rhs)
                assertRaises(NotImplementedError, df.query, ex, engine=engine,
                             parser=parser, local_dict={'strings': df.strings})
        else:
            res = df.query('"a" == strings', engine=engine, parser=parser)
            assert_frame_equal(res, expect)

            res = df.query('strings == "a"', engine=engine, parser=parser)
            assert_frame_equal(res, expect)
            assert_frame_equal(res, df[df.strings.isin(['a'])])

            expect = df[df.strings != 'a']
            res = df.query('strings != "a"', engine=engine, parser=parser)
            assert_frame_equal(res, expect)

            res = df.query('"a" != strings', engine=engine, parser=parser)
            assert_frame_equal(res, expect)
            assert_frame_equal(res, df[~df.strings.isin(['a'])])
Example #3
0
    def test_index_resolvers_come_after_columns_with_the_same_name(self):
        n = 1  # noqa
        a = np.r_[20:101:20]

        df = DataFrame({'index': a, 'b': np.random.randn(a.size)})
        df.index.name = 'index'
        result = df.query('index > 5', engine=self.engine, parser=self.parser)
        expected = df[df['index'] > 5]
        assert_frame_equal(result, expected)

        df = DataFrame({'index': a,
                        'b': np.random.randn(a.size)})
        result = df.query('ilevel_0 > 5', engine=self.engine,
                          parser=self.parser)
        expected = df.loc[df.index[df.index > 5]]
        assert_frame_equal(result, expected)

        df = DataFrame({'a': a, 'b': np.random.randn(a.size)})
        df.index.name = 'a'
        result = df.query('a > 5', engine=self.engine, parser=self.parser)
        expected = df[df.a > 5]
        assert_frame_equal(result, expected)

        result = df.query('index > 5', engine=self.engine, parser=self.parser)
        expected = df.loc[df.index[df.index > 5]]
        assert_frame_equal(result, expected)
Example #4
0
    def test_nested_scope(self):
        engine = self.engine
        parser = self.parser

        skip_if_no_pandas_parser(parser)

        df = DataFrame(np.random.randn(5, 3))
        df2 = DataFrame(np.random.randn(5, 3))
        expected = df[(df > 0) & (df2 > 0)]

        result = df.query('(@df > 0) & (@df2 > 0)', engine=engine,
                          parser=parser)
        assert_frame_equal(result, expected)

        result = pd.eval('df[df > 0 and df2 > 0]', engine=engine,
                         parser=parser)
        assert_frame_equal(result, expected)

        result = pd.eval('df[df > 0 and df2 > 0 and df[df > 0] > 0]',
                         engine=engine, parser=parser)
        expected = df[(df > 0) & (df2 > 0) & (df[df > 0] > 0)]
        assert_frame_equal(result, expected)

        result = pd.eval('df[(df>0) & (df2>0)]', engine=engine, parser=parser)
        expected = df.query('(@df>0) & (@df2>0)', engine=engine, parser=parser)
        assert_frame_equal(result, expected)
Example #5
0
    def test_nested_scope(self):
        from pandas.core.computation.ops import UndefinedVariableError
        engine = self.engine
        parser = self.parser
        # smoke test
        x = 1  # noqa
        result = pd.eval('x + 1', engine=engine, parser=parser)
        assert result == 2

        df = DataFrame(np.random.randn(5, 3))
        df2 = DataFrame(np.random.randn(5, 3))

        # don't have the pandas parser
        with pytest.raises(SyntaxError):
            df.query('(@df>0) & (@df2>0)', engine=engine, parser=parser)

        with pytest.raises(UndefinedVariableError):
            df.query('(df>0) & (df2>0)', engine=engine, parser=parser)

        expected = df[(df > 0) & (df2 > 0)]
        result = pd.eval('df[(df > 0) & (df2 > 0)]', engine=engine,
                         parser=parser)
        assert_frame_equal(expected, result)

        expected = df[(df > 0) & (df2 > 0) & (df[df > 0] > 0)]
        result = pd.eval('df[(df > 0) & (df2 > 0) & (df[df > 0] > 0)]',
                         engine=engine, parser=parser)
        assert_frame_equal(expected, result)
Example #6
0
    def test_nested_raises_on_local_self_reference(self):
        from pandas.core.computation.ops import UndefinedVariableError

        df = DataFrame(np.random.randn(5, 3))

        # can't reference ourself b/c we're a local so @ is necessary
        with pytest.raises(UndefinedVariableError):
            df.query('df > 0', engine=self.engine, parser=self.parser)
Example #7
0
 def test_query_undefined_local(self):
     from pandas.computation.ops import UndefinedVariableError
     engine, parser = self.engine, self.parser
     skip_if_no_pandas_parser(parser)
     df = DataFrame(np.random.rand(10, 2), columns=list('ab'))
     with tm.assertRaisesRegexp(UndefinedVariableError,
                                "local variable 'c' is not defined"):
         df.query('a == @c', engine=engine, parser=parser)
Example #8
0
    def test_query(self):
        engine, parser = self.engine, self.parser
        df = DataFrame(np.random.randn(10, 3), columns=['a', 'b', 'c'])

        assert_frame_equal(df.query('a < b', engine=engine, parser=parser),
                           df[df.a < df.b])
        assert_frame_equal(df.query('a + b > b * c', engine=engine,
                                    parser=parser),
                           df[df.a + df.b > df.b * df.c])
Example #9
0
def update_progress(
        status: dict,
        settings: dict,
        progress: pd.DataFrame,
        queue: pd.DataFrame,
        passengers: pd.DataFrame,
        **kwargs
) -> pd.DataFrame:
    """
    :param status:
        The current status of the simulation
    :param settings:
        Configuration settings for the current trial
    :param progress:
        The current progress data frame for the snapshot, which will be
        replaced by the one returned
    :param queue:
        The queue data frame for the trial
    :param passengers:
        The passengers data frame for the trial
    """

    row = dict(
        progress=status['progress'],
        time=status['time']
    )

    aisle_vacancies = []

    for queue_index in range(queue.shape[0]):
        queue_item = queue.loc[queue_index]
        passenger_index = queue_item['passenger']

        if queue_item['aisle'] >= 0:
            aisle_vacancies.append(0 if passenger_index is None else 1)

        if passenger_index is not None:
            key = 'p_{}'.format(passenger_index)
            row[key] = 'Q:{}'.format(queue_item['aisle'])

    seated = passengers.query('delay_interchange > 0')
    for passenger_index, passenger in seated.iterrows():
        key = 'p_{}'.format(passenger_index)
        row[key] = '{}:{}'.format(
            'O' if passenger['seated'] else 'I',
            passenger['aisle']
        )

    seated = passengers.query('seated and delay_interchange == 0')
    for passenger_index, passenger in seated.iterrows():
        key = 'p_{}'.format(passenger_index)
        row[key] = 'S:{}'.format(passenger['aisle'])

    row['aisle_density'] = sum(aisle_vacancies)/len(aisle_vacancies)

    return progress.append(row, ignore_index=True)
Example #10
0
    def test_query_undefined_local(self):
        from pandas.core.computation.ops import UndefinedVariableError
        engine, parser = self.engine, self.parser
        skip_if_no_pandas_parser(parser)

        df = DataFrame(np.random.rand(10, 2), columns=list('ab'))
        msg = "local variable 'c' is not defined"

        with pytest.raises(UndefinedVariableError, match=msg):
            df.query('a == @c', engine=engine, parser=parser)
Example #11
0
    def test_query_doesnt_pickup_local(self):
        from pandas.core.computation.ops import UndefinedVariableError

        engine, parser = self.engine, self.parser
        n = m = 10
        df = DataFrame(np.random.randint(m, size=(n, 3)), columns=list('abc'))

        # we don't pick up the local 'sin'
        with pytest.raises(UndefinedVariableError):
            df.query('sin > 5', engine=engine, parser=parser)
Example #12
0
 def test_date_index_query_with_NaT_duplicates(self):
     engine, parser = self.engine, self.parser
     n = 10
     df = DataFrame(np.random.randn(n, 3))
     df['dates1'] = date_range('1/1/2012', periods=n)
     df['dates3'] = date_range('1/1/2014', periods=n)
     df.loc[np.random.rand(n) > 0.5, 'dates1'] = pd.NaT
     df.set_index('dates1', inplace=True, drop=True)
     with pytest.raises(NotImplementedError):
         df.query('index < 20130101 < dates3', engine=engine, parser=parser)
Example #13
0
    def test_query_builtin(self):
        from pandas.core.computation.engines import NumExprClobberingError
        engine, parser = self.engine, self.parser

        n = m = 10
        df = DataFrame(np.random.randint(m, size=(n, 3)), columns=list('abc'))

        df.index.name = 'sin'
        msg = 'Variables in expression.+'
        with pytest.raises(NumExprClobberingError, match=msg):
            df.query('sin > 5', engine=engine, parser=parser)
Example #14
0
    def test_object_array_eq_ne(self, parser, engine):
        df = DataFrame({'a': list('aaaabbbbcccc'),
                        'b': list('aabbccddeeff'),
                        'c': np.random.randint(5, size=12),
                        'd': np.random.randint(9, size=12)})
        res = df.query('a == b', parser=parser, engine=engine)
        exp = df[df.a == df.b]
        assert_frame_equal(res, exp)

        res = df.query('a != b', parser=parser, engine=engine)
        exp = df[df.a != df.b]
        assert_frame_equal(res, exp)
Example #15
0
    def test_date_query_with_non_date(self):
        engine, parser = self.engine, self.parser

        n = 10
        df = DataFrame({'dates': date_range('1/1/2012', periods=n),
                        'nondate': np.arange(n)})

        ops = '==', '!=', '<', '>', '<=', '>='

        for op in ops:
            with tm.assertRaises(TypeError):
                df.query('dates %s nondate' % op, parser=parser, engine=engine)
Example #16
0
    def test_local_syntax(self):
        skip_if_no_pandas_parser(self.parser)

        engine, parser = self.engine, self.parser
        df = DataFrame(np.random.randn(100, 10), columns=list('abcdefghij'))
        b = 1
        expect = df[df.a < b]
        result = df.query('a < @b', engine=engine, parser=parser)
        assert_frame_equal(result, expect)

        expect = df[df.a < df.b]
        result = df.query('a < b', engine=engine, parser=parser)
        assert_frame_equal(result, expect)
Example #17
0
    def test_query_index_with_name(self):
        engine, parser = self.engine, self.parser
        df = DataFrame(np.random.randint(10, size=(10, 3)),
                       index=Index(range(10), name='blob'),
                       columns=['a', 'b', 'c'])
        res = df.query('(blob < 5) & (a < b)', engine=engine, parser=parser)
        expec = df[(df.index < 5) & (df.a < df.b)]
        assert_frame_equal(res, expec)

        res = df.query('blob < b', engine=engine, parser=parser)
        expec = df[df.index < df.b]

        assert_frame_equal(res, expec)
Example #18
0
    def test_query_index_without_name(self):
        engine, parser = self.engine, self.parser
        df = DataFrame(np.random.randint(10, size=(10, 3)),
                       index=range(10), columns=['a', 'b', 'c'])

        # "index" should refer to the index
        res = df.query('index < b', engine=engine, parser=parser)
        expec = df[df.index < df.b]
        assert_frame_equal(res, expec)

        # test against a scalar
        res = df.query('index < 5', engine=engine, parser=parser)
        expec = df[df.index < 5]
        assert_frame_equal(res, expec)
Example #19
0
    def test_local_variable_with_in(self):
        engine, parser = self.engine, self.parser
        skip_if_no_pandas_parser(parser)
        a = Series(np.random.randint(3, size=15), name='a')
        b = Series(np.random.randint(10, size=15), name='b')
        df = DataFrame({'a': a, 'b': b})

        expected = df.loc[(df.b - 1).isin(a)]
        result = df.query('b - 1 in a', engine=engine, parser=parser)
        assert_frame_equal(expected, result)

        b = Series(np.random.randint(10, size=15), name='b')
        expected = df.loc[(b - 1).isin(a)]
        result = df.query('@b - 1 in a', engine=engine, parser=parser)
        assert_frame_equal(expected, result)
Example #20
0
    def test_date_query_with_non_date(self):
        engine, parser = self.engine, self.parser

        n = 10
        df = DataFrame({'dates': date_range('1/1/2012', periods=n),
                        'nondate': np.arange(n)})

        result = df.query('dates == nondate', parser=parser, engine=engine)
        assert len(result) == 0

        result = df.query('dates != nondate', parser=parser, engine=engine)
        assert_frame_equal(result, df)

        for op in ['<', '>', '<=', '>=']:
            with pytest.raises(TypeError):
                df.query('dates %s nondate' % op, parser=parser, engine=engine)
Example #21
0
    def open(self, slug):
        columns = json.loads(MyBucket.get('{}-columns'.format(slug)).data)
        fields = columns
        if self.get_argument('fields', None):
            fields = self.get_argument('fields').split(',')

        self.write_message({'type': 'columns', 'data': fields})

        filters = [i[0] for i in self.request.arguments.iteritems()
                   if len(i[0].split('filter__')) > 1]

        df = DataFrame(MyBucket.get(slug).data, columns=fields)
        if len(filters) >= 1:
            for f in filters:
                df = df.query(df_generate(df, self.get_argument, f))

        ca = None
        for e in MyAdminBucket.get('element').data:
            if e['slug'] == slug:
                ca = e['categories']

        categories = []
        for i in df.to_dict(outtype='records'):
            if ca:
                categories.append(i[ca])
            self.write_message({'type': 'data', 'data': i})

        self.write_message({'type': 'categories', 'data': categories})
        self.write_message({'type': 'close'})
Example #22
0
def QA_fetch_stock_min_adv(code, start, end, type_='1min', if_drop_index=False, collections=QA_Setting.client.quantaxis.stock_min):
    '获取股票分钟线'
    if type_ in ['1min', '1m']:
        type_ = '1min'
    elif type_ in ['5min', '5m']:
        type_ = '5min'
    elif type_ in ['15min', '15m']:
        type_ = '15min'
    elif type_ in ['30min', '30m']:
        type_ = '30min'
    elif type_ in ['60min', '60m']:
        type_ = '60min'
    __data = []
    for item in collections.find({
        'code': str(code), "time_stamp": {
            "$gte": QA_util_time_stamp(start),
            "$lte": QA_util_time_stamp(end)
        }, 'type': type_
    }):

        __data.append([str(item['code']), float(item['open']), float(item['high']), float(
            item['low']), float(item['close']), float(item['vol']), item['datetime'], item['time_stamp'], item['date']])

    __data = DataFrame(__data, columns=[
        'code', 'open', 'high', 'low', 'close', 'volume', 'datetime', 'time_stamp', 'date'])

    __data['datetime'] = pd.to_datetime(__data['datetime'])
    return QA_DataStruct_Stock_min(__data.query('volume>1').set_index(['datetime', 'code'], drop=if_drop_index))
Example #23
0
    def post(self, slug):
        columns = json.loads(MyBucket.get('{}-columns'.format(slug)).data)
        fields = columns
        if self.get_argument('fields', None):
            fields = self.get_argument('fields').split(',')

        filters = [i[0] for i in self.request.arguments.iteritems()
                   if len(i[0].split('filter__')) > 1]

        fields_json = json.dumps(fields)
        filters_json = json.dumps({f: self.get_argument(f) for f in filters})
        if MyCache.get(str(slug)) and\
                MyCache.get('{}-columns'.format(slug)) == fields_json and\
                MyCache.get('{}-fulters'.format(slug)) == filters_json:
            self.write(MyCache.get(str(slug)))
            self.finish()

        MyCache.set('{}-columns'.format(slug), fields_json)
        MyCache.set('{}-filters'.format(slug), filters_json)

        df = DataFrame(MyBucket.get(slug).data, columns=fields)
        if len(filters) >= 1:
            for f in filters:
                df = df.query(df_generate(df, self.get_argument, f))
        convert = df.to_dict(outtype='records')

        write = json.dumps({'columns': fields, 'json': convert})
        MyCache.set(str(slug), write)
        self.write(write)
        self.finish()
    def from_contract_description(cls, contracts: pd.DataFrame, position, premium, option_type=None,
                                  strike_price=None, underlying_asset=None, expiry=None, quantity=1):
        queries = []

        if option_type is not None:
            if option_type == OptionType.Call:
                queries.append("Right=='C'")
            elif option_type == OptionType.Put:
                queries.append("Right=='P'")

        if strike_price is not None:
            queries.append("Strike==" + str(strike_price))

        if underlying_asset is not None:
            queries.append("Symbol=='{}'".format(underlying_asset))

        if expiry is not None:
            queries.append("Symbol=='{}'".format(expiry))

        query = None
        for q in queries:
            if query is None:
                query = q
            else:
                query = query + " and " + q

        selected_contract = contracts.query(query)
        if selected_contract.shape[0] > 1:
            raise ValueError()
        else:
            return cls.from_ConId(contracts, selected_contract.index[0], position, premium, quantity)
Example #25
0
 def test_query_single_element_booleans(self, parser, engine):
     columns = 'bid', 'bidsize', 'ask', 'asksize'
     data = np.random.randint(2, size=(1, len(columns))).astype(bool)
     df = DataFrame(data, columns=columns)
     res = df.query('bid & ask', engine=engine, parser=parser)
     expected = df[df.bid & df.ask]
     assert_frame_equal(res, expected)
Example #26
0
 def test_query_with_nested_special_character(self, parser, engine):
     skip_if_no_pandas_parser(parser)
     df = DataFrame({'a': ['a', 'b', 'test & test'],
                     'b': [1, 2, 3]})
     res = df.query('a == "test & test"', parser=parser, engine=engine)
     expec = df[df.a == 'test & test']
     assert_frame_equal(res, expec)
Example #27
0
def create_trackway_info(
        trackway_name: str,
        trackway_df: pd.DataFrame,
        **kwargs
) -> dict:
    """
    Creates information about the trackway
    """

    pes_df = trackway_df.query('is_pes == True')
    manus_df = trackway_df.query('is_pes == False')

    return dict(
        trackway=trackway_name,
        pes_count=len(pes_df),
        manus_count=len(manus_df)
    )
Example #28
0
 def f(self, query):
     """
     Filter data based on query
     query
         A valid pandas dataframe query
     """
     df = DataFrame(self._cash)
     return df.query(query)
Example #29
0
 def f(self, f):
     """
     Filter trades based on conditions
     f
         Any valid pandas dataframe query
     """
     df = DataFrame(self._trades)
     return df.query(f)
Example #30
0
 def test_at_inside_string(self):
     engine, parser = self.engine, self.parser
     skip_if_no_pandas_parser(parser)
     c = 1  # noqa
     df = DataFrame({'a': ['a', 'a', 'b', 'b', '@c', '@c']})
     result = df.query('a == "@c"', engine=engine, parser=parser)
     expected = df[df.a == "@c"]
     assert_frame_equal(result, expected)