Exemple #1
0
    def __init__(self,
                 pickle_protocol=pickle.DEFAULT_PROTOCOL,
                 namespace=None,
                 **kwargs):
        """Initialize Redis DB

        Args:
            pickle_protocol (int, optional): https://docs.python.org/3.7/library/pickle.html#data-stream-format
            namespace (str, optional): If set, key names in the Redis DB will
                be preceeded by "<namespace>:".
            **kwargs (optional): All these will be sent to redis.Redis(). See
                https://github.com/andymccurdy/redis-py for more info.
        """
        from redis import Redis

        super().__init__()
        self._pickle_protocol = pickle_protocol
        self._namespace = namespace

        # Quick check so the number of databases is sufficient
        if "db" in kwargs and kwargs["db"] > 0:
            redis_kwargs = kwargs.copy()
            redis_kwargs["db"] = 0
            redis = Redis(**redis_kwargs)
            dbs = redis.config_get("databases")
            if int(dbs["databases"]) <= kwargs["db"]:
                redis.close()
                raise ValueError(
                    f"Tried to open Redis DB #{kwargs['db']}, but there are only {dbs['databases']} databases"
                )
            redis.close()

        self._redis = Redis(**kwargs)
def create_payment_request(payment_request=None):  # noqa: E501
    """createPaymentRequest

    Creates a payment request for a user. API for: ACMESky # noqa: E501

    :param payment_request: 
    :type payment_request: dict | bytes

    :rtype: PaymentCreationResponse
    """
    if connexion.request.is_json:
        payment_request = PaymentRequest.from_dict(
            connexion.request.get_json())  # noqa: E501

    # This line simulates a network slowdown (comment if not needed).
    time.sleep(5)
    """ Generate an unique transaction id and associate the id with the payment request
    """
    transaction_id = str(uuid.uuid1())
    redis_connection = Redis(host="payment_provider_redis", port=6379, db=0)
    redis_connection.set(transaction_id, json.dumps(payment_request.to_dict()))
    redis_connection.close()
    """ Return the url where the user can pay
    """
    frontend_url = environ.get("PAYMENT_PROVIDER_FRONTEND",
                               "http://0.0.0.0:4002")
    redirect_page = f'{frontend_url}/?transaction_id={transaction_id}'  # The transaction id will be used to retrieve the informations saved on redis
    return PaymentCreationResponse(redirect_page=redirect_page,
                                   transaction_id=transaction_id)
Exemple #3
0
    def image_search(self, request, pk=None):
        redis_connection = Redis()
        course = self.get_object()

        search_result = redis_connection.get(hash(course))
        if not search_result:
            search_string = str(course)
            search_token = re.findall(
                r'vqd=\'(.*)\';',
                requests.post("https://duckduckgo.com/",
                              data={
                                  'q': search_string
                              }).text)[0]
            search_result = requests.get(
                "https://duckduckgo.com/i.js?l=it-it&o=json&vqd=%s&q=%s&f=,,,&p=2"
                % (search_token, search_string),
                headers={
                    'Dnt': '1',
                    'X-Requested-With': 'XMLHttpRequest',
                    'User-Agent':
                    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
                    'Accept': 'application/json, text/javascript, */*; q=0.01',
                    'Referer': 'https://duckduckgo.com/',
                    'Authority': 'duckduckgo.com',
                    'Host': 'duckduckgo.com',
                    'Connection': 'keep-alive',
                    'Cache-Control': 'no-cache',
                }).text
            redis_connection.append(hash(course), search_result)
        redis_connection.close()
        return Response(json.loads(search_result)['results'][0])
class RedisMessageSubscriber(implements(MessageSubscriber)):
    def __init__(self):
        self.__client = Redis(connection_pool=settings.REDIS_CONNECTION_POOL)
        self.__stop = False

    async def subscribe_message(self, topic, message_handler):
        p = self.__client.pubsub()
        p.subscribe(topic)
        while self.stop is False:
            await asyncio.sleep(1)

            message = p.get_message(ignore_subscribe_messages=True)
            if message is None:
                continue

            try:
                message_handler.handle(pickle.loads(message['data']))
            except Exception as ex:
                # TODO: refactor it
                # TODO: replace print() with logger
                print(''.join(
                    traceback.format_exception(None, ex, ex.__traceback__)))

    @property
    def stop(self):
        return self.__stop

    def close(self):
        self.__stop = True
        self.__client.close()
def send_payment_information(payment_information=None):  # noqa: E501
    """sendPaymentInformation

    Sends the information received by the user for verification purposes. API for: Payment Provider # noqa: E501

    :param payment_information: 
    :type payment_information: dict | bytes

    :rtype: None
    """
    if connexion.request.is_json:
        payment_information = PaymentInformation.from_dict(connexion.request.get_json())  # noqa: E501

    """ Open the connection to redis to get the right process instance id in the Camunda Engine (saved by a worker)
    """
    redis_connection = Redis(host="acmesky_redis", port=6379, db=0)
    process_instance_id = redis_connection.get(payment_information.transaction_id).decode("utf-8")
    redis_connection.close()

    """ Send a correlate message to Camunda with the information about the payment status. The message is for a specific
        process instance, identified by the process_instance_id variable 
    """
    r = send_string_as_correlate_message("payment_status",
                                         [("payment_status", json.dumps(payment_information.to_dict()))],
                                         process_instance_id)
    if r.status_code >= 300:
        logging.error(f"Fail to send message to Camunda. Response: {r.text}")
    return None, r.status_code
Exemple #6
0
def main_currency():
    """main f-n -- engine of the update currency"""
    r = Redis(host='redis',
              username='******',
              password='******',
              port=6379,
              db=0)

    subscribers = r.get('subscribers')
    logger.info(subscribers)
    if subscribers:
        response = requests.get(url=MAIN_URL)
        data = response.json()['rates']
        text = f'<u>LAST UPDATE: {datetime.now(tz=timezone("Europe/Kiev")):%Y-%m-%d -- %H:%M:%S}</u>\n\n'

        for i in sorted(data,
                        key=lambda key: key['pair']):  # sorted Alphabetical
            if i['pair'] in [
                    'BTC_ETH', 'ETH_BTC', 'BTC_USDT', 'ETH_UAH', 'USDT_UAH',
                    'ETH_USDT', 'BTC_UAH'
            ]:
                text += f"<b>{i['pair']}</b> -- {i['price']}\n"
        list_to_send = subscribers.decode("utf-8").split()
        logger.info(list_to_send)

        loop.run_until_complete(send(r, list_to_send,
                                     text))  # async run send f-n
    else:
        pass
    r.close()
Exemple #7
0
def update_charts(signal, instance, **kwargs):
    channel_layer = layers.get_channel_layer()
    now = localtime()
    user = instance.user.name
    value = str(instance.customer_balance)

    db = Redis(host='localhost')
    redis_data = get_or_create_redis(user, db)
    redis_time = get_or_create_redis(str(user) + 'time', db)
    result = PushList(redis_data)
    time = PushList(redis_time, type=str)

    result.add(value)
    time.add(now.tm_hour)

    db.delete(user)
    db.delete(user + 'time')
    db.rpush(user, *result.data)
    db.rpush(user + 'time', *time.data)

    db.close()

    async_to_sync(channel_layer.group_send)(user, {
        'type': 'chat_message',
        'text': {
            'time': time.data,
            'data': result.data
        }
    })
Exemple #8
0
def redis_reset_startup(redis_con: redis.Redis, process_name: str,
                        key_run: str, key_time: str, key_time_ete: str,
                        key_exception: str, key_current: str,
                        key_total: str) -> None:
    """Check if Redis key for running is set and when ok than reset the keys stored in redis database

    :param redis_con: an instance of Redis connection
    :param process_name: the name of the action that will be started or not
    :param key_run: Redis key for run indication
    :param key_time: Redis key to store time information (start time or runtime)
    :param key_time_ete: Redis key to store start time of time calculation
    :param key_exception: Redis key where exception information is stored
    :param key_current: Redis key containing progressed items count
    :param key_total: Redis key containing total to progress item count
    """
    if int(redis_con.get(key_run) or 0) == 1:
        redis_con.close()
        flask.abort(400, process_name + " already running")

    # (re)set Redis keys
    redis_con.set(key_run, 1)
    redis_con.set(key_time, int(datetime.timestamp(datetime.now())))
    redis_con.delete(key_time_ete)
    redis_con.delete(key_exception)
    redis_con.set(key_current, 0)
    redis_con.set(key_total, 0)
Exemple #9
0
class TestInvertedIndex:
    def setup_class(self):
        self.client = Redis(decode_responses=True)

        self.laptops = laptops = InvertedIndex(self.client)
        laptops.add_index("MacBook Pro", "Apple", "MacOS", "13inch")
        laptops.add_index("MacBook Air", "Apple", "MacOS", "13inch")

        laptops.add_index("X1 Carbon", "ThinkPad", "Windows", "13inch")

        laptops.add_index("T450", "ThinkPad", "Windows", "14inch")

        laptops.add_index("XPS", "DELL", "Windows", "13inch")

    def test_get_keywords(self):
        assert {"Apple", "MacOS",
                "13inch"} == self.laptops.get_keywords("MacBook Pro")

    def test_get_items(self):
        assert self.laptops.get_items("13inch") == {
            "MacBook Pro", "MacBook Air", "X1 Carbon", "XPS"
        }
        assert self.laptops.get_items(
            "13inch", "Apple") == {"MacBook Pro", "MacBook Air"}

    def teardown_class(self):
        self.client.close()
Exemple #10
0
class RedisZ(object):
    def __init__(self):
        self.redis_conn = Redis(connection_pool=conn_pool)

    def __enter__(self):
        return self.redis_conn

    def __exit__(self, exc, value, traceback):
        self.redis_conn.close()
Exemple #11
0
def redis_available():
    params = dict(host=os.getenv('REDIS_HOST') or '127.0.0.1',
                  port=int(os.getenv('REDIS_PORT') or 6379))
    conn = Redis(**params)
    try:
        conn.keys('*')
        conn.close()
        return True
    except Exception:
        return False
 def set_record(self, key, value):
     """
     Function to set record to database.
     It has nothing to do with states.
     Made for backend compatibility
     """
     connection = Redis(connection_pool=self.redis)
     connection.set(self.prefix + str(key), json.dumps(value))
     connection.close()
     return True
 def delete_record(self, key):
     """
     Function to delete record from database.
     It has nothing to do with states.
     Made for backend compatibility
     """
     connection = Redis(connection_pool=self.redis)
     connection.delete(self.prefix + str(key))
     connection.close()
     return True
 def get_record(self, key):
     """
     Function to get record from database.
     It has nothing to do with states.
     Made for backend compatibility
     """
     connection = Redis(connection_pool=self.redis)
     result = connection.get(self.prefix + str(key))
     connection.close()
     if result: return json.loads(result)
     return
Exemple #15
0
class RedisPlieLine:
    conn = None

    def open_spider(self, spider):
        self.conn = Redis(host='127.0.0.1', port=6379)

    def close_spider(self, spider):
        self.conn.close()

    def process_item(self, item, spider):
        self.conn.lpush(jdsn.Jdssspider().key, str(item))
 def start_requests(self):
     redis = Redis(host='www.daoyang.top', port=6379, db=0)
     while redis.llen(REDIS_DS_KEY) > 0:
         spider.logger.debug("REDIS POP BEFORE")
         email = redis.lpop(REDIS_DS_KEY)
         redis.close()
         spider.logger.debug("REDIS POP AFTER")
         if email:
             yield scrapy.Request(bytes.decode(email), callback=self.parse)
             spider.logger.debug("REDIS YIELD YIELD YIELD YIELD")
         else:
             break
Exemple #17
0
class ZhongwenPlieLine:
    conn = None

    def open_spider(self, spider):
        self.conn = Redis(host='127.0.0.1', port=6379)

    def close_spider(self, spider):
        self.conn.close()

    def process_item(self, item, spider):
        # self.conn.lpush(zhongwentwo.FenghuangSpider().Key, str(item))
        self.conn.lpush(f'<<{item["title"]}>>{item["author"]}', str(item))
class RedisClient:
    def __init__(self, options):
        self.connection_options = options

    def __enter__(self):
        try:
            self.connection = Redis(**self.connection_options)
            return self.connection
        except Exception as e:
            raise Exception(f'ERROR: Cannot connect to Redis: {str(e)}')

    def __exit__(self, exc_type, exc_value, traceback):
        self.connection.close()
Exemple #19
0
class PttSpiderPipeline(object):
    def open_spider(self, spider):
        # init connection with MongoDB and Redis
        self.client = MongoClient(MONGO_HOST)
        self.db = self.client[PTT_DB]
        self.r = Redis(host=REDIS_HOST)

    def close_spider(self, spider):
        # remove job_id from redis, since the job is done
        self.r.delete(spider.job_id)
        # close connection while closing spider
        self.client.close()
        self.r.close()

    def process_item(self, item, spider):
        # check needed fields exists otherwise drop item
        needed_fields = {
            'authorId', "authorName", "title", "publishedTime", "content",
            "canonicalUrl"
        }
        missing_fields = needed_fields - set(item.keys())
        if missing_fields != set():
            raise DropItem(missing_fields)
        # init collection
        article_col = self.db['article']
        comment_col = self.db['comment']
        # use authorId and publishedTime as unique ID for document
        author_and_time = {
            "authorId": item.pop('authorId'),
            "publishedTime": item.pop('publishedTime')
        }
        # extract comments from article and update unique ID for comments
        comments = item.pop('comments', None)
        article = dict(item)
        now = dt.now()
        article['updateTime'] = now
        # update article and comments to MongoDB
        article_col.update_one(author_and_time, {
            "$set": article,
            "$setOnInsert": {
                'createdTime': now
            }
        },
                               upsert=True)
        if comments is not None:
            comments = [{**c, **author_and_time} for c in comments]
            comment_col.remove(author_and_time)
            comment_col.insert_many(comments)
        # record url in redis
        self.r.sadd(spider.job_id, article['canonicalUrl'])
        return item
def get_payment_details(transaction_id):  # noqa: E501
    """Your GET endpoint

    Gets the information for the payment request for a user. API for: User # noqa: E501

    :param transaction_id: ID of transaction
    :type transaction_id: 

    :rtype: PaymentRequest
    """
    redis_connection = Redis(host="payment_provider_redis", port=6379, db=0)
    payment_request = json.loads(redis_connection.get(transaction_id))
    redis_connection.close()
    return PaymentRequest.from_dict(payment_request)
 def restore_redis():
     from redis import Redis
     r = Redis(host='0.0.0.0', db=5)
     updated_conf = {
         "HOST": "0.0.0.0",
         "PORT": "6000",
         "USER": "******",
         "PASSWORD": "******",
         "NAME": "postgres"
     }
     r.set('tenant_1_innovaccer_xyz_incare_orm_config_django_orm_default', json.dumps(
         updated_conf
     ))
     r.close()
Exemple #22
0
class SimpleRedisDispatcher(AbstractRedisDispatcher):
    redis_args = None
    redis_kwargs = None

    def __init__(self, *redis_args, **redis_kwargs):
        self.redis_args = redis_args
        self.redis_kwargs = redis_kwargs
        AbstractRedisDispatcher.__init__(self)
        self.redis = Redis(*self.redis_args, **self.redis_kwargs)

    def get_redis(self):
        return self.redis

    def close(self):
        self.redis.close()
def is_backend_running() -> bool:
    try:
        conn = Redis(host=get_redis_host(),
                     port=int(get_redis_port()),
                     db=int(get_redis_dbnum()),
                     password=get_redis_password())
        conn.client_list()  # Must perform an operation to check connection.
    except ConnectionError as e:
        print("Failed to connect to Redis instance at %s", get_redis_host())
        print(repr(e))
        return False

    conn.close()  # type: ignore

    return True
Exemple #24
0
class RedisRpcServer(implements(RpcServer)):
    def __init__(self):
        self.__client = Redis(connection_pool=settings.REDIS_CONNECTION_POOL)
        self.__stop = False

    async def register_handler(self, topic, request_handler):
        while self.stop is False:
            request = None
            try:
                if self.__client.llen(name=topic) < 1:
                    await asyncio.sleep(1)
                    continue

                request = self.__client.rpop(topic)
                request = pickle.loads(request)
                """
                when request successfully unmarshalled, process request
                with request_handler
                """
                result = request_handler.handle(request.params)
                """
                send back result to client
                """
                self.__client.rpush(
                    request.id,
                    pickle.dumps(RpcResponse(result=result, id=request.id)))
                self.__client.expire(request.id, time=15)
            except CancelledError:
                self.close()
                break
            except Exception as ex:
                # TODO: refactor it
                # TODO: replace print() with logger
                print(''.join(
                    traceback.format_exception(None, ex, ex.__traceback__)))

                self.__client.rpush(
                    request.id,
                    pickle.dumps(RpcResponse(result=ex, id=request.id)))
                self.__client.expire(request.id, time=15)

    @property
    def stop(self):
        return self.__stop

    def close(self):
        self.__stop = True
        self.__client.close()
class TestAutoComplete:
    def setup_class(self):
        self.client = Redis(decode_responses=True)

    def test_auto_complete(self):
        auto_complete = AutoComplete(self.client)
        auto_complete.feed("黄晓明", 3)
        auto_complete.feed("黄建宏", 2)
        auto_complete.feed("黄晓军", 1)

        assert auto_complete.hint("黄") == ["黄晓明", "黄建宏", "黄晓军"]
        assert auto_complete.hint("黄晓") == ["黄晓明", "黄晓军"]

    def teardown_class(self):
        keys = [k for k in self.client.keys()]
        self.client.delete(*keys)
        self.client.close()
    def update_redis(*args, **kwargs):

        from redis import Redis
        r = Redis(host='0.0.0.0', db=5)
        updated_conf = {
            "HOST": "0.0.0.0",
            "PORT": "6001",
            "USER": "******",
            "PASSWORD": "******",
            "NAME": "postgres"
        }
        r.set('tenant_1_innovaccer_xyz_incare_orm_config_django_orm_default', json.dumps(
            updated_conf
        ))
        r.close()
        expected_result.clear()
        print("new expected_result is ", expected_result)
def insert():
    import json
    from redis import Redis
    from urllib import parse

    # 获取redis连接对象
    redis = Redis(host="114.116.126.177", password="******")
    # redis = Redis()
    with open("kind.json", mode="r", encoding="utf-8") as f:
        datas = json.loads(f.read())

    for data in datas:
        print(data)
        redis.lpush("hy:start_urls",
                    f'http://so.huangye88.com/?kw={data}&type=company&')

    # 关闭redis
    redis.close()
Exemple #28
0
def insert():
    import json
    from redis import Redis
    from urllib import parse

    # 获取redis连接对象
    redis = Redis(host="114.116.126.177", password="******")
    # redis = Redis()
    with open("kind.json", mode="r", encoding="utf-8") as f:
        datas = json.loads(f.read())

    for data in datas:
        print(data)
        redis.lpush("lbw:start_urls",
                    f'http://b2b.liebiao.com/sou-gs/{data}/?pn=1')

    # 关闭redis
    redis.close()
Exemple #29
0
def test_data_retrieval_within_the_minute():
    """check update doesn't occur within the 1 minute
       delete data and assert no data is retreivable
    """

    redis_conn = Redis(app.config["REDIS_URL"])

    # first call
    requests.post(url=UPDATE_URL)
    res = requests.get(url=GET_MOVIES_URL)
    assert res.status_code == 200

    redis_conn.delete("ghibli")

    # second call
    res = requests.get(url=GET_MOVIES_URL)
    assert res.status_code != 200

    redis_conn.close()
class GatherMetricToStorage(object):
    def __init__(self, duration, recsqant):
        """
        Initialize storage connection
        """
        echo(" * Initializing metric fetch system...")
        self.baseobjc = Redis(host="127.0.0.1", port=6379)
        self.duration = duration
        self.recsqant = recsqant

    def jsonify_system_live_updating_metrics(self):
        """
        Convert metric data to a JSON-friendly format
        """
        timestmp = str(time()).split(".")[0]
        hashiden = sha256(timestmp.encode()).hexdigest()
        keyvalpr = {
            timestmp:
            json.dumps({
                "hashiden": hashiden,
                "liveupdt": LiveUpdatingElements().return_live_data()
            })
        }
        return keyvalpr

    def continuously_store_data(self):
        """
        Periodically push passive metrics to Redis store
        """
        self.baseobjc.flushall()
        try:
            while True:
                if self.baseobjc.dbsize() == self.recsqant:
                    self.baseobjc.keys().sort()
                    self.baseobjc.delete(self.baseobjc.keys()[0])
                self.baseobjc.mset(self.jsonify_system_live_updating_metrics())
                # echo(" * [" + ctime() + "] Stored system metrics now...")
                sleep(self.duration)
        except KeyboardInterrupt as expt:
            self.baseobjc.close()
            echo("\n" + " * Closing storage connection...")
            exit()