def serialize_value(value: Any):
hook = RedisHook(redis_conn_id=XComRedisBackend.CONN_ID)
key = str(uuid4())
# We use the default serializer, which pickles or JSONs
hook.get_conn().set(key, pickle.dumps(value))
# Add prefix to make it clear where the value is stored.
value = XComRedisBackend.PREFIX + key
return BaseXCom.serialize_value(value)
def test_get_conn(self):
hook = RedisHook(redis_conn_id='redis_default')
self.assertEqual(hook.redis, None)
self.assertEqual(hook.host, None, 'host initialised as None.')
self.assertEqual(hook.port, None, 'port initialised as None.')
self.assertEqual(hook.password, None, 'password initialised as None.')
self.assertEqual(hook.db, None, 'db initialised as None.')
self.assertIs(hook.get_conn(), hook.get_conn(), 'Connection initialized only if None.')
def test_get_conn(self):
hook = RedisHook(redis_conn_id='redis_default')
assert hook.redis is None
assert hook.host is None, 'host initialised as None.'
assert hook.port is None, 'port initialised as None.'
assert hook.password is None, 'password initialised as None.'
assert hook.db is None, 'db initialised as None.'
assert hook.get_conn() is hook.get_conn(), 'Connection initialized only if None.'
def serialize_value(value: Any):
"""
Docstring goes here
"""
hook = RedisHook()
hook.get_conn()
redis = hook.redis
key = f"data_{uuid.uuid4()}"
xcom = {key: json.dumps(value)}
redis.mset(xcom)
return BaseXCom.serialize_value(key)
def deserialize_value(result) -> Any:
"""
Docstring goes here
"""
result = BaseXCom.deserialize_value(result)
if isinstance(result, str):
hook = RedisHook()
hook.get_conn()
redis = hook.redis
xcom = redis.mget(result)
result = eval(xcom[0])
return result
def test_poke(self):
hook = RedisHook(redis_conn_id='redis_default')
redis = hook.get_conn()
redis.set('test_key', 'test_value')
self.assertTrue(self.sensor.poke(None), "Key exists on first call.")
redis.delete('test_key')
self.assertFalse(self.sensor.poke(None), "Key does NOT exists on second call.")
def test_poke_true(self):
sensor = RedisPubSubSensor(task_id='test_task',
dag=self.dag,
channels='test',
redis_conn_id='redis_default')
hook = RedisHook(redis_conn_id='redis_default')
redis = hook.get_conn()
redis.publish('test', 'message')
result = sensor.poke(self.mock_context)
self.assertFalse(result)
result = sensor.poke(self.mock_context)
self.assertTrue(result)
context_calls = [
call.xcom_push(key='message',
value={
'type': 'message',
'pattern': None,
'channel': b'test',
'data': b'message'
})
]
self.assertTrue(self.mock_context['ti'].method_calls == context_calls,
"context calls should be same")
result = sensor.poke(self.mock_context)
self.assertFalse(result)
def test_execute_hello(self):
operator = RedisPublishOperator(
task_id='test_task',
dag=self.dag,
message='hello',
channel=self.channel,
redis_conn_id='redis_default',
)
hook = RedisHook(redis_conn_id='redis_default')
pubsub = hook.get_conn().pubsub()
pubsub.subscribe(self.channel)
operator.execute(self.mock_context)
context_calls = []
assert self.mock_context[
'ti'].method_calls == context_calls, "context calls should be same"
message = pubsub.get_message()
assert message['type'] == 'subscribe'
message = pubsub.get_message()
assert message['type'] == 'message'
assert message['data'] == b'hello'
pubsub.unsubscribe(self.channel)
def test_real_get_and_set(self):
hook = RedisHook(redis_conn_id='redis_default')
redis = hook.get_conn()
self.assertTrue(redis.set('test_key', 'test_value'), 'Connection to Redis with SET works.')
self.assertEqual(redis.get('test_key'), b'test_value', 'Connection to Redis with GET works.')
self.assertEqual(redis.delete('test_key'), 1, 'Connection to Redis with DELETE works.')
def test_get_conn_with_extra_config(self, mock_get_connection, mock_redis):
connection = mock_get_connection.return_value
hook = RedisHook()
hook.get_conn()
mock_redis.assert_called_once_with(
host=connection.host,
password=connection.password,
port=connection.port,
db=connection.extra_dejson["db"],
ssl=connection.extra_dejson["ssl"],
ssl_cert_reqs=connection.extra_dejson["ssl_cert_reqs"],
ssl_ca_certs=connection.extra_dejson["ssl_ca_certs"],
ssl_keyfile=connection.extra_dejson["ssl_keyfile"],
ssl_cert_file=connection.extra_dejson["ssl_cert_file"],
ssl_check_hostname=connection.extra_dejson["ssl_check_hostname"])
def deserialize_value(result) -> Any:
result = BaseXCom.deserialize_value(result)
prefix = XComRedisBackend.PREFIX
if isinstance(result, str) and result.startswith(prefix):
key = result.replace(prefix, "")
hook = RedisHook(redis_conn_id=XComRedisBackend.CONN_ID)
result = hook.get_conn().get(key)
result = pickle.loads(result)
return result
def execute(self, context: Dict) -> None:
"""
Publish the message to Redis channel
:param context: the context object
:type context: dict
"""
redis_hook = RedisHook(redis_conn_id=self.redis_conn_id)
self.log.info('Sending messsage %s to Redis on channel %s', self.message, self.channel)
result = redis_hook.get_conn().publish(channel=self.channel, message=self.message)
self.log.info('Result of publishing %s', result)
def execute(self, context):
oracle = OracleHelper(self.oracle_conn_id)
redis = RedisHook(self.redis_conn_id)
self.log.info(f"Executing SQL:{self.sql}")
self.log.info("Extracting data from Oracle")
conn_redis = redis.get_conn()
records = oracle.get_rows_with_bind(sql=self.sql,
bind=self.dict_bind)
self.log.info("Inserting rows into Redis")
pipe = conn_redis.pipeline()
[pipe.lpush(self.name_redis_key, str(row)) for row in records]
pipe.execute()
self.log.info(f"Inserted {len(records)} rows.")
def test_real_ping(self):
hook = RedisHook(redis_conn_id='redis_default')
redis = hook.get_conn()
self.assertTrue(redis.ping(), 'Connection to Redis with PING works.')
def test_get_conn_password_stays_none(self):
hook = RedisHook(redis_conn_id='redis_default')
hook.get_conn()
self.assertEqual(hook.password, None)
def test_get_conn_password_stays_none(self):
hook = RedisHook(redis_conn_id='redis_default')
hook.get_conn()
assert hook.password is None