def collect_process_label_reddit_data(**kwargs):
hook = AwsLambdaHook('redditbitcoin',
region_name='eu-west-1',
log_type='None', qualifier='$LATEST',
invocation_type='RequestResponse',
config=None, aws_conn_id='aws_default_FreddieReid')
response_1 = hook.invoke_lambda(payload='null')
log.info("collected r/bitcoin data and added sentiment labels")
print('Response--->', response_1)
def reddit_to_dynamo_db(**kwargs):
hook = AwsLambdaHook('reddit_to_dynamo',
region_name='eu-west-1',
log_type='None', qualifier='$LATEST',
invocation_type='RequestResponse',
config=None, aws_conn_id='aws_default_FreddieReid')
response_1 = hook.invoke_lambda(payload='null')
log.info("sent twitter data to dynamo db")
print('Response--->', response_1)
def lambda_preprocessing(**kwargs):
hook = AwsLambdaHook('twitterpreprocessing',
region_name='eu-west-1',
log_type='None', qualifier='$LATEST',
invocation_type='RequestResponse',
config=None, aws_conn_id='aws_default_FreddieReid')
response_1 = hook.invoke_lambda(payload='null')
log.info("preprocessed the twitter data")
print('Response--->', response_1)
def create_sentiment_labels(**kwargs):
hook = AwsLambdaHook('textblob',
region_name='eu-west-1',
log_type='None', qualifier='$LATEST',
invocation_type='RequestResponse',
config=None, aws_conn_id='aws_default_FreddieReid')
response_1 = hook.invoke_lambda(payload='null')
log.info("added sentiment labels to twitter data")
print('Response--->', response_1)
def trigger_download_lambda(payload):
"""Sends Message to AWS lambda to perform download"""
hook = AwsLambdaHook(
aws_conn_id=DOWNLOAD_OPERATOR_USE_CONNECTION,
function_name=DOWNLOAD_OPERATOR_USE_LAMBDA_NAME,
region_name=DOWNLOAD_OPERATOR_USE_LAMBDA_REGION,
)
resp = hook.invoke_lambda(payload=payload)
return resp
def create_thumbnail(ds, **kwargs):
hook = AwsLambdaHook('LAMBDA_FN_NAME', #LAMBDA_FN_NAME
log_type='None',qualifier='$LATEST',
invocation_type='RequestResponse',
config=None,aws_conn_id='aws_default')
response_1 = hook.invoke_lambda(payload=json.dumps(kwargs['dag_run'].conf))
payload = json.loads(response_1['Payload'].read().decode())
kwargs['ti'].xcom_push(key="ThumbnailDetails", value=payload)
def test_invoke_lambda_function(self):
hook = AwsLambdaHook(aws_conn_id='aws_default',
function_name="test_function", region_name="us-east-1")
with patch.object(hook.conn, 'invoke') as mock_invoke:
payload = '{"hello": "airflow"}'
hook.invoke_lambda(payload=payload)
mock_invoke.asset_called_once_with(
FunctionName="test_functin",
InvocationType="RequestResponse",
LogType="None",
Payload=payload,
Qualifier="$LATEST"
)
def test_invoke_lambda_function(self):
hook = AwsLambdaHook(aws_conn_id='aws_default',
function_name="test_function",
region_name="us-east-1")
hook.get_conn().create_function(
FunctionName='test_function',
Runtime='python2.7',
Role='test-iam-role',
Handler='lambda_function.lambda_handler',
Code={
'ZipFile': self.lambda_function(),
},
Description='test lambda function',
Timeout=3,
MemorySize=128,
Publish=True,
)
payload = {'hello': 'airflow'}
response = hook.invoke_lambda(payload=json.dumps(payload))
self.assertEqual(response["StatusCode"], 202)
def test_get_conn_returns_a_boto3_connection(self):
hook = AwsLambdaHook(aws_conn_id='aws_default',
function_name="test_function",
region_name="us-east-1")
self.assertIsNotNone(hook.get_conn())
