CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
import os
import requests
from requests.adapters import HTTPAdapter
from urllib3 import Retry
from helpers.logging import get_logger
from helpers.util import filesizeformat
FETCH_TIMEOUT = os.environ.get("FETCH_TIMEOUT", 60) # in seconds
logger = get_logger("fetch-service")
class ResourceFetchException(Exception):
pass
def requests_retry_session(
retries=3, backoff_factor=0.3, status_forcelist=(500, 502, 504), session=None,
):
"""
Drop-in replacement for requests.get with support for retries. Uses an exponential
backoff algorithm to sleep between retry attempts.
If the backoff_factor is 0.1, will sleep for [0.0s, 0.2s, 0.4s, ...] between retries.
:param retries: Total number of retries to allow.
from json import loads, dumps
from helpers.progress import ProgressPercentage
from helpers import botoutils as bu
from helpers import images, logging
class C:
UNPROCESSED = str(os.getenv("UNPROCESSED"))
IMAGES = str(os.getenv("IMAGES"))
TEMP = str(os.getenv("TEMP"))
THUMB_SIZE = int(os.getenv("THUMB_SIZE"))
IMAGE_EXTENSIONS = ["jpg", "jpeg"]
logger = logging.get_logger("process_images_lambda")
def lambda_handler(event, context):
s3 = boto3.client("s3")
s3r = boto3.resource("s3")
processed = []
# SQS triggers Lambda with event containing message payload and S3 records
# Process messages by printing out body and optional author name
for message in event["Records"]:
records = loads(message["body"]).get("Records")
if not records:
return {
"statusCode":
200,
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
from config import MetricHandler, MetricHandlerException # noqa
from helpers.logging import get_logger # noqa
from helpers.util import abspath, required_keys # noqa
from services.location_service import LocationService # noqa
sns = boto3.client("sns")
SENTRY_DSN = os.environ["SENTRY_DSN"]
SNS_RESULT_TOPIC_ARN = os.environ["SNS_RESULT_TOPIC_ARN"]
sentry_sdk.init(dsn=SENTRY_DSN, integrations=[AwsLambdaIntegration()
]) # AWS Lambda integration
logger = get_logger("worker-handler")
fetch_service = LocationService()
def lambda_handler(event, context):
"""
:param event: AWS Lambda uses this parameter to pass in event data to the handler.
For details, see: https://docs.aws.amazon.com/lambda/latest/dg/with-sns.html
:param context: AWS Lambda uses this parameter to provide runtime information to your handler.
For details, see: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
:return:
"""
logger.debug(f"Received event: {event['Records'][0]['Sns']['MessageId']}")
message = event["Records"][0]["Sns"]["Message"]
CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
import os
import json_api_doc
from helpers.logging import get_logger
from helpers.util import urljoin
from services.fetch_service import fetch_resource
SERVICE_API_ENDPOINT = os.environ["SERVICE_API_ENDPOINT"]
SERVICE_API_KEY = os.environ.get("SERVICE_API_KEY", None) # ApiKey (optional)
logger = get_logger("location-service")
class LocationServiceException(Exception):
pass
class LocationService:
headers = {"Accept": "application/vnd.api+json"}
def __init__(self):
if SERVICE_API_KEY:
self.headers["ApiKey"] = SERVICE_API_KEY
self.endpoint = SERVICE_API_ENDPOINT
def get_by_id(
from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
from config import MetricHandler # noqa
from helpers.logging import get_logger # noqa
from helpers.util import required_keys # noqa
sns = boto3.client("sns")
SENTRY_DSN = os.environ["SENTRY_DSN"]
SNS_WORKER_TOPIC_ARN = os.environ["SNS_WORKER_TOPIC_ARN"]
sentry_sdk.init(dsn=SENTRY_DSN, integrations=[AwsLambdaIntegration()]) # AWS Lambda integration
logger = get_logger("manager-handler")
def lambda_handler(event, context):
"""
:param event: AWS Lambda uses this parameter to pass in event data to the handler.
For details, see: https://docs.aws.amazon.com/lambda/latest/dg/with-sns.html
:param context: AWS Lambda uses this parameter to provide runtime information to your handler.
For details, see: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
:return:
"""
logger.debug(f"Received event: {event['Records'][0]['Sns']['MessageId']}")
message = event["Records"][0]["Sns"]["Message"]
decoded = json.loads(message)