def test_log_info(capsys):
logger = logging.get_logger(__name__, config)
logger = logging.get_logger(__name__, config)
logger.info("test_log_info")
captured = capsys.readouterr()
record_object = json.loads(captured.out)
assert record_object["message"] == "test_log_info"
assert record_object["logger"] == "test_logging"
assert record_object["level"] == "info"
assert record_object["timestamp"] != ""
def __init__(self):
configParser = ConfigParser()
self.log = logging.get_logger(__name__, config=configParser)
self.config = configParser.app_cfg
try:
self.rabbitClient = rabbit.RabbitClient()
except pika.exceptions.AMQPConnectionError as error:
self.log.error("Connection to RabbitMQ failed.")
raise error
def __init__(self):
config_parser = ConfigParser()
self.config = config_parser.app_cfg
self.log = logging.get_logger(__name__, config=config_parser)
self.threads = []
try:
self.rabbit_client = RabbitClient()
except AMQPConnectionError as error:
self.log.error("Connection to RabbitMQ failed.")
raise error
def test_logger_name(caplog, capsys):
logger = logging.get_logger("test_naam", config)
logger.info("test")
out, err = capsys.readouterr()
record_object = json.loads(out)
assert record_object["message"] == "test"
assert record_object["logger"] == "test_naam"
assert record_object["level"] == "info"
assert record_object["timestamp"] != ""
def test_log_critical(caplog, capsys):
logger = logging.get_logger(__name__, config)
logger.critical("test_log_critical")
out, err = capsys.readouterr()
record_object = json.loads(out)
assert record_object["message"] == "test_log_critical"
assert record_object["logger"] == "test_logging"
assert record_object["level"] == "critical"
assert record_object["timestamp"] != ""
def __init__(self):
configParser = ConfigParser()
self.log = logging.get_logger(__name__, config=configParser)
self.config = configParser.app_cfg
self.ftp_client = FTPClient(configParser)
self.mh_client = MediahavenClient(configParser)
self.event_parser = EventParser()
try:
self.rabbit_client = RabbitClient()
except AMQPConnectionError as error:
self.log.error("Connection to RabbitMQ failed.")
raise error
def __init__(self):
configParser = ConfigParser()
self.log = logging.get_logger(__name__, config=configParser)
self.rabbitConfig = configParser.app_cfg["rabbitmq"]
self.credentials = pika.PlainCredentials(self.rabbitConfig["username"],
self.rabbitConfig["password"])
self.connection = pika.BlockingConnection(
pika.ConnectionParameters(
host=self.rabbitConfig["host"],
port=self.rabbitConfig["port"],
credentials=self.credentials,
))
self.channel = self.connection.channel()
self.prefetch_count = int(self.rabbitConfig["prefetch_count"])
def __init__(self):
configParser = ConfigParser()
self.config = configParser.app_cfg
self.log = logging.get_logger(__name__, config=configParser)
self.mh_client = MediahavenClient(self.config)
try:
self.rabbit_client = RabbitClient()
except AMQPConnectionError as error:
self.log.error("Connection to RabbitMQ failed.")
raise error
self.pid_service = PIDService(self.config["pid-service"]["URL"])
self.essence_linked_rk = self.config["rabbitmq"][
"essence_linked_routing_key"]
self.essence_unlinked_rk = self.config["rabbitmq"][
"essence_unlinked_routing_key"]
self.object_deleted_rk = self.config["rabbitmq"][
"object_deleted_routing_key"]
self.get_metadata_rk = self.config["rabbitmq"][
"get_metadata_routing_key"]
@author: tina
"""
from sys import exit
import json
import uuid
import configparser
from json import JSONDecodeError
import pika
from retry import retry
from viaa.observability import logging
from viaa.configuration import ConfigParser
from s3_io.create_url_to_filesystem_task import process
config = ConfigParser()
config_ = configparser.ConfigParser()
logger = logging.get_logger('s3io', config)
swarmurl = config.app_cfg['castor']['swarmurl']
@retry(pika.exceptions.AMQPConnectionError,
delay=5,
tries=-1,
backoff=2,
jitter=(1, 3))
def __main__():
"""
Description:
- Cunsumes from rabbitMQ queue and creates a job,
msg ack on async task create
def __init__(self, configParser: ConfigParser = None):
self.log = logging.get_logger(__name__, config=configParser)
self.cfg: dict = configParser.app_cfg
self.token_info = None
self.url = f'{self.cfg["mediahaven"]["host"]}/media/'
Created on Wed Jan 8 16:25:28 2020
@author: tina
"""
from viaa.observability import logging
from viaa.configuration import ConfigParser
from celery import Celery
from kombu import Exchange, Queue
from s3_io.s3io_tools import SwarmS3Client, SwarmIo
import s3_io.celeryconfig as celeryconfig
from s3_io.remote_curl import RemoteCurl
app = Celery('s3io', )
app.config_from_object(celeryconfig)
config = ConfigParser()
logger = logging.get_logger('s3io')
app.config_from_object(celeryconfig)
app.conf.task_queues = (Queue('s3io',
Exchange('py-worker-s3io'),
routing_key='s3io'), )
app.conf.task_default_queue = 's3io'
app.conf.task_default_exchange_type = 'direct'
app.conf.task_default_routing_key = 's3io'
s3access_key = config.app_cfg['S3_TO_FTP']['s3access_key']
s3secret_key = config.app_cfg['S3_TO_FTP']['s3secret_key']
swarmurl = config.app_cfg['castor']['swarmurl']
@app.task(max_retries=5, bind=True)
def swarm_to_ftp(self, **body):
@author: tina
"""
import sys
import threading
import atexit
from s3_io.event_consumer import __main__ as Consume
from s3_io.s3io_api import __main__ as Api
from s3_io.s3io_tasks import app
from viaa.observability import logging
from viaa.configuration import ConfigParser
from celery.signals import setup_logging, task_postrun, task_prerun
from celery.result import AsyncResult
config = ConfigParser()
logger = logging.get_logger('s3io.scalable_worker', config)
@task_postrun.connect
def log_task_complete(sender, task_id, task, args, **kwargs):
"""Runs on task complete """
# add_rabbithandler()
result = AsyncResult(task_id).result
if result is None:
result = 'NO_RESULT_FOUND'
@task_prerun.connect
def log_task_Started(sender, task_id, task, args, **kwargs):
"""RUNS ON TASK START"""
try:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Apr 4 11:00:16 2020
@author: tina
"""
import paramiko
from viaa.observability import logging
from viaa.configuration import ConfigParser
config = ConfigParser()
logger = logging.get_logger('s3_auditlog_parser', config)
def remote_get(url, dest_path):
"""Description:
- NOT USED atm
- Downlod url to dest_path, using paramiko and curl
Arguments:
- dest_path: string
- url : string
"""
remote_client = paramiko.SSHClient()
- Uses celery signature:
s3_io.s3io_tasks.swarm_to_remote.s(body=msg)
0k2699098k-left.mp4
@author: tina
"""
import uuid
from viaa.observability import logging
from viaa.configuration import ConfigParser
from s3_io.s3io_tasks import swarm_to_remote
config = ConfigParser()
logger = logging.get_logger('s3io.task_creator')
extra = {'app_name': 's3io'}
rnd = str(uuid.uuid4().hex)
debug_msg = {
"service_type": "celery",
"service_name": "s3_to_filesystem",
"service_version": "0.1",
"x-request-id": rnd,
"source": {
"domain": {
"name": "s3-qas.viaa.be"
},
"bucket": {
"name": "mam-highresvideo"
},
def __init__(self, configParser: ConfigParser = None):
self.log = logging.get_logger(__name__, config=configParser)
self.cfg: dict = configParser.app_cfg
self.host = self.__set_host()
self.conn = self.__connect()
import functools
import time
from viaa.observability import logging
log = logging.get_logger(__name__)
log.setLevel("DEBUG")
class RetryException(Exception):
""" Exception raised when an action needs to be retried
in combination with _retry decorator"""
pass
DELAY = 1
BACKOFF = 2
NUMBER_OF_TRIES = 5
def retry(exceptions):
def decorator_retry(func):
@functools.wraps(func)
def wrapper(self, *args, **kwargs):
delay = DELAY
tries = NUMBER_OF_TRIES
while tries:
tries -= 1
try:
return func(self, *args, **kwargs)
@author: tina
"""
import sys
import threading
import atexit
from s3_io.event_consumer import __main__ as Consume
from s3_io.s3io_api import __main__ as Api
from s3_io.s3io_tasks import app
from viaa.observability import logging
from viaa.configuration import ConfigParser
from celery.signals import setup_logging, task_postrun, task_prerun
from celery.result import AsyncResult
config = ConfigParser()
logger = logging.get_logger('s3io.worker', config)
@task_postrun.connect
def log_task_complete(sender, task_id, task, args, **kwargs):
"""Runs on task complete """
# add_rabbithandler()
result = AsyncResult(task_id).result
if result is None:
result = 'NO_RESULT_FOUND'
@task_prerun.connect
def log_task_Started(sender, task_id, task, args, **kwargs):
"""RUNS ON TASK START"""
try:
r=RemoteCurl(url="http://10.50.152.194:80/tests3vents/0k2699098k-left.mp4", dest_path='/mnt/temptina/tmp/test 123456.x
```
@author: tina
"""
import os
import time
import threading
from functools import update_wrapper
import paramiko
import requests
from viaa.observability import logging
from requests.exceptions import HTTPError
from viaa.configuration import ConfigParser
config = ConfigParser()
logger = logging.get_logger('s3io.remote_curl')
def decorator(func_n):
"Make function d a decorator: d wraps a function fn."
def _d(f_n):
return update_wrapper(func_n(f_n), f_n)
update_wrapper(_d, func_n)
return _d
@decorator
def timeit(func_name):
"""time a function, used as decorator"""
from fastapi import APIRouter, BackgroundTasks, Depends
from viaa.configuration import ConfigParser
from viaa.observability import logging
from app.core.event_handler import handle_event
from app.core.events_parser import parse_premis_events
from app.models.premis_events import PremisEvents
from app.models.xml_body import XmlBody
router = APIRouter()
config = ConfigParser()
log = logging.get_logger(__name__, config=config)
@router.post("/", status_code=202)
async def handle_events(
background_tasks: BackgroundTasks,
premis_events: PremisEvents = Depends(
XmlBody(PremisEvents, parse_premis_events)),
):
"""
Returns OK if the xml parsing didn't crash.
"""
events = premis_events.events
archived_events = [
event for event in events if event.is_valid and event.has_valid_outcome
]
log.info(
@author: tina
"""
import sys
import threading
import atexit
from s3_io.event_consumer import __main__ as Consume
from s3_io.s3io_api import __main__ as Api
from s3_io.s3io_tasks import app
from viaa.observability import logging
from viaa.configuration import ConfigParser
from celery.signals import setup_logging, task_postrun, task_prerun
from celery.result import AsyncResult
config = ConfigParser()
logger = logging.get_logger('s3io.input', config)
@task_postrun.connect
def log_task_complete(sender, task_id, task, args, **kwargs):
"""Runs on task complete """
# add_rabbithandler()
result = AsyncResult(task_id).result
if result is None:
result = 'NO_RESULT_FOUND'
@task_prerun.connect
def log_task_Started(sender, task_id, task, args, **kwargs):
"""RUNS ON TASK START"""
try:
