# SPDX - License - Identifier: GPL-3.0-or-later
# ############################################################################### #
"""
Represents the messages passed between AMQ queues
"""
import json
import logging
import attr
from model.message.validation import stages
from utils.project.static_content import LOG_FORMAT
from utils.project.structure import get_log_file
logging.basicConfig(filename=get_log_file('job.log'),
level=logging.INFO,
format=LOG_FORMAT)
# pylint:disable=too-many-instance-attributes
@attr.s
class Message:
"""
A class that represents an AMQ Message.
Messages can be serialized and deserialized for sending messages to and from AMQ
"""
description = attr.ib(default=None)
facility = attr.ib(default="ISIS")
run_number = attr.ib(default=None)
instrument = attr.ib(default=None)
# ############################################################################### #
"""
Module to perform ICAT client functionality
Functions for login and query available from class
"""
import logging
import icat
from utils.settings import ICAT_SETTINGS
from utils.clients.abstract_client import AbstractClient
from utils.clients.connection_exception import ConnectionException
from utils.project.structure import get_log_file
from utils.project.static_content import LOG_FORMAT
logging.basicConfig(filename=get_log_file('icat_client.log'), level=logging.INFO, format=LOG_FORMAT)
class ICATClient(AbstractClient):
"""
This class provides a layer of abstraction from Python ICAT.
Only allowing logging in and querying.
"""
def __init__(self, credentials=None):
if not credentials:
credentials = ICAT_SETTINGS
super(ICATClient, self).__init__(credentials) # pylint:disable=super-with-arguments
self.client = icat.Client(self.credentials.host)
def connect(self):
"""
# Autoreduction Repository : https://github.com/ISISScientificComputing/autoreduce
#
# Copyright © 2020 ISIS Rutherford Appleton Laboratory UKRI
# SPDX - License - Identifier: GPL-3.0-or-later
# ############################################################################### #
"""
Creates a database session for the reduction database
"""
import logging
from model.database import DjangoORM
from utils.clients.connection_exception import ConnectionException
from utils.project.static_content import LOG_FORMAT
from utils.project.structure import get_log_file
logging.basicConfig(filename=get_log_file('django_database_client.log'), level=logging.INFO, format=LOG_FORMAT)
class DatabaseClient:
"""
Single access point for the mysql database
"""
# ORM object that describes the reduction jobs (data) in the database
data_model = None
# ORM object that describes the variables relating to reduction jobs in the database
variable_model = None
def connect(self):
"""
Get the connection to the database service and set the model variables
# ############################################################################### #
"""
Module to perform ICAT client functionality
Functions for login and query available from class
"""
import logging
import icat
from utils.settings import ICAT_SETTINGS
from utils.clients.abstract_client import AbstractClient
from utils.clients.connection_exception import ConnectionException
from utils.project.structure import get_log_file
from utils.project.static_content import LOG_FORMAT
logging.basicConfig(filename=get_log_file('icat_client.log'),
level=logging.INFO,
format=LOG_FORMAT)
class ICATClient(AbstractClient):
"""
This class provides a layer of abstraction from Python ICAT.
Only allowing logging in and querying.
"""
def __init__(self, credentials=None):
if not credentials:
credentials = ICAT_SETTINGS
super(ICATClient, self).__init__(credentials) # pylint:disable=super-with-arguments
self.client = icat.Client(self.credentials.host)
"""
Client class for retrieving files via SFTP from servers (e.g. CEPH)
"""
import os.path
import logging
import re
import pysftp
from utils.clients.abstract_client import AbstractClient
from utils.clients.connection_exception import ConnectionException
from utils.test_settings import SFTP_SETTINGS
from utils.project.structure import get_log_file
from utils.project.static_content import LOG_FORMAT
logging.basicConfig(filename=get_log_file('sftp_client.log'), level=logging.INFO,
format=LOG_FORMAT)
class SFTPClient(AbstractClient):
"""
This class allows files to be retrieved from SFTP servers
"""
def __init__(self, credentials=None):
if not credentials:
credentials = SFTP_SETTINGS
super(SFTPClient, self).__init__(credentials) # pylint:disable=super-with-arguments
self._connection = None
def connect(self):
"""
"""
Creates a database session for the reduction database
"""
import logging
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import create_engine, MetaData, Table
from sqlalchemy.orm import sessionmaker, relationship
from utils.settings import MYSQL_SETTINGS
from utils.clients.abstract_client import AbstractClient
from utils.clients.connection_exception import ConnectionException
from utils.project.structure import get_log_file
from utils.project.static_content import LOG_FORMAT
logging.basicConfig(filename=get_log_file('database_client.log'),
level=logging.INFO,
format=LOG_FORMAT)
class DatabaseClient(AbstractClient):
"""
Single access point for the mysql database
"""
def __init__(self, credentials=None):
if not credentials:
credentials = MYSQL_SETTINGS
super(DatabaseClient, self).__init__(credentials) # pylint:disable=super-with-arguments
self._connection = None
self._meta_data = None
self._engine = None
import traceback
from pathlib import Path
from git import Git, exc
from utils.project.static_content import LOG_FORMAT
from utils.project.structure import get_log_file
ISIS_MOUNT_PATH = Path("/isis")
AUTOREDUCTION_PATH = Path("user/scripts/autoreduction")
REDUCE_FILES_TO_SAVE = ["reduce.py", "reduce_vars.py"]
# STORAGE_DIR is the git repository dir that has been configured to point to the correct remote
STORAGE_DIR = Path("~/autoreduction_scripts").expanduser().absolute()
logging.basicConfig(filename=get_log_file('backup_reduction_scripts.log'), level=logging.INFO, format=LOG_FORMAT)
log = logging.getLogger(__file__)
log.addHandler(logging.StreamHandler())
def check_if_git_directory(path: Path):
"""
Ensures that the directory is a git repo.
If it is not the function will raise
:param path: The path to the directory
"""
repo = Git(path.absolute())
repo.status()
"""
import logging
import time
import uuid
import stomp
from stomp.exception import ConnectFailedException
from model.message.message import Message
from utils.clients.abstract_client import AbstractClient
from utils.clients.connection_exception import ConnectionException
from utils.settings import ACTIVEMQ_SETTINGS
from utils.project.structure import get_log_file
from utils.project.static_content import LOG_FORMAT
logging.basicConfig(filename=get_log_file('queue_client.log'),
level=logging.INFO,
format=LOG_FORMAT)
class QueueClient(AbstractClient):
"""
Class for client to access messaging service via python
"""
def __init__(self, credentials=None, consumer_name='queue_client'):
if not credentials:
credentials = ACTIVEMQ_SETTINGS
super(QueueClient, self).__init__(credentials) # pylint:disable=super-with-arguments
self._connection = None
self._consumer_name = consumer_name
self._autoreduce_queues = self.credentials.all_subscriptions
import os
import h5py
from filelock import (FileLock, Timeout)
from model.message.message import Message
from monitors.settings import (LAST_RUNS_CSV, CYCLE_FOLDER)
from utils.clients.queue_client import QueueClient
from utils.project.structure import get_log_file
from utils.project.static_content import LOG_FORMAT
# Setup logging
EORM_LOG = logging.getLogger('run_detection')
EORM_LOG.setLevel(logging.INFO)
FH = logging.FileHandler(get_log_file('run_detection.log'))
CH = logging.StreamHandler()
FORMATTER = logging.Formatter(LOG_FORMAT)
FH.setFormatter(FORMATTER)
FH.setFormatter(FORMATTER)
EORM_LOG.addHandler(FH)
EORM_LOG.addHandler(CH)
class InstrumentMonitorError(Exception):
"""
Any fatal exception that occurs during execution of the
instrument monitor
"""
def test_get_log_file(self):
actual = get_log_file('test.log')
self.assertEqual(os.path.split(actual)[-1], 'test.log')