def setUp(self) -> None:
if Path(ARCHIVE_ROOT).exists():
rmtree(ARCHIVE_ROOT)
self.data_archive = DataArchive(["test"], 19, 20)
self.expected_cycle_path = Path(get_project_root(), "data-archive", "NDXtest", "Instrument", "data")
self.expected_script_path = Path(
Path(get_project_root(), "data-archive", "NDXtest", "user", "scripts", "autoreduction"))
def test_get_project_root(self):
path = get_project_root()
expected = [
'build', 'docker_reduction', 'documentation', 'scripts', 'queue_processors', 'utils', 'WebApp', 'monitors'
]
actual = os.listdir(path)
for directory in expected:
self.assertIn(directory, actual)
def add_webapp_path():
"""
Get path to the autoreduce_webapp and add this path to the system path
This will enable importing of the autoreduce_webapp (and it's models)
"""
path = os.path.join(get_project_root(), 'WebApp', 'autoreduce_webapp')
if path not in sys.path:
sys.path.append(path)
def setUp(self):
""" Start all external services """
# Get all clients
self.database_client = DatabaseClient()
self.database_client.connect()
self.queue_client = QueueClient(ACTIVEMQ_SETTINGS)
self.queue_client.connect()
# Create test archive and add data
self.data_archive_creator = DataArchiveCreator(os.path.join(
get_project_root()),
overwrite=True)
self.archive_explorer = ArchiveExplorer(
os.path.join(get_project_root(), 'data-archive'))
# Add placeholder variables:
# these are used to ensure runs are deleted even if test fails before completion
self.instrument = None
self.rb_number = None
self.run_number = None
def test_add_webapp_path(self):
"""
Test: The correct path is added to sys.path
When: add_webapp_path is called
"""
path = sys.path
expected = os.path.join(get_project_root(), 'WebApp',
'autoreduce_webapp')
DjangoORM.add_webapp_path()
self.assertTrue(expected in path)
sys.path.remove(expected) # Cleanup test
def add_webapp_path():
"""
Get path to the autoreduce_webapp and add this path to the system path
This will enable importing of the autoreduce_webapp (and it's models)
"""
path = os.path.join(get_project_root(), 'WebApp', 'autoreduce_webapp')
if path not in sys.path:
# Can't properly replicate this in a testing environment due to travis
# constraint on sys path access and mocking would break underlying dependency
# but it's simple enough we don't need to worry about testing it
sys.path.append(path) # pragma: no cover
def test_add_webapp_path_already_exist(self):
"""
Test: The webapp path is not added to sys.path
When: webapp path already exists and add_webapp_path is called
"""
webapp_path = os.path.join(get_project_root(), 'WebApp',
'autoreduce_webapp')
sys.path.append(webapp_path)
expected = sys.path.count(webapp_path)
DjangoORM.add_webapp_path()
self.assertEqual(expected, sys.path.count(webapp_path))
sys.path.remove(webapp_path) # Cleanup test
def __init__(self,
data_filepath: str,
server_dir: str,
rb_number: str = None):
self.data_filename: str = self._get_only_data_file_name(data_filepath)
self.rb_number = rb_number # Used when searching for full Experiment graph
self.server_dir = server_dir
self.file_extensions = ["png", "jpg", "bmp", "gif", "tiff"]
# Directory to place fetched data files / images
self.static_graph_dir = os.path.join(get_project_root(), 'WebApp',
'autoreduce_webapp', 'static',
'graphs')
def __init__(self,
instrument_name,
run_number,
server_dir,
rb_number=None):
self.instrument_name = instrument_name
self.rb_number = rb_number # Used when searching for full Experiment graph
self.run_number = run_number
self.server_dir = server_dir
self.file_extensions = ["png", "jpg", "bmp", "gif", "tiff"]
# Directory to place fetched data files / images
self.static_graph_dir = os.path.join(get_project_root(), 'WebApp',
'autoreduce_webapp', 'static',
'graphs')
def _make_reduction_directory(instrument, rb_number, run_number):
"""
Make a directory in the expected location for reduced runs to be written to
"""
reduced_dir = os.path.join(get_project_root(), 'reduced-data')
reduced_inst = os.path.join(reduced_dir, str(instrument))
reduced_rb = os.path.join(reduced_inst,
'RB{}'.format(str(rb_number)))
reduced_auto = os.path.join(reduced_rb, 'autoreduced')
reduced_run = os.path.join(reduced_auto, str(run_number))
os.mkdir(reduced_dir)
os.mkdir(reduced_inst)
os.mkdir(reduced_rb)
os.mkdir(reduced_auto)
os.mkdir(reduced_run)
def test_add_webapp_path_not_already_exist(self):
"""
Test: The webapp path is added to sys.path
When: webapp path does not already exist and add_webapp_path is called
"""
expected = os.path.join(get_project_root(), 'WebApp',
'autoreduce_webapp')
old_sys_path = sys.path.copy()
if expected in sys.path:
# Remove all expected from sys.path
sys.path = list(filter(lambda a: a != expected, sys.path))
DjangoORM.add_webapp_path()
self.assertIn(expected, sys.path)
sys.path = old_sys_path # Cleanup test
def setUp(self):
"""
Create a few test PlotHandler objects
"""
self.expected_file_extension_regex = '(png|jpg|bmp|gif|tiff)'
self.expected_mari_file_regex = f'MAR(I)?1234.*.{self.expected_file_extension_regex}'
self.expected_wish_file_regex = f"WISH1234.*.{self.expected_file_extension_regex}"
self.expected_wish_instrument_name = "WISH"
self.expected_mari_instrument_name = "MARI"
self.expected_mari_rb_number = 12345678
self.expected_mari_run_number = 1234
self.expected_mari_rb_folder = "/instrument/MARI/RBNumber/RB12345678/1234/autoreduced/"
self.test_plot_handler = PlotHandler(
instrument_name=self.expected_mari_instrument_name,
rb_number=self.expected_mari_rb_number,
run_number=self.expected_mari_run_number,
server_dir=self.expected_mari_rb_folder)
self.expected_static_graph_dir = os.path.join(get_project_root(),
'WebApp',
'autoreduce_webapp',
'static', 'graphs')
def test_copy_tree_folder(self):
"""
Test: Expected directory structure found
When: Called for a given file
"""
directory_to_copy = mkdtemp(prefix='test-dir')
with open(os.path.join(directory_to_copy, 'test-file.txt'), 'w+') as test_file:
test_file.write('test content')
ppa = PostProcessAdmin(self.message, None)
ppa._copy_tree(directory_to_copy, os.path.join(get_project_root(), 'test-dir'))
self.assertTrue(os.path.exists(os.path.join(get_project_root(), 'test-dir')))
self.assertTrue(os.path.isdir(os.path.join(get_project_root(), 'test-dir')))
self.assertTrue(os.path.exists(os.path.join(get_project_root(), 'test-dir',
'test-file.txt')))
self.assertTrue(os.path.isfile(os.path.join(get_project_root(), 'test-dir',
'test-file.txt')))
shutil.rmtree(os.path.join(get_project_root(), 'test-dir'))
def setUp(self):
"""
Create a few test PlotHandler objects
"""
self.expected_file_extension_regex = '(png|jpg|bmp|gif|tiff)'
self.expected_wish_data_filename = "WISH1234"
self.expected_wish_file_regex = f"{self.expected_wish_data_filename}.*.{self.expected_file_extension_regex}"
self.expected_mari_data_filename = "MARI1234"
self.expected_mari_file_regex = f'{self.expected_mari_data_filename}.*.{self.expected_file_extension_regex}'
self.input_data_filepath = "\\\\isis\\inst$\\NDXMARI\\Instrument\\data\\cycle_test\\MARI1234.nxs"
self.expected_mari_rb_number = 12345678
self.expected_mari_rb_folder = "/instrument/MARI/RBNumber/RB12345678/1234/autoreduced"
self.test_plot_handler = PlotHandler(
data_filepath=self.input_data_filepath,
server_dir=self.expected_mari_rb_folder,
rb_number=self.expected_mari_rb_number)
self.expected_static_graph_dir = os.path.join(get_project_root(),
'WebApp',
'autoreduce_webapp',
'static', 'graphs')
# ############################################################################### #
# Autoreduction Repository : https://github.com/ISISScientificComputing/autoreduce
#
# Copyright © 2020 ISIS Rutherford Appleton Laboratory UKRI
# SPDX - License - Identifier: GPL-3.0-or-later
# ############################################################################### #
# pylint: skip-file
import logging.handlers
import os
from utils.project.structure import get_project_root
LOGGING_LEVEL = logging.WARNING
LOGGING_LOC = os.path.join(get_project_root(), 'logs',
'isisicat_prefix_mappings.log')
logger = logging.getLogger('IsisICATPrefixMappings')
logger.setLevel(LOGGING_LEVEL)
handler = logging.handlers.RotatingFileHandler(LOGGING_LOC,
maxBytes=104857600,
backupCount=20)
handler.setLevel(LOGGING_LEVEL)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
# ############################################################################### #
# pylint: skip-file
import os
from utils.project.structure import get_project_root
FACILITY = 'ISIS'
MYSQL = {
'HOST': 'localhost:3306',
'USER': '******',
'PASSWD': 'pass',
'DB': 'autoreduction'
}
# Logging
LOG_FILE = os.path.join(get_project_root(), 'logs', 'queue_processor.log')
DEBUG = False
if DEBUG:
LOG_LEVEL = 'DEBUG'
else:
LOG_LEVEL = 'INFO'
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'verbose': {
'format':
"[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s",
'datefmt': "%d/%b/%Y %H:%M:%S"
"""
Settings for connecting to the test services that run locally
"""
import configparser
import os
from utils.project.structure import get_project_root
from utils.clients.settings.client_settings_factory import ClientSettingsFactory
VALID_INSTRUMENTS = [
'ENGINX', 'GEM', 'HRPD', 'MAPS', 'MARI', 'MUSR', 'OSIRIS', 'POLARIS',
'POLREF', 'WISH'
]
CONFIG = configparser.ConfigParser()
INI_FILE = os.path.join(get_project_root(), 'utils', 'credentials.ini')
CONFIG.read(INI_FILE)
def get_str(section, key):
return str(
CONFIG.get(section, key, raw=True)
) # raw=True to allow strings with special characters to be passed
SETTINGS_FACTORY = ClientSettingsFactory()
ICAT_SETTINGS = SETTINGS_FACTORY.create('icat',
username=get_str('ICAT', 'user'),
password=get_str('ICAT', 'password'),
host=get_str('ICAT', 'host'),
def _delete_reduction_directory():
""" Delete the temporary reduction directory"""
shutil.rmtree(os.path.join(get_project_root(), 'reduced-data'))
# ############################################################################### #
# Autoreduction Repository : https://github.com/ISISScientificComputing/autoreduce
#
# Copyright © 2019 ISIS Rutherford Appleton Laboratory UKRI
# SPDX - License - Identifier: GPL-3.0-or-later
# ############################################################################### #
# pylint: skip-file
"""
Settings for ActiveMQ and reduction variables
"""
import os
from paths.path_manipulation import append_path
from utils.project.structure import get_project_root
# MISC
# "scripts_directory": "/isis/NDX%s/user/scripts/autoreduction",
# "ceph_directory": "/instrument/%s/RBNumber/RB%s/autoreduced/%s",
MISC = {
"script_timeout": 3600, # The max time to wait for a user script to finish running (seconds)
"mantid_path": "/opt/Mantid/lib",
"scripts_directory": append_path(get_project_root(), ['data-archive', 'NDX%s', 'user', 'scripts', 'autoreduction']),
"post_process_directory": append_path(os.path.dirname(os.path.realpath(__file__)), ["post_process_admin.py"]),
"ceph_directory": append_path(get_project_root(), ['reduced-data', '%s', 'RB%s', 'autoreduced', '%s']),
"temp_root_directory": "/autoreducetmp",
"excitation_instruments": ["LET", "MARI", "MAPS", "MERLIN", "WISH", "GEM"]
}
import logging
import os
import sys
from django.shortcuts import redirect
from django.core.exceptions import PermissionDenied
from django.shortcuts import render
# The below is a template on the repository
# pylint: disable=relative-import
from .settings import (DEVELOPMENT_MODE, INSTALLED_APPS, LOGIN_URL, OUTDATED_BROWSERS, UOWS_LOGIN_URL,
USER_ACCESS_CHECKS)
# pylint: disable=relative-import
from .icat_cache import ICATCache
from utils.project.structure import get_project_root
sys.path.append(os.path.join(get_project_root(), 'WebApp', 'autoreduce_webapp'))
from reduction_viewer.models import ReductionRun, Experiment
from reduction_viewer.models import Notification, Setting
LOGGER = logging.getLogger(__name__)
def has_valid_login(request):
"""
Check that the user is correctly logged in and their session is still considered valid
"""
LOGGER.debug("Checking if user is authenticated")
if DEVELOPMENT_MODE:
LOGGER.debug("DEVELOPMENT_MODE True so allowing access")
return True
Settings for ActiveMQ and reduction variables
"""
import os
from paths.path_manipulation import append_path
from utils.project.structure import get_project_root
# MISC
# "scripts_directory": "/isis/NDX%s/user/scripts/autoreduction",
# "ceph_directory": "/instrument/%s/RBNumber/RB%s/autoreduced/%s",
MISC = {
"script_timeout":
3600, # The max time to wait for a user script to finish running (seconds)
"mantid_path":
"/opt/Mantid/lib",
"scripts_directory":
append_path(get_project_root(),
['data-archive', 'NDX%s', 'user', 'scripts', 'autoreduction']),
"post_process_directory":
append_path(os.path.dirname(os.path.realpath(__file__)),
["post_process_admin.py"]),
"ceph_directory":
append_path(get_project_root(),
['reduced-data', '%s', 'RB%s', 'autoreduced', '%s']),
"temp_root_directory":
"/autoreducetmp",
"flat_output_instruments":
["LET", "MARI", "MAPS", "MERLIN", "WISH", "GEM"]
}
Deals with communication with ICAT service
"""
import logging
import sys
import os
import datetime
import icat
from django.utils.encoding import smart_str
# The below is a template on the repository
# pylint: disable=relative-import
from .settings import ICAT, BASE_DIR
from utils.project.structure import get_project_root
sys.path.append(os.path.join(get_project_root(), 'WebApp',
'autoreduce_webapp'))
from reduction_viewer.models import Setting
LOGGER = logging.getLogger(__name__)
sys.path.insert(0, BASE_DIR)
class ICATCommunication(object):
"""
Handles communication with the ICAT service
"""
def __init__(self, **kwargs):
if 'URL' not in kwargs:
kwargs['URL'] = ICAT['URL']
