def setUp(self) -> None: if Path(ARCHIVE_ROOT).exists(): rmtree(ARCHIVE_ROOT) self.data_archive = DataArchive(["test"], 19, 20) self.expected_cycle_path = Path(get_project_root(), "data-archive", "NDXtest", "Instrument", "data") self.expected_script_path = Path( Path(get_project_root(), "data-archive", "NDXtest", "user", "scripts", "autoreduction"))
def test_get_project_root(self): path = get_project_root() expected = [ 'build', 'docker_reduction', 'documentation', 'scripts', 'queue_processors', 'utils', 'WebApp', 'monitors' ] actual = os.listdir(path) for directory in expected: self.assertIn(directory, actual)
def add_webapp_path(): """ Get path to the autoreduce_webapp and add this path to the system path This will enable importing of the autoreduce_webapp (and it's models) """ path = os.path.join(get_project_root(), 'WebApp', 'autoreduce_webapp') if path not in sys.path: sys.path.append(path)
def setUp(self): """ Start all external services """ # Get all clients self.database_client = DatabaseClient() self.database_client.connect() self.queue_client = QueueClient(ACTIVEMQ_SETTINGS) self.queue_client.connect() # Create test archive and add data self.data_archive_creator = DataArchiveCreator(os.path.join( get_project_root()), overwrite=True) self.archive_explorer = ArchiveExplorer( os.path.join(get_project_root(), 'data-archive')) # Add placeholder variables: # these are used to ensure runs are deleted even if test fails before completion self.instrument = None self.rb_number = None self.run_number = None
def test_add_webapp_path(self): """ Test: The correct path is added to sys.path When: add_webapp_path is called """ path = sys.path expected = os.path.join(get_project_root(), 'WebApp', 'autoreduce_webapp') DjangoORM.add_webapp_path() self.assertTrue(expected in path) sys.path.remove(expected) # Cleanup test
def add_webapp_path(): """ Get path to the autoreduce_webapp and add this path to the system path This will enable importing of the autoreduce_webapp (and it's models) """ path = os.path.join(get_project_root(), 'WebApp', 'autoreduce_webapp') if path not in sys.path: # Can't properly replicate this in a testing environment due to travis # constraint on sys path access and mocking would break underlying dependency # but it's simple enough we don't need to worry about testing it sys.path.append(path) # pragma: no cover
def test_add_webapp_path_already_exist(self): """ Test: The webapp path is not added to sys.path When: webapp path already exists and add_webapp_path is called """ webapp_path = os.path.join(get_project_root(), 'WebApp', 'autoreduce_webapp') sys.path.append(webapp_path) expected = sys.path.count(webapp_path) DjangoORM.add_webapp_path() self.assertEqual(expected, sys.path.count(webapp_path)) sys.path.remove(webapp_path) # Cleanup test
def __init__(self, data_filepath: str, server_dir: str, rb_number: str = None): self.data_filename: str = self._get_only_data_file_name(data_filepath) self.rb_number = rb_number # Used when searching for full Experiment graph self.server_dir = server_dir self.file_extensions = ["png", "jpg", "bmp", "gif", "tiff"] # Directory to place fetched data files / images self.static_graph_dir = os.path.join(get_project_root(), 'WebApp', 'autoreduce_webapp', 'static', 'graphs')
def __init__(self, instrument_name, run_number, server_dir, rb_number=None): self.instrument_name = instrument_name self.rb_number = rb_number # Used when searching for full Experiment graph self.run_number = run_number self.server_dir = server_dir self.file_extensions = ["png", "jpg", "bmp", "gif", "tiff"] # Directory to place fetched data files / images self.static_graph_dir = os.path.join(get_project_root(), 'WebApp', 'autoreduce_webapp', 'static', 'graphs')
def _make_reduction_directory(instrument, rb_number, run_number): """ Make a directory in the expected location for reduced runs to be written to """ reduced_dir = os.path.join(get_project_root(), 'reduced-data') reduced_inst = os.path.join(reduced_dir, str(instrument)) reduced_rb = os.path.join(reduced_inst, 'RB{}'.format(str(rb_number))) reduced_auto = os.path.join(reduced_rb, 'autoreduced') reduced_run = os.path.join(reduced_auto, str(run_number)) os.mkdir(reduced_dir) os.mkdir(reduced_inst) os.mkdir(reduced_rb) os.mkdir(reduced_auto) os.mkdir(reduced_run)
def test_add_webapp_path_not_already_exist(self): """ Test: The webapp path is added to sys.path When: webapp path does not already exist and add_webapp_path is called """ expected = os.path.join(get_project_root(), 'WebApp', 'autoreduce_webapp') old_sys_path = sys.path.copy() if expected in sys.path: # Remove all expected from sys.path sys.path = list(filter(lambda a: a != expected, sys.path)) DjangoORM.add_webapp_path() self.assertIn(expected, sys.path) sys.path = old_sys_path # Cleanup test
def setUp(self): """ Create a few test PlotHandler objects """ self.expected_file_extension_regex = '(png|jpg|bmp|gif|tiff)' self.expected_mari_file_regex = f'MAR(I)?1234.*.{self.expected_file_extension_regex}' self.expected_wish_file_regex = f"WISH1234.*.{self.expected_file_extension_regex}" self.expected_wish_instrument_name = "WISH" self.expected_mari_instrument_name = "MARI" self.expected_mari_rb_number = 12345678 self.expected_mari_run_number = 1234 self.expected_mari_rb_folder = "/instrument/MARI/RBNumber/RB12345678/1234/autoreduced/" self.test_plot_handler = PlotHandler( instrument_name=self.expected_mari_instrument_name, rb_number=self.expected_mari_rb_number, run_number=self.expected_mari_run_number, server_dir=self.expected_mari_rb_folder) self.expected_static_graph_dir = os.path.join(get_project_root(), 'WebApp', 'autoreduce_webapp', 'static', 'graphs')
def test_copy_tree_folder(self): """ Test: Expected directory structure found When: Called for a given file """ directory_to_copy = mkdtemp(prefix='test-dir') with open(os.path.join(directory_to_copy, 'test-file.txt'), 'w+') as test_file: test_file.write('test content') ppa = PostProcessAdmin(self.message, None) ppa._copy_tree(directory_to_copy, os.path.join(get_project_root(), 'test-dir')) self.assertTrue(os.path.exists(os.path.join(get_project_root(), 'test-dir'))) self.assertTrue(os.path.isdir(os.path.join(get_project_root(), 'test-dir'))) self.assertTrue(os.path.exists(os.path.join(get_project_root(), 'test-dir', 'test-file.txt'))) self.assertTrue(os.path.isfile(os.path.join(get_project_root(), 'test-dir', 'test-file.txt'))) shutil.rmtree(os.path.join(get_project_root(), 'test-dir'))
def setUp(self): """ Create a few test PlotHandler objects """ self.expected_file_extension_regex = '(png|jpg|bmp|gif|tiff)' self.expected_wish_data_filename = "WISH1234" self.expected_wish_file_regex = f"{self.expected_wish_data_filename}.*.{self.expected_file_extension_regex}" self.expected_mari_data_filename = "MARI1234" self.expected_mari_file_regex = f'{self.expected_mari_data_filename}.*.{self.expected_file_extension_regex}' self.input_data_filepath = "\\\\isis\\inst$\\NDXMARI\\Instrument\\data\\cycle_test\\MARI1234.nxs" self.expected_mari_rb_number = 12345678 self.expected_mari_rb_folder = "/instrument/MARI/RBNumber/RB12345678/1234/autoreduced" self.test_plot_handler = PlotHandler( data_filepath=self.input_data_filepath, server_dir=self.expected_mari_rb_folder, rb_number=self.expected_mari_rb_number) self.expected_static_graph_dir = os.path.join(get_project_root(), 'WebApp', 'autoreduce_webapp', 'static', 'graphs')
# ############################################################################### # # Autoreduction Repository : https://github.com/ISISScientificComputing/autoreduce # # Copyright © 2020 ISIS Rutherford Appleton Laboratory UKRI # SPDX - License - Identifier: GPL-3.0-or-later # ############################################################################### # # pylint: skip-file import logging.handlers import os from utils.project.structure import get_project_root LOGGING_LEVEL = logging.WARNING LOGGING_LOC = os.path.join(get_project_root(), 'logs', 'isisicat_prefix_mappings.log') logger = logging.getLogger('IsisICATPrefixMappings') logger.setLevel(LOGGING_LEVEL) handler = logging.handlers.RotatingFileHandler(LOGGING_LOC, maxBytes=104857600, backupCount=20) handler.setLevel(LOGGING_LEVEL) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) logger.addHandler(handler)
# ############################################################################### # # pylint: skip-file import os from utils.project.structure import get_project_root FACILITY = 'ISIS' MYSQL = { 'HOST': 'localhost:3306', 'USER': '******', 'PASSWD': 'pass', 'DB': 'autoreduction' } # Logging LOG_FILE = os.path.join(get_project_root(), 'logs', 'queue_processor.log') DEBUG = False if DEBUG: LOG_LEVEL = 'DEBUG' else: LOG_LEVEL = 'INFO' LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'formatters': { 'verbose': { 'format': "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s", 'datefmt': "%d/%b/%Y %H:%M:%S"
""" Settings for connecting to the test services that run locally """ import configparser import os from utils.project.structure import get_project_root from utils.clients.settings.client_settings_factory import ClientSettingsFactory VALID_INSTRUMENTS = [ 'ENGINX', 'GEM', 'HRPD', 'MAPS', 'MARI', 'MUSR', 'OSIRIS', 'POLARIS', 'POLREF', 'WISH' ] CONFIG = configparser.ConfigParser() INI_FILE = os.path.join(get_project_root(), 'utils', 'credentials.ini') CONFIG.read(INI_FILE) def get_str(section, key): return str( CONFIG.get(section, key, raw=True) ) # raw=True to allow strings with special characters to be passed SETTINGS_FACTORY = ClientSettingsFactory() ICAT_SETTINGS = SETTINGS_FACTORY.create('icat', username=get_str('ICAT', 'user'), password=get_str('ICAT', 'password'), host=get_str('ICAT', 'host'),
def _delete_reduction_directory(): """ Delete the temporary reduction directory""" shutil.rmtree(os.path.join(get_project_root(), 'reduced-data'))
# ############################################################################### # # Autoreduction Repository : https://github.com/ISISScientificComputing/autoreduce # # Copyright © 2019 ISIS Rutherford Appleton Laboratory UKRI # SPDX - License - Identifier: GPL-3.0-or-later # ############################################################################### # # pylint: skip-file """ Settings for ActiveMQ and reduction variables """ import os from paths.path_manipulation import append_path from utils.project.structure import get_project_root # MISC # "scripts_directory": "/isis/NDX%s/user/scripts/autoreduction", # "ceph_directory": "/instrument/%s/RBNumber/RB%s/autoreduced/%s", MISC = { "script_timeout": 3600, # The max time to wait for a user script to finish running (seconds) "mantid_path": "/opt/Mantid/lib", "scripts_directory": append_path(get_project_root(), ['data-archive', 'NDX%s', 'user', 'scripts', 'autoreduction']), "post_process_directory": append_path(os.path.dirname(os.path.realpath(__file__)), ["post_process_admin.py"]), "ceph_directory": append_path(get_project_root(), ['reduced-data', '%s', 'RB%s', 'autoreduced', '%s']), "temp_root_directory": "/autoreducetmp", "excitation_instruments": ["LET", "MARI", "MAPS", "MERLIN", "WISH", "GEM"] }
import logging import os import sys from django.shortcuts import redirect from django.core.exceptions import PermissionDenied from django.shortcuts import render # The below is a template on the repository # pylint: disable=relative-import from .settings import (DEVELOPMENT_MODE, INSTALLED_APPS, LOGIN_URL, OUTDATED_BROWSERS, UOWS_LOGIN_URL, USER_ACCESS_CHECKS) # pylint: disable=relative-import from .icat_cache import ICATCache from utils.project.structure import get_project_root sys.path.append(os.path.join(get_project_root(), 'WebApp', 'autoreduce_webapp')) from reduction_viewer.models import ReductionRun, Experiment from reduction_viewer.models import Notification, Setting LOGGER = logging.getLogger(__name__) def has_valid_login(request): """ Check that the user is correctly logged in and their session is still considered valid """ LOGGER.debug("Checking if user is authenticated") if DEVELOPMENT_MODE: LOGGER.debug("DEVELOPMENT_MODE True so allowing access") return True
Settings for ActiveMQ and reduction variables """ import os from paths.path_manipulation import append_path from utils.project.structure import get_project_root # MISC # "scripts_directory": "/isis/NDX%s/user/scripts/autoreduction", # "ceph_directory": "/instrument/%s/RBNumber/RB%s/autoreduced/%s", MISC = { "script_timeout": 3600, # The max time to wait for a user script to finish running (seconds) "mantid_path": "/opt/Mantid/lib", "scripts_directory": append_path(get_project_root(), ['data-archive', 'NDX%s', 'user', 'scripts', 'autoreduction']), "post_process_directory": append_path(os.path.dirname(os.path.realpath(__file__)), ["post_process_admin.py"]), "ceph_directory": append_path(get_project_root(), ['reduced-data', '%s', 'RB%s', 'autoreduced', '%s']), "temp_root_directory": "/autoreducetmp", "flat_output_instruments": ["LET", "MARI", "MAPS", "MERLIN", "WISH", "GEM"] }
Deals with communication with ICAT service """ import logging import sys import os import datetime import icat from django.utils.encoding import smart_str # The below is a template on the repository # pylint: disable=relative-import from .settings import ICAT, BASE_DIR from utils.project.structure import get_project_root sys.path.append(os.path.join(get_project_root(), 'WebApp', 'autoreduce_webapp')) from reduction_viewer.models import Setting LOGGER = logging.getLogger(__name__) sys.path.insert(0, BASE_DIR) class ICATCommunication(object): """ Handles communication with the ICAT service """ def __init__(self, **kwargs): if 'URL' not in kwargs: kwargs['URL'] = ICAT['URL']