__copyright__ = 'Copyright 2013 Vanderbilt University. All Rights Reserved' import os import sys import logging from datetime import datetime, timedelta import processors import modules import XnatUtils import task import cluster import bin from task import Task, ClusterTask, XnatTask from dax_settings import DAX_Settings DAX_SETTINGS = DAX_Settings() UPDATE_PREFIX = 'updated--' UPDATE_FORMAT = "%Y-%m-%d %H:%M:%S" BUILD_SUFFIX = 'BUILD_RUNNING.txt' UPDATE_SUFFIX = 'UPDATE_RUNNING.txt' LAUNCH_SUFFIX = 'LAUNCHER_RUNNING.txt' #Logger to print logs LOGGER = logging.getLogger('dax') def str_to_timedelta(delta_str): if len(delta_str) <= 1: raise ValueError('invalid timedelta string value')
__copyright__ = 'Copyright 2013 Vanderbilt University. All Rights Reserved' import os import sys import logging from datetime import datetime, timedelta import processors import modules import XnatUtils import task import cluster import bin from task import Task from dax_settings import DAX_Settings DAX_SETTINGS = DAX_Settings() RESULTS_DIR = DAX_SETTINGS.get_results_dir() DEFAULT_ROOT_JOB_DIR = DAX_SETTINGS.get_root_job_dir() DEFAULT_QUEUE_LIMIT = DAX_SETTINGS.get_queue_limit() DEFAULT_MAX_AGE = DAX_SETTINGS.get_max_age() UPDATE_PREFIX = 'updated--' UPDATE_FORMAT = "%Y-%m-%d %H:%M:%S" BUILD_SUFFIX = 'BUILD_RUNNING.txt' UPDATE_SUFFIX = 'UPDATE_RUNNING.txt' LAUNCH_SUFFIX = 'LAUNCHER_RUNNING.txt' #Logger to print logs LOGGER = logging.getLogger('dax') class Launcher(object):
""" Task object to generate / manage assessors and cluster """ import os import shutil import errno import time import logging from datetime import date import cluster from cluster import PBS from dax_settings import DAX_Settings DAX_SETTINGS = DAX_Settings() #Logger to print logs LOGGER = logging.getLogger('dax') # Job Statuses NO_DATA = 'NO_DATA' # assessor that doesn't have data to run (for session assessor): E.G: dtiqa multi but no dti present. NEED_TO_RUN = 'NEED_TO_RUN' # assessor that is ready to be launch on the cluster (ACCRE). All the input data for the process to run are there. NEED_INPUTS = 'NEED_INPUTS' # assessor where input data are missing from a scan, multiple scans or other assessor. JOB_RUNNING = 'JOB_RUNNING' # the job has been submitted on the cluster and is running right now. JOB_FAILED = 'JOB_FAILED' # the job failed on the cluster. READY_TO_UPLOAD = 'READY_TO_UPLOAD' # Job done, waiting for the Spider to upload the results UPLOADING = 'UPLOADING' # in the process of uploading the resources on XNAT. COMPLETE = 'COMPLETE' # the assessors contains all the files. The upload and the job are done. READY_TO_COMPLETE = 'READY_TO_COMPLETE' # the job finished and upload is complete DOES_NOT_EXIST = 'DOES_NOT_EXIST' OPEN_STATUS_LIST = [NEED_TO_RUN, UPLOADING, JOB_RUNNING, READY_TO_COMPLETE, JOB_FAILED] JOB_BUILT = 'JOB_BUILT'
Cluster functionality """ #!/usr/bin/env python # -*- coding: utf-8 -*- __copyright__ = 'Copyright 2013 Vanderbilt University. All Rights Reserved' import os import time import logging import subprocess from datetime import datetime from subprocess import CalledProcessError from dax_settings import DAX_Settings DAX_SETTINGS = DAX_Settings() DEFAULT_EMAIL_OPTS = DAX_SETTINGS.get_email_opts() JOB_TEMPLATE = DAX_SETTINGS.get_job_template() CMD_SUBMIT = DAX_SETTINGS.get_cmd_submit() CMD_COUNT_NB_JOBS = DAX_SETTINGS.get_cmd_count_nb_jobs() CMD_GET_JOB_STATUS = DAX_SETTINGS.get_cmd_get_job_status() CMD_GET_JOB_WALLTIME = DAX_SETTINGS.get_cmd_get_job_walltime() CMD_GET_JOB_MEMORY = DAX_SETTINGS.get_cmd_get_job_memory() CMD_GET_JOB_NODE = DAX_SETTINGS.get_cmd_get_job_node() RUNNING_STATUS = DAX_SETTINGS.get_running_status() QUEUE_STATUS = DAX_SETTINGS.get_queue_status() COMPLETE_STATUS = DAX_SETTINGS.get_complete_status() PREFIX_JOBID = DAX_SETTINGS.get_prefix_jobid() SUFFIX_JOBID = DAX_SETTINGS.get_suffix_jobid() MAX_TRACE_DAYS = 30
""" Module classes for Scan and Sessions """ import os import shutil import smtplib import logging import XnatUtils from datetime import datetime from email.mime.text import MIMEText from dax_settings import DAX_Settings DAX_SETTINGS = DAX_Settings() SMTP_HOST = DAX_SETTINGS.get_smtp_host() SMTP_FROM = DAX_SETTINGS.get_smtp_from() SMTP_PASS = DAX_SETTINGS.get_smtp_pass() #Logger for logs LOGGER = logging.getLogger('dax') class Module(object): """ Object Module to create a module for DAX Module runs directly during a build on a session or scan to generate inputs data for scans/sessions """ def __init__(self, mod_name, directory, email, text_report): """ Entry point of the Base Module Class. :param mod_name: Name of the module :param directory: Temp directory to store data :param email: email address to send report :param text_report: string to write at the beggining of the report email :return: None """
""" File containing functions called by dax executables """ #!/usr/bin/env python # -*- coding: utf-8 -*- import os import imp import redcap import logging from datetime import datetime import log import XnatUtils from dax_settings import DAX_Settings DAX_SETTINGS = DAX_Settings() API_URL = DAX_SETTINGS.get_api_url() API_KEY_DAX = DAX_SETTINGS.get_api_key_dax() REDCAP_VAR = DAX_SETTINGS.get_dax_manager_data_dictionary() def set_logger(logfile, debug): """ Set the logging depth :param logfile: File to log output to :param debug: Should debug depth be used? :return: logger object """ #Logger for logs if debug: logger = log.setup_debug_logger('dax', logfile) else:
""" Task object to generate / manage assessors and cluster """ import os import time import logging from datetime import date import cluster from cluster import PBS from dax_settings import DAX_Settings DAX_SETTINGS = DAX_Settings() RESULTS_DIR = DAX_SETTINGS.get_results_dir() DEFAULT_EMAIL_OPTS = DAX_SETTINGS.get_email_opts() JOB_EXTENSION_FILE = DAX_SETTINGS.get_job_extension_file() # Logger to print logs LOGGER = logging.getLogger("dax") # Job Statuses NO_DATA = ( "NO_DATA" ) # assessor that doesn't have data to run (for session assessor): E.G: dtiqa multi but no dti present. NEED_TO_RUN = ( "NEED_TO_RUN" ) # assessor that is ready to be launch on the cluster (ACCRE). All the input data for the process to run are there. NEED_INPUTS = "NEED_INPUTS" # assessor where input data are missing from a scan, multiple scans or other assessor. JOB_RUNNING = "JOB_RUNNING" # the job has been submitted on the cluster and is running right now. JOB_FAILED = "JOB_FAILED" # the job failed on the cluster. READY_TO_UPLOAD = "READY_TO_UPLOAD" # Job done, waiting for the Spider to upload the results UPLOADING = "UPLOADING" # in the process of uploading the resources on XNAT.
""" File containing functions called by dax executables """ #!/usr/bin/env python # -*- coding: utf-8 -*- import os import imp import redcap import logging from datetime import datetime import log import XnatUtils from dax_settings import DAX_Settings DAX_SETTINGS = DAX_Settings() API_URL = DAX_SETTINGS.get_api_url() API_KEY_DAX = DAX_SETTINGS.get_api_key_dax() REDCAP_VAR = DAX_SETTINGS.get_dax_manager_config() def set_logger(logfile, debug): """ Set the logging depth :param logfile: File to log output to :param debug: Should debug depth be used? :return: logger object """ #Logger for logs if debug: logger = log.setup_debug_logger('dax', logfile) else: