Exemple #1
0
    def __init__(self, globalWorkingDir, localWorkingDir, outputs=None, job=None, esJobManager=None, outputDir=None, rank=None, logger=None):
        threading.Thread.__init__(self)
        self.__globalWorkingDir = globalWorkingDir
        self.__localWorkingDir = localWorkingDir
        self.__currentDir = None
        self.__rank = rank
        if logger and False:
            self.__tmpLog = logger
        else:
            curdir = _abspath (self.__localWorkingDir)
            wkdirname = "rank_%s" % str(self.__rank)
            wkdir  = _abspath (_join(curdir,wkdirname))
            self.__tmpLog = Logger.Logger(filename=os.path.join(wkdir, 'Droid.log'))
        self.__job = job
        self.__esJobManager = esJobManager
        self.__stop = threading.Event()
        self.__isFinished = False
        self.__tmpLog.info("Rank %s: Global working dir: %s" % (self.__rank, self.__globalWorkingDir))
        os.environ['PilotHomeDir'] = os.path.dirname(self.__globalWorkingDir)

        self.__jobId = None
        self.__copyOutputToGlobal = False
        self.__outputDir = outputDir

        self.__hostname = socket.getfqdn()

        self.__outputs = outputs
        self.__threadpool = None
        self.setup(job)
Exemple #2
0
 def initWorkingDir(self):
     # Create separate working directory for each rank
     curdir = _abspath(self.__localWorkingDir)
     wkdirname = "rank_%s" % str(self.__rank)
     wkdir = _abspath(_join(curdir, wkdirname))
     if not os.path.exists(wkdir):
         os.makedirs(wkdir)
     os.chdir(wkdir)
     self.__currentDir = wkdir
Exemple #3
0
 def initWorkingDir(self):
     # Create separate working directory for each rank
     curdir = _abspath (self.localWorkingDir)
     wkdirname = "rank_%s" % str(self.rank)
     wkdir  = _abspath (_join(curdir,wkdirname))
     if not os.path.exists(wkdir):
          os.makedirs (wkdir)
     os.chdir (wkdir)
     self.currentDir = wkdir
Exemple #4
0
    def __init__(self):
        # robot is expected to be run from the NVDA repo root directory. We want all repo specific
        # paths to be relative to this. This would allow us to change where it is run from if we decided to.
        self.repoRoot = _abspath("./")
        self.stagingDir = _tempFile.gettempdir()
        opSys.directory_should_exist(self.stagingDir)

        self.whichNVDA = builtIn.get_variable_value("${whichNVDA}", "source")
        self._installFilePath = builtIn.get_variable_value(
            "${installDir}", None)
        self.NVDAInstallerCommandline = None
        if self.whichNVDA == "source":
            self._runNVDAFilePath = _pJoin(self.repoRoot, "runnvda.bat")
            self.baseNVDACommandline = self._runNVDAFilePath
        elif self.whichNVDA == "installed":
            self._runNVDAFilePath = self.findInstalledNVDAPath()
            self.baseNVDACommandline = f'"{str(self._runNVDAFilePath)}"'
            if self._installFilePath is not None:
                self.NVDAInstallerCommandline = f'"{str(self._installFilePath)}"'
        else:
            raise AssertionError(
                "RobotFramework should be run with argument: '-v whichNVDA:[source|installed]'"
            )

        self.profileDir = _pJoin(self.stagingDir, "nvdaProfile")
        self.logPath = _pJoin(self.profileDir, 'nvda.log')
        self.preservedLogsDir = _pJoin(
            builtIn.get_variable_value("${OUTPUT DIR}"), "nvdaTestRunLogs")
Exemple #5
0
def _invirt():
    """Make sure we have bootstrapped."""
    with settings(hide('running', 'stdout', 'stderr')):
        current_py = local('which python', capture=True)
        virt_py = _abspath('.virt/bin/python')
        if not current_py == virt_py:
            puts(c.red('Not in virtualenv! Please run ". bootstrap".'))
            sys.exit(1)
Exemple #6
0
def _invirt():
    """Make sure we have bootstrapped."""
    with settings(hide('running', 'stdout', 'stderr')):
        current_py = local('which python', capture=True)
        virt_py = _abspath('.virt/bin/python')
        if not current_py == virt_py:
            puts(c.red('Not in virtualenv! Please run ". bootstrap".'))
            sys.exit(1)
Exemple #7
0
def create_worker_dir(top_wkdir):
    """create tmp wkdir for forked worker under top_wkdir"""
    #changing the workdir to tmp location  
    import os
    from os.path import abspath as _abspath, join as _join
    ppid = os.getppid()
    pid  = os.getpid()
    curdir = _abspath (os.curdir)
    wkdir  = _abspath (_join (top_wkdir,str(pid)))
    _info("curdir=[%s]", curdir)
    _info("top_wkdir=[%s]", top_wkdir)
    _info("wkdir=[%s]", wkdir)
    if os.path.exists(wkdir):
        import shutil
        shutil.rmtree (wkdir)
    os.makedirs (wkdir)
        
    return wkdir
Exemple #8
0
def create_worker_dir(top_wkdir):
    """create tmp wkdir for forked worker under top_wkdir"""
    #changing the workdir to tmp location  
    import os
    from os.path import abspath as _abspath, join as _join
    ppid = os.getppid()
    pid  = os.getpid()
    curdir = _abspath (os.curdir)
    wkdir  = _abspath (_join (top_wkdir,str(pid)))
    _info("curdir=[%s]", curdir)
    _info("top_wkdir=[%s]", top_wkdir)
    _info("wkdir=[%s]", wkdir)
    if os.path.exists(wkdir):
        import shutil
        shutil.rmtree (wkdir)
    os.makedirs (wkdir)
        
    return wkdir
Exemple #9
0
def main(globalWorkDir, localWorkDir):
    comm = MPI.COMM_WORLD
    mpirank = comm.Get_rank()

    # Create separate working directory for each rank
    from os.path import abspath as _abspath, join as _join
    curdir = _abspath (localWorkDir)
    wkdirname = "rank_%s" % str(mpirank)
    wkdir  = _abspath (_join(curdir,wkdirname))
    if not os.path.exists(wkdir):
        os.makedirs (wkdir)
    os.chdir (wkdir)

    if mpirank==0:
        yoda = Yoda.Yoda(globalWorkDir, localWorkDir)
        yoda.run()
    else:
        droid = Droid.Droid(globalWorkDir, localWorkDir)
        droid.run()
Exemple #10
0
def main(globalWorkDir, localWorkDir):
    comm = MPI.COMM_WORLD
    mpirank = comm.Get_rank()

    # Create separate working directory for each rank
    from os.path import abspath as _abspath, join as _join
    curdir = _abspath(localWorkDir)
    wkdirname = "rank_%s" % str(mpirank)
    wkdir = _abspath(_join(curdir, wkdirname))
    if not os.path.exists(wkdir):
        os.makedirs(wkdir)
    os.chdir(wkdir)

    if mpirank == 0:
        yoda = Yoda.Yoda(globalWorkDir, localWorkDir)
        yoda.run()
    else:
        droid = Droid.Droid(globalWorkDir, localWorkDir)
        droid.run()
    def _getPath(fileName: str = None) -> any:
        if _isValid(fileName):
            filePath = _abspath(fileName)
            args.full_path = True

            for bms in getBookmarks():
                if bms[1] == filePath:
                    raise Exception(bms[0] +" Already Bookmarked")
            
            return filePath

        else:
            raise FileNotFoundError()
Exemple #12
0
	def __init__(self):
		# robot is expected to be run from the NVDA repo root directory. We want all repo specific
		# paths to be relative to this. This would allow us to change where it is run from if we decided to.
		self.repoRoot = _abspath("./")
		self.stagingDir = _tempFile.gettempdir()
		opSys.directory_should_exist(self.stagingDir)

		self.whichNVDA = builtIn.get_variable_value("${whichNVDA}", "source")
		if self.whichNVDA == "source":
			self._runNVDAFilePath = _pJoin(self.repoRoot, "source/nvda.pyw")
			self.baseNVDACommandline = f"pyw -3.7-32 {self._runNVDAFilePath}"
		elif self.whichNVDA == "installed":
			self._runNVDAFilePath = _pJoin(_expandvars('%PROGRAMFILES%'), 'nvda', 'nvda.exe')
			self.baseNVDACommandline = f'"{str(self._runNVDAFilePath)}"'
		else:
			raise AssertionError("RobotFramework should be run with argument: '-v whichNVDA [source|installed]'")

		self.profileDir = _pJoin(self.stagingDir, "nvdaProfile")
		self.logPath = _pJoin(self.profileDir, 'nvda.log')
		self.preservedLogsDir = _pJoin(
			builtIn.get_variable_value("${OUTPUT DIR}"),
			"nvdaTestRunLogs"
		)
Exemple #13
0
def to_abspath(*paths):
    return _abspath(_join(*paths))
Exemple #14
0
# -*- coding: utf-8 -*-

from __future__ import absolute_import

# set the warning format to be on a single line
import warnings as _warnings
from os import name as _name
from os.path import abspath as _abspath
from os.path import dirname as _dirname

from cobra import flux_analysis, io
from cobra.core import (
    DictList, Gene, Metabolite, Model, Object, Reaction, Species)
from cobra.util.version_info import show_versions

__version__ = "0.11.2"

# set the warning format to be prettier and fit on one line
_cobra_path = _dirname(_abspath(__file__))
if _name == "posix":
    _warning_base = "%s:%s \x1b[1;31m%s\x1b[0m: %s\n"  # colors
else:
    _warning_base = "%s:%s %s: %s\n"


def _warn_format(message, category, filename, lineno, file=None, line=None):
    shortname = filename.replace(_cobra_path, "cobra", 1)
    return _warning_base % (shortname, lineno, category.__name__, message)
_warnings.formatwarning = _warn_format
Exemple #15
0
"""
Lind Static Resources
====

Lind static resources is a package with static files containing pre-computed
experimental designs for the package Lind. These static designs offer optional
functionality for users of Lind.

"""

from os.path import dirname as _dirname, abspath as _abspath

static_files_abs_path = _dirname(_abspath(__file__))

from ._version import get_versions
__version__ = get_versions()['version']
del get_versions
Exemple #16
0
def main(globalWorkDir,
         localWorkDir,
         nonMPIMode=False,
         outputDir=None,
         dumpEventOutputs=True):
    sys.path.append(os.path.dirname(os.path.abspath(__file__)))

    if nonMPIMode:
        comm = None
        mpirank = 0
        mpisize = 1
    else:
        try:
            from mpi4py import MPI
            comm = MPI.COMM_WORLD
            mpirank = comm.Get_rank()
            mpisize = comm.Get_size()
        except:
            print "Failed to load mpi4py: %s" % (traceback.format_exc())
            sys.exit(1)

    # Create separate working directory for each rank
    from os.path import abspath as _abspath, join as _join
    curdir = _abspath(localWorkDir)
    wkdirname = "rank_%s" % str(mpirank)
    wkdir = _abspath(_join(curdir, wkdirname))
    if not os.path.exists(wkdir):
        os.makedirs(wkdir)
    os.chdir(wkdir)

    print "GlobalWorkDir: %s" % globalWorkDir
    print "LocalWorkDir: %s" % localWorkDir
    print "OutputDir: %s" % outputDir
    print "RANK: %s" % mpirank

    if mpirank == 0:
        try:
            from pandayoda.yodacore import Yoda
            yoda = Yoda.Yoda(globalWorkDir,
                             localWorkDir,
                             rank=0,
                             nonMPIMode=nonMPIMode,
                             outputDir=outputDir,
                             dumpEventOutputs=dumpEventOutputs)
            yoda.start()

            from pandayoda.yodaexe import Droid
            reserveCores = 1
            droid = Droid.Droid(globalWorkDir,
                                localWorkDir,
                                rank=0,
                                nonMPIMode=True,
                                reserveCores=reserveCores,
                                outputDir=outputDir)
            droid.start()

            i = 30
            while True:
                print "Rank %s: Yoda isAlive %s" % (mpirank, yoda.isAlive())
                print "Rank %s: Droid isAlive %s" % (mpirank, droid.isAlive())

                if yoda and yoda.isAlive():
                    time.sleep(60)
                else:
                    break
            print "Rank %s: Yoda finished" % (mpirank)
        except:
            print "Rank %s: Yoda failed: %s" % (mpirank,
                                                traceback.format_exc())
        sys.exit(0)
        #os._exit(0)
    else:
        try:
            status = 0
            from pandayoda.yodaexe import Droid
            droid = Droid.Droid(globalWorkDir,
                                localWorkDir,
                                rank=mpirank,
                                nonMPIMode=nonMPIMode,
                                outputDir=outputDir)
            droid.start()
            while (droid and droid.isAlive()):
                droid.join(timeout=1)
            # parent process
            #pid, status = os.waitpid(child_pid, 0)
            print "Rank %s: Droid finished status: %s" % (mpirank, status)
        except:
            print "Rank %s: Droid failed: %s" % (mpirank,
                                                 traceback.format_exc())
        #sys.exit(0)
    return mpirank
Exemple #17
0
from os.path import abspath as _abspath
from sys import path as _path
_one_folder_up = _abspath(__file__).rsplit('/', 2)[0]
_path.insert(1, _one_folder_up)

# if __name__ == "__main__":
#     from os.path import abspath as _abspath
#     import imp as _imp
#     _current_folder_init = _abspath(__file__).rsplit('/', 1)[0]+ "/__init__.py"
#     _imp.load_source("__init__", _current_folder_init)
Exemple #18
0
# -*- coding: utf-8 -*-
from os.path import (abspath as _abspath, dirname as _dirname, join as _join)

BASEDIR = _dirname(_dirname(_abspath(__file__)))
DATADIR = _join(BASEDIR, 'tests', 'data')
Exemple #19
0
"""The PiRan module."""
from os.path import isfile, getsize as get_len_of_file, abspath as _abspath, dirname
from os import remove
from sys import maxsize, stdout, stderr
from ctypes import CDLL, cast as c_cast, c_char_p
from typing import Optional, IO, Union, cast

__author__ = "Firefnix"
__license__ = "GPL-3-or-later"
__version__ = "1.0"

location = _abspath(dirname(__file__))
del dirname

def _required_digits(max_value: int) -> int:
    return len(str(max_value + 1))

def _assert_uint(arg: object) -> None:
    if (not isinstance(arg, int)) or arg < 0:
        raise ValueError(f"Must be an positive integer (is {arg})")

class Random:
    """Use random numbers, bytes and chars."""

    def __init__(self, pi_file_name: str = f"{location}pi",
        cursor: Union[str, int, None] = None) -> None:

        self._cursor_file_name: str
        if isfile(pi_file_name) is False:
            raise FileNotFoundError(f"{_abspath(pi_file_name)} not found")
Exemple #20
0
from __future__ import absolute_import

# set the warning format to be on a single line
import warnings as _warnings
from os import name as _name
from os.path import abspath as _abspath
from os.path import dirname as _dirname

from cobra.core import (Configuration, DictList, Gene, Metabolite, Model,
                        Object, Reaction, Species)
from cobra import flux_analysis
from cobra import io
from cobra.util import show_versions

__version__ = "0.14.2"

# set the warning format to be prettier and fit on one line
_cobra_path = _dirname(_abspath(__file__))
if _name == "posix":
    _warning_base = "%s:%s \x1b[1;31m%s\x1b[0m: %s\n"  # colors
else:
    _warning_base = "%s:%s %s: %s\n"


def _warn_format(message, category, filename, lineno, file=None, line=None):
    shortname = filename.replace(_cobra_path, "cobra", 1)
    return _warning_base % (shortname, lineno, category.__name__, message)


_warnings.formatwarning = _warn_format
Exemple #21
0
import numpy as np
import mdtraj as md
import nglview
from os.path import dirname as _dirname, join as _join, abspath as _abspath
from copy import deepcopy
from simtk.openmm import app

test_systems_path = _abspath(
    _join(_dirname(__file__), '../examples/testsystems'))


def copy_molcomplex(molcomplex):

    tmp_complex = deepcopy(molcomplex)
    tmp_complex.modeller = app.Modeller(molcomplex.topology,
                                        molcomplex.positions)
    tmp_complex.topology = tmp_complex.modeller.getTopology()

    return tmp_complex


def make_view(topology, positions):

    if type(positions) == list:
        positions = np.array([tmp_pos._value
                              for tmp_pos in positions]) * positions[0].unit

    topology_mdtraj = md.Topology.from_openmm(topology)
    positions_mdtraj = positions._value
    aux_traj = md.Trajectory(positions_mdtraj, topology_mdtraj)
Exemple #22
0
# -*- coding: utf-8 -*-

from __future__ import absolute_import

# set the warning format to be on a single line
import warnings as _warnings
from os import name as _name
from os.path import abspath as _abspath
from os.path import dirname as _dirname

# set the warning format to be prettier and fit on one line
_metabolicmodelgapfilling_path = _dirname(_abspath(__file__))
if _name == "posix":
    _warning_base = "%s:%s \x1b[1;31m%s\x1b[0m: %s\n"  # colors
else:
    _warning_base = "%s:%s %s: %s\n"


def _warn_format(message, category, filename, lineno, file=None, line=None):
    shortname = filename.replace(_metabolicmodelgapfilling_path,
                                 "MetabolicModelGapfilling", 1)
    return _warning_base % (shortname, lineno, category.__name__, message)


_warnings.formatwarning = _warn_format

import sys

from MetabolicModelGapfilling.core import (BaseModule, GapfillingModule)

__version__ = "0.1.0"
Exemple #23
0
from os.path import join as _join, dirname as _dirname
from os.path import abspath as _abspath
import pandas as _pandas

datadir = _abspath(_join(_dirname(__file__), '..', 'ohmyyeast-data'))


def load_optpub():
    _cols = [
        'AID', 'e2', 'e3', 'e4', 'e5', 'e6', 'e7', 'e8', 'e9', 'e10', 'S',
        'k_SA', 'K_SA', 'k_BA', 'K_BA', 'ki_BA', 'Ki_BA', 'k_CA', 'K_CA',
        'ki_CA', 'Ki_CA', 'k_AA', 'K_AA', 'ki_AA', 'Ki_AA', 'k_AB', 'K_AB',
        'ki_AB', 'Ki_AB', 'k_CB', 'K_CB', 'ki_CB', 'Ki_CB', 'k_BB', 'K_BB',
        'ki_BB', 'Ki_BB', 'k_BC', 'K_BC', 'ki_BC', 'Ki_BC', 'k_AC', 'K_AC',
        'ki_AC', 'Ki_AC', 'k_CC', 'K_CC', 'ki_CC', 'Ki_CC', 'ki_EA', 'Ki_EA',
        'ki_EB', 'Ki_EB', 'ki_EC', 'Ki_EC', 'k_FA', 'K_FA', 'k_FB', 'K_FB',
        'k_FC', 'K_FC', 'FUS1MAX', 'Ke50_FUS1', 'kd', 'ks', 'n', 'At', 'Bt',
        'Ct', 'E', 'F', 'fVal'
    ]

    myfile = _join(datadir, 'opt/pub', 'reduced.mat.csv')
    dataset = _pandas.read_csv(myfile, names=_cols)

    for edge in ['e2', 'e3', 'e4', 'e5', 'e6', 'e7', 'e8', 'e9', 'e10']:
        dataset[edge] = dataset[edge].astype(int)

    return dataset


def load_optsnu():
    myfile = _join(datadir, 'opt/snu', 'reduced.mat.csv')
Exemple #24
0
def main():
    start_g = time.time()
    start_g_str = time.asctime(time.localtime(start_g))
    comm = MPI.COMM_WORLD
    rank = comm.Get_rank()
    max_rank = comm.Get_size()
    print("%s | Rank: %s from %s ranks started" %
          (start_g_str, rank, max_rank))

    # Get a file name with job descriptions
    input_file = sys.argv[1]

    #Read file into an array removing new line.
    try:
        file = open(input_file)
        temp = file.read().splitlines()
        file.close()
    except IOError as e:
        print("I/O error({0}): {1}".format(e.errno, e.strerror))
        print("Exit from rank %s" % (rank))
        return

    # Number of commands in the input file
    n_comm = len(temp)
    #print("Number of commands: %s" % n_comm)

    # Safeguard here. Just in case things are messed up.
    if n_comm != max_rank:
        print(
            "Rank %s: problem with input file! Number of commands not equal to number of ranks! Exiting"
            % (rank))
        exit()

    #PandaID of the job for the command
    wkdirname = temp[rank].split("!!")[0].strip()
    # Get the corresponding command from input file
    my_command = temp[rank].split("!!")[1]
    #filename for stdout & srderr
    payloadstdout_s = temp[rank].split("!!")[2]
    payloadstderr_s = temp[rank].split("!!")[3]

    PanDA_jobId = wkdirname[wkdirname.find("_") + 1:]

    # create working directory for a given rank
    curdir = _abspath(os.curdir)

    # wkdirname = "PandaJob_%s" % my_job_index
    wkdir = _abspath(_join(curdir, wkdirname))
    if not os.path.exists(wkdir):
        os.mkdir(wkdir)
    '''
        print "Directory ", wkdir, " created"
    else:
        print "Directory ", wkdir, " exists"
    '''
    # Go to local working directory (job rank)
    os.chdir(wkdir)

    # If rank == 0 then setup MySQL db, all others wait for db to start
    if rank % 32 == 0:
        print("Rank %s is going to start MySQL db" % (rank))
        print("Preparing environment")
        dbsetup_comm_arr = [
            'cp /projects/sciteam/balh/panda/db_setup.sh .',
            'sh db_setup.sh &>dbsetup.log &'
        ]
        dbsetup_comm = "\n".join(dbsetup_comm_arr)
        p = subprocess.Popen(dbsetup_comm, shell=True)
        time.sleep(90)

        print("Going to check MySQL server status")
        output = subprocess.Popen("ps -C mysqld -o pid=",
                                  stdout=subprocess.PIPE,
                                  shell=True).communicate()[0]
        print("MySQL server pid: %s" % output.rstrip())
        if len(output) > 0:
            print("MySQL database is running, going to send GO! broadcast")

            output = subprocess.Popen("echo $HOSTNAME",
                                      stdout=subprocess.PIPE,
                                      shell=True).communicate()[0]
            print("MySQL database is running on %s" % output.rstrip())
        else:
            print("MySQL database is not running, exiting")
    else:
        print(
            "Rank %s is going to sleep a little bit to allow MySQL db to start"
            % (rank))
        time.sleep(90 + random.randint(10, 100))

    mysql_host = "0.0.0.0"
    coral_cdbserver_comm = "export CDBSERVER=%s;" % mysql_host
    my_command = coral_cdbserver_comm + my_command
    #    pcmd = subprocess.call(coral_cdbserver_comm, shell=True)

    payloadstdout = open(payloadstdout_s, "w")
    payloadstderr = open(payloadstderr_s, "w")

    localtime = time.asctime(time.localtime(time.time()))
    print("%s | %s Rank: %s Starting payload: %s" %
          (localtime, PanDA_jobId, rank, my_command))
    t0 = os.times()
    exit_code = subprocess.call(my_command,
                                stdout=payloadstdout,
                                stderr=payloadstderr,
                                shell=True)
    t1 = os.times()
    localtime = time.asctime(time.localtime(time.time()))
    t = map(lambda x, y: x - y, t1, t0)
    t_tot = reduce(lambda x, y: x + y, t[2:3])

    report = open("rank_report.txt", "w")
    report.write("cpuConsumptionTime: %s\n" % (t_tot))
    report.write("exitCode: %s" % exit_code)
    report.close()

    # Finished all steps here
    localtime = time.asctime(time.localtime(time.time()))
    end_all = time.time()
    print(
        "%s | %s Rank: %s Finished. Program run took.  %s min. Exit code: %s cpuConsumptionTime: %s"
        % (localtime, PanDA_jobId, rank,
           (end_all - start_g) / 60., exit_code, t_tot))

    return 0
Exemple #25
0
from os.path import join as _join

try:
    import winsound as _winsound
except ImportError:
    _winsound = None
try:
    # Python2
    import Tkinter as _tk
except ImportError:
    # Python3
    import tkinter as _tk

__all__ = ['YouDidntSayTheMagicWord', 'getpass', 'access_main_program']

_PATH = _dirname(_abspath(__file__))


class YouDidntSayTheMagicWord(Exception):
    """
        Custom YouDidntSayTheMagicWord exception that spams the console with
        "YOU DIDN'T SAY THE MAGIC WORD!" while displaying an animated gif of
        Dennis Nedry wagging his finger with "Uh uh uh, you didn't say the
        magic word." repeating in the background. Why? Because.

    """
    def __init__(self):
        self._console_spam()
        _sleep(0.8)
        self._audio_spam()
        self._gif_spam()
Exemple #26
0
# $Id: $

# Make sure Python loads the modules of this package via absolute paths.
from os.path import abspath as _abspath
__path__[0] = _abspath(__path__[0])

Exemple #27
0
def main(globalWorkDir, localWorkDir, nonMPIMode=False):
    sys.path.append(os.path.dirname(os.path.abspath(__file__)))

    if nonMPIMode:
        comm = None
        mpirank = 0
        mpisize = 1
    else:
        try:
            from mpi4py import MPI
            comm = MPI.COMM_WORLD
            mpirank = comm.Get_rank()
            mpisize = comm.Get_size()
        except:
            print "Failed to load mpi4py: %s" % (traceback.format_exc())
            sys.exit(1)

    # Create separate working directory for each rank
    from os.path import abspath as _abspath, join as _join
    curdir = _abspath (localWorkDir)
    wkdirname = "rank_%s" % str(mpirank)
    wkdir  = _abspath (_join(curdir,wkdirname))
    if not os.path.exists(wkdir):
        os.makedirs (wkdir)
    os.chdir (wkdir)

    print "GlobalWorkDir: %s" % globalWorkDir
    print "LocalWorkDir: %s" % localWorkDir
    print "RANK: %s" % mpirank
    if mpirank==0:
        try:
            from pandayoda.yodacore import Yoda
            yoda = Yoda.Yoda(globalWorkDir, localWorkDir, rank=0, nonMPIMode=nonMPIMode)
            yoda.start()

            from pandayoda.yodaexe import Droid
            reserveCores = 0
            if mpisize > 500:
                reserveCores = 4
            elif mpisize > 200:
                reserveCores = 3
            elif mpisize > 100:
                reserveCores = 2
            elif mpisize > 50:
                reserveCores = 1
            droid = Droid.Droid(globalWorkDir, localWorkDir, rank=0, nonMPIMode=True, reserveCores=reserveCores)
            droid.start()

            while True:
                yoda.join(timeout=1)
                if yoda and yoda.isAlive():
                    pass
                else:
                    break
            print "Rank %s: Yoda finished" % (mpirank)
        except:
            print "Rank %s: Yoda failed: %s" % (mpirank, traceback.format_exc())
        sys.exit(0)
    else:
        try:
            status = 0
            from pandayoda.yodaexe import Droid
            droid = Droid.Droid(globalWorkDir, localWorkDir, rank=mpirank, nonMPIMode=nonMPIMode)
            droid.start()
            while (droid and droid.isAlive()):
                droid.join(timeout=1)
            # parent process
            #pid, status = os.waitpid(child_pid, 0)
            print "Rank %s: Droid finished status: %s" % (mpirank, status)
        except:
            print "Rank %s: Droid failed: %s" % (mpirank, traceback.format_exc())
        #sys.exit(0)
    return mpirank
Exemple #28
0
# We need to execute this file directly, so we always run the import hack
from os.path import abspath as _abspath
import imp as _imp
_current_folder_init = _abspath(__file__).rsplit('/', 1)[0] + "/__init__.py"
_imp.load_source("__init__", _current_folder_init)

from kombu.exceptions import OperationalError
from celery import Celery, states
from celery.states import SUCCESS

STARTED_OR_WAITING = [states.PENDING, states.RECEIVED, states.STARTED]

FAILED = [states.REVOKED, states.RETRY, states.FAILURE]

try:
    with open("/home/ubuntu/manager_ip", 'r') as f:
        manager_ip = f.read()
except IOError:
    print "could not load the manager ip file, defaulting to 127.0.0.1"
    manager_ip = "127.0.0.1"

celery_app = Celery("data_processing_tasks",
                    broker='pyamqp://guest@%s//' % manager_ip,
                    backend='rpc://',
                    task_publish_retry=False,
                    task_track_started=True)

# Load Django
from config import load_django

################################################################################
Exemple #29
0
from robot.libraries.BuiltIn import BuiltIn
from robot.libraries.OperatingSystem import OperatingSystem
from robot.libraries.Process import Process
from systemTestUtils import _blockUntilConditionMet

builtIn = BuiltIn()  # type: BuiltIn
process = builtIn.get_library_instance('Process')  # type: Process
opSys = builtIn.get_library_instance('OperatingSystem')  # type: OperatingSystem

spyServerPort = 8270  # is `registered by IANA` for remote server usage. Two ASCII values:'RF'
spyServerURI = 'http://127.0.0.1:{}'.format(spyServerPort)
spyAlias = "nvdaSpy"

# robot is expected to be run from the NVDA repo root directory. We want all repo specific
# paths to be relative to this. This would allow us to change where it is run from if we decided to.
repoRoot = _abspath("./")
whichNVDA = builtIn.get_variable_value("${whichNVDA}", "source")
if whichNVDA == "source":
	NVDACommandPathToCheckExists = _pJoin(repoRoot, "source/nvda.pyw")
	baseNVDACommandline = "pythonw "+NVDACommandPathToCheckExists
elif whichNVDA == "installed":
	NVDACommandPathToCheckExists = _pJoin(_expandvars('%PROGRAMFILES%'),'nvda','nvda.exe')
	baseNVDACommandline='"%s"' % NVDACommandPathToCheckExists
else:
	raise AssertionError("robot should be given argument `-v whichNVDA [source|installed]")

# Paths
systemTestSourceDir = _pJoin(repoRoot, "tests", "system")
tempDir = tempfile.gettempdir()
opSys.directory_should_exist(tempDir)
nvdaProfileWorkingDir = _pJoin(tempDir, "nvdaProfile")
Exemple #30
0
from robot.libraries.OperatingSystem import OperatingSystem
from robot.libraries.Process import Process
from systemTestUtils import _blockUntilConditionMet

builtIn = BuiltIn()  # type: BuiltIn
process = builtIn.get_library_instance('Process')  # type: Process
opSys = builtIn.get_library_instance(
    'OperatingSystem')  # type: OperatingSystem

spyServerPort = 8270  # is `registered by IANA` for remote server usage. Two ASCII values:'RF'
spyServerURI = 'http://127.0.0.1:{}'.format(spyServerPort)
spyAlias = "nvdaSpy"

# robot is expected to be run from the NVDA repo root directory. We want all repo specific
# paths to be relative to this. This would allow us to change where it is run from if we decided to.
repoRoot = _abspath("./")
whichNVDA = builtIn.get_variable_value("${whichNVDA}", "source")
if whichNVDA == "source":
    NVDACommandPathToCheckExists = _pJoin(repoRoot, "source/nvda.pyw")
    baseNVDACommandline = "pythonw " + NVDACommandPathToCheckExists
elif whichNVDA == "installed":
    NVDACommandPathToCheckExists = _pJoin(_expandvars('%PROGRAMFILES%'),
                                          'nvda', 'nvda.exe')
    baseNVDACommandline = '"%s"' % NVDACommandPathToCheckExists
else:
    raise AssertionError(
        "robot should be given argument `-v whichNVDA [source|installed]")

# Paths
systemTestSourceDir = _pJoin(repoRoot, "tests", "system")
tempDir = tempfile.gettempdir()
Exemple #31
0
def main(globalWorkDir, localWorkDir, nonMPIMode=False, outputDir=None, dumpEventOutputs=True):
    sys.path.append(os.path.dirname(os.path.abspath(__file__)))

    if nonMPIMode:
        comm = None
        mpirank = 0
        mpisize = 1
    else:
        try:
            comm = MPI.COMM_WORLD
            mpirank = comm.Get_rank()
            mpisize = comm.Get_size()
        except:
            logger.exception("Failed to load mpi4py")
            raise

    # Create separate working directory for each rank
    from os.path import abspath as _abspath, join as _join

    curdir = _abspath(localWorkDir)
    wkdirname = "rank_%s" % str(mpirank)
    wkdir = _abspath(_join(curdir, wkdirname))
    if not os.path.exists(wkdir):
        os.makedirs(wkdir)
    os.chdir(wkdir)

    logger.info("GlobalWorkDir: %s" % globalWorkDir)
    logger.info("LocalWorkDir: %s" % localWorkDir)
    logger.info("OutputDir: %s" % outputDir)
    logger.info("RANK: %s" % mpirank)

    if mpirank == 0:
        try:
            from pandayoda.yodacore import Yoda

            yoda = Yoda.Yoda(
                globalWorkDir,
                localWorkDir,
                rank=0,
                nonMPIMode=nonMPIMode,
                outputDir=outputDir,
                dumpEventOutputs=dumpEventOutputs,
            )
            yoda.start()

            from pandayoda.yodaexe import Droid

            if nonMPIMode:
                reserveCores = 0
            else:
                reserveCores = 1
            droid = Droid.Droid(
                globalWorkDir, localWorkDir, rank=0, nonMPIMode=True, reserveCores=reserveCores, outputDir=outputDir
            )
            droid.start()

            i = 30
            while True:
                logger.info("Rank %s: Yoda isAlive %s" % (mpirank, yoda.isAlive()))
                logger.info("Rank %s: Droid isAlive %s" % (mpirank, droid.isAlive()))

                if yoda and yoda.isAlive():
                    time.sleep(60)
                else:
                    break
            logger.info("Rank %s: Yoda finished" % (mpirank))
        except:
            logger.exception("Rank %s: Yoda failed" % mpirank)
            raise
        # os._exit(0)
    else:
        try:
            status = 0
            from pandayoda.yodaexe import Droid

            droid = Droid.Droid(globalWorkDir, localWorkDir, rank=mpirank, nonMPIMode=nonMPIMode, outputDir=outputDir)
            droid.start()
            while droid and droid.isAlive():
                droid.join(timeout=1)
            # parent process
            # pid, status = os.waitpid(child_pid, 0)
            logger.info("Rank %s: Droid finished status: %s" % (mpirank, status))
        except:
            logger.exception("Rank %s: Droid failed" % mpirank)
            raise
    return mpirank
Exemple #32
0
# Make sure Python loads the modules of this package via absolute paths.
from os.path import abspath as _abspath
from gppylib.commands.base import WorkerPool
from gppylib import gplog
__path__[0] = _abspath(__path__[0])

logger = gplog.get_default_logger()


class Operation(object):
    """
    An Operation (abstract class) is one atomic unit of work.
    
    An implementation of Operation is fully defined by:
        __init__(self, opts, args) - opts, args are intended to be validated and marshaled, as appropriate
        execute(self) - the actual Operation logic commences
        cleanup(self) - [optional] any cleanup operations; e.g. stopping a WorkerPool
        undo(self) - ?
        progress(self) - ?
        validate(self) - ?
        cancel(self) - ?

    By decomposing work into these Operation classes whenever possible, 
    we should be able to reap several benefits: 
        1. Unit testing - It's easier to build unit tests around functions 
                          that are not tightly bound to a monolithic class
        2. Composition  - We should be able to compose Operations in a useful manner. 
                          Consider ParallelOperation, RemoteOperation, SerialOperation, etc.
        3. Readability  - We no longer have to create nested, segment-facing scripts 
                          to perform work on segment hosts: e.g. figuring out how 
                          gpstart -> startSegments -> gpsegstart.py propagates information is 
Exemple #33
0
from os.path import abspath as _abspath, dirname as _dirname
_PROJECT_ROOT = _dirname(_abspath(__file__))

ADMINS = (
    # ('Your Name', '*****@*****.**'),
)

MANAGERS = ADMINS

LOGIN_URL = '/openid/login/'
LOGIN_REDIRECT_URL = '/publish/'

OPENID_USE_AS_ADMIN_LOGIN = True

DATABASES = {
	'default': {
		'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
		'NAME': 'data',					  # Or path to database file if using sqlite3.
		'USER': '',					  # Not used with sqlite3.
		'PASSWORD': '',				  # Not used with sqlite3.
		'HOST': '',					  # Set to empty string for localhost. Not used with sqlite3.
		'PORT': '',					  # Set to empty string for default. Not used with sqlite3.
	}
}

USE_TZ=True

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same