def __init__(self, globalWorkingDir, localWorkingDir, outputs=None, job=None, esJobManager=None, outputDir=None, rank=None, logger=None):
threading.Thread.__init__(self)
self.__globalWorkingDir = globalWorkingDir
self.__localWorkingDir = localWorkingDir
self.__currentDir = None
self.__rank = rank
if logger and False:
self.__tmpLog = logger
else:
curdir = _abspath (self.__localWorkingDir)
wkdirname = "rank_%s" % str(self.__rank)
wkdir = _abspath (_join(curdir,wkdirname))
self.__tmpLog = Logger.Logger(filename=os.path.join(wkdir, 'Droid.log'))
self.__job = job
self.__esJobManager = esJobManager
self.__stop = threading.Event()
self.__isFinished = False
self.__tmpLog.info("Rank %s: Global working dir: %s" % (self.__rank, self.__globalWorkingDir))
os.environ['PilotHomeDir'] = os.path.dirname(self.__globalWorkingDir)
self.__jobId = None
self.__copyOutputToGlobal = False
self.__outputDir = outputDir
self.__hostname = socket.getfqdn()
self.__outputs = outputs
self.__threadpool = None
self.setup(job)
def initWorkingDir(self):
# Create separate working directory for each rank
curdir = _abspath(self.__localWorkingDir)
wkdirname = "rank_%s" % str(self.__rank)
wkdir = _abspath(_join(curdir, wkdirname))
if not os.path.exists(wkdir):
os.makedirs(wkdir)
os.chdir(wkdir)
self.__currentDir = wkdir
def initWorkingDir(self):
# Create separate working directory for each rank
curdir = _abspath (self.localWorkingDir)
wkdirname = "rank_%s" % str(self.rank)
wkdir = _abspath (_join(curdir,wkdirname))
if not os.path.exists(wkdir):
os.makedirs (wkdir)
os.chdir (wkdir)
self.currentDir = wkdir
def __init__(self):
# robot is expected to be run from the NVDA repo root directory. We want all repo specific
# paths to be relative to this. This would allow us to change where it is run from if we decided to.
self.repoRoot = _abspath("./")
self.stagingDir = _tempFile.gettempdir()
opSys.directory_should_exist(self.stagingDir)
self.whichNVDA = builtIn.get_variable_value("${whichNVDA}", "source")
self._installFilePath = builtIn.get_variable_value(
"${installDir}", None)
self.NVDAInstallerCommandline = None
if self.whichNVDA == "source":
self._runNVDAFilePath = _pJoin(self.repoRoot, "runnvda.bat")
self.baseNVDACommandline = self._runNVDAFilePath
elif self.whichNVDA == "installed":
self._runNVDAFilePath = self.findInstalledNVDAPath()
self.baseNVDACommandline = f'"{str(self._runNVDAFilePath)}"'
if self._installFilePath is not None:
self.NVDAInstallerCommandline = f'"{str(self._installFilePath)}"'
else:
raise AssertionError(
"RobotFramework should be run with argument: '-v whichNVDA:[source|installed]'"
)
self.profileDir = _pJoin(self.stagingDir, "nvdaProfile")
self.logPath = _pJoin(self.profileDir, 'nvda.log')
self.preservedLogsDir = _pJoin(
builtIn.get_variable_value("${OUTPUT DIR}"), "nvdaTestRunLogs")
def _invirt():
"""Make sure we have bootstrapped."""
with settings(hide('running', 'stdout', 'stderr')):
current_py = local('which python', capture=True)
virt_py = _abspath('.virt/bin/python')
if not current_py == virt_py:
puts(c.red('Not in virtualenv! Please run ". bootstrap".'))
sys.exit(1)
def _invirt():
"""Make sure we have bootstrapped."""
with settings(hide('running', 'stdout', 'stderr')):
current_py = local('which python', capture=True)
virt_py = _abspath('.virt/bin/python')
if not current_py == virt_py:
puts(c.red('Not in virtualenv! Please run ". bootstrap".'))
sys.exit(1)
def create_worker_dir(top_wkdir):
"""create tmp wkdir for forked worker under top_wkdir"""
#changing the workdir to tmp location
import os
from os.path import abspath as _abspath, join as _join
ppid = os.getppid()
pid = os.getpid()
curdir = _abspath (os.curdir)
wkdir = _abspath (_join (top_wkdir,str(pid)))
_info("curdir=[%s]", curdir)
_info("top_wkdir=[%s]", top_wkdir)
_info("wkdir=[%s]", wkdir)
if os.path.exists(wkdir):
import shutil
shutil.rmtree (wkdir)
os.makedirs (wkdir)
return wkdir
def create_worker_dir(top_wkdir):
"""create tmp wkdir for forked worker under top_wkdir"""
#changing the workdir to tmp location
import os
from os.path import abspath as _abspath, join as _join
ppid = os.getppid()
pid = os.getpid()
curdir = _abspath (os.curdir)
wkdir = _abspath (_join (top_wkdir,str(pid)))
_info("curdir=[%s]", curdir)
_info("top_wkdir=[%s]", top_wkdir)
_info("wkdir=[%s]", wkdir)
if os.path.exists(wkdir):
import shutil
shutil.rmtree (wkdir)
os.makedirs (wkdir)
return wkdir
def main(globalWorkDir, localWorkDir):
comm = MPI.COMM_WORLD
mpirank = comm.Get_rank()
# Create separate working directory for each rank
from os.path import abspath as _abspath, join as _join
curdir = _abspath (localWorkDir)
wkdirname = "rank_%s" % str(mpirank)
wkdir = _abspath (_join(curdir,wkdirname))
if not os.path.exists(wkdir):
os.makedirs (wkdir)
os.chdir (wkdir)
if mpirank==0:
yoda = Yoda.Yoda(globalWorkDir, localWorkDir)
yoda.run()
else:
droid = Droid.Droid(globalWorkDir, localWorkDir)
droid.run()
def main(globalWorkDir, localWorkDir):
comm = MPI.COMM_WORLD
mpirank = comm.Get_rank()
# Create separate working directory for each rank
from os.path import abspath as _abspath, join as _join
curdir = _abspath(localWorkDir)
wkdirname = "rank_%s" % str(mpirank)
wkdir = _abspath(_join(curdir, wkdirname))
if not os.path.exists(wkdir):
os.makedirs(wkdir)
os.chdir(wkdir)
if mpirank == 0:
yoda = Yoda.Yoda(globalWorkDir, localWorkDir)
yoda.run()
else:
droid = Droid.Droid(globalWorkDir, localWorkDir)
droid.run()
def _getPath(fileName: str = None) -> any:
if _isValid(fileName):
filePath = _abspath(fileName)
args.full_path = True
for bms in getBookmarks():
if bms[1] == filePath:
raise Exception(bms[0] +" Already Bookmarked")
return filePath
else:
raise FileNotFoundError()
def __init__(self):
# robot is expected to be run from the NVDA repo root directory. We want all repo specific
# paths to be relative to this. This would allow us to change where it is run from if we decided to.
self.repoRoot = _abspath("./")
self.stagingDir = _tempFile.gettempdir()
opSys.directory_should_exist(self.stagingDir)
self.whichNVDA = builtIn.get_variable_value("${whichNVDA}", "source")
if self.whichNVDA == "source":
self._runNVDAFilePath = _pJoin(self.repoRoot, "source/nvda.pyw")
self.baseNVDACommandline = f"pyw -3.7-32 {self._runNVDAFilePath}"
elif self.whichNVDA == "installed":
self._runNVDAFilePath = _pJoin(_expandvars('%PROGRAMFILES%'), 'nvda', 'nvda.exe')
self.baseNVDACommandline = f'"{str(self._runNVDAFilePath)}"'
else:
raise AssertionError("RobotFramework should be run with argument: '-v whichNVDA [source|installed]'")
self.profileDir = _pJoin(self.stagingDir, "nvdaProfile")
self.logPath = _pJoin(self.profileDir, 'nvda.log')
self.preservedLogsDir = _pJoin(
builtIn.get_variable_value("${OUTPUT DIR}"),
"nvdaTestRunLogs"
)
def to_abspath(*paths):
return _abspath(_join(*paths))
# -*- coding: utf-8 -*-
from __future__ import absolute_import
# set the warning format to be on a single line
import warnings as _warnings
from os import name as _name
from os.path import abspath as _abspath
from os.path import dirname as _dirname
from cobra import flux_analysis, io
from cobra.core import (
DictList, Gene, Metabolite, Model, Object, Reaction, Species)
from cobra.util.version_info import show_versions
__version__ = "0.11.2"
# set the warning format to be prettier and fit on one line
_cobra_path = _dirname(_abspath(__file__))
if _name == "posix":
_warning_base = "%s:%s \x1b[1;31m%s\x1b[0m: %s\n" # colors
else:
_warning_base = "%s:%s %s: %s\n"
def _warn_format(message, category, filename, lineno, file=None, line=None):
shortname = filename.replace(_cobra_path, "cobra", 1)
return _warning_base % (shortname, lineno, category.__name__, message)
_warnings.formatwarning = _warn_format
""" Lind Static Resources ==== Lind static resources is a package with static files containing pre-computed experimental designs for the package Lind. These static designs offer optional functionality for users of Lind. """ from os.path import dirname as _dirname, abspath as _abspath static_files_abs_path = _dirname(_abspath(__file__)) from ._version import get_versions __version__ = get_versions()['version'] del get_versions
def main(globalWorkDir,
localWorkDir,
nonMPIMode=False,
outputDir=None,
dumpEventOutputs=True):
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
if nonMPIMode:
comm = None
mpirank = 0
mpisize = 1
else:
try:
from mpi4py import MPI
comm = MPI.COMM_WORLD
mpirank = comm.Get_rank()
mpisize = comm.Get_size()
except:
print "Failed to load mpi4py: %s" % (traceback.format_exc())
sys.exit(1)
# Create separate working directory for each rank
from os.path import abspath as _abspath, join as _join
curdir = _abspath(localWorkDir)
wkdirname = "rank_%s" % str(mpirank)
wkdir = _abspath(_join(curdir, wkdirname))
if not os.path.exists(wkdir):
os.makedirs(wkdir)
os.chdir(wkdir)
print "GlobalWorkDir: %s" % globalWorkDir
print "LocalWorkDir: %s" % localWorkDir
print "OutputDir: %s" % outputDir
print "RANK: %s" % mpirank
if mpirank == 0:
try:
from pandayoda.yodacore import Yoda
yoda = Yoda.Yoda(globalWorkDir,
localWorkDir,
rank=0,
nonMPIMode=nonMPIMode,
outputDir=outputDir,
dumpEventOutputs=dumpEventOutputs)
yoda.start()
from pandayoda.yodaexe import Droid
reserveCores = 1
droid = Droid.Droid(globalWorkDir,
localWorkDir,
rank=0,
nonMPIMode=True,
reserveCores=reserveCores,
outputDir=outputDir)
droid.start()
i = 30
while True:
print "Rank %s: Yoda isAlive %s" % (mpirank, yoda.isAlive())
print "Rank %s: Droid isAlive %s" % (mpirank, droid.isAlive())
if yoda and yoda.isAlive():
time.sleep(60)
else:
break
print "Rank %s: Yoda finished" % (mpirank)
except:
print "Rank %s: Yoda failed: %s" % (mpirank,
traceback.format_exc())
sys.exit(0)
#os._exit(0)
else:
try:
status = 0
from pandayoda.yodaexe import Droid
droid = Droid.Droid(globalWorkDir,
localWorkDir,
rank=mpirank,
nonMPIMode=nonMPIMode,
outputDir=outputDir)
droid.start()
while (droid and droid.isAlive()):
droid.join(timeout=1)
# parent process
#pid, status = os.waitpid(child_pid, 0)
print "Rank %s: Droid finished status: %s" % (mpirank, status)
except:
print "Rank %s: Droid failed: %s" % (mpirank,
traceback.format_exc())
#sys.exit(0)
return mpirank
from os.path import abspath as _abspath
from sys import path as _path
_one_folder_up = _abspath(__file__).rsplit('/', 2)[0]
_path.insert(1, _one_folder_up)
# if __name__ == "__main__":
# from os.path import abspath as _abspath
# import imp as _imp
# _current_folder_init = _abspath(__file__).rsplit('/', 1)[0]+ "/__init__.py"
# _imp.load_source("__init__", _current_folder_init)
# -*- coding: utf-8 -*- from os.path import (abspath as _abspath, dirname as _dirname, join as _join) BASEDIR = _dirname(_dirname(_abspath(__file__))) DATADIR = _join(BASEDIR, 'tests', 'data')
"""The PiRan module."""
from os.path import isfile, getsize as get_len_of_file, abspath as _abspath, dirname
from os import remove
from sys import maxsize, stdout, stderr
from ctypes import CDLL, cast as c_cast, c_char_p
from typing import Optional, IO, Union, cast
__author__ = "Firefnix"
__license__ = "GPL-3-or-later"
__version__ = "1.0"
location = _abspath(dirname(__file__))
del dirname
def _required_digits(max_value: int) -> int:
return len(str(max_value + 1))
def _assert_uint(arg: object) -> None:
if (not isinstance(arg, int)) or arg < 0:
raise ValueError(f"Must be an positive integer (is {arg})")
class Random:
"""Use random numbers, bytes and chars."""
def __init__(self, pi_file_name: str = f"{location}pi",
cursor: Union[str, int, None] = None) -> None:
self._cursor_file_name: str
if isfile(pi_file_name) is False:
raise FileNotFoundError(f"{_abspath(pi_file_name)} not found")
from __future__ import absolute_import
# set the warning format to be on a single line
import warnings as _warnings
from os import name as _name
from os.path import abspath as _abspath
from os.path import dirname as _dirname
from cobra.core import (Configuration, DictList, Gene, Metabolite, Model,
Object, Reaction, Species)
from cobra import flux_analysis
from cobra import io
from cobra.util import show_versions
__version__ = "0.14.2"
# set the warning format to be prettier and fit on one line
_cobra_path = _dirname(_abspath(__file__))
if _name == "posix":
_warning_base = "%s:%s \x1b[1;31m%s\x1b[0m: %s\n" # colors
else:
_warning_base = "%s:%s %s: %s\n"
def _warn_format(message, category, filename, lineno, file=None, line=None):
shortname = filename.replace(_cobra_path, "cobra", 1)
return _warning_base % (shortname, lineno, category.__name__, message)
_warnings.formatwarning = _warn_format
import numpy as np
import mdtraj as md
import nglview
from os.path import dirname as _dirname, join as _join, abspath as _abspath
from copy import deepcopy
from simtk.openmm import app
test_systems_path = _abspath(
_join(_dirname(__file__), '../examples/testsystems'))
def copy_molcomplex(molcomplex):
tmp_complex = deepcopy(molcomplex)
tmp_complex.modeller = app.Modeller(molcomplex.topology,
molcomplex.positions)
tmp_complex.topology = tmp_complex.modeller.getTopology()
return tmp_complex
def make_view(topology, positions):
if type(positions) == list:
positions = np.array([tmp_pos._value
for tmp_pos in positions]) * positions[0].unit
topology_mdtraj = md.Topology.from_openmm(topology)
positions_mdtraj = positions._value
aux_traj = md.Trajectory(positions_mdtraj, topology_mdtraj)
# -*- coding: utf-8 -*-
from __future__ import absolute_import
# set the warning format to be on a single line
import warnings as _warnings
from os import name as _name
from os.path import abspath as _abspath
from os.path import dirname as _dirname
# set the warning format to be prettier and fit on one line
_metabolicmodelgapfilling_path = _dirname(_abspath(__file__))
if _name == "posix":
_warning_base = "%s:%s \x1b[1;31m%s\x1b[0m: %s\n" # colors
else:
_warning_base = "%s:%s %s: %s\n"
def _warn_format(message, category, filename, lineno, file=None, line=None):
shortname = filename.replace(_metabolicmodelgapfilling_path,
"MetabolicModelGapfilling", 1)
return _warning_base % (shortname, lineno, category.__name__, message)
_warnings.formatwarning = _warn_format
import sys
from MetabolicModelGapfilling.core import (BaseModule, GapfillingModule)
__version__ = "0.1.0"
from os.path import join as _join, dirname as _dirname
from os.path import abspath as _abspath
import pandas as _pandas
datadir = _abspath(_join(_dirname(__file__), '..', 'ohmyyeast-data'))
def load_optpub():
_cols = [
'AID', 'e2', 'e3', 'e4', 'e5', 'e6', 'e7', 'e8', 'e9', 'e10', 'S',
'k_SA', 'K_SA', 'k_BA', 'K_BA', 'ki_BA', 'Ki_BA', 'k_CA', 'K_CA',
'ki_CA', 'Ki_CA', 'k_AA', 'K_AA', 'ki_AA', 'Ki_AA', 'k_AB', 'K_AB',
'ki_AB', 'Ki_AB', 'k_CB', 'K_CB', 'ki_CB', 'Ki_CB', 'k_BB', 'K_BB',
'ki_BB', 'Ki_BB', 'k_BC', 'K_BC', 'ki_BC', 'Ki_BC', 'k_AC', 'K_AC',
'ki_AC', 'Ki_AC', 'k_CC', 'K_CC', 'ki_CC', 'Ki_CC', 'ki_EA', 'Ki_EA',
'ki_EB', 'Ki_EB', 'ki_EC', 'Ki_EC', 'k_FA', 'K_FA', 'k_FB', 'K_FB',
'k_FC', 'K_FC', 'FUS1MAX', 'Ke50_FUS1', 'kd', 'ks', 'n', 'At', 'Bt',
'Ct', 'E', 'F', 'fVal'
]
myfile = _join(datadir, 'opt/pub', 'reduced.mat.csv')
dataset = _pandas.read_csv(myfile, names=_cols)
for edge in ['e2', 'e3', 'e4', 'e5', 'e6', 'e7', 'e8', 'e9', 'e10']:
dataset[edge] = dataset[edge].astype(int)
return dataset
def load_optsnu():
myfile = _join(datadir, 'opt/snu', 'reduced.mat.csv')
def main():
start_g = time.time()
start_g_str = time.asctime(time.localtime(start_g))
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
max_rank = comm.Get_size()
print("%s | Rank: %s from %s ranks started" %
(start_g_str, rank, max_rank))
# Get a file name with job descriptions
input_file = sys.argv[1]
#Read file into an array removing new line.
try:
file = open(input_file)
temp = file.read().splitlines()
file.close()
except IOError as e:
print("I/O error({0}): {1}".format(e.errno, e.strerror))
print("Exit from rank %s" % (rank))
return
# Number of commands in the input file
n_comm = len(temp)
#print("Number of commands: %s" % n_comm)
# Safeguard here. Just in case things are messed up.
if n_comm != max_rank:
print(
"Rank %s: problem with input file! Number of commands not equal to number of ranks! Exiting"
% (rank))
exit()
#PandaID of the job for the command
wkdirname = temp[rank].split("!!")[0].strip()
# Get the corresponding command from input file
my_command = temp[rank].split("!!")[1]
#filename for stdout & srderr
payloadstdout_s = temp[rank].split("!!")[2]
payloadstderr_s = temp[rank].split("!!")[3]
PanDA_jobId = wkdirname[wkdirname.find("_") + 1:]
# create working directory for a given rank
curdir = _abspath(os.curdir)
# wkdirname = "PandaJob_%s" % my_job_index
wkdir = _abspath(_join(curdir, wkdirname))
if not os.path.exists(wkdir):
os.mkdir(wkdir)
'''
print "Directory ", wkdir, " created"
else:
print "Directory ", wkdir, " exists"
'''
# Go to local working directory (job rank)
os.chdir(wkdir)
# If rank == 0 then setup MySQL db, all others wait for db to start
if rank % 32 == 0:
print("Rank %s is going to start MySQL db" % (rank))
print("Preparing environment")
dbsetup_comm_arr = [
'cp /projects/sciteam/balh/panda/db_setup.sh .',
'sh db_setup.sh &>dbsetup.log &'
]
dbsetup_comm = "\n".join(dbsetup_comm_arr)
p = subprocess.Popen(dbsetup_comm, shell=True)
time.sleep(90)
print("Going to check MySQL server status")
output = subprocess.Popen("ps -C mysqld -o pid=",
stdout=subprocess.PIPE,
shell=True).communicate()[0]
print("MySQL server pid: %s" % output.rstrip())
if len(output) > 0:
print("MySQL database is running, going to send GO! broadcast")
output = subprocess.Popen("echo $HOSTNAME",
stdout=subprocess.PIPE,
shell=True).communicate()[0]
print("MySQL database is running on %s" % output.rstrip())
else:
print("MySQL database is not running, exiting")
else:
print(
"Rank %s is going to sleep a little bit to allow MySQL db to start"
% (rank))
time.sleep(90 + random.randint(10, 100))
mysql_host = "0.0.0.0"
coral_cdbserver_comm = "export CDBSERVER=%s;" % mysql_host
my_command = coral_cdbserver_comm + my_command
# pcmd = subprocess.call(coral_cdbserver_comm, shell=True)
payloadstdout = open(payloadstdout_s, "w")
payloadstderr = open(payloadstderr_s, "w")
localtime = time.asctime(time.localtime(time.time()))
print("%s | %s Rank: %s Starting payload: %s" %
(localtime, PanDA_jobId, rank, my_command))
t0 = os.times()
exit_code = subprocess.call(my_command,
stdout=payloadstdout,
stderr=payloadstderr,
shell=True)
t1 = os.times()
localtime = time.asctime(time.localtime(time.time()))
t = map(lambda x, y: x - y, t1, t0)
t_tot = reduce(lambda x, y: x + y, t[2:3])
report = open("rank_report.txt", "w")
report.write("cpuConsumptionTime: %s\n" % (t_tot))
report.write("exitCode: %s" % exit_code)
report.close()
# Finished all steps here
localtime = time.asctime(time.localtime(time.time()))
end_all = time.time()
print(
"%s | %s Rank: %s Finished. Program run took. %s min. Exit code: %s cpuConsumptionTime: %s"
% (localtime, PanDA_jobId, rank,
(end_all - start_g) / 60., exit_code, t_tot))
return 0
from os.path import join as _join
try:
import winsound as _winsound
except ImportError:
_winsound = None
try:
# Python2
import Tkinter as _tk
except ImportError:
# Python3
import tkinter as _tk
__all__ = ['YouDidntSayTheMagicWord', 'getpass', 'access_main_program']
_PATH = _dirname(_abspath(__file__))
class YouDidntSayTheMagicWord(Exception):
"""
Custom YouDidntSayTheMagicWord exception that spams the console with
"YOU DIDN'T SAY THE MAGIC WORD!" while displaying an animated gif of
Dennis Nedry wagging his finger with "Uh uh uh, you didn't say the
magic word." repeating in the background. Why? Because.
"""
def __init__(self):
self._console_spam()
_sleep(0.8)
self._audio_spam()
self._gif_spam()
# $Id: $ # Make sure Python loads the modules of this package via absolute paths. from os.path import abspath as _abspath __path__[0] = _abspath(__path__[0])
def main(globalWorkDir, localWorkDir, nonMPIMode=False):
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
if nonMPIMode:
comm = None
mpirank = 0
mpisize = 1
else:
try:
from mpi4py import MPI
comm = MPI.COMM_WORLD
mpirank = comm.Get_rank()
mpisize = comm.Get_size()
except:
print "Failed to load mpi4py: %s" % (traceback.format_exc())
sys.exit(1)
# Create separate working directory for each rank
from os.path import abspath as _abspath, join as _join
curdir = _abspath (localWorkDir)
wkdirname = "rank_%s" % str(mpirank)
wkdir = _abspath (_join(curdir,wkdirname))
if not os.path.exists(wkdir):
os.makedirs (wkdir)
os.chdir (wkdir)
print "GlobalWorkDir: %s" % globalWorkDir
print "LocalWorkDir: %s" % localWorkDir
print "RANK: %s" % mpirank
if mpirank==0:
try:
from pandayoda.yodacore import Yoda
yoda = Yoda.Yoda(globalWorkDir, localWorkDir, rank=0, nonMPIMode=nonMPIMode)
yoda.start()
from pandayoda.yodaexe import Droid
reserveCores = 0
if mpisize > 500:
reserveCores = 4
elif mpisize > 200:
reserveCores = 3
elif mpisize > 100:
reserveCores = 2
elif mpisize > 50:
reserveCores = 1
droid = Droid.Droid(globalWorkDir, localWorkDir, rank=0, nonMPIMode=True, reserveCores=reserveCores)
droid.start()
while True:
yoda.join(timeout=1)
if yoda and yoda.isAlive():
pass
else:
break
print "Rank %s: Yoda finished" % (mpirank)
except:
print "Rank %s: Yoda failed: %s" % (mpirank, traceback.format_exc())
sys.exit(0)
else:
try:
status = 0
from pandayoda.yodaexe import Droid
droid = Droid.Droid(globalWorkDir, localWorkDir, rank=mpirank, nonMPIMode=nonMPIMode)
droid.start()
while (droid and droid.isAlive()):
droid.join(timeout=1)
# parent process
#pid, status = os.waitpid(child_pid, 0)
print "Rank %s: Droid finished status: %s" % (mpirank, status)
except:
print "Rank %s: Droid failed: %s" % (mpirank, traceback.format_exc())
#sys.exit(0)
return mpirank
# We need to execute this file directly, so we always run the import hack
from os.path import abspath as _abspath
import imp as _imp
_current_folder_init = _abspath(__file__).rsplit('/', 1)[0] + "/__init__.py"
_imp.load_source("__init__", _current_folder_init)
from kombu.exceptions import OperationalError
from celery import Celery, states
from celery.states import SUCCESS
STARTED_OR_WAITING = [states.PENDING, states.RECEIVED, states.STARTED]
FAILED = [states.REVOKED, states.RETRY, states.FAILURE]
try:
with open("/home/ubuntu/manager_ip", 'r') as f:
manager_ip = f.read()
except IOError:
print "could not load the manager ip file, defaulting to 127.0.0.1"
manager_ip = "127.0.0.1"
celery_app = Celery("data_processing_tasks",
broker='pyamqp://guest@%s//' % manager_ip,
backend='rpc://',
task_publish_retry=False,
task_track_started=True)
# Load Django
from config import load_django
################################################################################
from robot.libraries.BuiltIn import BuiltIn
from robot.libraries.OperatingSystem import OperatingSystem
from robot.libraries.Process import Process
from systemTestUtils import _blockUntilConditionMet
builtIn = BuiltIn() # type: BuiltIn
process = builtIn.get_library_instance('Process') # type: Process
opSys = builtIn.get_library_instance('OperatingSystem') # type: OperatingSystem
spyServerPort = 8270 # is `registered by IANA` for remote server usage. Two ASCII values:'RF'
spyServerURI = 'http://127.0.0.1:{}'.format(spyServerPort)
spyAlias = "nvdaSpy"
# robot is expected to be run from the NVDA repo root directory. We want all repo specific
# paths to be relative to this. This would allow us to change where it is run from if we decided to.
repoRoot = _abspath("./")
whichNVDA = builtIn.get_variable_value("${whichNVDA}", "source")
if whichNVDA == "source":
NVDACommandPathToCheckExists = _pJoin(repoRoot, "source/nvda.pyw")
baseNVDACommandline = "pythonw "+NVDACommandPathToCheckExists
elif whichNVDA == "installed":
NVDACommandPathToCheckExists = _pJoin(_expandvars('%PROGRAMFILES%'),'nvda','nvda.exe')
baseNVDACommandline='"%s"' % NVDACommandPathToCheckExists
else:
raise AssertionError("robot should be given argument `-v whichNVDA [source|installed]")
# Paths
systemTestSourceDir = _pJoin(repoRoot, "tests", "system")
tempDir = tempfile.gettempdir()
opSys.directory_should_exist(tempDir)
nvdaProfileWorkingDir = _pJoin(tempDir, "nvdaProfile")
from robot.libraries.OperatingSystem import OperatingSystem
from robot.libraries.Process import Process
from systemTestUtils import _blockUntilConditionMet
builtIn = BuiltIn() # type: BuiltIn
process = builtIn.get_library_instance('Process') # type: Process
opSys = builtIn.get_library_instance(
'OperatingSystem') # type: OperatingSystem
spyServerPort = 8270 # is `registered by IANA` for remote server usage. Two ASCII values:'RF'
spyServerURI = 'http://127.0.0.1:{}'.format(spyServerPort)
spyAlias = "nvdaSpy"
# robot is expected to be run from the NVDA repo root directory. We want all repo specific
# paths to be relative to this. This would allow us to change where it is run from if we decided to.
repoRoot = _abspath("./")
whichNVDA = builtIn.get_variable_value("${whichNVDA}", "source")
if whichNVDA == "source":
NVDACommandPathToCheckExists = _pJoin(repoRoot, "source/nvda.pyw")
baseNVDACommandline = "pythonw " + NVDACommandPathToCheckExists
elif whichNVDA == "installed":
NVDACommandPathToCheckExists = _pJoin(_expandvars('%PROGRAMFILES%'),
'nvda', 'nvda.exe')
baseNVDACommandline = '"%s"' % NVDACommandPathToCheckExists
else:
raise AssertionError(
"robot should be given argument `-v whichNVDA [source|installed]")
# Paths
systemTestSourceDir = _pJoin(repoRoot, "tests", "system")
tempDir = tempfile.gettempdir()
def main(globalWorkDir, localWorkDir, nonMPIMode=False, outputDir=None, dumpEventOutputs=True):
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
if nonMPIMode:
comm = None
mpirank = 0
mpisize = 1
else:
try:
comm = MPI.COMM_WORLD
mpirank = comm.Get_rank()
mpisize = comm.Get_size()
except:
logger.exception("Failed to load mpi4py")
raise
# Create separate working directory for each rank
from os.path import abspath as _abspath, join as _join
curdir = _abspath(localWorkDir)
wkdirname = "rank_%s" % str(mpirank)
wkdir = _abspath(_join(curdir, wkdirname))
if not os.path.exists(wkdir):
os.makedirs(wkdir)
os.chdir(wkdir)
logger.info("GlobalWorkDir: %s" % globalWorkDir)
logger.info("LocalWorkDir: %s" % localWorkDir)
logger.info("OutputDir: %s" % outputDir)
logger.info("RANK: %s" % mpirank)
if mpirank == 0:
try:
from pandayoda.yodacore import Yoda
yoda = Yoda.Yoda(
globalWorkDir,
localWorkDir,
rank=0,
nonMPIMode=nonMPIMode,
outputDir=outputDir,
dumpEventOutputs=dumpEventOutputs,
)
yoda.start()
from pandayoda.yodaexe import Droid
if nonMPIMode:
reserveCores = 0
else:
reserveCores = 1
droid = Droid.Droid(
globalWorkDir, localWorkDir, rank=0, nonMPIMode=True, reserveCores=reserveCores, outputDir=outputDir
)
droid.start()
i = 30
while True:
logger.info("Rank %s: Yoda isAlive %s" % (mpirank, yoda.isAlive()))
logger.info("Rank %s: Droid isAlive %s" % (mpirank, droid.isAlive()))
if yoda and yoda.isAlive():
time.sleep(60)
else:
break
logger.info("Rank %s: Yoda finished" % (mpirank))
except:
logger.exception("Rank %s: Yoda failed" % mpirank)
raise
# os._exit(0)
else:
try:
status = 0
from pandayoda.yodaexe import Droid
droid = Droid.Droid(globalWorkDir, localWorkDir, rank=mpirank, nonMPIMode=nonMPIMode, outputDir=outputDir)
droid.start()
while droid and droid.isAlive():
droid.join(timeout=1)
# parent process
# pid, status = os.waitpid(child_pid, 0)
logger.info("Rank %s: Droid finished status: %s" % (mpirank, status))
except:
logger.exception("Rank %s: Droid failed" % mpirank)
raise
return mpirank
# Make sure Python loads the modules of this package via absolute paths.
from os.path import abspath as _abspath
from gppylib.commands.base import WorkerPool
from gppylib import gplog
__path__[0] = _abspath(__path__[0])
logger = gplog.get_default_logger()
class Operation(object):
"""
An Operation (abstract class) is one atomic unit of work.
An implementation of Operation is fully defined by:
__init__(self, opts, args) - opts, args are intended to be validated and marshaled, as appropriate
execute(self) - the actual Operation logic commences
cleanup(self) - [optional] any cleanup operations; e.g. stopping a WorkerPool
undo(self) - ?
progress(self) - ?
validate(self) - ?
cancel(self) - ?
By decomposing work into these Operation classes whenever possible,
we should be able to reap several benefits:
1. Unit testing - It's easier to build unit tests around functions
that are not tightly bound to a monolithic class
2. Composition - We should be able to compose Operations in a useful manner.
Consider ParallelOperation, RemoteOperation, SerialOperation, etc.
3. Readability - We no longer have to create nested, segment-facing scripts
to perform work on segment hosts: e.g. figuring out how
gpstart -> startSegments -> gpsegstart.py propagates information is
from os.path import abspath as _abspath, dirname as _dirname
_PROJECT_ROOT = _dirname(_abspath(__file__))
ADMINS = (
# ('Your Name', '*****@*****.**'),
)
MANAGERS = ADMINS
LOGIN_URL = '/openid/login/'
LOGIN_REDIRECT_URL = '/publish/'
OPENID_USE_AS_ADMIN_LOGIN = True
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
'NAME': 'data', # Or path to database file if using sqlite3.
'USER': '', # Not used with sqlite3.
'PASSWORD': '', # Not used with sqlite3.
'HOST': '', # Set to empty string for localhost. Not used with sqlite3.
'PORT': '', # Set to empty string for default. Not used with sqlite3.
}
}
USE_TZ=True
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same