def function(self, f):
"""
setter for function that carefully takes care of
namespace, avoiding __main__ as a module
"""
m = inspect.getmodule(f)
try:
self.path = clean_path(os.path.dirname(os.path.abspath(
inspect.getsourcefile(f))))
except TypeError:
self.path = ''
# if module is not __main__, all is good
if m.__name__ != "__main__":
self._f = f
else:
# determine real module name
mn = os.path.splitext(os.path.basename(m.__file__))[0]
# make sure module is present
__import__(mn)
# get module
mod = sys.modules[mn]
# set function from module
self._f = getattr(mod, f.__name__)
def _main():
"""
Parse the command line inputs and call _run_job
"""
# Get command line arguments
parser = argparse.ArgumentParser(
description="This wrapper script will run \
a pickled Python function on \
some pickled retrieved data \
via 0MQ. You almost never \
want to run this yourself.")
parser.add_argument('home_address', help='IP address of submitting host.')
parser.add_argument(
'module_dir',
help='Directory that contains module containing pickled\
function. This will get added to PYTHONPATH \
temporarily.')
args = parser.parse_args()
# Make warnings from built-in warnings module get formatted more nicely
logging.captureWarnings(True)
logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' +
'%(message)s'),
level=logging.INFO)
logger = logging.getLogger(__name__)
logger.info("Appended {0} to PYTHONPATH".format(args.module_dir))
sys.path.append(clean_path(args.module_dir))
logger.debug("Job ID: %i\tHome address: %s\tModule dir: %s",
os.environ['JOB_ID'], args.home_address, args.module_dir)
# Process the database and get job started
_run_job(os.environ['JOB_ID'], args.home_address)
def _main():
"""
Parse the command line inputs and call _run_job
"""
# Get command line arguments
parser = argparse.ArgumentParser(description="This wrapper script will run \
a pickled Python function on \
some pickled retrieved data \
via 0MQ. You almost never \
want to run this yourself.")
parser.add_argument('home_address',
help='IP address of submitting host.')
parser.add_argument('module_dir',
help='Directory that contains module containing pickled\
function. This will get added to PYTHONPATH \
temporarily.')
args = parser.parse_args()
# Make warnings from built-in warnings module get formatted more nicely
logging.captureWarnings(True)
logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' +
'%(message)s'), level=logging.INFO)
logger = logging.getLogger(__name__)
logger.info("Appended {0} to PYTHONPATH".format(args.module_dir))
sys.path.append(clean_path(args.module_dir))
logger.debug("Job ID: %i\tHome address: %s\tModule dir: %s",
os.environ['JOB_ID'],
args.home_address, args.module_dir)
# Process the database and get job started
_run_job(os.environ['JOB_ID'], args.home_address)
def main(argv=None):
"""
Parse the command line inputs and start web monitor.
:param argv: List of arguments, as if specified on the command-line.
If None, ``sys.argv`` is used instead.
:type argv: list of str
"""
# Get command line arguments
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(description="Provides a web interface to \
0MQ job monitor.")
parser.add_argument(
'module_dir',
help='Directory that contains module containing pickled\
function. This will get added to PYTHONPATH \
temporarily.',
nargs='+')
parser.add_argument('-p',
'--port',
help='Port for server to listen on.',
type=int,
default=8076)
parser.add_argument('--version',
action='version',
version='%(prog)s {0}'.format(__version__))
args = parser.parse_args(argv)
# Make warnings from built-in warnings module get formatted more nicely
logging.captureWarnings(True)
logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' +
'%(message)s'))
logger = logging.getLogger(__name__)
# Append module directories to path
for module_dir in args.module_dir:
logger.info("Appended {0} to PYTHONPATH".format(module_dir))
sys.path.append(clean_path(module_dir))
# Start server
hostname = gethostname()
if not isinstance(hostname, bytes):
hostname = hostname.encode()
cherrypy.quickstart(WebMonitor(),
config={
b'global': {
b'server.socket_port': args.port,
b'server.socket_host': hostname
}
})
def main(argv=None):
"""
Parse the command line inputs and start web monitor.
:param argv: List of arguments, as if specified on the command-line.
If None, ``sys.argv`` is used instead.
:type argv: list of str
"""
# Get command line arguments
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(
description="Provides a web interface to \
0MQ job monitor."
)
parser.add_argument(
"module_dir",
help="Directory that contains module containing pickled\
function. This will get added to PYTHONPATH \
temporarily.",
nargs="+",
)
parser.add_argument("-p", "--port", help="Port for server to listen on.", type=int, default=8076)
parser.add_argument("--version", action="version", version="%(prog)s {0}".format(__version__))
args = parser.parse_args(argv)
# Make warnings from built-in warnings module get formatted more nicely
logging.captureWarnings(True)
logging.basicConfig(format=("%(asctime)s - %(name)s - %(levelname)s - " + "%(message)s"))
logger = logging.getLogger(__name__)
# Append module directories to path
for module_dir in args.module_dir:
logger.info("Appended {0} to PYTHONPATH".format(module_dir))
sys.path.append(clean_path(module_dir))
# Start server
hostname = gethostname()
if not isinstance(hostname, bytes):
hostname = hostname.encode()
cherrypy.quickstart(
WebMonitor(), config={b"global": {b"server.socket_port": args.port, b"server.socket_host": hostname}}
)
def __init__(self, f, args, kwlist=None, cleanup=True, mem_free="1G",
name='gridmap_job', num_slots=1, queue=DEFAULT_QUEUE):
"""
Initializes a new Job.
:param f: a function, which should be executed.
:type f: function
:param args: argument list of function f
:type args: list
:param kwlist: dictionary of keyword arguments for f
:type kwlist: dict
:param cleanup: flag that determines the cleanup of input and log file
:type cleanup: boolean
:param mem_free: Estimate of how much memory this job will need (for
scheduling)
:type mem_free: str
:param name: Name to give this job
:type name: str
:param num_slots: Number of slots this job should use.
:type num_slots: int
:param queue: SGE queue to schedule job on.
:type queue: str
"""
self.track_mem = []
self.track_cpu = []
self.heart_beat = None
self.traceback = None
self.host_name = ''
self.timestamp = None
self.log_stdout_fn = ''
self.log_stderr_fn = ''
self.home_address = ''
self.num_resubmits = 0
self.cause_of_death = ''
self.path = None
self._f = None
self.function = f
self.args = args
self.id = -1
self.kwlist = kwlist if kwlist is not None else {}
self.cleanup = cleanup
self.ret = _JOB_NOT_FINISHED
self.num_slots = num_slots
self.mem_free = mem_free
self.white_list = []
self.name = name.replace(' ', '_')
self.queue = queue
# Save copy of environment variables
self.environment = {}
for env_var, value in os.environ.items():
try:
if not isinstance(env_var, bytes):
env_var = env_var.encode()
if not isinstance(value, bytes):
value = value.encode()
except UnicodeEncodeError:
logger = logging.getLogger(__name__)
logger.warning('Skipping non-ASCII environment variable.')
else:
self.environment[env_var] = value
self.working_dir = clean_path(os.getcwd())
