def function(self, f): """ setter for function that carefully takes care of namespace, avoiding __main__ as a module """ m = inspect.getmodule(f) try: self.path = clean_path(os.path.dirname(os.path.abspath( inspect.getsourcefile(f)))) except TypeError: self.path = '' # if module is not __main__, all is good if m.__name__ != "__main__": self._f = f else: # determine real module name mn = os.path.splitext(os.path.basename(m.__file__))[0] # make sure module is present __import__(mn) # get module mod = sys.modules[mn] # set function from module self._f = getattr(mod, f.__name__)
def _main(): """ Parse the command line inputs and call _run_job """ # Get command line arguments parser = argparse.ArgumentParser( description="This wrapper script will run \ a pickled Python function on \ some pickled retrieved data \ via 0MQ. You almost never \ want to run this yourself.") parser.add_argument('home_address', help='IP address of submitting host.') parser.add_argument( 'module_dir', help='Directory that contains module containing pickled\ function. This will get added to PYTHONPATH \ temporarily.') args = parser.parse_args() # Make warnings from built-in warnings module get formatted more nicely logging.captureWarnings(True) logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' + '%(message)s'), level=logging.INFO) logger = logging.getLogger(__name__) logger.info("Appended {0} to PYTHONPATH".format(args.module_dir)) sys.path.append(clean_path(args.module_dir)) logger.debug("Job ID: %i\tHome address: %s\tModule dir: %s", os.environ['JOB_ID'], args.home_address, args.module_dir) # Process the database and get job started _run_job(os.environ['JOB_ID'], args.home_address)
def _main(): """ Parse the command line inputs and call _run_job """ # Get command line arguments parser = argparse.ArgumentParser(description="This wrapper script will run \ a pickled Python function on \ some pickled retrieved data \ via 0MQ. You almost never \ want to run this yourself.") parser.add_argument('home_address', help='IP address of submitting host.') parser.add_argument('module_dir', help='Directory that contains module containing pickled\ function. This will get added to PYTHONPATH \ temporarily.') args = parser.parse_args() # Make warnings from built-in warnings module get formatted more nicely logging.captureWarnings(True) logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' + '%(message)s'), level=logging.INFO) logger = logging.getLogger(__name__) logger.info("Appended {0} to PYTHONPATH".format(args.module_dir)) sys.path.append(clean_path(args.module_dir)) logger.debug("Job ID: %i\tHome address: %s\tModule dir: %s", os.environ['JOB_ID'], args.home_address, args.module_dir) # Process the database and get job started _run_job(os.environ['JOB_ID'], args.home_address)
def main(argv=None): """ Parse the command line inputs and start web monitor. :param argv: List of arguments, as if specified on the command-line. If None, ``sys.argv`` is used instead. :type argv: list of str """ # Get command line arguments if argv is None: argv = sys.argv parser = argparse.ArgumentParser(description="Provides a web interface to \ 0MQ job monitor.") parser.add_argument( 'module_dir', help='Directory that contains module containing pickled\ function. This will get added to PYTHONPATH \ temporarily.', nargs='+') parser.add_argument('-p', '--port', help='Port for server to listen on.', type=int, default=8076) parser.add_argument('--version', action='version', version='%(prog)s {0}'.format(__version__)) args = parser.parse_args(argv) # Make warnings from built-in warnings module get formatted more nicely logging.captureWarnings(True) logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' + '%(message)s')) logger = logging.getLogger(__name__) # Append module directories to path for module_dir in args.module_dir: logger.info("Appended {0} to PYTHONPATH".format(module_dir)) sys.path.append(clean_path(module_dir)) # Start server hostname = gethostname() if not isinstance(hostname, bytes): hostname = hostname.encode() cherrypy.quickstart(WebMonitor(), config={ b'global': { b'server.socket_port': args.port, b'server.socket_host': hostname } })
def main(argv=None): """ Parse the command line inputs and start web monitor. :param argv: List of arguments, as if specified on the command-line. If None, ``sys.argv`` is used instead. :type argv: list of str """ # Get command line arguments if argv is None: argv = sys.argv parser = argparse.ArgumentParser( description="Provides a web interface to \ 0MQ job monitor." ) parser.add_argument( "module_dir", help="Directory that contains module containing pickled\ function. This will get added to PYTHONPATH \ temporarily.", nargs="+", ) parser.add_argument("-p", "--port", help="Port for server to listen on.", type=int, default=8076) parser.add_argument("--version", action="version", version="%(prog)s {0}".format(__version__)) args = parser.parse_args(argv) # Make warnings from built-in warnings module get formatted more nicely logging.captureWarnings(True) logging.basicConfig(format=("%(asctime)s - %(name)s - %(levelname)s - " + "%(message)s")) logger = logging.getLogger(__name__) # Append module directories to path for module_dir in args.module_dir: logger.info("Appended {0} to PYTHONPATH".format(module_dir)) sys.path.append(clean_path(module_dir)) # Start server hostname = gethostname() if not isinstance(hostname, bytes): hostname = hostname.encode() cherrypy.quickstart( WebMonitor(), config={b"global": {b"server.socket_port": args.port, b"server.socket_host": hostname}} )
def __init__(self, f, args, kwlist=None, cleanup=True, mem_free="1G", name='gridmap_job', num_slots=1, queue=DEFAULT_QUEUE): """ Initializes a new Job. :param f: a function, which should be executed. :type f: function :param args: argument list of function f :type args: list :param kwlist: dictionary of keyword arguments for f :type kwlist: dict :param cleanup: flag that determines the cleanup of input and log file :type cleanup: boolean :param mem_free: Estimate of how much memory this job will need (for scheduling) :type mem_free: str :param name: Name to give this job :type name: str :param num_slots: Number of slots this job should use. :type num_slots: int :param queue: SGE queue to schedule job on. :type queue: str """ self.track_mem = [] self.track_cpu = [] self.heart_beat = None self.traceback = None self.host_name = '' self.timestamp = None self.log_stdout_fn = '' self.log_stderr_fn = '' self.home_address = '' self.num_resubmits = 0 self.cause_of_death = '' self.path = None self._f = None self.function = f self.args = args self.id = -1 self.kwlist = kwlist if kwlist is not None else {} self.cleanup = cleanup self.ret = _JOB_NOT_FINISHED self.num_slots = num_slots self.mem_free = mem_free self.white_list = [] self.name = name.replace(' ', '_') self.queue = queue # Save copy of environment variables self.environment = {} for env_var, value in os.environ.items(): try: if not isinstance(env_var, bytes): env_var = env_var.encode() if not isinstance(value, bytes): value = value.encode() except UnicodeEncodeError: logger = logging.getLogger(__name__) logger.warning('Skipping non-ASCII environment variable.') else: self.environment[env_var] = value self.working_dir = clean_path(os.getcwd())