def __init__(self, serverUrl, serverPass, golemOutputPath, pickleScratch, thisLibraryPath, pyPath = "/hpc/bin/python", pickleOut = None, taskSize = 10): """ Constructor for a Golemizer. Parameters: serverUrl - String representing the URL of the server (including port number). serverPass - String representing the server password. golemOutputPath - String representing the file system path containing the golem_n folders used as working directories for the worker nodes. pickleScratch - String representing the file system path where Golemizer should write the pickled input, and possibly other files. Needs to be reachable, via the same path, on all workers. thisLibraryPath - String representing the file system path, accessible from all machines, to golemize.py. pyPath - String representing the file system path to the installation of Python on the workers. Default: "/hpc/bin/python" pickleOut - Alternate output directory for pickle files. The working directory for the Golem nodes will be used if this is not provided. Default: "None", which is equivalent to "./" or anything that evaluates to false taskSize - Number of tasks per execution of the software on a worker. Larger values (possibly significantly) reduce job distribution overhead. Smaller values can be distributed more evenly. """ self.masterPath = golem.canonizeMaster(serverUrl, False) + "/jobs/" self.serverPass = serverPass self.golemOutPath = golemOutputPath #self.golemIds = ["{0:02d}".format(id) for id in golemIdSeq] self.pickleInputShare = pickleScratch self.pyPath = pyPath self.thisLibraryPath = thisLibraryPath if pickleOut: self.jobOutputPath = pickleOut else: self.jobOutputPath = "./" self.taskSize = taskSize
def main(argv): """ Handles command line arguments and uses them to start a job (or job batch) and wait for it to finish. Intended to be used interactively from the __name__ == "__main__" check. """ parser = optparse.OptionParser() parser.add_option("-p", "--password", dest="password", help="Specify the password for connecting to the server.", default="") parser.add_option("-e", "--echo", dest="echo", action="store_true", default=False, help = "Not yet implemented") flags, args = parser.parse_args(argv[1:4]) #because "late params" are actually arguments to the target script password = flags.password args = args + argv[4:] if len(args) < 3: print "Not enough arguments." printUsage() sys.exit(status=493) master = args[0] master = golem.canonizeMaster(master) url = master+"/jobs/" command = args[1] cmdArgs = args[2:] if command=="run": response, content = golem.runOneLine(int(cmdArgs[0]), cmdArgs[1:], password, url) elif command=="runlist": response, content = golem.runList(open(cmdArgs[0]), password, url) elif command=="runoneach": response, content = golem.runOnEach([{"Args": cmdArgs}],password,url) else: raise ValueError("golemBlocking can only handle the commands 'run', 'runlist', and 'runoneach'.") id = jobIdFromResponse(content) try: stall(id, url) except KeyboardInterrupt: golem.stopJob(id, password, url) print "Job halted." jobid_dat = open("JOBID.DAT", "w") jobid_dat.write(id) jobid_dat.flush() jobid_dat.close()
def __init__(self, serverUrl, serverPass, golemOutputPath, pickleScratch, thisLibraryPath, pyPath="/hpc/bin/python", pickleOut=None, taskSize=10): """ Constructor for a Golemizer. Parameters: serverUrl - String representing the URL of the server (including port number). serverPass - String representing the server password. golemOutputPath - String representing the file system path containing the golem_n folders used as working directories for the worker nodes. pickleScratch - String representing the file system path where Golemizer should write the pickled input, and possibly other files. Needs to be reachable, via the same path, on all workers. thisLibraryPath - String representing the file system path, accessible from all machines, to golemize.py. pyPath - String representing the file system path to the installation of Python on the workers. Default: "/hpc/bin/python" pickleOut - Alternate output directory for pickle files. The working directory for the Golem nodes will be used if this is not provided. Default: "None", which is equivalent to "./" or anything that evaluates to false taskSize - Number of tasks per execution of the software on a worker. Larger values (possibly significantly) reduce job distribution overhead. Smaller values can be distributed more evenly. """ self.masterPath = golem.canonizeMaster(serverUrl, False) + "/jobs/" self.serverPass = serverPass self.golemOutPath = golemOutputPath #self.golemIds = ["{0:02d}".format(id) for id in golemIdSeq] self.pickleInputShare = pickleScratch self.pyPath = pyPath self.thisLibraryPath = thisLibraryPath if pickleOut: self.jobOutputPath = pickleOut else: self.jobOutputPath = "./" self.taskSize = taskSize
def main(argv): """ Handles command line arguments and uses them to start a job (or job batch) and wait for it to finish. Intended to be used interactively from the __name__ == "__main__" check. """ parser = optparse.OptionParser() parser.add_option( "-p", "--password", dest="password", help="Specify the password for connecting to the server.", default="") parser.add_option("-e", "--echo", dest="echo", action="store_true", default=False, help="Not yet implemented") # because "late params" are actually arguments to the target script flags, args = parser.parse_args(argv[1:4]) password = flags.password args = args + argv[4:] if len(args) < 3: print "Not enough arguments." printUsage() sys.exit(status=493) master = args[0] master = golem.canonizeMaster(master) url = master + "/jobs/" command = args[1] cmdArgs = args[2:] if command == "run": response, content = golem.runOneLine(int(cmdArgs[0]), cmdArgs[1:], password, url) elif command == "runlist": response, content = golem.runList(open(cmdArgs[0]), password, url) elif command == "runoneach": response, content = golem.runOnEach([{"Args": cmdArgs}], password, url) else: raise ValueError( "golemBlocking can only handle the commands 'run', 'runlist', and 'runoneach'." ) id = jobIdFromResponse(content) try: stall(id, url) except KeyboardInterrupt: golem.stopJob(id, password, url) print "Job halted." jobid_dat = open("JOBID.DAT", "w") jobid_dat.write(id) jobid_dat.flush() jobid_dat.close()