def put_du_scp(self, du): logger.debug("Copy DU using SCP") du_items = du.list() for i in du_items.keys(): local_filename = du_items[i]["local"] remote_path = os.path.join(self.path, str(du.id), os.path.basename(local_filename)) logger.debug("Put file: %s to %s"%(i, remote_path)) if local_filename.startswith("ssh://"): # check if remote path is directory if self.__is_remote_directory(local_filename): logger.warning("Path %s is a directory. Ignored."%local_filename) continue #self.__third_party_transfer(i.local_url, remote_path) else: try: if stat.S_ISDIR(os.stat(local_filename).st_mode): logger.warning("Path %s is a directory. Ignored."%local_filename) continue except: pass result = urlparse.urlparse(local_filename) source_host = result.netloc source_path = result.path logger.debug(str((source_host, source_path, self.host, remote_path))) if source_host == "" or source_host==None: cmd = "scp "+ source_path + " " + self.host + ":" + remote_path else: cmd = "scp "+ source_host+":"+source_path + " " + self.host + ":" + remote_path rc = os.system(cmd) logger.debug("Command: %s Return code: %d"%(cmd, rc) )
def put_du_scp(self, du): logger.debug("Copy DU using SCP") du_items = du.list() for i in du_items.keys(): local_filename = du_items[i]["local"] remote_path = os.path.join(self.path, str(du.id), os.path.basename(local_filename)) logger.debug("Put file: %s to %s"%(i, remote_path)) if local_filename.startswith("ssh://"): # check if remote path is directory if self.__is_remote_directory(local_filename): logger.warning("Path %s is a directory. Ignored."%local_filename) continue #self.__third_party_transfer(i.local_url, remote_path) else: try: if stat.S_ISDIR(os.stat(local_filename).st_mode): logger.warning("Path %s is a directory. Ignored."%local_filename) continue except: pass result = urlparse.urlparse(local_filename) source_host = result.netloc source_path = result.path source_user = result.username logger.debug(str((source_host, source_path, self.host, remote_path))) self.__run_scp_command(self.userkey, source_user, source_host, source_path, self.user, self.host, remote_path)
def get_state(self): result = State.UNKNOWN try: self.instance.update() result=self.instance.state except: logger.warning("Instance not reachable/active yet...") return result
def get_state(self): result = State.UNKNOWN try: self.instance.update() result = self.instance.state except: logger.warning("Instance not reachable/active yet...") return result
def put_pd(self, pd): for i in pd.list_data_units(): remote_path = os.path.join(self.__get_pd_path(pd.id), os.path.basename(i.local_url)) logger.debug("Put file: %s to %s"%(i.local_url, remote_path)) if i.local_url.startswith("file://") or i.local_url.startswith("/"): if stat.S_ISDIR(os.stat(i.local_url).st_mode): logger.warning("Path %s is a directory. Ignored."%i.local_url) continue self.__webhdfs.copyFromLocal(i.local_url, remote_path) else: logger.error("File URLs: %s not supported"%i.local_url)
def get_state(self): result = State.UNKNOWN try: if self.subprocess_handle != None: rc = self.subprocess_handle.poll() if rc==None: result = State.RUNNING elif rc!=0: result = State.FAILED elif rc==0: result = State.DONE except: logger.warning("Instance not reachable/active yet...") return result
def put_pd(self, pd): for i in pd.list_data_units(): remote_path = os.path.join(self.__get_pd_path(pd.id), os.path.basename(i.local_url)) logger.debug("Put file: %s to %s" % (i.local_url, remote_path)) if i.local_url.startswith("file://") or i.local_url.startswith( "/"): if stat.S_ISDIR(os.stat(i.local_url).st_mode): logger.warning("Path %s is a directory. Ignored." % i.local_url) continue self.__webhdfs.copyFromLocal(i.local_url, remote_path) else: logger.error("File URLs: %s not supported" % i.local_url)
def put_du_paramiko(self, du): logger.debug("Copy DU using Paramiko") for i in du.list_data_unit_items(): remote_path = os.path.join(self.path, str(du.id), os.path.basename(i.local_url)) logger.debug("Put file: %s to %s"%(i.local_url, remote_path)) if i.local_url.startswith("ssh://"): # check if remote path is directory if self.__is_remote_directory(i.local_url): logger.warning("Path %s is a directory. Ignored."%i.local_url) continue self.__third_party_transfer(i.local_url, remote_path) else: if stat.S_ISDIR(os.stat(i.local_url).st_mode): logger.warning("Path %s is a directory. Ignored."%i.local_url) continue #self.__sftp.put(i.local_url, remote_path, self.put_progress, True) ssh_client, sftp_client = self.__create_sftp_client() sftp_client.put(i.local_url, remote_path) sftp_client.close() ssh_client.close()
def run(self): """ Start VMs""" # Submit job working_directory = os.getcwd() if "working_directory" in self.pilot_compute_description: working_directory=self.pilot_compute_description["working_directory"] TRIAL_MAX=3 trials=0 while trials < TRIAL_MAX: try: args = [] #args.append(self.job_description.executable) #args.extend(self.job_description.arguments) args.extend(["python", "-m", "bigjob.bigjob_agent"]) args.extend(self.job_description.arguments) logger.debug("Execute: " + str(args)) self.subprocess_handle=subprocess.Popen(args=args, stdout=self.job_output, stderr=self.job_error, cwd=working_directory, shell=False) if self.subprocess_handle.poll() != None and self.subprocess_handle.poll()!=0: logger.warning("Submission failed.") trials = trials + 1 time.sleep(30) continue else: break except: exc_type, exc_value, exc_traceback = sys.exc_info() logger.warning("Submission failed: " + str(exc_value)) #self.__print_traceback() trials = trials + 1 time.sleep(3) if trials == TRIAL_MAX: raise Exception("Submission of agent failed.") logger.debug("Job State : %s" % (self.get_state()))
def put_du_paramiko(self, du): logger.debug("Copy DU using Paramiko") for i in du.list_data_unit_items(): remote_path = os.path.join(self.path, str(du.id), os.path.basename(i.local_url)) logger.debug("Put file: %s to %s" % (i.local_url, remote_path)) if i.local_url.startswith("ssh://"): # check if remote path is directory if self.__is_remote_directory(i.local_url): logger.warning("Path %s is a directory. Ignored." % i.local_url) continue self.__third_party_transfer(i.local_url, remote_path) else: if stat.S_ISDIR(os.stat(i.local_url).st_mode): logger.warning("Path %s is a directory. Ignored." % i.local_url) continue #self.__sftp.put(i.local_url, remote_path, self.put_progress, True) ssh_client, sftp_client = self.__create_sftp_client() sftp_client.put(i.local_url, remote_path) sftp_client.close() ssh_client.close()
def put_du_scp(self, du): logger.debug("Copy DU using SCP") du_items = du.list() for i in du_items.keys(): local_filename = du_items[i]["local"] remote_path = os.path.join(self.path, str(du.id), os.path.basename(local_filename)) logger.debug("Put file: %s to %s" % (i, remote_path)) if local_filename.startswith("ssh://"): # check if remote path is directory if self.__is_remote_directory(local_filename): logger.warning("Path %s is a directory. Ignored." % local_filename) continue #self.__third_party_transfer(i.local_url, remote_path) else: try: if stat.S_ISDIR(os.stat(local_filename).st_mode): logger.warning("Path %s is a directory. Ignored." % local_filename) continue except: pass result = urlparse.urlparse(local_filename) source_host = result.netloc source_path = result.path logger.debug( str((source_host, source_path, self.host, remote_path))) if source_host == "" or source_host == None: cmd = "scp " + source_path + " " + self.host + ":" + remote_path else: cmd = "scp " + source_host + ":" + source_path + " " + self.host + ":" + remote_path rc = os.system(cmd) logger.debug("Command: %s Return code: %d" % (cmd, rc))
def run(self): """ Start VM and start BJ agent via SSH on VM """ """ Map fields of Pilot description to EC2 API { "vm_id":"ami-d7f742be", "vm_ssh_username":"******", "vm_ssh_keyname":"MyKey", "vm_ssh_keyfile":"<path>", "vm_type":"t1.micro", "access_key_id":"xxx", "secret_access_key":"xxx" } """ reservation = self.ec2_conn.run_instances(self.pilot_compute_description["vm_id"], key_name=self.pilot_compute_description["vm_ssh_keyname"], instance_type=self.pilot_compute_description["vm_type"], security_groups=[SECURITY_GROUP]) self.instance = reservation.instances[0] self.instance_id = self.instance.id logger.debug("Started EC2/Eucalyptus/Nova instance: %s"%self.instance_id) time.sleep(5) self.wait_for_running() if self.resource_url.scheme != "euca+ssh" and self.resource_url.scheme != "nova+ssh": self.ec2_conn.create_tags([self.instance_id], {"Name": self.id}) self.network_ip = self.instance.ip_address url = "ssh://" + str(self.network_ip) logger.debug("Connect to: %s"%(url)) # Submit job ctx = saga.Context("SSH") #ctx.type = saga.Context.SSH ctx.user_id = self.pilot_compute_description["vm_ssh_username"] ctx.user_key = self.pilot_compute_description["vm_ssh_keyfile"] session = saga.Session() session.add_context(ctx) TRIAL_MAX=30 trials=0 while trials < TRIAL_MAX: try: js = saga.job.Service(url, session=session) logger.debug("Job Description Type: " + str(type(self.job_description))) job = js.create_job(self.job_description) logger.debug("Attempt: %d, submit pilot job to: %s "%(trials,str(url))) job.run() if job.get_state()==saga.job.FAILED: logger.warning("Submission failed.") trials = trials + 1 time.sleep(30) continue else: break except: exc_type, exc_value, exc_traceback = sys.exc_info() logger.warning("Submission failed: " + str(exc_value)) #self.__print_traceback() trials = trials + 1 time.sleep(30) if trials == TRIAL_MAX: raise Exception("Submission of agent failed.") logger.debug("Job State : %s" % (job.get_state()))
try: import saga except: logging.debug("SAGA not imported. ") logging.debug(str(sys.path)) from threadpool import * # BigJob/Pilot framework classes from bigjob import logger try: from pilot.impl.pilotdata_manager import PilotData, DataUnit, PilotDataService except: logger.warning( "Pilot Data classes could not be loaded. File movement will not work!") logger.debug("Python Version: " + str(sys.version_info)) if sys.version_info < (2, 5): sys.stderr.write("Python 2.4 - Warning: Not all functionalities working\n") if sys.version_info < (2, 4): sys.stderr.write("Warning: Using unsupported Python version\n") if sys.version_info < (2, 3): sys.stderr.write("Warning: Python versions <2.3 not supported\n") sys.exit(-1) import subprocess """ Config parameters (will move to config file in future) """ CONFIG_FILE = "bigjob_agent.conf" THREAD_POOL_SIZE = 4 APPLICATION_NAME = "bigjob"
def run(self): """ Start VM and start BJ agent via SSH on VM """ """ Map fields of Pilot description to EC2 API { "vm_id":"ami-d7f742be", "vm_ssh_username":"******", "vm_ssh_keyname":"MyKey", "vm_ssh_keyfile":"<path>", "vm_type":"t1.micro", "access_key_id":"xxx", "secret_access_key":"xxx" } """ reservation = self.ec2_conn.run_instances( self.pilot_compute_description["vm_id"], key_name=self.pilot_compute_description["vm_ssh_keyname"], instance_type=self.pilot_compute_description["vm_type"], security_groups=[SECURITY_GROUP]) self.instance = reservation.instances[0] self.instance_id = self.instance.id logger.debug("Started EC2/Eucalyptus/Nova instance: %s" % self.instance_id) time.sleep(5) self.wait_for_running() if self.resource_url.scheme != "euca+ssh" and self.resource_url.scheme != "nova+ssh": self.ec2_conn.create_tags([self.instance_id], {"Name": self.id}) self.network_ip = self.instance.ip_address url = "ssh://" + str(self.network_ip) logger.debug("Connect to: %s" % (url)) # Submit job ctx = saga.Context("SSH") #ctx.type = saga.Context.SSH ctx.user_id = self.pilot_compute_description["vm_ssh_username"] ctx.user_key = self.pilot_compute_description["vm_ssh_keyfile"] session = saga.Session() session.add_context(ctx) TRIAL_MAX = 30 trials = 0 while trials < TRIAL_MAX: try: js = saga.job.Service(url, session=session) logger.debug("Job Description Type: " + str(type(self.job_description))) job = js.create_job(self.job_description) logger.debug("Attempt: %d, submit pilot job to: %s " % (trials, str(url))) job.run() if job.get_state() == saga.job.FAILED: logger.warning("Submission failed.") trials = trials + 1 time.sleep(30) continue else: break except: exc_type, exc_value, exc_traceback = sys.exc_info() logger.warning("Submission failed: " + str(exc_value)) #self.__print_traceback() trials = trials + 1 time.sleep(30) if trials == TRIAL_MAX: raise Exception("Submission of agent failed.") logger.debug("Job State : %s" % (job.get_state()))
try: import saga except: logging.debug("SAGA not imported. ") logging.debug(str(sys.path)) from threadpool import * # BigJob/Pilot framework classes from bigjob import logger try: from pilot.impl.pilotdata_manager import PilotData, DataUnit, PilotDataService except: logger.warning("Pilot Data classes could not be loaded. File movement will not work!") logger.debug("Python Version: " + str(sys.version_info)) if sys.version_info < (2, 5): sys.stderr.write("Python 2.4 - Warning: Not all functionalities working\n") if sys.version_info < (2, 4): sys.stderr.write("Warning: Using unsupported Python version\n") if sys.version_info < (2, 3): sys.stderr.write("Warning: Python versions <2.3 not supported\n") sys.exit(-1) import subprocess """ Config parameters (will move to config file in future) """ CONFIG_FILE="bigjob_agent.conf" THREAD_POOL_SIZE=4