def put_du_scp(self, du):
logger.debug("Copy DU using SCP")
du_items = du.list()
for i in du_items.keys():
local_filename = du_items[i]["local"]
remote_path = os.path.join(self.path, str(du.id), os.path.basename(local_filename))
logger.debug("Put file: %s to %s"%(i, remote_path))
if local_filename.startswith("ssh://"):
# check if remote path is directory
if self.__is_remote_directory(local_filename):
logger.warning("Path %s is a directory. Ignored."%local_filename)
continue
#self.__third_party_transfer(i.local_url, remote_path)
else:
try:
if stat.S_ISDIR(os.stat(local_filename).st_mode):
logger.warning("Path %s is a directory. Ignored."%local_filename)
continue
except:
pass
result = urlparse.urlparse(local_filename)
source_host = result.netloc
source_path = result.path
logger.debug(str((source_host, source_path, self.host, remote_path)))
if source_host == "" or source_host==None:
cmd = "scp "+ source_path + " " + self.host + ":" + remote_path
else:
cmd = "scp "+ source_host+":"+source_path + " " + self.host + ":" + remote_path
rc = os.system(cmd)
logger.debug("Command: %s Return code: %d"%(cmd, rc) )
def put_du_scp(self, du):
logger.debug("Copy DU using SCP")
du_items = du.list()
for i in du_items.keys():
local_filename = du_items[i]["local"]
remote_path = os.path.join(self.path, str(du.id), os.path.basename(local_filename))
logger.debug("Put file: %s to %s"%(i, remote_path))
if local_filename.startswith("ssh://"):
# check if remote path is directory
if self.__is_remote_directory(local_filename):
logger.warning("Path %s is a directory. Ignored."%local_filename)
continue
#self.__third_party_transfer(i.local_url, remote_path)
else:
try:
if stat.S_ISDIR(os.stat(local_filename).st_mode):
logger.warning("Path %s is a directory. Ignored."%local_filename)
continue
except:
pass
result = urlparse.urlparse(local_filename)
source_host = result.netloc
source_path = result.path
source_user = result.username
logger.debug(str((source_host, source_path, self.host, remote_path)))
self.__run_scp_command(self.userkey, source_user, source_host, source_path, self.user, self.host, remote_path)
def get_state(self):
result = State.UNKNOWN
try:
self.instance.update()
result=self.instance.state
except:
logger.warning("Instance not reachable/active yet...")
return result
def get_state(self):
result = State.UNKNOWN
try:
self.instance.update()
result = self.instance.state
except:
logger.warning("Instance not reachable/active yet...")
return result
def put_pd(self, pd):
for i in pd.list_data_units():
remote_path = os.path.join(self.__get_pd_path(pd.id), os.path.basename(i.local_url))
logger.debug("Put file: %s to %s"%(i.local_url, remote_path))
if i.local_url.startswith("file://") or i.local_url.startswith("/"):
if stat.S_ISDIR(os.stat(i.local_url).st_mode):
logger.warning("Path %s is a directory. Ignored."%i.local_url)
continue
self.__webhdfs.copyFromLocal(i.local_url, remote_path)
else:
logger.error("File URLs: %s not supported"%i.local_url)
def get_state(self):
result = State.UNKNOWN
try:
if self.subprocess_handle != None:
rc = self.subprocess_handle.poll()
if rc==None:
result = State.RUNNING
elif rc!=0:
result = State.FAILED
elif rc==0:
result = State.DONE
except:
logger.warning("Instance not reachable/active yet...")
return result
def put_pd(self, pd):
for i in pd.list_data_units():
remote_path = os.path.join(self.__get_pd_path(pd.id),
os.path.basename(i.local_url))
logger.debug("Put file: %s to %s" % (i.local_url, remote_path))
if i.local_url.startswith("file://") or i.local_url.startswith(
"/"):
if stat.S_ISDIR(os.stat(i.local_url).st_mode):
logger.warning("Path %s is a directory. Ignored." %
i.local_url)
continue
self.__webhdfs.copyFromLocal(i.local_url, remote_path)
else:
logger.error("File URLs: %s not supported" % i.local_url)
def put_du_paramiko(self, du):
logger.debug("Copy DU using Paramiko")
for i in du.list_data_unit_items():
remote_path = os.path.join(self.path, str(du.id), os.path.basename(i.local_url))
logger.debug("Put file: %s to %s"%(i.local_url, remote_path))
if i.local_url.startswith("ssh://"):
# check if remote path is directory
if self.__is_remote_directory(i.local_url):
logger.warning("Path %s is a directory. Ignored."%i.local_url)
continue
self.__third_party_transfer(i.local_url, remote_path)
else:
if stat.S_ISDIR(os.stat(i.local_url).st_mode):
logger.warning("Path %s is a directory. Ignored."%i.local_url)
continue
#self.__sftp.put(i.local_url, remote_path, self.put_progress, True)
ssh_client, sftp_client = self.__create_sftp_client()
sftp_client.put(i.local_url, remote_path)
sftp_client.close()
ssh_client.close()
def run(self):
""" Start VMs"""
# Submit job
working_directory = os.getcwd()
if "working_directory" in self.pilot_compute_description:
working_directory=self.pilot_compute_description["working_directory"]
TRIAL_MAX=3
trials=0
while trials < TRIAL_MAX:
try:
args = []
#args.append(self.job_description.executable)
#args.extend(self.job_description.arguments)
args.extend(["python", "-m", "bigjob.bigjob_agent"])
args.extend(self.job_description.arguments)
logger.debug("Execute: " + str(args))
self.subprocess_handle=subprocess.Popen(args=args,
stdout=self.job_output,
stderr=self.job_error,
cwd=working_directory,
shell=False)
if self.subprocess_handle.poll() != None and self.subprocess_handle.poll()!=0:
logger.warning("Submission failed.")
trials = trials + 1
time.sleep(30)
continue
else:
break
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
logger.warning("Submission failed: " + str(exc_value))
#self.__print_traceback()
trials = trials + 1
time.sleep(3)
if trials == TRIAL_MAX:
raise Exception("Submission of agent failed.")
logger.debug("Job State : %s" % (self.get_state()))
def put_du_paramiko(self, du):
logger.debug("Copy DU using Paramiko")
for i in du.list_data_unit_items():
remote_path = os.path.join(self.path, str(du.id),
os.path.basename(i.local_url))
logger.debug("Put file: %s to %s" % (i.local_url, remote_path))
if i.local_url.startswith("ssh://"):
# check if remote path is directory
if self.__is_remote_directory(i.local_url):
logger.warning("Path %s is a directory. Ignored." %
i.local_url)
continue
self.__third_party_transfer(i.local_url, remote_path)
else:
if stat.S_ISDIR(os.stat(i.local_url).st_mode):
logger.warning("Path %s is a directory. Ignored." %
i.local_url)
continue
#self.__sftp.put(i.local_url, remote_path, self.put_progress, True)
ssh_client, sftp_client = self.__create_sftp_client()
sftp_client.put(i.local_url, remote_path)
sftp_client.close()
ssh_client.close()
def put_du_scp(self, du):
logger.debug("Copy DU using SCP")
du_items = du.list()
for i in du_items.keys():
local_filename = du_items[i]["local"]
remote_path = os.path.join(self.path, str(du.id),
os.path.basename(local_filename))
logger.debug("Put file: %s to %s" % (i, remote_path))
if local_filename.startswith("ssh://"):
# check if remote path is directory
if self.__is_remote_directory(local_filename):
logger.warning("Path %s is a directory. Ignored." %
local_filename)
continue
#self.__third_party_transfer(i.local_url, remote_path)
else:
try:
if stat.S_ISDIR(os.stat(local_filename).st_mode):
logger.warning("Path %s is a directory. Ignored." %
local_filename)
continue
except:
pass
result = urlparse.urlparse(local_filename)
source_host = result.netloc
source_path = result.path
logger.debug(
str((source_host, source_path, self.host, remote_path)))
if source_host == "" or source_host == None:
cmd = "scp " + source_path + " " + self.host + ":" + remote_path
else:
cmd = "scp " + source_host + ":" + source_path + " " + self.host + ":" + remote_path
rc = os.system(cmd)
logger.debug("Command: %s Return code: %d" % (cmd, rc))
def run(self):
""" Start VM and start BJ agent via SSH on VM """
""" Map fields of Pilot description to EC2 API
{ "vm_id":"ami-d7f742be",
"vm_ssh_username":"******",
"vm_ssh_keyname":"MyKey",
"vm_ssh_keyfile":"<path>",
"vm_type":"t1.micro",
"access_key_id":"xxx",
"secret_access_key":"xxx"
}
"""
reservation = self.ec2_conn.run_instances(self.pilot_compute_description["vm_id"],
key_name=self.pilot_compute_description["vm_ssh_keyname"],
instance_type=self.pilot_compute_description["vm_type"],
security_groups=[SECURITY_GROUP])
self.instance = reservation.instances[0]
self.instance_id = self.instance.id
logger.debug("Started EC2/Eucalyptus/Nova instance: %s"%self.instance_id)
time.sleep(5)
self.wait_for_running()
if self.resource_url.scheme != "euca+ssh" and self.resource_url.scheme != "nova+ssh":
self.ec2_conn.create_tags([self.instance_id], {"Name": self.id})
self.network_ip = self.instance.ip_address
url = "ssh://" + str(self.network_ip)
logger.debug("Connect to: %s"%(url))
# Submit job
ctx = saga.Context("SSH")
#ctx.type = saga.Context.SSH
ctx.user_id = self.pilot_compute_description["vm_ssh_username"]
ctx.user_key = self.pilot_compute_description["vm_ssh_keyfile"]
session = saga.Session()
session.add_context(ctx)
TRIAL_MAX=30
trials=0
while trials < TRIAL_MAX:
try:
js = saga.job.Service(url, session=session)
logger.debug("Job Description Type: " + str(type(self.job_description)))
job = js.create_job(self.job_description)
logger.debug("Attempt: %d, submit pilot job to: %s "%(trials,str(url)))
job.run()
if job.get_state()==saga.job.FAILED:
logger.warning("Submission failed.")
trials = trials + 1
time.sleep(30)
continue
else:
break
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
logger.warning("Submission failed: " + str(exc_value))
#self.__print_traceback()
trials = trials + 1
time.sleep(30)
if trials == TRIAL_MAX:
raise Exception("Submission of agent failed.")
logger.debug("Job State : %s" % (job.get_state()))
try:
import saga
except:
logging.debug("SAGA not imported. ")
logging.debug(str(sys.path))
from threadpool import *
# BigJob/Pilot framework classes
from bigjob import logger
try:
from pilot.impl.pilotdata_manager import PilotData, DataUnit, PilotDataService
except:
logger.warning(
"Pilot Data classes could not be loaded. File movement will not work!")
logger.debug("Python Version: " + str(sys.version_info))
if sys.version_info < (2, 5):
sys.stderr.write("Python 2.4 - Warning: Not all functionalities working\n")
if sys.version_info < (2, 4):
sys.stderr.write("Warning: Using unsupported Python version\n")
if sys.version_info < (2, 3):
sys.stderr.write("Warning: Python versions <2.3 not supported\n")
sys.exit(-1)
import subprocess
""" Config parameters (will move to config file in future) """
CONFIG_FILE = "bigjob_agent.conf"
THREAD_POOL_SIZE = 4
APPLICATION_NAME = "bigjob"
def run(self):
""" Start VM and start BJ agent via SSH on VM """
""" Map fields of Pilot description to EC2 API
{ "vm_id":"ami-d7f742be",
"vm_ssh_username":"******",
"vm_ssh_keyname":"MyKey",
"vm_ssh_keyfile":"<path>",
"vm_type":"t1.micro",
"access_key_id":"xxx",
"secret_access_key":"xxx"
}
"""
reservation = self.ec2_conn.run_instances(
self.pilot_compute_description["vm_id"],
key_name=self.pilot_compute_description["vm_ssh_keyname"],
instance_type=self.pilot_compute_description["vm_type"],
security_groups=[SECURITY_GROUP])
self.instance = reservation.instances[0]
self.instance_id = self.instance.id
logger.debug("Started EC2/Eucalyptus/Nova instance: %s" %
self.instance_id)
time.sleep(5)
self.wait_for_running()
if self.resource_url.scheme != "euca+ssh" and self.resource_url.scheme != "nova+ssh":
self.ec2_conn.create_tags([self.instance_id], {"Name": self.id})
self.network_ip = self.instance.ip_address
url = "ssh://" + str(self.network_ip)
logger.debug("Connect to: %s" % (url))
# Submit job
ctx = saga.Context("SSH")
#ctx.type = saga.Context.SSH
ctx.user_id = self.pilot_compute_description["vm_ssh_username"]
ctx.user_key = self.pilot_compute_description["vm_ssh_keyfile"]
session = saga.Session()
session.add_context(ctx)
TRIAL_MAX = 30
trials = 0
while trials < TRIAL_MAX:
try:
js = saga.job.Service(url, session=session)
logger.debug("Job Description Type: " +
str(type(self.job_description)))
job = js.create_job(self.job_description)
logger.debug("Attempt: %d, submit pilot job to: %s " %
(trials, str(url)))
job.run()
if job.get_state() == saga.job.FAILED:
logger.warning("Submission failed.")
trials = trials + 1
time.sleep(30)
continue
else:
break
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
logger.warning("Submission failed: " + str(exc_value))
#self.__print_traceback()
trials = trials + 1
time.sleep(30)
if trials == TRIAL_MAX:
raise Exception("Submission of agent failed.")
logger.debug("Job State : %s" % (job.get_state()))
try:
import saga
except:
logging.debug("SAGA not imported. ")
logging.debug(str(sys.path))
from threadpool import *
# BigJob/Pilot framework classes
from bigjob import logger
try:
from pilot.impl.pilotdata_manager import PilotData, DataUnit, PilotDataService
except:
logger.warning("Pilot Data classes could not be loaded. File movement will not work!")
logger.debug("Python Version: " + str(sys.version_info))
if sys.version_info < (2, 5):
sys.stderr.write("Python 2.4 - Warning: Not all functionalities working\n")
if sys.version_info < (2, 4):
sys.stderr.write("Warning: Using unsupported Python version\n")
if sys.version_info < (2, 3):
sys.stderr.write("Warning: Python versions <2.3 not supported\n")
sys.exit(-1)
import subprocess
""" Config parameters (will move to config file in future) """
CONFIG_FILE="bigjob_agent.conf"
THREAD_POOL_SIZE=4
