def ec2_retrieve_user_data(self):
try:
user_data_url = self.config.ec2_url
# touch the file so that it exists with proper permissions
vm_utils.touch(self.config.userdata_file, mode=0600)
# Now retrieve userdata into the file
self.config.userdata_file, _ = urllib.urlretrieve(user_data_url, self.config.userdata_file)
except Exception, ex:
raise UserDataError("Error retrieving User Data(context type: EC2): %s\n" % str(ex))
def retrieve_instance_userdata(self):
self.config.log.log_info(
'Retrieving instance_userdata for contextualize_protocol %s' %
self.config.contextualization_type)
try:
# touch the file so that it exists with proper permissions
vm_utils.touch(self.config.userdata_file, mode=0600)
# Now retrieve userdata into the file
request = urllib2.Request(self.config.instance_userdata_url)
response = urllib2.urlopen(request)
userdata = response.read()
except Exception, ex:
raise UserDataError(
"Error retrieving instance userdata contextualize_protocol %s: %s\n"
% (self.config.contextualization_type, str(ex)))
def retrieve_instance_userdata(self):
self.config.log.log_info(
'Retrieving instance_userdata for contextualize_protocol %s' %
self.config.contextualization_type)
try:
userdata_base_url = self.config.instance_userdata_url
# touch the file so that it exists with proper permissions
vm_utils.touch(self.config.userdata_file, mode=0600)
# Now retrieve userdata
userdata = {}
for attribute in self.userdata_attributes:
request = urllib2.Request(
'%s/%s' % (userdata_base_url, attribute), None,
{'Metadata-Flavor': 'Google'})
response = urllib2.urlopen(request)
userdata[attribute] = response.read()
except Exception, ex:
raise UserDataError(
"Error retrieving instance userdata contextualize_protocol %s: %s\n"
% (self.config.contextualization_type, str(ex)))
def one_retrieve_user_data(self):
try:
# Mount cdrom drive... OpenNebula contextualization unmounts it
self.config.log.log_info('Mounting opennebula context disk %s' % opennebula_context_disk())
mount_cmd = ["mount", "-t", "iso9660", opennebula_context_disk(), "/mnt"]
subprocess.call(mount_cmd)
# copy the OpenNebula userdata file
self.config.log.log_info('Reading context file: %s' % self.config.one_user_data_file)
vm_utils.touch(self.config.userdata_file, mode=0600)
fd = open(self.config.userdata_file, 'w')
user_data = base64.b64decode(one_ec2_user_data(self.config.one_user_data_file))
self.config.log.log_info('Writing User data')
# Only write to logs for debugging purposes.
# Writing ec2_user_data to logs is a security issue.
#self.config.log.log_info('Writing User data %s' % user_data)
fd.write(user_data)
fd.close()
umount_cmd = ["umount", "/mnt"]
subprocess.call(umount_cmd)
except Exception, ex:
raise UserDataError("Error retrieving User Data (context type: OPENNEBULA): %s\n" % str(ex))
def retrieve_instance_userdata(self):
self.config.log.log_info(
'Retrieving instance_userdata for contextualize_protocol %s' %
self.config.contextualization_type)
context_disk = self.one_context_disk()
try:
# Mount cdrom drive... OpenNebula contextualization unmounts it
self.config.log.log_info('Mounting opennebula context disk %s' %
context_disk)
mount_cmd = ["mount", "-t", "iso9660", context_disk, "/mnt"]
subprocess.call(mount_cmd)
# copy the OpenNebula userdata file
self.config.log.log_info('Reading context file: %s' %
self.config.one_userdata_file)
vm_utils.touch(self.config.userdata_file, mode=0600)
userdata = base64.b64decode(self.read_one_userdata())
umount_cmd = ["umount", "/mnt"]
subprocess.call(umount_cmd)
except Exception, ex:
raise UserDataError(
"Error retrieving instance userdata contextualize_protocol %s: %s\n"
% (self.config.contextualization_type, str(ex)))
def extract_user_data(self):
"""
The user data has the following format:
pilot ini file####extra args####compressed glidein proxy
OR
ini file
"""
delimiter = "####"
try:
# Split the user data
userdata = open(self.config.userdata_file, 'r').read()
if userdata.find(delimiter) > 0:
log_msg = "Delimiter found. Assuming we have been launched from glideinWMS"
self.config.log.log_info(self.template % log_msg)
# split userdata into usable chunks
userdata = userdata.split(delimiter)
# get and process the pilot ini file
log_msg = "Extracting ini file from the USERDATA"
self.config.log.log_info(self.template % log_msg)
pilot_ini = base64.b64decode(userdata[0])
vm_utils.touch(self.config.ini_file, mode=0600)
ini_fd = open(self.config.ini_file, 'w')
ini_fd.write(pilot_ini)
ini_fd.close()
ini = ini_handler.Ini(self.config.ini_file)
# get the initial set of arguments for the glidein_startup.sh script
log_msg = "Extracting arguments from the USERDATA"
self.config.log.log_info(self.template % log_msg)
self.config.pilot_args = ini.get("glidein_startup", "args")
log_msg = "pilot_args : %s" % self.config.pilot_args
self.config.log.log_info(self.template % log_msg)
# set the factory location so that we can download glidein_startup.sh
self.config.factory_url = ini.get("glidein_startup", "webbase")
log_msg = "factory_url : %s" % self.config.factory_url
self.config.log.log_info(self.template % log_msg)
# now get and add the extra args to the main arg list
extra_args = userdata[1]
extra_args = extra_args.replace("\\", "")
# add the extra arguments to the arg list for glidein_startup.sh
self.config.pilot_args += " %s" % extra_args
log_msg = "Full config.pilot_args : %s" % self.config.pilot_args
self.config.log.log_info(self.template % log_msg)
# check to see if the "don't shutdown" flag has been set
self.config.disable_shutdown = False
if ini.has_option("vm_properties", "disable_shutdown"):
self.config.disable_shutdown = smart_bool(
ini.get("vm_properties", "disable_shutdown"))
log_msg = "config.disable_shutdown : %s" % self.config.disable_shutdown
self.config.log.log_info(self.template % log_msg)
# set the max_lifetime if available
if ini.has_option("vm_properties", "max_lifetime"):
self.config.max_lifetime = ini.get("vm_properties",
"max_lifetime")
log_msg = "config.max_lifetime : %s" % self.config.max_lifetime
self.config.log.log_info(self.template % log_msg)
# get the proxy file name from the ini file
proxy_file_name = ini.get("glidein_startup", "proxy_file_name")
log_msg = "proxy_file_name : %s" % proxy_file_name
self.config.log.log_info(self.template % log_msg)
# set the full path to the proxy. The proxy will be written to this path
self.config.proxy_file = "%s/%s" % (self.config.home_dir,
proxy_file_name)
log_msg = "config.proxy_file : %s" % self.config.proxy_file
self.config.log.log_info(self.template % log_msg)
# Get the compressed proxy and write it to a tmp file.
# The tmp file name is completely predictable, but this
# isn't an interactive node and the lifetime of the node is
# very short so the risk of attack on this vector is minimal
# HTCondor will attach proxy file passed using the
# ec2_userdata_file at the end. Accessing it as the last
# token rather than a fixed positional token gives us
# the flexibility to append custom info after the second
# token, i.e the extra args token
log_msg = "Extracting pilot proxy: from the USERDATA"
self.config.log.log_info(self.template % log_msg)
compressed_proxy = base64.b64decode(userdata[-1])
fd = os.open("%s.gz" % self.config.proxy_file,
os.O_CREAT | os.O_WRONLY, 0600)
try:
os.write(fd, compressed_proxy)
finally:
os.close(fd)
# modified by Anthony Tiradani from tmp to gz
proxy_fd = gzip.open("%s.gz" % self.config.proxy_file, 'rb')
proxy_content = proxy_fd.read()
proxy_fd.close()
fd = os.open(self.config.proxy_file, os.O_CREAT | os.O_WRONLY,
0600)
try:
os.write(fd, proxy_content)
finally:
os.close(fd)
else:
# the only thing expected here is a simple ini file containing:
#
# [vm_properties]
# disable_shutdown = False
log_msg = "Delimiter not found. Assuming we have been launched manually"
self.config.log.log_info(self.template % log_msg)
vm_utils.touch(self.config.ini_file, mode=0600)
fd = open(self.config.ini_file, 'w')
fd.write(userdata)
fd.close()
ini = ini_handler.Ini(self.config.ini_file)
contents = ini.dump()
log_msg = "Ini file contents:\n%s" % contents
self.config.log.log_info(self.template % log_msg)
if ini.has_option("vm_properties", "disable_shutdown"):
self.config.disable_shutdown = smart_bool(
ini.get("vm_properties", "disable_shutdown"))
self.config.debug = True
elif ini.has_option("DEFAULT", "disable_shutdown"):
self.config.disable_shutdown = smart_bool(
ini.get("vm_properties", "disable_shutdown"))
self.config.debug = True
else:
raise UserDataError(
"Invalid ini file in user data.\nUser data contents:\n%s"
% userdata)
except Exception, ex:
raise PilotError("Error extracting User Data: %s\n" % str(ex))
def extract_user_data(self):
"""
The user data has the following format:
pilot ini file####extra args####compressed glidein proxy
OR
ini file
"""
delimiter = "####"
try:
# Split the user data
userdata = open(self.config.userdata_file, 'r').read()
if userdata.find(delimiter) > 0:
log_msg = "Delimiter found. Assuming we have been launched from glideinWMS"
self.config.log.log_info(self.template % log_msg)
# split userdata into usable chunks
userdata = userdata.split(delimiter)
# get and process the pilot ini file
log_msg = "Extracting ini file from the EC2_USER_DATA"
self.config.log.log_info(self.template % log_msg)
pilot_ini = base64.b64decode(userdata[0])
vm_utils.touch(self.config.ini_file, mode=0600)
ini_fd = open(self.config.ini_file, 'w')
ini_fd.write(pilot_ini)
ini_fd.close()
ini = ini_handler.Ini(self.config.ini_file)
# get the initial set of arguments for the glidein_startup.sh script
log_msg = "Extracting arguments from the EC2_USER_DATA"
self.config.log.log_info(self.template % log_msg)
self.config.pilot_args = ini.get("glidein_startup", "args")
log_msg = "pilot_args : %s" % self.config.pilot_args
self.config.log.log_info(self.template % log_msg)
# set the factory location so that we can download glidein_startup.sh
self.config.factory_url = ini.get("glidein_startup", "webbase")
log_msg = "factory_url : %s" % self.config.factory_url
self.config.log.log_info(self.template % log_msg)
# now get and add the extra args to the main arg list
extra_args = userdata[1]
extra_args = extra_args.replace("\\", "")
# add the extra arguments to the arg list for glidein_startup.sh
self.config.pilot_args += " %s" % extra_args
log_msg = "Full config.pilot_args : %s" % self.config.pilot_args
self.config.log.log_info(self.template % log_msg)
# check to see if the "don't shutdown" flag has been set
self.config.disable_shutdown = False
if ini.has_option("vm_properties", "disable_shutdown"):
self.config.disable_shutdown = smart_bool(ini.get("vm_properties", "disable_shutdown"))
log_msg = "config.disable_shutdown : %s" % self.config.disable_shutdown
self.config.log.log_info(self.template % log_msg)
# set the max_lifetime if available
if ini.has_option("vm_properties", "max_lifetime"):
self.config.max_lifetime = ini.get("vm_properties", "max_lifetime")
log_msg = "config.max_lifetime : %s" % self.config.max_lifetime
self.config.log.log_info(self.template % log_msg)
# get the proxy file name from the ini file
proxy_file_name = ini.get("glidein_startup", "proxy_file_name")
log_msg = "proxy_file_name : %s" % proxy_file_name
self.config.log.log_info(self.template % log_msg)
# set the full path to the proxy. The proxy will be written to this path
self.config.proxy_file = "%s/%s" % (self.config.home_dir, proxy_file_name)
log_msg = "config.proxy_file : %s" % self.config.proxy_file
self.config.log.log_info(self.template % log_msg)
# Get the compressed proxy and write it to a tmp file.
# The tmp file name is completely predictable, but this
# isn't an interactive node and the lifetime of the node is
# very short so the risk of attack on this vector is minimal
# HTCondor will attach proxy file passed using the
# ec2_userdata_file at the end. Accessing it as the last
# token rather than a fixed positional token gives us
# the flexibility to append custom info after the second
# token, i.e the extra args token
log_msg = "Extracting pilot proxy: from the EC2_USER_DATA"
self.config.log.log_info(self.template % log_msg)
compressed_proxy = base64.b64decode(userdata[-1:])
fd = os.open("%s.gz" % self.config.proxy_file,
os.O_CREAT|os.O_WRONLY, 0600)
try:
os.write(fd, compressed_proxy)
finally:
os.close(fd)
# modified by Anthony Tiradani from tmp to gz
proxy_fd = gzip.open("%s.gz" % self.config.proxy_file, 'rb')
proxy_content = proxy_fd.read()
proxy_fd.close()
fd = os.open(self.config.proxy_file, os.O_CREAT|os.O_WRONLY, 0600)
try:
os.write(fd, proxy_content)
finally:
os.close(fd)
else:
# the only thing expected here is a simple ini file containing:
#
# [vm_properties]
# disable_shutdown = False
log_msg = "Delimiter not found. Assuming we have been launched manually"
self.config.log.log_info(self.template % log_msg)
vm_utils.touch(self.config.ini_file, mode=0600)
fd = open(self.config.ini_file, 'w')
fd.write(userdata)
fd.close()
ini = ini_handler.Ini(self.config.ini_file)
contents = ini.dump()
log_msg = "Ini file contents:\n%s" % contents
self.config.log.log_info(self.template % log_msg)
if ini.has_option("vm_properties", "disable_shutdown"):
self.config.disable_shutdown = smart_bool(ini.get("vm_properties", "disable_shutdown"))
self.config.debug = True
elif ini.has_option("DEFAULT", "disable_shutdown"):
self.config.disable_shutdown = smart_bool(ini.get("vm_properties", "disable_shutdown"))
self.config.debug = True
else:
raise UserDataError("Invalid ini file in user data.\nUser data contents:\n%s" % userdata)
except Exception, ex:
raise PilotError("Error extracting User Data: %s\n" % str(ex))
def cernvm_retrieve_user_data(self):
try:
shutil.copy(self.config.cernvm_user_data_file, self.config.userdata_file)
vm_utils.touch(self.config.userdata_file, mode=0600)
except Exception, ex:
raise UserDataError("Error retrieving User Data (context type: CERNVM): %s\n" % str(ex))
