def __init__(self, command, wait=True, fail_fast=False, cwd=None):
self.exit_code = 0
self.command = command
self.log_fp = StringIO()
self.wait = wait
self.fail_fast = fail_fast
self.run(cwd=cwd)
def __init__(self, command, wait=True, fail_fast=False, cwd=None):
self.exit_code = 0
self.command = command
self.log_fp = StringIO()
self.wait = wait
self.fail_fast = fail_fast
self.run(cwd=cwd)
class ShellCommand(object):
def __init__(self, command, wait=True, fail_fast=False, cwd=None):
self.exit_code = 0
self.command = command
self.log_fp = StringIO()
self.wait = wait
self.fail_fast = fail_fast
self.run(cwd=cwd)
def run(self, cwd=None):
mssapi.log.info('running:%s' % self.command)
self.process = subprocess.Popen(self.command, shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=cwd)
if(self.wait):
while self.process.poll() is None:
time.sleep(1)
t = self.process.communicate()
self.log_fp.write(t[0])
self.log_fp.write(t[1])
mssapi.log.info(self.log_fp.getvalue())
self.exit_code = self.process.returncode
if self.fail_fast and self.exit_code != 0:
raise Exception("Command " + self.command +
" failed with status " + self.exit_code)
return self.exit_code
def setReadOnly(self, value):
raise AttributeError
def getStatus(self):
return self.exit_code
status = property(getStatus, setReadOnly, None,
'The exit code for the command')
def getOutput(self):
return self.log_fp.getvalue()
output = property(getOutput, setReadOnly, None,
'The STDIN and STDERR output of the command')
def write_mime_multipart(content,
compress=False,
deftype='text/plain',
delimiter=':'):
"""Description:
:param content: A list of tuples of name-content pairs. This is used
instead of a dict to ensure that scripts run in order
:type list of tuples:
:param compress: Use gzip to compress the scripts, defaults to no compression
:type bool:
:param deftype: The type that should be assumed if nothing else can be figured out
:type str:
:param delimiter: mime delimiter
:type str:
:return: Final mime multipart
:rtype: str:
"""
wrapper = email.mime.multipart.MIMEMultipart()
for name, con in content:
definite_type = guess_mime_type(con, deftype)
maintype, subtype = definite_type.split('/', 1)
if maintype == 'text':
mime_con = email.mime.text.MIMEText(con, _subtype=subtype)
else:
mime_con = email.mime.base.MIMEBase(maintype, subtype)
mime_con.set_payload(con)
# Encode the payload using Base64
email.encoders.encode_base64(mime_con)
mime_con.add_header('Content-Disposition', 'attachment', filename=name)
wrapper.attach(mime_con)
rcontent = wrapper.as_string()
if compress:
buf = StringIO()
gz = gzip.GzipFile(mode='wb', fileobj=buf)
try:
gz.write(rcontent)
finally:
gz.close()
rcontent = buf.getvalue()
return rcontent
class ShellCommand(object):
def __init__(self, command, wait=True, fail_fast=False, cwd=None):
self.exit_code = 0
self.command = command
self.log_fp = StringIO()
self.wait = wait
self.fail_fast = fail_fast
self.run(cwd=cwd)
def run(self, cwd=None):
mssapi.log.info('running:%s' % self.command)
self.process = subprocess.Popen(self.command,
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=cwd)
if (self.wait):
while self.process.poll() is None:
time.sleep(1)
t = self.process.communicate()
self.log_fp.write(t[0])
self.log_fp.write(t[1])
mssapi.log.info(self.log_fp.getvalue())
self.exit_code = self.process.returncode
if self.fail_fast and self.exit_code != 0:
raise Exception("Command " + self.command +
" failed with status " + self.exit_code)
return self.exit_code
def setReadOnly(self, value):
raise AttributeError
def getStatus(self):
return self.exit_code
status = property(getStatus, setReadOnly, None,
'The exit code for the command')
def getOutput(self):
return self.log_fp.getvalue()
output = property(getOutput, setReadOnly, None,
'The STDIN and STDERR output of the command')
def load_credential_file(self, path):
"""Load a credential file as is setup like the Java utilities"""
c_data = StringIO()
c_data.write("[Credentials]\n")
for line in open(path, "r").readlines():
c_data.write(
line.replace("AWSAccessKeyId", "aws_access_key_id").replace(
"AWSSecretKey", "aws_secret_access_key"))
c_data.seek(0)
self.readfp(c_data)
def dump_safe(self, fp=None):
if not fp:
fp = StringIO()
for section in self.sections():
fp.write('[%s]\n' % section)
for option in self.options(section):
if option == 'aws_secret_access_key':
fp.write('%s = xxxxxxxxxxxxxxxxxx\n' % option)
else:
fp.write('%s = %s\n' % (option, self.get(section, option)))
def write_mime_multipart(content, compress=False, deftype='text/plain', delimiter=':'):
"""Description:
:param content: A list of tuples of name-content pairs. This is used
instead of a dict to ensure that scripts run in order
:type list of tuples:
:param compress: Use gzip to compress the scripts, defaults to no compression
:type bool:
:param deftype: The type that should be assumed if nothing else can be figured out
:type str:
:param delimiter: mime delimiter
:type str:
:return: Final mime multipart
:rtype: str:
"""
wrapper = email.mime.multipart.MIMEMultipart()
for name, con in content:
definite_type = guess_mime_type(con, deftype)
maintype, subtype = definite_type.split('/', 1)
if maintype == 'text':
mime_con = email.mime.text.MIMEText(con, _subtype=subtype)
else:
mime_con = email.mime.base.MIMEBase(maintype, subtype)
mime_con.set_payload(con)
# Encode the payload using Base64
email.encoders.encode_base64(mime_con)
mime_con.add_header('Content-Disposition', 'attachment', filename=name)
wrapper.attach(mime_con)
rcontent = wrapper.as_string()
if compress:
buf = StringIO()
gz = gzip.GzipFile(mode='wb', fileobj=buf)
try:
gz.write(rcontent)
finally:
gz.close()
rcontent = buf.getvalue()
return rcontent
def set_contents_from_string(self,
s,
headers=None,
replace=True,
cb=None,
num_cb=10,
policy=None,
md5=None,
if_generation=None):
"""
Store an object in GCS using the name of the Key object as the
key in GCS and the string 's' as the contents.
See set_contents_from_file method for details about the
parameters.
:type headers: dict
:param headers: Additional headers to pass along with the
request to AWS.
:type replace: bool
:param replace: If True, replaces the contents of the file if
it already exists.
:type cb: function
:param cb: a callback function that will be called to report
progress on the upload. The callback should accept
two integer parameters, the first representing the
number of bytes that have been successfully
transmitted to GCS and the second representing the
size of the to be transmitted object.
:type cb: int
:param num_cb: (optional) If a callback is specified with
the cb parameter this parameter determines the
granularity of the callback by defining
the maximum number of times the callback will
be called during the file transfer.
:type policy: :class:`mssapi.gs.acl.CannedACLStrings`
:param policy: A canned ACL policy that will be applied to the
new key in GCS.
:type md5: A tuple containing the hexdigest version of the MD5
checksum of the file as the first element and the
Base64-encoded version of the plain checksum as the
second element. This is the same format returned by
the compute_md5 method.
:param md5: If you need to compute the MD5 for any reason prior
to upload, it's silly to have to do it twice so this
param, if present, will be used as the MD5 values
of the file. Otherwise, the checksum will be computed.
:type if_generation: int
:param if_generation: (optional) If set to a generation number, the
object will only be written to if its current generation number is
this value. If set to the value 0, the object will only be written
if it doesn't already exist.
"""
# Clear out any previously computed md5 hashes, since we are setting the content.
self.md5 = None
self.base64md5 = None
fp = StringIO(get_utf8_value(s))
r = self.set_contents_from_file(fp,
headers,
replace,
cb,
num_cb,
policy,
md5,
if_generation=if_generation)
fp.close()
return r
def dump(self):
s = StringIO()
self.write(s)
print(s.getvalue())
def set_contents_from_string(self, s, headers=None, replace=True,
cb=None, num_cb=10, policy=None, md5=None,
if_generation=None):
"""
Store an object in GCS using the name of the Key object as the
key in GCS and the string 's' as the contents.
See set_contents_from_file method for details about the
parameters.
:type headers: dict
:param headers: Additional headers to pass along with the
request to AWS.
:type replace: bool
:param replace: If True, replaces the contents of the file if
it already exists.
:type cb: function
:param cb: a callback function that will be called to report
progress on the upload. The callback should accept
two integer parameters, the first representing the
number of bytes that have been successfully
transmitted to GCS and the second representing the
size of the to be transmitted object.
:type cb: int
:param num_cb: (optional) If a callback is specified with
the cb parameter this parameter determines the
granularity of the callback by defining
the maximum number of times the callback will
be called during the file transfer.
:type policy: :class:`mssapi.gs.acl.CannedACLStrings`
:param policy: A canned ACL policy that will be applied to the
new key in GCS.
:type md5: A tuple containing the hexdigest version of the MD5
checksum of the file as the first element and the
Base64-encoded version of the plain checksum as the
second element. This is the same format returned by
the compute_md5 method.
:param md5: If you need to compute the MD5 for any reason prior
to upload, it's silly to have to do it twice so this
param, if present, will be used as the MD5 values
of the file. Otherwise, the checksum will be computed.
:type if_generation: int
:param if_generation: (optional) If set to a generation number, the
object will only be written to if its current generation number is
this value. If set to the value 0, the object will only be written
if it doesn't already exist.
"""
# Clear out any previously computed md5 hashes, since we are setting the content.
self.md5 = None
self.base64md5 = None
fp = StringIO(get_utf8_value(s))
r = self.set_contents_from_file(fp, headers, replace, cb, num_cb,
policy, md5,
if_generation=if_generation)
fp.close()
return r
def parseString(self, content):
return self.parser.parse(StringIO(content))