def get_contents_as_string(self,
headers=None,
cb=None,
num_cb=10,
torrent=False):
"""
Retrieve file data from the Key, and return contents as a string.
:type headers: dict
:param headers: ignored in this subclass.
:type cb: function
:param cb: ignored in this subclass.
:type cb: int
:param num_cb: ignored in this subclass.
:type cb: int
:param num_cb: ignored in this subclass.
:type torrent: bool
:param torrent: ignored in this subclass.
:rtype: string
:returns: The contents of the file as a string
"""
fp = StringIO()
self.get_contents_to_file(fp)
return fp.getvalue()
def __init__(self, command, wait=True, fail_fast=False, cwd=None):
self.exit_code = 0
self.command = command
self.log_fp = StringIO()
self.wait = wait
self.fail_fast = fail_fast
self.run(cwd=cwd)
def test_four_part_file(self):
key_name = "k"
contents = "01234567890123456789"
sfp = StringIO(contents)
# upload 20 bytes in 4 parts of 5 bytes each
mpu = self.bucket.initiate_multipart_upload(key_name)
mpu.upload_part_from_file(sfp, part_num=1, size=5)
mpu.upload_part_from_file(sfp, part_num=2, size=5)
mpu.upload_part_from_file(sfp, part_num=3, size=5)
mpu.upload_part_from_file(sfp, part_num=4, size=5)
sfp.close()
etags = {}
pn = 0
for part in mpu:
pn += 1
self.assertEqual(5, part.size)
etags[pn] = part.etag
self.assertEqual(pn, 4)
# etags for 01234
self.assertEqual(etags[1], etags[3])
# etags for 56789
self.assertEqual(etags[2], etags[4])
# etag 01234 != etag 56789
self.assertNotEqual(etags[1], etags[2])
# parts are too small to compete as each part must
# be a min of 5MB so so we'll assume that is enough
# testing and abort the upload.
mpu.cancel_upload()
def get_contents_as_string(self, headers=None, cb=None, num_cb=10,
torrent=False):
"""
Retrieve file data from the Key, and return contents as a string.
:type headers: dict
:param headers: ignored in this subclass.
:type cb: function
:param cb: ignored in this subclass.
:type cb: int
:param num_cb: ignored in this subclass.
:type cb: int
:param num_cb: ignored in this subclass.
:type torrent: bool
:param torrent: ignored in this subclass.
:rtype: string
:returns: The contents of the file as a string
"""
fp = StringIO()
self.get_contents_to_file(fp)
return fp.getvalue()
def test_upload_part_by_size(self):
key_name = "k"
contents = "01234567890123456789"
sfp = StringIO(contents)
# upload 20 bytes in 4 parts of 5 bytes each
mpu = self.bucket.initiate_multipart_upload(key_name)
mpu.upload_part_from_file(sfp, part_num=1, size=5)
mpu.upload_part_from_file(sfp, part_num=2, size=5)
mpu.upload_part_from_file(sfp, part_num=3, size=5)
mpu.upload_part_from_file(sfp, part_num=4, size=5)
sfp.close()
etags = {}
pn = 0
for part in mpu:
pn += 1
self.assertEqual(5, part.size)
etags[pn] = part.etag
self.assertEqual(pn, 4)
# etags for 01234
self.assertEqual(etags[1], etags[3])
# etags for 56789
self.assertEqual(etags[2], etags[4])
# etag 01234 != etag 56789
self.assertNotEqual(etags[1], etags[2])
# parts are too small to complete as each part must
# be a min of 5MB so so we'll assume that is enough
# testing and abort the upload.
mpu.cancel_upload()
def start(self):
self.stop()
ec2 = boto.connect_ec2()
ami = ec2.get_all_images(image_ids = [str(self.ami_id)])[0]
groups = ec2.get_all_security_groups(groupnames=[str(self.security_group)])
if not self._config:
self.load_config()
if not self._config.has_section("Credentials"):
self._config.add_section("Credentials")
self._config.set("Credentials", "aws_access_key_id", ec2.aws_access_key_id)
self._config.set("Credentials", "aws_secret_access_key", ec2.aws_secret_access_key)
if not self._config.has_section("Pyami"):
self._config.add_section("Pyami")
if self._manager.domain:
self._config.set('Pyami', 'server_sdb_domain', self._manager.domain.name)
self._config.set("Pyami", 'server_sdb_name', self.name)
cfg = StringIO()
self._config.write(cfg)
cfg = cfg.getvalue()
r = ami.run(min_count=1,
max_count=1,
key_name=self.key_name,
security_groups = groups,
instance_type = self.instance_type,
placement = self.zone,
user_data = cfg)
i = r.instances[0]
self.instance_id = i.id
self.put()
if self.elastic_ip:
ec2.associate_address(self.instance_id, self.elastic_ip)
def __init__(self, connection=None, nodename=None):
"""Initialize this Item"""
self._connection = connection
self._nodename = nodename
self._nodepath = []
self._curobj = None
self._xml = StringIO()
def do_start(self):
ami_id = self.sd.get('ami_id')
instance_type = self.sd.get('instance_type', 'm1.small')
security_group = self.sd.get('security_group', 'default')
if not ami_id:
self.parser.error(
'ami_id option is required when starting the service')
ec2 = boto.connect_ec2()
if not self.sd.has_section('Credentials'):
self.sd.add_section('Credentials')
self.sd.set('Credentials', 'aws_access_key_id',
ec2.aws_access_key_id)
self.sd.set('Credentials', 'aws_secret_access_key',
ec2.aws_secret_access_key)
s = StringIO()
self.sd.write(s)
rs = ec2.get_all_images([ami_id])
img = rs[0]
r = img.run(user_data=s.getvalue(),
key_name=self.options.keypair,
max_count=self.options.num_instances,
instance_type=instance_type,
security_groups=[security_group])
print('Starting AMI: %s' % ami_id)
print('Reservation %s contains the following instances:' % r.id)
for i in r.instances:
print('\t%s' % i.id)
class ResponseGroup(xml.sax.ContentHandler):
"""A Generic "Response Group", which can
be anything from the entire list of Items to
specific response elements within an item"""
def __init__(self, connection=None, nodename=None):
"""Initialize this Item"""
self._connection = connection
self._nodename = nodename
self._nodepath = []
self._curobj = None
self._xml = StringIO()
def __repr__(self):
return '<%s: %s>' % (self.__class__.__name__, self.__dict__)
#
# Attribute Functions
#
def get(self, name):
return self.__dict__.get(name)
def set(self, name, value):
self.__dict__[name] = value
def to_xml(self):
return "<%s>%s</%s>" % (self._nodename, self._xml.getvalue(), self._nodename)
#
# XML Parser functions
#
def startElement(self, name, attrs, connection):
self._xml.write("<%s>" % name)
self._nodepath.append(name)
if len(self._nodepath) == 1:
obj = ResponseGroup(self._connection)
self.set(name, obj)
self._curobj = obj
elif self._curobj:
self._curobj.startElement(name, attrs, connection)
return None
def endElement(self, name, value, connection):
self._xml.write("%s</%s>" % (cgi.escape(value).replace("&amp;", "&"), name))
if len(self._nodepath) == 0:
return
obj = None
curval = self.get(name)
if len(self._nodepath) == 1:
if value or not curval:
self.set(name, value)
if self._curobj:
self._curobj = None
#elif len(self._nodepath) == 2:
#self._curobj = None
elif self._curobj:
self._curobj.endElement(name, value, connection)
self._nodepath.pop()
return None
def test_complete_japanese(self):
key_name = u"テスト"
mpu = self.bucket.initiate_multipart_upload(key_name)
fp = StringIO("small file")
mpu.upload_part_from_file(fp, part_num=1)
fp.close()
cmpu = mpu.complete_upload()
self.assertEqual(cmpu.key_name, key_name)
self.assertNotEqual(cmpu.etag, None)
def test_complete_japanese(self):
key_name = u"テスト"
mpu = self.bucket.initiate_multipart_upload(key_name)
fp = StringIO("small file")
mpu.upload_part_from_file(fp, part_num=1)
fp.close()
cmpu = mpu.complete_upload()
self.assertEqual(cmpu.key_name, key_name)
self.assertNotEqual(cmpu.etag, None)
def setUp(self):
self.file_contents = StringIO()
file_contents = StringIO(
'[Boto]\n'
'https_validate_certificates = true\n'
'other = false\n'
'http_socket_timeout = 1\n'
'[Credentials]\n'
'aws_access_key_id=foo\n'
'aws_secret_access_key=bar\n'
)
self.config = config.Config(fp=file_contents)
def test_download_to_fileobj(self):
http_response = mock.Mock(read=mock.Mock(return_value='xyz'))
response = GlacierResponse(http_response, None)
response['TreeHash'] = 'tree_hash'
self.api.get_job_output.return_value = response
fileobj = StringIO()
self.job.archive_size = 3
with mock.patch('boto.glacier.job.tree_hash_from_str') as t:
t.return_value = 'tree_hash'
self.job.download_to_fileobj(fileobj)
fileobj.seek(0)
self.assertEqual(http_response.read.return_value, fileobj.read())
def test_download_to_fileobj(self):
http_response=mock.Mock(read=mock.Mock(return_value='xyz'))
response = GlacierResponse(http_response, None)
response['TreeHash'] = 'tree_hash'
self.api.get_job_output.return_value = response
fileobj = StringIO()
self.job.archive_size = 3
with mock.patch('boto.glacier.job.tree_hash_from_str') as t:
t.return_value = 'tree_hash'
self.job.download_to_fileobj(fileobj)
fileobj.seek(0)
self.assertEqual(http_response.read.return_value, fileobj.read())
def decode(self, value):
try:
msg = {}
fp = StringIO(value)
line = fp.readline()
while line:
delim = line.find(':')
key = line[0:delim]
value = line[delim + 1:].strip()
msg[key.strip()] = value.strip()
line = fp.readline()
except:
raise SQSDecodeError('Unable to decode message', self)
return msg
def decode(self, value):
try:
msg = {}
fp = StringIO(value)
line = fp.readline()
while line:
delim = line.find(':')
key = line[0:delim]
value = line[delim+1:].strip()
msg[key.strip()] = value.strip()
line = fp.readline()
except:
raise SQSDecodeError('Unable to decode message', self)
return msg
def load_credential_file(self, path):
"""Load a credential file as is setup like the Java utilities"""
c_data = StringIO()
c_data.write("[Credentials]\n")
for line in open(path, "r").readlines():
c_data.write(line.replace("AWSAccessKeyId", "aws_access_key_id").replace("AWSSecretKey", "aws_secret_access_key"))
c_data.seek(0)
self.readfp(c_data)
def test_sign_canned_policy_pk_file_like(self):
"""
Test signing the canned policy from amazon's cloudfront documentation
with a file-like object (not a subclass of 'file' type)
"""
expected = ("Nql641NHEUkUaXQHZINK1FZ~SYeUSoBJMxjdgqrzIdzV2gyEXPDN"
"v0pYdWJkflDKJ3xIu7lbwRpSkG98NBlgPi4ZJpRRnVX4kXAJK6td"
"Nx6FucDB7OVqzcxkxHsGFd8VCG1BkC-Afh9~lOCMIYHIaiOB6~5j"
"t9w2EOwi6sIIqrg_")
pk_file = StringIO()
pk_file.write(self.pk_str)
pk_file.seek(0)
sig = self.dist._sign_string(self.canned_policy, private_key_file=pk_file)
encoded_sig = self.dist._url_base64_encode(sig)
self.assertEqual(expected, encoded_sig)
def __init__(self, connection=None, nodename=None):
"""Initialize this Item"""
self._connection = connection
self._nodename = nodename
self._nodepath = []
self._curobj = None
self._xml = StringIO()
def test_etag_of_parts(self):
key_name = "etagtest"
mpu = self.bucket.initiate_multipart_upload(key_name)
fp = StringIO("small file")
# upload 2 parts and save each part
uparts = []
uparts.append(mpu.upload_part_from_file(fp, part_num=1, size=5))
uparts.append(mpu.upload_part_from_file(fp, part_num=2))
fp.close()
# compare uploaded parts etag to listed parts
pn = 0
for lpart in mpu:
self.assertEqual(uparts[pn].etag, lpart.etag)
pn += 1
# Can't complete 2 small parts so just clean up.
mpu.cancel_upload()
def test_etag_of_parts(self):
key_name = "etagtest"
mpu = self.bucket.initiate_multipart_upload(key_name)
fp = StringIO("small file")
# upload 2 parts and save each part
uparts = []
uparts.append(mpu.upload_part_from_file(fp, part_num=1, size=5))
uparts.append(mpu.upload_part_from_file(fp, part_num=2))
fp.close()
# compare uploaded parts etag to listed parts
pn = 0
for lpart in mpu:
self.assertEqual(uparts[pn].etag, lpart.etag)
pn += 1
# Can't complete 2 small parts so just clean up.
mpu.cancel_upload()
class ShellCommand(object):
def __init__(self, command, wait=True, fail_fast=False, cwd=None):
self.exit_code = 0
self.command = command
self.log_fp = StringIO()
self.wait = wait
self.fail_fast = fail_fast
self.run(cwd=cwd)
def run(self, cwd=None):
boto.log.info('running:%s' % self.command)
self.process = subprocess.Popen(self.command, shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=cwd)
if(self.wait):
while self.process.poll() is None:
time.sleep(1)
t = self.process.communicate()
self.log_fp.write(t[0])
self.log_fp.write(t[1])
boto.log.info(self.log_fp.getvalue())
self.exit_code = self.process.returncode
if self.fail_fast and self.exit_code != 0:
raise Exception("Command " + self.command +
" failed with status " + self.exit_code)
return self.exit_code
def setReadOnly(self, value):
raise AttributeError
def getStatus(self):
return self.exit_code
status = property(getStatus, setReadOnly, None,
'The exit code for the command')
def getOutput(self):
return self.log_fp.getvalue()
output = property(getOutput, setReadOnly, None,
'The STDIN and STDERR output of the command')
def write_mime_multipart(content,
compress=False,
deftype='text/plain',
delimiter=':'):
"""Description:
:param content: A list of tuples of name-content pairs. This is used
instead of a dict to ensure that scripts run in order
:type list of tuples:
:param compress: Use gzip to compress the scripts, defaults to no compression
:type bool:
:param deftype: The type that should be assumed if nothing else can be figured out
:type str:
:param delimiter: mime delimiter
:type str:
:return: Final mime multipart
:rtype: str:
"""
wrapper = email.mime.multipart.MIMEMultipart()
for name, con in content:
definite_type = guess_mime_type(con, deftype)
maintype, subtype = definite_type.split('/', 1)
if maintype == 'text':
mime_con = email.mime.text.MIMEText(con, _subtype=subtype)
else:
mime_con = email.mime.base.MIMEBase(maintype, subtype)
mime_con.set_payload(con)
# Encode the payload using Base64
email.encoders.encode_base64(mime_con)
mime_con.add_header('Content-Disposition', 'attachment', filename=name)
wrapper.attach(mime_con)
rcontent = wrapper.as_string()
if compress:
buf = StringIO()
gz = gzip.GzipFile(mode='wb', fileobj=buf)
try:
gz.write(rcontent)
finally:
gz.close()
rcontent = buf.getvalue()
return rcontent
def test_500_retry(self, sleep_mock):
self.set_http_response(status_code=500)
b = Bucket(self.service_connection, 'mybucket')
k = b.new_key('test_failure')
fail_file = StringIO('This will attempt to retry.')
with self.assertRaises(BotoServerError):
k.send_file(fail_file)
def dump_safe(self, fp=None):
if not fp:
fp = StringIO()
for section in self.sections():
fp.write('[%s]\n' % section)
for option in self.options(section):
if option == 'aws_secret_access_key':
fp.write('%s = xxxxxxxxxxxxxxxxxx\n' % option)
else:
fp.write('%s = %s\n' % (option, self.get(section, option)))
def test_set_contents_with_md5(self):
content="01234567890123456789"
sfp = StringIO(content)
# fp is set at 0 for just opened (for read) files.
# set_contents should write full content to key.
k = self.bucket.new_key("k")
good_md5 = k.compute_md5(sfp)
k.set_contents_from_file(sfp, md5=good_md5)
kn = self.bucket.new_key("k")
ks = kn.get_contents_as_string().decode('utf-8')
self.assertEqual(ks, content)
# set fp to 5 and only set 5 bytes. this should
# write the value "56789" to the key.
sfp.seek(5)
k = self.bucket.new_key("k")
good_md5 = k.compute_md5(sfp, size=5)
k.set_contents_from_file(sfp, size=5, md5=good_md5)
self.assertEqual(sfp.tell(), 10)
kn = self.bucket.new_key("k")
ks = kn.get_contents_as_string().decode('utf-8')
self.assertEqual(ks, content[5:10])
# let's try a wrong md5 by just altering it.
k = self.bucket.new_key("k")
sfp.seek(0)
hexdig, base64 = k.compute_md5(sfp)
bad_md5 = (hexdig, base64[3:])
try:
k.set_contents_from_file(sfp, md5=bad_md5)
self.fail("should fail with bad md5")
except S3ResponseError:
pass
def test_set_contents_as_file(self):
content="01234567890123456789"
sfp = StringIO(content)
# fp is set at 0 for just opened (for read) files.
# set_contents should write full content to key.
k = self.bucket.new_key("k")
k.set_contents_from_file(sfp)
self.assertEqual(k.size, 20)
kn = self.bucket.new_key("k")
ks = kn.get_contents_as_string().decode('utf-8')
self.assertEqual(ks, content)
# set fp to 5 and set contents. this should
# set "567890123456789" to the key
sfp.seek(5)
k = self.bucket.new_key("k")
k.set_contents_from_file(sfp)
self.assertEqual(k.size, 15)
kn = self.bucket.new_key("k")
ks = kn.get_contents_as_string().decode('utf-8')
self.assertEqual(ks, content[5:])
# set fp to 5 and only set 5 bytes. this should
# write the value "56789" to the key.
sfp.seek(5)
k = self.bucket.new_key("k")
k.set_contents_from_file(sfp, size=5)
self.assertEqual(k.size, 5)
self.assertEqual(sfp.tell(), 10)
kn = self.bucket.new_key("k")
ks = kn.get_contents_as_string().decode('utf-8')
self.assertEqual(ks, content[5:10])
def setUp(self):
self.size_patch = mock.patch('os.path.getsize')
self.getsize = self.size_patch.start()
self.api = mock.Mock()
self.vault = vault.Vault(self.api, None)
self.vault.name = 'myvault'
self.mock_open = mock.mock_open()
stringio = StringIO('content')
self.mock_open.return_value.read = stringio.read
def load_credential_file(self, path):
"""Load a credential file as is setup like the Java utilities"""
c_data = StringIO()
c_data.write("[Credentials]\n")
for line in open(path, "r").readlines():
c_data.write(line.replace("AWSAccessKeyId", "aws_access_key_id").replace("AWSSecretKey", "aws_secret_access_key"))
c_data.seek(0)
self.readfp(c_data)
def write_mime_multipart(content, compress=False, deftype='text/plain', delimiter=':'):
"""Description:
:param content: A list of tuples of name-content pairs. This is used
instead of a dict to ensure that scripts run in order
:type list of tuples:
:param compress: Use gzip to compress the scripts, defaults to no compression
:type bool:
:param deftype: The type that should be assumed if nothing else can be figured out
:type str:
:param delimiter: mime delimiter
:type str:
:return: Final mime multipart
:rtype: str:
"""
wrapper = email.mime.multipart.MIMEMultipart()
for name, con in content:
definite_type = guess_mime_type(con, deftype)
maintype, subtype = definite_type.split('/', 1)
if maintype == 'text':
mime_con = email.mime.text.MIMEText(con, _subtype=subtype)
else:
mime_con = email.mime.base.MIMEBase(maintype, subtype)
mime_con.set_payload(con)
# Encode the payload using Base64
email.encoders.encode_base64(mime_con)
mime_con.add_header('Content-Disposition', 'attachment', filename=name)
wrapper.attach(mime_con)
rcontent = wrapper.as_string()
if compress:
buf = StringIO()
gz = gzip.GzipFile(mode='wb', fileobj=buf)
try:
gz.write(rcontent)
finally:
gz.close()
rcontent = buf.getvalue()
return rcontent
def test_get_contents_with_md5(self):
content="01234567890123456789"
sfp = StringIO(content)
k = self.bucket.new_key("k")
k.set_contents_from_file(sfp)
kn = self.bucket.new_key("k")
s = kn.get_contents_as_string().decode('utf-8')
self.assertEqual(kn.md5, k.md5)
self.assertEqual(s, content)
def test_compute_hash_tempfile_py3(self):
# Note the missing 'b' in the mode!
with tempfile.TemporaryFile(mode='w+') as f:
with self.assertRaises(ValueError):
compute_hashes_from_fileobj(f, chunk_size=512)
# What about file-like objects without a mode? If it has an
# encoding we use it, otherwise attempt UTF-8 encoding to
# bytes for hashing.
f = StringIO('test data' * 500)
compute_hashes_from_fileobj(f, chunk_size=512)
def test_should_not_raise_kms_related_integrity_errors(self):
self.set_http_response(
status_code=200,
header=[('x-amz-server-side-encryption-aws-kms-key-id', 'key'),
('etag', 'not equal to key.md5')])
bucket = Bucket(self.service_connection, 'mybucket')
key = bucket.new_key('test_kms')
file_content = StringIO('Some content to upload.')
# Should not raise errors related to integrity checks:
key.send_file(file_content)
def dump_safe(self, fp=None):
if not fp:
fp = StringIO()
for section in self.sections():
fp.write('[%s]\n' % section)
for option in self.options(section):
if option == 'aws_secret_access_key':
fp.write('%s = xxxxxxxxxxxxxxxxxx\n' % option)
else:
fp.write('%s = %s\n' % (option, self.get(section, option)))
def test_set_contents_from_file_dataloss(self):
# Create an empty stringio and write to it.
content = "abcde"
sfp = StringIO()
sfp.write(content)
# Try set_contents_from_file() without rewinding sfp
k = self.bucket.new_key("k")
try:
k.set_contents_from_file(sfp)
self.fail("forgot to rewind so should fail.")
except AttributeError:
pass
# call with rewind and check if we wrote 5 bytes
k.set_contents_from_file(sfp, rewind=True)
self.assertEqual(k.size, 5)
# check actual contents by getting it.
kn = self.bucket.new_key("k")
ks = kn.get_contents_as_string().decode('utf-8')
self.assertEqual(ks, content)
# finally, try with a 0 length string
sfp = StringIO()
k = self.bucket.new_key("k")
k.set_contents_from_file(sfp)
self.assertEqual(k.size, 0)
# check actual contents by getting it.
kn = self.bucket.new_key("k")
ks = kn.get_contents_as_string().decode('utf-8')
self.assertEqual(ks, "")
def _run(self, msg, vtimeout):
boto.log.info('Task[%s] - running:%s' % (self.name, self.command))
log_fp = StringIO()
process = subprocess.Popen(self.command,
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
nsecs = 5
current_timeout = vtimeout
while process.poll() is None:
boto.log.info('nsecs=%s, timeout=%s' % (nsecs, current_timeout))
if nsecs >= current_timeout:
current_timeout += vtimeout
boto.log.info('Task[%s] - setting timeout to %d seconds' %
(self.name, current_timeout))
if msg:
msg.change_visibility(current_timeout)
time.sleep(5)
nsecs += 5
t = process.communicate()
log_fp.write(t[0])
log_fp.write(t[1])
boto.log.info('Task[%s] - output: %s' % (self.name, log_fp.getvalue()))
self.last_executed = self.now
self.last_status = process.returncode
self.last_output = log_fp.getvalue()[0:1023]
def do_start(self):
ami_id = self.sd.get('ami_id')
instance_type = self.sd.get('instance_type', 'm1.small')
security_group = self.sd.get('security_group', 'default')
if not ami_id:
self.parser.error('ami_id option is required when starting the service')
ec2 = boto.connect_ec2()
if not self.sd.has_section('Credentials'):
self.sd.add_section('Credentials')
self.sd.set('Credentials', 'aws_access_key_id', ec2.aws_access_key_id)
self.sd.set('Credentials', 'aws_secret_access_key', ec2.aws_secret_access_key)
s = StringIO()
self.sd.write(s)
rs = ec2.get_all_images([ami_id])
img = rs[0]
r = img.run(user_data=s.getvalue(), key_name=self.options.keypair,
max_count=self.options.num_instances,
instance_type=instance_type,
security_groups=[security_group])
print('Starting AMI: %s' % ami_id)
print('Reservation %s contains the following instances:' % r.id)
for i in r.instances:
print('\t%s' % i.id)
def test_504_gateway_timeout(self, sleep_mock):
weird_timeout_body = "<Error><Code>GatewayTimeout</Code></Error>"
self.set_http_response(status_code=504, body=weird_timeout_body)
b = Bucket(self.service_connection, 'mybucket')
k = b.new_key('test_failure')
fail_file = StringIO('This will pretend to be chunk-able.')
k.should_retry = counter(k.should_retry)
self.assertEqual(k.should_retry.count, 0)
with self.assertRaises(BotoServerError):
k.send_file(fail_file)
self.assertTrue(k.should_retry.count, 1)
def _run(self, msg, vtimeout):
boto.log.info('Task[%s] - running:%s' % (self.name, self.command))
log_fp = StringIO()
process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
nsecs = 5
current_timeout = vtimeout
while process.poll() is None:
boto.log.info('nsecs=%s, timeout=%s' % (nsecs, current_timeout))
if nsecs >= current_timeout:
current_timeout += vtimeout
boto.log.info('Task[%s] - setting timeout to %d seconds' % (self.name, current_timeout))
if msg:
msg.change_visibility(current_timeout)
time.sleep(5)
nsecs += 5
t = process.communicate()
log_fp.write(t[0])
log_fp.write(t[1])
boto.log.info('Task[%s] - output: %s' % (self.name, log_fp.getvalue()))
self.last_executed = self.now
self.last_status = process.returncode
self.last_output = log_fp.getvalue()[0:1023]
def run(self):
boto.log.info('running:%s' % self.command)
log_fp = StringIO()
process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
while process.poll() is None:
time.sleep(1)
t = process.communicate()
log_fp.write(t[0])
log_fp.write(t[1])
boto.log.info(log_fp.getvalue())
boto.log.info('output: %s' % log_fp.getvalue())
return (process.returncode, log_fp.getvalue())
def check_no_resume(self, data, resume_set=set()):
fobj = StringIO(data.decode('utf-8'))
part_hash_map = {}
for part_index in resume_set:
start = self.part_size * part_index
end = start + self.part_size
part_data = data[start:end]
part_hash_map[part_index] = tree_hash(
chunk_hashes(part_data, self.chunk_size))
resume_file_upload(self.vault, sentinel.upload_id, self.part_size,
fobj, part_hash_map, self.chunk_size)
upload_part_calls, data_tree_hashes = calculate_mock_vault_calls(
data, self.part_size, self.chunk_size)
resume_upload_part_calls = [
call for part_index, call in enumerate(upload_part_calls)
if part_index not in resume_set
]
check_mock_vault_calls(self.vault, resume_upload_part_calls,
data_tree_hashes, len(data))
def test_sign_canned_policy_pk_file_like(self):
"""
Test signing the canned policy from amazon's cloudfront documentation
with a file-like object (not a subclass of 'file' type)
"""
expected = ("Nql641NHEUkUaXQHZINK1FZ~SYeUSoBJMxjdgqrzIdzV2gyEXPDN"
"v0pYdWJkflDKJ3xIu7lbwRpSkG98NBlgPi4ZJpRRnVX4kXAJK6td"
"Nx6FucDB7OVqzcxkxHsGFd8VCG1BkC-Afh9~lOCMIYHIaiOB6~5j"
"t9w2EOwi6sIIqrg_")
pk_file = StringIO()
pk_file.write(self.pk_str)
pk_file.seek(0)
sig = self.dist._sign_string(self.canned_policy,
private_key_file=pk_file)
encoded_sig = self.dist._url_base64_encode(sig)
self.assertEqual(expected, encoded_sig)
def run(self):
"""
Open a subprocess and run a command on the local host.
:rtype: tuple
:return: This function returns a tuple that contains an integer status
and a string with the combined stdout and stderr output.
"""
boto.log.info('running:%s' % self.command)
log_fp = StringIO()
process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
while process.poll() is None:
time.sleep(1)
t = process.communicate()
log_fp.write(t[0])
log_fp.write(t[1])
boto.log.info(log_fp.getvalue())
boto.log.info('output: %s' % log_fp.getvalue())
return (process.returncode, log_fp.getvalue())
def test_1_basic(self):
c = boto.connect_sqs()
# create a queue so we can test BigMessage
queue_name = 'test%d' % int(time.time())
timeout = 60
queue = c.create_queue(queue_name, timeout)
self.addCleanup(c.delete_queue, queue, True)
queue.set_message_class(BigMessage)
# create a bucket with the same name to store the message in
s3 = boto.connect_s3()
bucket = s3.create_bucket(queue_name)
self.addCleanup(s3.delete_bucket, queue_name)
time.sleep(30)
# now add a message
msg_body = 'This is a test of the big message'
fp = StringIO(msg_body)
s3_url = 's3://%s' % queue_name
message = queue.new_message(fp, s3_url=s3_url)
queue.write(message)
time.sleep(30)
s3_object_name = message.s3_url.split('/')[-1]
# Make sure msg body is in bucket
self.assertTrue(bucket.lookup(s3_object_name))
m = queue.read()
self.assertEqual(m.get_body().decode('utf-8'), msg_body)
m.delete()
time.sleep(30)
# Make sure msg is deleted from bucket
self.assertIsNone(bucket.lookup(s3_object_name))
def run(self):
"""
Open a subprocess and run a command on the local host.
:rtype: tuple
:return: This function returns a tuple that contains an integer status
and a string with the combined stdout and stderr output.
"""
boto.log.info('running:%s' % self.command)
log_fp = StringIO()
process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
while process.poll() is None:
time.sleep(1)
t = process.communicate()
log_fp.write(t[0])
log_fp.write(t[1])
boto.log.info(log_fp.getvalue())
boto.log.info('output: %s' % log_fp.getvalue())
return (process.returncode, log_fp.getvalue())
def bundle(self, bucket=None, prefix=None, key_file=None, cert_file=None,
size=None, ssh_key=None, fp=None, clear_history=True):
iobject = IObject()
if not bucket:
bucket = iobject.get_string('Name of S3 bucket')
if not prefix:
prefix = iobject.get_string('Prefix for AMI file')
if not key_file:
key_file = iobject.get_filename('Path to RSA private key file')
if not cert_file:
cert_file = iobject.get_filename('Path to RSA public cert file')
if not size:
size = iobject.get_int('Size (in MB) of bundled image')
if not ssh_key:
ssh_key = self.server.get_ssh_key_file()
self.copy_x509(key_file, cert_file)
if not fp:
fp = StringIO()
fp.write('sudo mv %s /mnt/boto.cfg; ' % BotoConfigPath)
fp.write('mv ~/.ssh/authorized_keys /mnt/authorized_keys; ')
if clear_history:
fp.write('history -c; ')
fp.write(self.bundle_image(prefix, size, ssh_key))
fp.write('; ')
fp.write(self.upload_bundle(bucket, prefix, ssh_key))
fp.write('; ')
fp.write('sudo mv /mnt/boto.cfg %s; ' % BotoConfigPath)
fp.write('mv /mnt/authorized_keys ~/.ssh/authorized_keys')
command = fp.getvalue()
print('running the following command on the remote server:')
print(command)
t = self.ssh_client.run(command)
print('\t%s' % t[0])
print('\t%s' % t[1])
print('...complete!')
print('registering image...')
self.image_id = self.server.ec2.register_image(name=prefix, image_location='%s/%s.manifest.xml' % (bucket, prefix))
return self.image_id
def create(cls, config_file=None, logical_volume = None, cfg = None, **params):
"""
Create a new instance based on the specified configuration file or the specified
configuration and the passed in parameters.
If the config_file argument is not None, the configuration is read from there.
Otherwise, the cfg argument is used.
The config file may include other config files with a #import reference. The included
config files must reside in the same directory as the specified file.
The logical_volume argument, if supplied, will be used to get the current physical
volume ID and use that as an override of the value specified in the config file. This
may be useful for debugging purposes when you want to debug with a production config
file but a test Volume.
The dictionary argument may be used to override any EC2 configuration values in the
config file.
"""
if config_file:
cfg = Config(path=config_file)
if cfg.has_section('EC2'):
# include any EC2 configuration values that aren't specified in params:
for option in cfg.options('EC2'):
if option not in params:
params[option] = cfg.get('EC2', option)
getter = CommandLineGetter()
getter.get(cls, params)
region = params.get('region')
ec2 = region.connect()
cls.add_credentials(cfg, ec2.aws_access_key_id, ec2.aws_secret_access_key)
ami = params.get('ami')
kp = params.get('keypair')
group = params.get('group')
zone = params.get('zone')
# deal with possibly passed in logical volume:
if logical_volume != None:
cfg.set('EBS', 'logical_volume_name', logical_volume.name)
cfg_fp = StringIO()
cfg.write(cfg_fp)
# deal with the possibility that zone and/or keypair are strings read from the config file:
if isinstance(zone, Zone):
zone = zone.name
if isinstance(kp, KeyPair):
kp = kp.name
reservation = ami.run(min_count=1,
max_count=params.get('quantity', 1),
key_name=kp,
security_groups=[group],
instance_type=params.get('instance_type'),
placement = zone,
user_data = cfg_fp.getvalue())
l = []
i = 0
elastic_ip = params.get('elastic_ip')
instances = reservation.instances
if elastic_ip is not None and instances.__len__() > 0:
instance = instances[0]
print('Waiting for instance to start so we can set its elastic IP address...')
# Sometimes we get a message from ec2 that says that the instance does not exist.
# Hopefully the following delay will giv eec2 enough time to get to a stable state:
time.sleep(5)
while instance.update() != 'running':
time.sleep(1)
instance.use_ip(elastic_ip)
print('set the elastic IP of the first instance to %s' % elastic_ip)
for instance in instances:
s = cls()
s.ec2 = ec2
s.name = params.get('name') + '' if i==0 else str(i)
s.description = params.get('description')
s.region_name = region.name
s.instance_id = instance.id
if elastic_ip and i == 0:
s.elastic_ip = elastic_ip
s.put()
l.append(s)
i += 1
return l
def set_contents_from_string(self, s, headers=None, replace=True,
cb=None, num_cb=10, policy=None, md5=None,
if_generation=None):
"""
Store an object in GCS using the name of the Key object as the
key in GCS and the string 's' as the contents.
See set_contents_from_file method for details about the
parameters.
:type headers: dict
:param headers: Additional headers to pass along with the
request to AWS.
:type replace: bool
:param replace: If True, replaces the contents of the file if
it already exists.
:type cb: function
:param cb: a callback function that will be called to report
progress on the upload. The callback should accept
two integer parameters, the first representing the
number of bytes that have been successfully
transmitted to GCS and the second representing the
size of the to be transmitted object.
:type cb: int
:param num_cb: (optional) If a callback is specified with
the cb parameter this parameter determines the
granularity of the callback by defining
the maximum number of times the callback will
be called during the file transfer.
:type policy: :class:`boto.gs.acl.CannedACLStrings`
:param policy: A canned ACL policy that will be applied to the
new key in GCS.
:type md5: A tuple containing the hexdigest version of the MD5
checksum of the file as the first element and the
Base64-encoded version of the plain checksum as the
second element. This is the same format returned by
the compute_md5 method.
:param md5: If you need to compute the MD5 for any reason prior
to upload, it's silly to have to do it twice so this
param, if present, will be used as the MD5 values
of the file. Otherwise, the checksum will be computed.
:type if_generation: int
:param if_generation: (optional) If set to a generation number, the
object will only be written to if its current generation number is
this value. If set to the value 0, the object will only be written
if it doesn't already exist.
"""
# Clear out any previously computed md5 hashes, since we are setting the content.
self.md5 = None
self.base64md5 = None
fp = StringIO(get_utf8_value(s))
r = self.set_contents_from_file(fp, headers, replace, cb, num_cb,
policy, md5,
if_generation=if_generation)
fp.close()
return r
def test_file_callback(self):
def callback(wrote, total):
self.my_cb_cnt += 1
self.assertNotEqual(wrote, self.my_cb_last, "called twice with same value")
self.my_cb_last = wrote
# Zero bytes written => 1 call
self.my_cb_cnt = 0
self.my_cb_last = None
k = self.bucket.new_key("k")
k.BufferSize = 2
sfp = StringIO("")
k.set_contents_from_file(sfp, cb=callback, num_cb=10)
self.assertEqual(self.my_cb_cnt, 1)
self.assertEqual(self.my_cb_last, 0)
sfp.close()
# Read back zero bytes => 1 call
self.my_cb_cnt = 0
self.my_cb_last = None
s = k.get_contents_as_string(cb=callback)
self.assertEqual(self.my_cb_cnt, 1)
self.assertEqual(self.my_cb_last, 0)
content="01234567890123456789"
sfp = StringIO(content)
# expect 2 calls due start/finish
self.my_cb_cnt = 0
self.my_cb_last = None
k = self.bucket.new_key("k")
k.set_contents_from_file(sfp, cb=callback, num_cb=10)
self.assertEqual(self.my_cb_cnt, 2)
self.assertEqual(self.my_cb_last, 20)
# Read back all bytes => 2 calls
self.my_cb_cnt = 0
self.my_cb_last = None
s = k.get_contents_as_string(cb=callback).decode('utf-8')
self.assertEqual(self.my_cb_cnt, 2)
self.assertEqual(self.my_cb_last, 20)
self.assertEqual(s, content)
# rewind sfp and try upload again. -1 should call
# for every read/write so that should make 11 when bs=2
sfp.seek(0)
self.my_cb_cnt = 0
self.my_cb_last = None
k = self.bucket.new_key("k")
k.BufferSize = 2
k.set_contents_from_file(sfp, cb=callback, num_cb=-1)
self.assertEqual(self.my_cb_cnt, 11)
self.assertEqual(self.my_cb_last, 20)
# Read back all bytes => 11 calls
self.my_cb_cnt = 0
self.my_cb_last = None
s = k.get_contents_as_string(cb=callback, num_cb=-1).decode('utf-8')
self.assertEqual(self.my_cb_cnt, 11)
self.assertEqual(self.my_cb_last, 20)
self.assertEqual(s, content)
# no more than 1 times => 2 times
# last time always 20 bytes
sfp.seek(0)
self.my_cb_cnt = 0
self.my_cb_last = None
k = self.bucket.new_key("k")
k.BufferSize = 2
k.set_contents_from_file(sfp, cb=callback, num_cb=1)
self.assertTrue(self.my_cb_cnt <= 2)
self.assertEqual(self.my_cb_last, 20)
# no more than 1 times => 2 times
self.my_cb_cnt = 0
self.my_cb_last = None
s = k.get_contents_as_string(cb=callback, num_cb=1).decode('utf-8')
self.assertTrue(self.my_cb_cnt <= 2)
self.assertEqual(self.my_cb_last, 20)
self.assertEqual(s, content)
# no more than 2 times
# last time always 20 bytes
sfp.seek(0)
self.my_cb_cnt = 0
self.my_cb_last = None
k = self.bucket.new_key("k")
k.BufferSize = 2
k.set_contents_from_file(sfp, cb=callback, num_cb=2)
self.assertTrue(self.my_cb_cnt <= 2)
self.assertEqual(self.my_cb_last, 20)
# no more than 2 times
self.my_cb_cnt = 0
self.my_cb_last = None
s = k.get_contents_as_string(cb=callback, num_cb=2).decode('utf-8')
self.assertTrue(self.my_cb_cnt <= 2)
self.assertEqual(self.my_cb_last, 20)
self.assertEqual(s, content)
# no more than 3 times
# last time always 20 bytes
sfp.seek(0)
self.my_cb_cnt = 0
self.my_cb_last = None
k = self.bucket.new_key("k")
k.BufferSize = 2
k.set_contents_from_file(sfp, cb=callback, num_cb=3)
self.assertTrue(self.my_cb_cnt <= 3)
self.assertEqual(self.my_cb_last, 20)
# no more than 3 times
self.my_cb_cnt = 0
self.my_cb_last = None
s = k.get_contents_as_string(cb=callback, num_cb=3).decode('utf-8')
self.assertTrue(self.my_cb_cnt <= 3)
self.assertEqual(self.my_cb_last, 20)
self.assertEqual(s, content)
# no more than 4 times
# last time always 20 bytes
sfp.seek(0)
self.my_cb_cnt = 0
self.my_cb_last = None
k = self.bucket.new_key("k")
k.BufferSize = 2
k.set_contents_from_file(sfp, cb=callback, num_cb=4)
self.assertTrue(self.my_cb_cnt <= 4)
self.assertEqual(self.my_cb_last, 20)
# no more than 4 times
self.my_cb_cnt = 0
self.my_cb_last = None
s = k.get_contents_as_string(cb=callback, num_cb=4).decode('utf-8')
self.assertTrue(self.my_cb_cnt <= 4)
self.assertEqual(self.my_cb_last, 20)
self.assertEqual(s, content)
# no more than 6 times
# last time always 20 bytes
sfp.seek(0)
self.my_cb_cnt = 0
self.my_cb_last = None
k = self.bucket.new_key("k")
k.BufferSize = 2
k.set_contents_from_file(sfp, cb=callback, num_cb=6)
self.assertTrue(self.my_cb_cnt <= 6)
self.assertEqual(self.my_cb_last, 20)
# no more than 6 times
self.my_cb_cnt = 0
self.my_cb_last = None
s = k.get_contents_as_string(cb=callback, num_cb=6).decode('utf-8')
self.assertTrue(self.my_cb_cnt <= 6)
self.assertEqual(self.my_cb_last, 20)
self.assertEqual(s, content)
# no more than 10 times
# last time always 20 bytes
sfp.seek(0)
self.my_cb_cnt = 0
self.my_cb_last = None
k = self.bucket.new_key("k")
k.BufferSize = 2
k.set_contents_from_file(sfp, cb=callback, num_cb=10)
self.assertTrue(self.my_cb_cnt <= 10)
self.assertEqual(self.my_cb_last, 20)
# no more than 10 times
self.my_cb_cnt = 0
self.my_cb_last = None
s = k.get_contents_as_string(cb=callback, num_cb=10).decode('utf-8')
self.assertTrue(self.my_cb_cnt <= 10)
self.assertEqual(self.my_cb_last, 20)
self.assertEqual(s, content)
# no more than 1000 times
# last time always 20 bytes
sfp.seek(0)
self.my_cb_cnt = 0
self.my_cb_last = None
k = self.bucket.new_key("k")
k.BufferSize = 2
k.set_contents_from_file(sfp, cb=callback, num_cb=1000)
self.assertTrue(self.my_cb_cnt <= 1000)
self.assertEqual(self.my_cb_last, 20)
# no more than 1000 times
self.my_cb_cnt = 0
self.my_cb_last = None
s = k.get_contents_as_string(cb=callback, num_cb=1000).decode('utf-8')
self.assertTrue(self.my_cb_cnt <= 1000)
self.assertEqual(self.my_cb_last, 20)
self.assertEqual(s, content)
def get_userdata_string(self):
s = StringIO()
self.config.write(s)
return s.getvalue()
def dump(self):
s = StringIO()
self.write(s)
print(s.getvalue())
