def test_set_contents_from_file_dataloss(self):
# Create an empty stringio and write to it.
content = "abcde"
sfp = StringIO()
sfp.write(content)
# Try set_contents_from_file() without rewinding sfp
k = self.bucket.new_key("k")
try:
k.set_contents_from_file(sfp)
self.fail("forgot to rewind so should fail.")
except AttributeError:
pass
# call with rewind and check if we wrote 5 bytes
k.set_contents_from_file(sfp, rewind=True)
self.assertEqual(k.size, 5)
# check actual contents by getting it.
kn = self.bucket.new_key("k")
ks = kn.get_contents_as_string().decode('utf-8')
self.assertEqual(ks, content)
# finally, try with a 0 length string
sfp = StringIO()
k = self.bucket.new_key("k")
k.set_contents_from_file(sfp)
self.assertEqual(k.size, 0)
# check actual contents by getting it.
kn = self.bucket.new_key("k")
ks = kn.get_contents_as_string().decode('utf-8')
self.assertEqual(ks, "")
def setUp(self):
self.file_contents = StringIO()
file_contents = StringIO(
'[Boto]\n'
'https_validate_certificates = true\n'
'other = false\n'
'http_socket_timeout = 1\n'
'[Credentials]\n'
'aws_access_key_id=foo\n'
'aws_secret_access_key=bar\n'
)
self.config = config.Config(fp=file_contents)
def test_four_part_file(self):
key_name = "k"
contents = "01234567890123456789"
sfp = StringIO(contents)
# upload 20 bytes in 4 parts of 5 bytes each
mpu = self.bucket.initiate_multipart_upload(key_name)
mpu.upload_part_from_file(sfp, part_num=1, size=5)
mpu.upload_part_from_file(sfp, part_num=2, size=5)
mpu.upload_part_from_file(sfp, part_num=3, size=5)
mpu.upload_part_from_file(sfp, part_num=4, size=5)
sfp.close()
etags = {}
pn = 0
for part in mpu:
pn += 1
self.assertEqual(5, part.size)
etags[pn] = part.etag
self.assertEqual(pn, 4)
# etags for 01234
self.assertEqual(etags[1], etags[3])
# etags for 56789
self.assertEqual(etags[2], etags[4])
# etag 01234 != etag 56789
self.assertNotEqual(etags[1], etags[2])
# parts are too small to compete as each part must
# be a min of 5MB so so we'll assume that is enough
# testing and abort the upload.
mpu.cancel_upload()
def test_set_contents_as_file(self):
content="01234567890123456789"
sfp = StringIO(content)
# fp is set at 0 for just opened (for read) files.
# set_contents should write full content to key.
k = self.bucket.new_key("k")
k.set_contents_from_file(sfp)
self.assertEqual(k.size, 20)
kn = self.bucket.new_key("k")
ks = kn.get_contents_as_string().decode('utf-8')
self.assertEqual(ks, content)
# set fp to 5 and set contents. this should
# set "567890123456789" to the key
sfp.seek(5)
k = self.bucket.new_key("k")
k.set_contents_from_file(sfp)
self.assertEqual(k.size, 15)
kn = self.bucket.new_key("k")
ks = kn.get_contents_as_string().decode('utf-8')
self.assertEqual(ks, content[5:])
# set fp to 5 and only set 5 bytes. this should
# write the value "56789" to the key.
sfp.seek(5)
k = self.bucket.new_key("k")
k.set_contents_from_file(sfp, size=5)
self.assertEqual(k.size, 5)
self.assertEqual(sfp.tell(), 10)
kn = self.bucket.new_key("k")
ks = kn.get_contents_as_string().decode('utf-8')
self.assertEqual(ks, content[5:10])
def __init__(self, connection=None, nodename=None):
"""Initialize this Item"""
self._connection = connection
self._nodename = nodename
self._nodepath = []
self._curobj = None
self._xml = StringIO()
def __init__(self, command, wait=True, fail_fast=False, cwd=None):
self.exit_code = 0
self.command = command
self.log_fp = StringIO()
self.wait = wait
self.fail_fast = fail_fast
self.run(cwd=cwd)
def get_contents_as_string(self,
headers=None,
cb=None,
num_cb=10,
torrent=False):
"""
Retrieve file data from the Key, and return contents as a string.
:type headers: dict
:param headers: ignored in this subclass.
:type cb: function
:param cb: ignored in this subclass.
:type cb: int
:param num_cb: ignored in this subclass.
:type cb: int
:param num_cb: ignored in this subclass.
:type torrent: bool
:param torrent: ignored in this subclass.
:rtype: string
:returns: The contents of the file as a string
"""
fp = StringIO()
self.get_contents_to_file(fp)
return fp.getvalue()
def _run(self, msg, vtimeout):
boto.log.info('Task[%s] - running:%s' % (self.name, self.command))
log_fp = StringIO()
process = subprocess.Popen(self.command,
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
nsecs = 5
current_timeout = vtimeout
while process.poll() is None:
boto.log.info('nsecs=%s, timeout=%s' % (nsecs, current_timeout))
if nsecs >= current_timeout:
current_timeout += vtimeout
boto.log.info('Task[%s] - setting timeout to %d seconds' %
(self.name, current_timeout))
if msg:
msg.change_visibility(current_timeout)
time.sleep(5)
nsecs += 5
t = process.communicate()
log_fp.write(t[0])
log_fp.write(t[1])
boto.log.info('Task[%s] - output: %s' % (self.name, log_fp.getvalue()))
self.last_executed = self.now
self.last_status = process.returncode
self.last_output = log_fp.getvalue()[0:1023]
def test_set_contents_with_md5(self):
content="01234567890123456789"
sfp = StringIO(content)
# fp is set at 0 for just opened (for read) files.
# set_contents should write full content to key.
k = self.bucket.new_key("k")
good_md5 = k.compute_md5(sfp)
k.set_contents_from_file(sfp, md5=good_md5)
kn = self.bucket.new_key("k")
ks = kn.get_contents_as_string().decode('utf-8')
self.assertEqual(ks, content)
# set fp to 5 and only set 5 bytes. this should
# write the value "56789" to the key.
sfp.seek(5)
k = self.bucket.new_key("k")
good_md5 = k.compute_md5(sfp, size=5)
k.set_contents_from_file(sfp, size=5, md5=good_md5)
self.assertEqual(sfp.tell(), 10)
kn = self.bucket.new_key("k")
ks = kn.get_contents_as_string().decode('utf-8')
self.assertEqual(ks, content[5:10])
# let's try a wrong md5 by just altering it.
k = self.bucket.new_key("k")
sfp.seek(0)
hexdig, base64 = k.compute_md5(sfp)
bad_md5 = (hexdig, base64[3:])
try:
k.set_contents_from_file(sfp, md5=bad_md5)
self.fail("should fail with bad md5")
except S3ResponseError:
pass
def start(self):
self.stop()
ec2 = boto.connect_ec2()
ami = ec2.get_all_images(image_ids = [str(self.ami_id)])[0]
groups = ec2.get_all_security_groups(groupnames=[str(self.security_group)])
if not self._config:
self.load_config()
if not self._config.has_section("Credentials"):
self._config.add_section("Credentials")
self._config.set("Credentials", "aws_access_key_id", ec2.aws_access_key_id)
self._config.set("Credentials", "aws_secret_access_key", ec2.aws_secret_access_key)
if not self._config.has_section("Pyami"):
self._config.add_section("Pyami")
if self._manager.domain:
self._config.set('Pyami', 'server_sdb_domain', self._manager.domain.name)
self._config.set("Pyami", 'server_sdb_name', self.name)
cfg = StringIO()
self._config.write(cfg)
cfg = cfg.getvalue()
r = ami.run(min_count=1,
max_count=1,
key_name=self.key_name,
security_groups = groups,
instance_type = self.instance_type,
placement = self.zone,
user_data = cfg)
i = r.instances[0]
self.instance_id = i.id
self.put()
if self.elastic_ip:
ec2.associate_address(self.instance_id, self.elastic_ip)
def do_start(self):
ami_id = self.sd.get('ami_id')
instance_type = self.sd.get('instance_type', 'm1.small')
security_group = self.sd.get('security_group', 'default')
if not ami_id:
self.parser.error(
'ami_id option is required when starting the service')
ec2 = boto.connect_ec2()
if not self.sd.has_section('Credentials'):
self.sd.add_section('Credentials')
self.sd.set('Credentials', 'aws_access_key_id',
ec2.aws_access_key_id)
self.sd.set('Credentials', 'aws_secret_access_key',
ec2.aws_secret_access_key)
s = StringIO()
self.sd.write(s)
rs = ec2.get_all_images([ami_id])
img = rs[0]
r = img.run(user_data=s.getvalue(),
key_name=self.options.keypair,
max_count=self.options.num_instances,
instance_type=instance_type,
security_groups=[security_group])
print('Starting AMI: %s' % ami_id)
print('Reservation %s contains the following instances:' % r.id)
for i in r.instances:
print('\t%s' % i.id)
def test_500_retry(self, sleep_mock):
self.set_http_response(status_code=500)
b = Bucket(self.service_connection, 'mybucket')
k = b.new_key('test_failure')
fail_file = StringIO('This will attempt to retry.')
with self.assertRaises(BotoServerError):
k.send_file(fail_file)
def load_credential_file(self, path):
"""Load a credential file as is setup like the Java utilities"""
c_data = StringIO()
c_data.write("[Credentials]\n")
for line in open(path, "r").readlines():
c_data.write(line.replace("AWSAccessKeyId", "aws_access_key_id").replace("AWSSecretKey", "aws_secret_access_key"))
c_data.seek(0)
self.readfp(c_data)
def test_complete_japanese(self):
key_name = u"テスト"
mpu = self.bucket.initiate_multipart_upload(key_name)
fp = StringIO("small file")
mpu.upload_part_from_file(fp, part_num=1)
fp.close()
cmpu = mpu.complete_upload()
self.assertEqual(cmpu.key_name, key_name)
self.assertNotEqual(cmpu.etag, None)
def setUp(self):
self.size_patch = mock.patch('os.path.getsize')
self.getsize = self.size_patch.start()
self.api = mock.Mock()
self.vault = vault.Vault(self.api, None)
self.vault.name = 'myvault'
self.mock_open = mock.mock_open()
stringio = StringIO('content')
self.mock_open.return_value.read = stringio.read
def dump_safe(self, fp=None):
if not fp:
fp = StringIO()
for section in self.sections():
fp.write('[%s]\n' % section)
for option in self.options(section):
if option == 'aws_secret_access_key':
fp.write('%s = xxxxxxxxxxxxxxxxxx\n' % option)
else:
fp.write('%s = %s\n' % (option, self.get(section, option)))
def test_get_contents_with_md5(self):
content="01234567890123456789"
sfp = StringIO(content)
k = self.bucket.new_key("k")
k.set_contents_from_file(sfp)
kn = self.bucket.new_key("k")
s = kn.get_contents_as_string().decode('utf-8')
self.assertEqual(kn.md5, k.md5)
self.assertEqual(s, content)
def test_should_not_raise_kms_related_integrity_errors(self):
self.set_http_response(
status_code=200,
header=[('x-amz-server-side-encryption-aws-kms-key-id', 'key'),
('etag', 'not equal to key.md5')])
bucket = Bucket(self.service_connection, 'mybucket')
key = bucket.new_key('test_kms')
file_content = StringIO('Some content to upload.')
# Should not raise errors related to integrity checks:
key.send_file(file_content)
def test_compute_hash_tempfile_py3(self):
# Note the missing 'b' in the mode!
with tempfile.TemporaryFile(mode='w+') as f:
with self.assertRaises(ValueError):
compute_hashes_from_fileobj(f, chunk_size=512)
# What about file-like objects without a mode? If it has an
# encoding we use it, otherwise attempt UTF-8 encoding to
# bytes for hashing.
f = StringIO('test data' * 500)
compute_hashes_from_fileobj(f, chunk_size=512)
def test_download_to_fileobj(self):
http_response=mock.Mock(read=mock.Mock(return_value='xyz'))
response = GlacierResponse(http_response, None)
response['TreeHash'] = 'tree_hash'
self.api.get_job_output.return_value = response
fileobj = StringIO()
self.job.archive_size = 3
with mock.patch('boto.glacier.job.tree_hash_from_str') as t:
t.return_value = 'tree_hash'
self.job.download_to_fileobj(fileobj)
fileobj.seek(0)
self.assertEqual(http_response.read.return_value, fileobj.read())
def run(self):
boto.log.info('running:%s' % self.command)
log_fp = StringIO()
process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
while process.poll() is None:
time.sleep(1)
t = process.communicate()
log_fp.write(t[0])
log_fp.write(t[1])
boto.log.info(log_fp.getvalue())
boto.log.info('output: %s' % log_fp.getvalue())
return (process.returncode, log_fp.getvalue())
def decode(self, value):
try:
msg = {}
fp = StringIO(value)
line = fp.readline()
while line:
delim = line.find(':')
key = line[0:delim]
value = line[delim + 1:].strip()
msg[key.strip()] = value.strip()
line = fp.readline()
except:
raise SQSDecodeError('Unable to decode message', self)
return msg
def test_504_gateway_timeout(self, sleep_mock):
weird_timeout_body = "<Error><Code>GatewayTimeout</Code></Error>"
self.set_http_response(status_code=504, body=weird_timeout_body)
b = Bucket(self.service_connection, 'mybucket')
k = b.new_key('test_failure')
fail_file = StringIO('This will pretend to be chunk-able.')
k.should_retry = counter(k.should_retry)
self.assertEqual(k.should_retry.count, 0)
with self.assertRaises(BotoServerError):
k.send_file(fail_file)
self.assertTrue(k.should_retry.count, 1)
def test_etag_of_parts(self):
key_name = "etagtest"
mpu = self.bucket.initiate_multipart_upload(key_name)
fp = StringIO("small file")
# upload 2 parts and save each part
uparts = []
uparts.append(mpu.upload_part_from_file(fp, part_num=1, size=5))
uparts.append(mpu.upload_part_from_file(fp, part_num=2))
fp.close()
# compare uploaded parts etag to listed parts
pn = 0
for lpart in mpu:
self.assertEqual(uparts[pn].etag, lpart.etag)
pn += 1
# Can't complete 2 small parts so just clean up.
mpu.cancel_upload()
def test_sign_canned_policy_pk_file_like(self):
"""
Test signing the canned policy from amazon's cloudfront documentation
with a file-like object (not a subclass of 'file' type)
"""
expected = ("Nql641NHEUkUaXQHZINK1FZ~SYeUSoBJMxjdgqrzIdzV2gyEXPDN"
"v0pYdWJkflDKJ3xIu7lbwRpSkG98NBlgPi4ZJpRRnVX4kXAJK6td"
"Nx6FucDB7OVqzcxkxHsGFd8VCG1BkC-Afh9~lOCMIYHIaiOB6~5j"
"t9w2EOwi6sIIqrg_")
pk_file = StringIO()
pk_file.write(self.pk_str)
pk_file.seek(0)
sig = self.dist._sign_string(self.canned_policy,
private_key_file=pk_file)
encoded_sig = self.dist._url_base64_encode(sig)
self.assertEqual(expected, encoded_sig)
def bundle(self,
bucket=None,
prefix=None,
key_file=None,
cert_file=None,
size=None,
ssh_key=None,
fp=None,
clear_history=True):
iobject = IObject()
if not bucket:
bucket = iobject.get_string('Name of S3 bucket')
if not prefix:
prefix = iobject.get_string('Prefix for AMI file')
if not key_file:
key_file = iobject.get_filename('Path to RSA private key file')
if not cert_file:
cert_file = iobject.get_filename('Path to RSA public cert file')
if not size:
size = iobject.get_int('Size (in MB) of bundled image')
if not ssh_key:
ssh_key = self.server.get_ssh_key_file()
self.copy_x509(key_file, cert_file)
if not fp:
fp = StringIO()
fp.write('sudo mv %s /mnt/boto.cfg; ' % BotoConfigPath)
fp.write('mv ~/.ssh/authorized_keys /mnt/authorized_keys; ')
if clear_history:
fp.write('history -c; ')
fp.write(self.bundle_image(prefix, size, ssh_key))
fp.write('; ')
fp.write(self.upload_bundle(bucket, prefix, ssh_key))
fp.write('; ')
fp.write('sudo mv /mnt/boto.cfg %s; ' % BotoConfigPath)
fp.write('mv /mnt/authorized_keys ~/.ssh/authorized_keys')
command = fp.getvalue()
print('running the following command on the remote server:')
print(command)
t = self.ssh_client.run(command)
print('\t%s' % t[0])
print('\t%s' % t[1])
print('...complete!')
print('registering image...')
self.image_id = self.server.ec2.register_image(
name=prefix,
image_location='%s/%s.manifest.xml' % (bucket, prefix))
return self.image_id
def write_mime_multipart(content,
compress=False,
deftype='text/plain',
delimiter=':'):
"""Description:
:param content: A list of tuples of name-content pairs. This is used
instead of a dict to ensure that scripts run in order
:type list of tuples:
:param compress: Use gzip to compress the scripts, defaults to no compression
:type bool:
:param deftype: The type that should be assumed if nothing else can be figured out
:type str:
:param delimiter: mime delimiter
:type str:
:return: Final mime multipart
:rtype: str:
"""
wrapper = email.mime.multipart.MIMEMultipart()
for name, con in content:
definite_type = guess_mime_type(con, deftype)
maintype, subtype = definite_type.split('/', 1)
if maintype == 'text':
mime_con = email.mime.text.MIMEText(con, _subtype=subtype)
else:
mime_con = email.mime.base.MIMEBase(maintype, subtype)
mime_con.set_payload(con)
# Encode the payload using Base64
email.encoders.encode_base64(mime_con)
mime_con.add_header('Content-Disposition', 'attachment', filename=name)
wrapper.attach(mime_con)
rcontent = wrapper.as_string()
if compress:
buf = StringIO()
gz = gzip.GzipFile(mode='wb', fileobj=buf)
try:
gz.write(rcontent)
finally:
gz.close()
rcontent = buf.getvalue()
return rcontent
def run(self):
"""
Open a subprocess and run a command on the local host.
:rtype: tuple
:return: This function returns a tuple that contains an integer status
and a string with the combined stdout and stderr output.
"""
boto.log.info('running:%s' % self.command)
log_fp = StringIO()
process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
while process.poll() is None:
time.sleep(1)
t = process.communicate()
log_fp.write(t[0])
log_fp.write(t[1])
boto.log.info(log_fp.getvalue())
boto.log.info('output: %s' % log_fp.getvalue())
return (process.returncode, log_fp.getvalue())
def check_no_resume(self, data, resume_set=set()):
fobj = StringIO(data.decode('utf-8'))
part_hash_map = {}
for part_index in resume_set:
start = self.part_size * part_index
end = start + self.part_size
part_data = data[start:end]
part_hash_map[part_index] = tree_hash(
chunk_hashes(part_data, self.chunk_size))
resume_file_upload(self.vault, sentinel.upload_id, self.part_size,
fobj, part_hash_map, self.chunk_size)
upload_part_calls, data_tree_hashes = calculate_mock_vault_calls(
data, self.part_size, self.chunk_size)
resume_upload_part_calls = [
call for part_index, call in enumerate(upload_part_calls)
if part_index not in resume_set
]
check_mock_vault_calls(self.vault, resume_upload_part_calls,
data_tree_hashes, len(data))
def test_1_basic(self):
c = boto.connect_sqs()
# create a queue so we can test BigMessage
queue_name = 'test%d' % int(time.time())
timeout = 60
queue = c.create_queue(queue_name, timeout)
self.addCleanup(c.delete_queue, queue, True)
queue.set_message_class(BigMessage)
# create a bucket with the same name to store the message in
s3 = boto.connect_s3()
bucket = s3.create_bucket(queue_name)
self.addCleanup(s3.delete_bucket, queue_name)
time.sleep(30)
# now add a message
msg_body = 'This is a test of the big message'
fp = StringIO(msg_body)
s3_url = 's3://%s' % queue_name
message = queue.new_message(fp, s3_url=s3_url)
queue.write(message)
time.sleep(30)
s3_object_name = message.s3_url.split('/')[-1]
# Make sure msg body is in bucket
self.assertTrue(bucket.lookup(s3_object_name))
m = queue.read()
self.assertEqual(m.get_body().decode('utf-8'), msg_body)
m.delete()
time.sleep(30)
# Make sure msg is deleted from bucket
self.assertIsNone(bucket.lookup(s3_object_name))
