def main_chunked(chunkby=1): tic = timeit.default_timer() LOCALPATH = os.path.dirname(os.path.abspath(__file__)) + "/" count = 0 srm_bucket_list = [x for x in bucket.list(prefix='AUDIO_FILES/', delimiter='/') if ".wav" in x.name] N = len(srm_bucket_list) l = range(0, N) chunks = l[::chunkby] for i in chunks: if chunks[-1] != i: chunk = srm_bucket_list[i:i + chunkby] else: chunk = srm_bucket_list[i:N] from_s3_chunked(chunk, LOCALPATH) convert_audiofiles_chunked(LOCALPATH) transcribe_chunks(LOCALPATH) to_s3_chunked(LOCALPATH) count += len(chunk) toc = timeit.default_timer() t = toc - tic k = Key(bucket) time_output_string = str(t) + " (time elapsed in seconds) " + count + " (number audio files)" k.key = "SRM_AUDIO_FILES/transcribed/time_output.txt" k.set_contents_from_string(time_output_string)
def map(self):
mc=MongoClient('ec2-52-0-148-244.compute-1.amazonaws.com',27017)
dbmc=mc.genid
idoc=dbmc.gentable.find_one_and_update(filter={},update={ "$inc": { "score": 1 } },upsert=True);
k=Key(self.bucket)
y=stopwords.words('english')
i=1
strx=str(int(idoc['score']))
strz=None
filestring=""
for line in sys.stdin:
line = unicode(line, "utf-8","ignore")
pattern = re.compile(r'\b(' + r'|'.join(y) + r')\b\s*')
line = pattern.sub('', line)
tokenizer = RegexpTokenizer(r'\w+')
words=tokenizer.tokenize(line)
strz=strx+'a'+str(i)
k.key=strz
filestring=line+'\n'
k.set_contents_from_string(filestring)
for word in words:
word=word.encode(encoding='UTF-8',errors='ignore')
print '%s\t%s' % (word.strip(), strz)
i+=1
def test_key_overwrite_and_copy(self):
first_content = b"abcdefghijklm"
second_content = b"nopqrstuvwxyz"
k = Key(self.bucket, 'testkey')
k.set_contents_from_string(first_content)
# Wait for S3's eventual consistency (may not be necessary)
while self.bucket.get_key('testkey') is None:
time.sleep(5)
# Get the first version_id
first_key = self.bucket.get_key('testkey')
first_version_id = first_key.version_id
# Overwrite the key
k = Key(self.bucket, 'testkey')
k.set_contents_from_string(second_content)
# Wait for eventual consistency
while True:
second_key = self.bucket.get_key('testkey')
if second_key is None or second_key.version_id == first_version_id:
time.sleep(5)
else:
break
# Copy first key (no longer the current version) to a new key
source_key = self.bucket.get_key('testkey',
version_id=first_version_id)
source_key.copy(self.bucket, 'copiedkey')
while self.bucket.get_key('copiedkey') is None:
time.sleep(5)
copied_key = self.bucket.get_key('copiedkey')
copied_key_contents = copied_key.get_contents_as_string()
self.assertEqual(first_content, copied_key_contents)
def test_key_size_with_validate_keyword():
"""
Test key.size on boto behavior with validate keyword
Not validating keys will make key.size = None
Writing to unvalidated keys should update that objects size
"""
key_name = 'the-key'
conn = boto.connect_s3('the_key', 'the_secret')
bucket = conn.create_bucket("foobar")
if bucket.get_key(key_name) is not None:
bucket.delete_key(key_name)
for string in ['', '0', '0'*5, '0'*10]:
# test non-existent keys
bucket.get_key(key_name, validate=False).size.should.be.none
(lambda: bucket.get_key(key_name, validate=True).size).should.throw(AttributeError)
key = Key(bucket)
key.key = key_name
key.size.should.be.none
# when writing key, key object updates size
key.set_contents_from_string(string)
key.size.should.equal(len(string))
# validated keys will have size
bucket.get_key(key_name, validate=True).size.should.equal(len(string))
# unvalidated keys that do not write do not have size set
key2 = Key(bucket)
key2.key = key_name
key2.size.should.be.none
bucket.get_key(key_name, validate=False).size.should.be.none
bucket.delete_key(key_name)
def upload(self, data, key, content_type, headers=None, public=True):
'''Uploads a file to S3 as the given key.
:param data: the file data
:type data: a file-like object or a :class:`str`
:param str key: the name associated with the file (usually looks like a
path).
:param str content_type: The MIME type of the data.
:param headers: Any extra headers associated with the file that will be
sent any time the file is accessed.
:type headers: :class:`dict` or :const:`None`
:returns: the protocol-agnostic URL of the new file on S3.
:rtype: :class:`str`
'''
if not headers:
headers = {}
headers.update({
'Content-Type': content_type,
})
key = Key(self.bucket, key)
if hasattr(data, 'read'):
key.set_contents_from_file(data, headers=headers)
else:
key.set_contents_from_string(data, headers=headers)
if public:
key.set_acl('public-read')
return '//%s.s3.amazonaws.com/%s' % (self.bucket_name, key.name)
def send_to_s3(items=None, is_binary=False):
"""
For items in an iterable, send them to your s3 account
"""
conn, bucket = s3_init(AWS_ACCESS_KEY_ID, AWS_SECRET_KEY, BUCKET_NAME)
for label, data in items:
key = Key(bucket)
key.key = label
for item in bucket.list():
local_md5 = hashlib.md5(data).hexdigest()
if item.name == label:
key.open()
key.close(); #loads key.etag
# remote hash
remote_md5 = key.etag.replace('\"','') # clears quote marks
# If new backup is different than the last saved one, update it
if local_md5 != remote_md5:
if is_binary:
key.set_contents_from_filename(data)
else:
key.set_contents_from_string(data)
else:
if is_binary:
key.set_contents_from_filename(data)
else:
key.set_contents_from_string(data)
def test_key_with_strings(self):
"""
test simple key 'from_string' and 'as_string' functions
"""
key_name = "test-key"
test_string = os.urandom(1024)
# create the bucket
bucket = self._s3_connection.create_unique_bucket()
self.assertTrue(bucket is not None)
# create an empty key
write_key = Key(bucket)
# set the name
write_key.name = key_name
# self.assertFalse(write_key.exists())
# upload some data
write_key.set_contents_from_string(test_string)
self.assertTrue(write_key.exists())
# create another key with the same name
read_key = Key(bucket, key_name)
# read back the data
returned_string = read_key.get_contents_as_string()
self.assertEqual(returned_string, test_string, (len(returned_string), len(test_string)))
# delete the key
read_key.delete()
self.assertFalse(write_key.exists())
# delete the bucket
self._s3_connection.delete_bucket(bucket.name)
def _upload_file(file_path, bucket, key_name, headers={}, do_gzip=False):
k = Key(bucket=bucket, name=key_name)
for header, value in headers.items():
if (header, value) != ('Content-Encoding', 'gzip'):
k.set_metadata(header, value)
mimetype = mimetypes.guess_type(file_path)[0]
if mimetype:
k.set_metadata('Content-Type', mimetype)
with open(file_path) as f:
content = f.read()
if do_gzip:
k.set_metadata('Content-Encoding', 'gzip')
gzipped = StringIO()
with gzip.GzipFile(fileobj=gzipped, mode='w') as _gzip:
_gzip.write(content)
content = gzipped.getvalue()
try:
k.set_contents_from_string(content)
except S3ResponseError:
if not do_gzip:
k.set_contents_from_filename(file_path)
else:
raise
k.make_public()
return k
def addPhoto(photo, setTitle):
url = flickr.photos_getSizes(photo_id = photo.attrib['id'])
realUrl = None
for url in url.find('sizes').findall('size'):
if url.attrib['label'] == "Original":
realUrl = url.attrib['source']
if realUrl:
keyId = setTitle + "/" + photo.attrib['id'] + ".jpg"
dataKeyId = keyId + ".metadata"
# Upload photo
if bucket.get_key(keyId) is None:
print "%s not found on S3; uploading" % keyId
f, h = urllib.urlretrieve(realUrl, reporthook = makeFlickrCallback())
key = Key(bucket)
key.key = keyId
print "Uploading %s to %s/%s" % (photo.attrib['title'], bucket.name, key.key)
key.set_metadata('flickrInfo', key.key + ".metadata")
key.set_metadata('inFlickrSet', set.attrib['id'])
key.set_contents_from_filename(f, cb = makeBotoCallback())
os.unlink(f)
# Upload metadata
if bucket.get_key(dataKeyId) is None:
print "%s not found on S3, setting metadata" % dataKeyId
photoInfo = flickr.photos_getInfo(photo_id = photo.attrib['id'], format = "rest")
key = Key(bucket)
key.key = dataKeyId
key.set_contents_from_string(photoInfo)
def scrape_data_to_html():
timestamp = datetime.fromtimestamp(time.time()).strftime("%H:%M:%S on %A, %d %B, %Y")
all_drinkers = drinkers_table.scan()
drinkers = []
for drinker in all_drinkers:
if (drinker['code'] == None):
drinker['code'] = "UNKNOWN"
if (drinker['name'] == None):
drinker['name'] = "UNKNOWN"
if (drinker['volume_consumed'] == None):
drinker['volume_consumed'] = 0
if (drinker['number_of_drinks'] == None):
drinker['number_of_drinks'] = 0
d = {}
d['code'] = drinker['code']
d['name'] = drinker['name']
d['volume_consumed'] = drinker['volume_consumed']
d['number_of_drinks'] = drinker['number_of_drinks']
drinkers.append(d)
loader = FileLoader('templates')
template = loader.load_template('drinks.html.template')
webpage = template.render(locals())
bucket = s3.get_bucket('kegerator')
key = Key(bucket)
key.key = 'drinks.html'
key.content_type = 'text/html'
key.set_contents_from_string(webpage)
key.make_public()
def add_meow(request):
if request.method == "POST":
newfile = request.FILES.get('new_meow_image')
key = Key(bucket)
keyname = str(int(uuid.uuid4()))[:10] + newfile.name
key.key = keyname
key.set_contents_from_string(newfile.read())
key.make_public()
url = 'https://s3.amazonaws.com/kitty2013/' + keyname
user = request.user
new_meow_text = request.POST.get('new_meow')
new_meow = Meow(text=new_meow_text,
user=user, image_url=url)
new_meow.save()
# Find these values at https://twilio.com/user/account
tagged_username = request.POST.get('tag')
tagged_user = User.objects.filter(username=tagged_username)[0]
if(tagged_user):
cell = tagged_user.userprofile.cell_phone
account_sid = "ACb65bbe159b7b180894c412cd4d47d231"
auth_token = "19d4f44997dbabfb6b15add63408682f"
client = TwilioRestClient(account_sid, auth_token)
message = client.messages.create(to="+%d" % cell, from_="+16083716550",
body="Hey %s, %s just tagged you in a picture" % (tagged_user.username, user.username))
return redirect('/user/%s' % user.id)
raise Http404
def _save_to_s3(self, data):
assert len(data) > 0, "Need data to save!"
# TODO: store AWS credentials in a better way.
assert 'AWS_ACCESS_KEY_ID' in config, "Need AWS key!"
assert 'AWS_SECRET_ACCESS_KEY' in config, "Need AWS secret!"
assert 'MESSAGE_STORE_BUCKET_NAME' in config, \
"Need bucket name to store message data!"
# Boto pools connections at the class level
conn = S3Connection(config.get('AWS_ACCESS_KEY_ID'),
config.get('AWS_SECRET_ACCESS_KEY'))
bucket = conn.get_bucket(config.get('MESSAGE_STORE_BUCKET_NAME'))
# See if it alreays exists and has the same hash
data_obj = bucket.get_key(self.data_sha256)
if data_obj:
assert data_obj.get_metadata('data_sha256') == self.data_sha256, \
"Block hash doesn't match what we previously stored on s3!"
# log.info("Block already exists on S3.")
return
data_obj = Key(bucket)
# if metadata:
# assert type(metadata) is dict
# for k, v in metadata.iteritems():
# data_obj.set_metadata(k, v)
data_obj.set_metadata('data_sha256', self.data_sha256)
# data_obj.content_type = self.content_type # Experimental
data_obj.key = self.data_sha256
# log.info("Writing data to S3 with hash {0}".format(self.data_sha256))
# def progress(done, total):
# log.info("%.2f%% done" % (done/total * 100) )
# data_obj.set_contents_from_string(data, cb=progress)
data_obj.set_contents_from_string(data)
def _addImageToS3(bucket, name, data):
num_retries = 0
max_retries = 5
while True:
try:
key = Key(bucket, name)
key.set_metadata("Content-Type", "image/jpeg")
key.set_contents_from_string(data.getvalue(), policy="public-read")
key.close()
return key
except Exception as e:
logs.warning("S3 Exception: %s" % e)
num_retries += 1
if num_retries > max_retries:
msg = "Unable to connect to S3 after %d retries (%s)" % (max_retries, self.__class__.__name__)
logs.warning(msg)
raise Exception(msg)
logs.info("Retrying (%s)" % (num_retries))
time.sleep(0.5)
finally:
try:
if not key.closed:
key.close()
except Exception:
logs.warning("Error closing key")
def push_to_s3(filename=None, contents=None):
"""
Save a file to the the configured bucket with name and contents
specified in the call.
It compresses the data.
This sets the contents to be publicly readable, cacheable by
intermediaries with an expiration date a specified number
of hours from when this job is run. (See above.)
"""
out = StringIO.StringIO()
with gzip.GzipFile(fileobj=out, mode="w") as f:
f.write(contents)
conn = S3Connection(calling_format=OrdinaryCallingFormat())
bucket = conn.get_bucket(BUCKET)
k = Key(bucket)
k.key = filename
expires = datetime.utcnow() + timedelta(hours=HOURS_TO_EXPIRE)
expires = expires.strftime("%a, %d %b %Y %H:%M:%S GMT")
k.set_contents_from_string(out.getvalue(),
policy='public-read',
headers={
'Cache-Control': 'public',
'Content-Type': 'application/json',
'Content-Encoding': 'gzip',
'Expires': '{}'.format(expires)
})
k.set_acl('public-read')
def main(stream_url: str, stream_name: str, bucket_name: str, duration: str):
temp_file = 'temp.m4a'
print('beginning rip')
code = subprocess.call(['ffmpeg',
'-i', stream_url,
'-t', duration,
'-acodec', 'copy',
'-absf', 'aac_adtstoasc',
temp_file])
assert code == 0, 'stream rip failed with code ' + str(code)
print('connecting to s3')
conn = S3Connection(is_secure=False) # AWS uses invalid certs
bucket = conn.get_bucket(bucket_name)
print('writing recorded file to s3')
m4a = Key(bucket)
m4a.name = datetime.datetime.utcnow().strftime(stream_name + '--%Y-%m-%d.m4a')
m4a.content_type = MIME_TYPE
m4a.metadata = {'Content-Type': MIME_TYPE}
m4a.storage_class = 'STANDARD_IA'
m4a.set_contents_from_filename(temp_file)
m4a.close()
print('generating new feed.xml from s3 bucket list')
feed_xml = Key(bucket)
feed_xml.name = 'feed.xml'
feed_xml.content_type = 'application/rss+xml'
feed_xml.set_contents_from_string(
rss_xml(stream_name, bucket_name, bucket.list()))
feed_xml.close()
def save_files():
index_missing = 0
with open('reports.csv', mode='r') as infile:
reader = csv.reader(infile)
for row in reader:
if row[21] != 'list_of_links' and row[20] !=0:
print "Getting links for: " + row[5]
links = row[21].split()
for link in links:
try:
k = Key(bucket)
file_name = urllib2.unquote(link)
file_name = file_name[file_name.index('$file')+6:].replace(".pdf","")
file_name = re.sub(r'([^\s\w])+', '', file_name).replace(" ", "-").lower() + ".pdf"
file_name = row[1].replace("/", "-") + "/" + row[7] + "/" + file_name
print "Saving: " + file_name
if bucket.get_key(file_name):
print "File exists, skipping"
else:
f = urllib2.urlopen(link)
data = f.read()
f.close()
k = bucket.new_key(file_name)
k.set_contents_from_string(data)
except Exception, e:
print "Could not get because: " + str(e)
new_missing_row = [row[1], row[7], link, str(e)]
df_missing_files.loc[index_missing] = new_missing_row
index_missing +=1
def save_page(self, html):
if html and len(html) > 100:
if settings.BACKED_BY_AWS.get('pages_on_s3'):
k = Key(settings.S3_PAGES_BUCKET)
k.key = self.feed.s3_pages_key
k.set_metadata('Content-Encoding', 'gzip')
k.set_metadata('Content-Type', 'text/html')
k.set_metadata('Access-Control-Allow-Origin', '*')
out = StringIO.StringIO()
f = gzip.GzipFile(fileobj=out, mode='w')
f.write(html)
f.close()
compressed_html = out.getvalue()
k.set_contents_from_string(compressed_html)
k.set_acl('public-read')
try:
feed_page = MFeedPage.objects.get(feed_id=self.feed.pk)
feed_page.delete()
logging.debug(' --->> [%-30s] ~FYTransfering page data to S3...' % (self.feed))
except MFeedPage.DoesNotExist:
pass
self.feed.s3_page = True
self.feed.save()
else:
try:
feed_page = MFeedPage.objects.get(feed_id=self.feed.pk)
feed_page.page_data = html
feed_page.save()
except MFeedPage.DoesNotExist:
feed_page = MFeedPage.objects.create(feed_id=self.feed.pk, page_data=html)
return feed_page
def write_to_s3(self, prefix, fname, content, ctype='text/plain'):
"""
Write ``content`` into S3 at ``prefix``/``fname``. If ``self.dry_run``,
write to local disk instead. Return the resulting URL, either an S3
URL or a local 'file://' URL.
:param prefix: the prefix to write S3 files under, or local files under
:type prefix: str
:param fname: the file name to create
:type fname: str
:param content: the content to write into the file
:type content: str
:returns: URL to the created file
:rtype: str
"""
path = os.path.join(prefix, fname)
if self.dry_run:
path = os.path.abspath(path)
logger.warning("DRY RUN: Writing s3-bound content to %s", path)
dest_dir = os.path.dirname(path)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
with open(path, 'w') as fh:
fh.write(content)
return 'file://%s' % path
# else write to S3
logger.debug("Creating S3 key: %s (Content-Type: %s)", path, ctype)
k = Key(self.bucket)
k.content_type = ctype
k.key = path
k.set_contents_from_string(content)
url = self.url_for_s3(path)
logger.debug("Data written to %s", url)
return url
def output_s3(fmt, output, opts, version):
try:
from boto.s3.connection import S3Connection
from boto.s3.key import Key
except ImportError:
print >> sys.stderr, "Unable to publish to S3: Boto not installed."
return
# Verify the S3 configuration
bucket_name = opts.output_s3
access_key = opts.aws_access_key or os.environ.get("AWS_ACCESS_KEY_ID")
secret_key = opts.aws_secret_key or os.environ.get("AWS_SECRET_ACCESS_KEY")
if not access_key or not secret_key:
print >> sys.stderr, "We need an AWS access key and AWS secret key"
return
conn = S3Connection(access_key, secret_key)
bucket = conn.get_bucket(bucket_name)
k = Key(bucket)
if version == 2:
file_name = "regions."
else:
file_name = "regions-v" + version + "."
k.key = file_name + fmt
# Set a content type
content_types = {"json": "application/json", "xml": "text/xml"}
if fmt in content_types:
k.set_metadata("Content-Type", content_types[fmt])
print "Writing %s/%s" % (bucket_name, k.key)
k.set_contents_from_string(output)
def upload(self, filepath, data, format):
k = Key(self.bucket)
k.key = filepath
if format == "txt":
k.set_contents_from_string(data)
elif format == "json":
k.set_contents_from_string(json.dumps(data))
def uploadPublicImageFromString(self, s3KeyName, imageString):
key = Key(self._bucket)
key.key = s3KeyName
# todo: check content-type
key.set_metadata("Content-Type", "image/jpeg")
key.set_contents_from_string(imageString)
key.set_acl("public-read")
def test_transport_addr_policy(self):
bucket = self.conn.create_bucket(self.bucket_name)
key_name = str(uuid.uuid4())
key = Key(bucket, key_name)
key.set_contents_from_string(self.data)
## anyone may GET this object
policy = '''
{"Version":"2008-10-17","Statement":[{"Sid":"Stmtaaa0","Effect":"Allow","Principal":"*","Action":["s3:GetObject"],"Resource":"arn:aws:s3:::%s/*","Condition":{"Bool":{"aws:SecureTransport":false}}}]}
''' % bucket.name
self.assertTrue(bucket.set_policy(policy, headers={'content-type':'application/json'}))
key.get_contents_as_string()
## policy accepts anyone who comes with http
conn = httplib.HTTPConnection(self.host, self.port)
headers = { "Host" : "%s.s3.amazonaws.com" % bucket.name }
conn.request('GET', ("/%s" % key_name) , None, headers)
response = conn.getresponse()
self.assertEqual(response.status, 200)
self.assertEqual(response.read(), key.get_contents_as_string())
## anyone without https may not do any operation
policy = '''
{"Version":"2008-10-17","Statement":[{"Sid":"Stmtaaa0","Effect":"Deny","Principal":"*","Action":"*","Resource":"arn:aws:s3:::%s/*","Condition":{"Bool":{"aws:SecureTransport":false}}}]}
''' % bucket.name
self.assertTrue(bucket.set_policy(policy, headers={'content-type':'application/json'}))
## policy accepts anyone who comes with http
conn = httplib.HTTPConnection(self.host, self.port)
headers = { "Host" : "%s.s3.amazonaws.com" % bucket.name }
conn.request('GET', ("/%s" % key_name) , None, headers)
response = conn.getresponse()
self.assertEqual(response.status, 403)
self.assertEqual(response.reason, 'Forbidden')
def s3UploadDataPublic(data, remote_file_name):
log("s3 upload data as '%s'" % remote_file_name)
bucket = s3PubBucket()
k = Key(bucket)
k.key = remote_file_name
k.set_contents_from_string(data)
k.make_public()
def write_to_s3(bucket, key, string):
emr = EMRJobRunner()
c = emr.fs.make_s3_conn()
b = c.get_bucket(bucket)
k = Key(b)
k.key = key
k.set_contents_from_string(string)
def getUploadLink(self):
k = Key(bucket)
k.key = 'test-key' #Should be replaced dynamically with e.g. the uuid
k.set_contents_from_string('Hello World, I am Fred from Fribourg!')
expires_in_seconds = 1800
url = k.generate_url(expires_in_seconds)
return url
def store_in_s3(filename, content,context):
s3 = boto.connect_s3(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
bucket = s3.get_bucket('xz820s')
k = Key(bucket)
k.key = filename
k.set_contents_from_string(content)
k.set_acl("public-read")
def dump(self, key, value, public=False):
"""
Dump file to S3.
Optionally make public
"""
assert isinstance(key, basestring), u'Key must be a string'
k = Key(self.bucket)
k.key = key
try:
k.set_metadata(u'Content-Type', u'application/json')
k.set_contents_from_string(json.dumps(value, sort_keys=True, indent=4, separators=(u',', u': ')))
# set file permissions
if public:
k.set_acl(u'public-read')
except Exception as e:
print e
return False
else:
# now update the cache
if self._keys is not None:
self._keys.add(key)
return True
def test_get_object_acl(self):
bucket = self.conn.create_bucket(self.bucket_name)
k = Key(bucket)
k.key = self.key_name
k.set_contents_from_string(self.data)
self.assertEqual(k.get_contents_as_string(), self.data)
self.assertEqual(k.get_acl().to_xml(), self.defaultAcl(self.user1))
def write(self, filepath, buffer):
""" Write a buffer for a single file.
"""
k = Key(self.bucket)
k.key = filepath
k.set_contents_from_string(buffer.getvalue())
self.bucket.set_acl("public-read", k.key)
def upload_files(method):
global BUCKET
files = []
for file_name in os.listdir(FINISHED_PATH):
if file_name.endswith('.PNG') or file_name.endswith('.png'):
files.append(file_name)
conn = boto.connect_s3(opts.accesskey,opts.secret)
bucket = conn.create_bucket(BUCKET)
i = 1
for file_name in files:
out(str(i)+'/'+str(len(files))+' | Uploading: '+file_name)
k = Key(bucket)
if method == 'overwrite':
k.key = file_name
elif method == 'prefix':
k.key = 'opt_'+file_name
elif method == 'newdir':
k.key = 'S3crush/'+file_name
k.set_contents_from_string(open(FINISHED_PATH+file_name,'r').read())
out(str(i)+'/'+str(len(files))+' | -> Upload finished: '+file_name)
i += 1
def test_ranged_get():
conn = boto.connect_s3()
bucket = conn.create_bucket('mybucket')
key = Key(bucket)
key.key = 'bigkey'
rep = b"0123456789"
key.set_contents_from_string(rep * 10)
# Implicitly bounded range requests.
key.get_contents_as_string(headers={
'Range': 'bytes=0-'
}).should.equal(rep * 10)
key.get_contents_as_string(headers={
'Range': 'bytes=50-'
}).should.equal(rep * 5)
key.get_contents_as_string(headers={
'Range': 'bytes=99-'
}).should.equal(b'9')
# Explicitly bounded range requests starting from the first byte.
key.get_contents_as_string(headers={
'Range': 'bytes=0-0'
}).should.equal(b'0')
key.get_contents_as_string(headers={
'Range': 'bytes=0-49'
}).should.equal(rep * 5)
key.get_contents_as_string(headers={
'Range': 'bytes=0-99'
}).should.equal(rep * 10)
key.get_contents_as_string(headers={
'Range': 'bytes=0-100'
}).should.equal(rep * 10)
key.get_contents_as_string(headers={
'Range': 'bytes=0-700'
}).should.equal(rep * 10)
# Explicitly bounded range requests starting from the / a middle byte.
key.get_contents_as_string(headers={
'Range': 'bytes=50-54'
}).should.equal(rep[:5])
key.get_contents_as_string(headers={
'Range': 'bytes=50-99'
}).should.equal(rep * 5)
key.get_contents_as_string(headers={
'Range': 'bytes=50-100'
}).should.equal(rep * 5)
key.get_contents_as_string(headers={
'Range': 'bytes=50-700'
}).should.equal(rep * 5)
# Explicitly bounded range requests starting from the last byte.
key.get_contents_as_string(headers={
'Range': 'bytes=99-99'
}).should.equal(b'9')
key.get_contents_as_string(headers={
'Range': 'bytes=99-100'
}).should.equal(b'9')
key.get_contents_as_string(headers={
'Range': 'bytes=99-700'
}).should.equal(b'9')
# Suffix range requests.
key.get_contents_as_string(headers={
'Range': 'bytes=-1'
}).should.equal(b'9')
key.get_contents_as_string(headers={
'Range': 'bytes=-60'
}).should.equal(rep * 6)
key.get_contents_as_string(headers={
'Range': 'bytes=-100'
}).should.equal(rep * 10)
key.get_contents_as_string(headers={
'Range': 'bytes=-101'
}).should.equal(rep * 10)
key.get_contents_as_string(headers={
'Range': 'bytes=-700'
}).should.equal(rep * 10)
key.size.should.equal(100)
def save(self):
conn = boto.connect_s3('the_key', 'the_secret')
bucket = conn.get_bucket('mybucket')
k = Key(bucket)
k.key = self.name
k.set_contents_from_string(self.value)
def write_photo(self, file_string):
"""
Accepts base64-encoded string from the admin form.
If s3 buckets are available, deploys to s3.
If not, deploys to www/live-data/img/.
"""
timestamp = int(time.mktime(datetime.datetime.now().timetuple()))
file_type, data = file_string.split(';')
prefix, data = data.split(',')
decoded_file = base64.b64decode(data)
file_name = '%s-%s' % (timestamp, self.file_name)
rendered_path = 'live-data/img'
if not os.path.exists('/tmp/%s' % app_config.PROJECT_SLUG):
os.makedirs('/tmp/%s' % app_config.PROJECT_SLUG)
# Write the initial file.
with open('/tmp/%s/%s' % (app_config.PROJECT_SLUG, file_name),
'wb') as writefile:
writefile.write(decoded_file)
# Write the rendered files.
for width in app_config.IMAGE_WIDTHS:
os.system('convert -resize %s "/tmp/%s/%s" "/tmp/%s/%spx-%s"' %
(width, app_config.PROJECT_SLUG, file_name,
app_config.PROJECT_SLUG, width, file_name))
# Glob for all of the files.
for resized_image in glob.glob('/tmp/%s/*px-%s' %
(app_config.PROJECT_SLUG, file_name)):
file_path, file_name = os.path.split(resized_image)
file_name, file_extension = os.path.splitext(file_name)
with open(resized_image, 'rb') as readfile:
decoded_file = readfile.read()
r = '%s/%s%s' % (rendered_path, file_name, file_extension)
width = resized_image.split('px')[0].split(file_path)[1].replace(
'/', '')
# Deployed
if app_config.S3_BUCKETS:
# Connect to S3.
s3 = boto.connect_s3()
r = '%s/%s' % (app_config.PROJECT_SLUG, r)
for bucket_name in app_config.S3_BUCKETS:
bucket = s3.get_bucket(bucket_name)
k = Key(bucket, r)
k.set_contents_from_string(
decoded_file, headers={'Cache-Control': 'max-age=5'})
k.set_acl('public-read')
setattr(
self, 'rendered_%s_path' % width,
'http://%s.s3.amazonaws.com/%s' %
(app_config.S3_BUCKETS[0], r))
# Local
else:
local_path = 'www/%s' % r
dirname = os.path.dirname(local_path)
if not os.path.exists(dirname):
os.makedirs(dirname)
with open(local_path, 'wb') as writefile:
writefile.write(decoded_file)
setattr(self, 'rendered_%s_path' % width,
'/%s/%s' % (app_config.PROJECT_SLUG, r))
os.system('rm -f %s' % resized_image)
def write_data(params):
keyname, stringdata = params
key = Key(bucket)
key.key = keyname
key.set_contents_from_string(stringdata)
return null
def s3UploadDataPublic(data, remote_file_name):
bucket = s3PubBucket()
k = Key(bucket)
k.key = remote_file_name
k.set_contents_from_string(data)
k.make_public()
def run_capture(hashtag, sessionData):
result = ''
#Twitter Keys
#app keys
twitter1 = sessionData['twitter1']
twitter2 = sessionData['twitter2']
#user keys
twitter3 = sessionData['twitter3']
twitter4 = sessionData['twitter4']
maxTweets = int(sessionData['maxTweets'])
viprOnline = sessionData['viprOnline']
if viprOnline == 'True':
print 'initializing ViPR system'
#Config info to find ViPR in the vLab
s3secret = sessionData['s3secret']
s3user = sessionData['s3user']
s3host = sessionData['s3host']
s3port = int(sessionData['s3port'])
s3bucket = sessionData['s3bucket']
print s3secret
print s3user
print s3host
conn = S3Connection(
aws_access_key_id=s3user,
aws_secret_access_key=s3secret,
host=s3host,
port=s3port,
calling_format=
'boto.s3.connection.ProtocolIndependentOrdinaryCallingFormat',
is_secure=False)
print 'Listing all buckets for this user'
print conn.get_all_buckets()
mybucket = conn.get_bucket(s3bucket)
api = TwitterAPI(twitter1, twitter2, twitter3, twitter4)
r = api.request('search/tweets', {'q': hashtag, 'count': maxTweets})
for item in r.get_iterator():
result = result + '<p>' + item['text'].encode('ascii',
'ignore') + '</p>'
result = result + '<p>' + '</p>'
result = result + '<p>' + item['user']['name'].encode(
'ascii', 'ignore') + '</p>'
result = result + '<p>' + '@' + item['user']['screen_name'].encode(
'ascii', 'ignore') + '</p>'
if viprOnline == 'True':
targetstring = '@' + item['user']['screen_name'].encode(
'ascii',
'ignore') + '\r\n' + item['user']['screen_name'].encode(
'ascii', 'ignore') + '\r\n' + item['text'].encode(
'ascii', 'ignore')
mykey = Key(mybucket)
mykey.key = '/user/hadoop/input/'
mykey.key = mykey.key + str(
time.time()) + item['user']['screen_name'].encode(
'ascii', 'ignore') + '.txt'
mykey.set_contents_from_string(targetstring)
result = Markup(result)
return result
clean_name = raw_name.split(',')
clean_name = clean_name[1][1:] + ' ' + clean_name[0]
field.append(clean_name)
print "field:", len(field)
salaries_order = []
salaries_map = {}
salaries_map['tournament'] = tournament
for index, row in salaries.iterrows():
pga_name = player_map['players'][row['PLAYER']]
salaries.set_value(index, 'PLAYER', pga_name)
cleaned_salary = row['SALARY'][1:].split(",")[0] + row['SALARY'][1:].split(
",")[1]
salaries_map[pga_name] = int(cleaned_salary)
if pga_name in field:
salaries_order.append(pga_name)
salaries_map['order'] = salaries_order
salaries_map = json.dumps(salaries_map)
k2 = Key(b)
k2.key = 'ff/salaries'
k2.set_contents_from_string(salaries_map)
k2.make_public()
k3 = Key(b)
k3.key = 'ff/' + str(year) + '/' + tournament + '/salaries'
k3.set_contents_from_string(salaries_map)
def test_bucket_name_with_dot():
conn = boto.connect_s3()
bucket = conn.create_bucket('firstname.lastname')
k = Key(bucket, 'somekey')
k.set_contents_from_string('somedata')
is_secure=False,
host=my_euca_host,
port=my_euca_port,
path=my_euca_path,
#provider='eucalyptus',
#bucket_class=Bucket,
calling_format=OrdinaryCallingFormat(),
debug=0)
bucket = conn.create_bucket('test')
from boto.s3.key import Key
k = Key(bucket)
k.key = 'foobar'
k.set_contents_from_string('hello world')
rs = conn.get_all_buckets()
for b in rs:
print b
#mehods
def create_bucket():
pass
def get_all_buckets():
pass
def store(name):
k = Key(bucket, prefix + name)
k.set_contents_from_string('somedata')
from boto.s3.key import Key
from csvkit.unicsv import UnicodeCSVDictWriter
AWS_KEY = os.environ['AWS_ACCESS_KEY']
AWS_SECRET = os.environ['AWS_SECRET_KEY']
url_pattern = '/CommitteeDetail.aspx?id=%s'
string_on_page = 'ctl00_ContentPlaceHolder1_CommitteeResultsLayout'
comm_scraper = CommitteeScraper(url_pattern=url_pattern,
string_on_page=string_on_page)
# comm_scraper.cache_storage = scrapelib.cache.FileCache('cache')
# comm_scraper.cache_write_only = False
committees = []
comms_outp = StringIO()
comm_header = [
'id', 'name', 'type', 'url', 'address', 'status', 'purpose',
'state_id', 'local_id', 'creation_date'
]
comm_writer = UnicodeCSVDictWriter(comms_outp, comm_header, delimiter='\t')
comm_writer.writeheader()
for committee in comm_scraper.scrape_all():
# Save to DB and maybe write as JSON?
committees.append(committee)
comm_writer.writerows(committees)
s3_conn = S3Connection(AWS_KEY, AWS_SECRET)
bucket = s3_conn.get_bucket('il-elections')
k = Key(bucket)
k.key = 'Committees.tsv'
k.set_contents_from_string(comms_outp.getvalue())
k.make_public()
def test_boto_without_vcr():
s3_conn = S3Connection()
s3_bucket = s3_conn.get_bucket('boto-demo-1394171994') # a bucket you can access
k = Key(s3_bucket)
k.key = 'test.txt'
k.set_contents_from_string('hello world i am a string')
#!/usr/bin/env python
import boto
from boto.s3.key import Key
import datetime
s3 = boto.s3.connect_to_region('ap-southeast-2')
dev_bucket = s3.get_bucket('mydetic-dev.ghosttrails.net')
k = Key(dev_bucket)
k.key = 'test1'
k.set_contents_from_string('foooooo')
#for bucket in s3.get_all_buckets():
# print bucket
import requests
import json
import time
from boto.s3.connection import S3Connection
from boto.s3.key import Key
for year in [2013, 2014, 2015]:
# get tournament schedule from Sports Data
sports_data_key = 'd6aw46aafm49s5xyc2bj8vwr'
full_request_string = 'http://api.sportsdatallc.org/golf-t1/schedule/pga/' + str(
year) + '/tournaments/schedule.json?api_key=' + sports_data_key
r = requests.get(full_request_string)
schedule = r.json()
schedule = json.dumps(schedule)
# save schedule to AWS S3
c = S3Connection('AKIAIQQ36BOSTXH3YEBA',
'cXNBbLttQnB9NB3wiEzOWLF13Xw8jKujvoFxmv3L')
b = c.get_bucket('public.tenthtee')
k = Key(b)
k.key = 'sportsData/' + str(year) + '/schedule.json'
k.set_contents_from_string(schedule)
time.sleep(5)
print str(year) + ': OK'
def report():
"""Add a base64 image"""
import base64
import uuid
import PIL
from PIL import Image, ExifTags
import tweepy
import json
from boto.s3.connection import S3Connection
from boto.s3.connection import OrdinaryCallingFormat
from boto.s3.key import Key
import datetime
import requests
img_data = request.values.get('img_data')
# try:
decoded = base64.b64decode(img_data)
filename = ("%s" % uuid.uuid4())[0:8]
with open("tmp/%s.png" % filename, 'wb') as f:
f.write(decoded)
im = Image.open("tmp/%s.png" % filename)
try:
for orientation in ExifTags.TAGS.keys():
if ExifTags.TAGS[orientation] == 'Orientation':
break
exif = dict(im._getexif().items())
if exif[orientation] == 3:
im = im.rotate(180, expand=True)
elif exif[orientation] == 6:
im = im.rotate(270, expand=True)
elif exif[orientation] == 8:
im = im.rotate(90, expand=True)
except (AttributeError, KeyError, IndexError):
# cases: image don't have getexif
pass
maxsize = (float(620), float(620))
size = (float(im.size[0]), float(im.size[1]))
scale = "down"
if size[0] < maxsize[0] or size[1] < maxsize[1]:
scale = "up"
if size[0] < size[1]:
newsize = (maxsize[0], maxsize[1] * (size[1] / size[0]))
else:
newsize = (maxsize[0] * (size[0] / size[1]), maxsize[1])
crop = (int(
(newsize[0] - maxsize[0]) / 2), int(
(newsize[1] - maxsize[1]) / 2), int((newsize[0] + maxsize[0]) / 2),
int((newsize[1] + maxsize[1]) / 2))
newsize = (int(newsize[0]), int(newsize[1]))
im = im.resize(newsize, PIL.Image.ANTIALIAS)
im = im.crop(crop)
page_type = get_page_type_from_request()
if page_type == 'VOTED':
img_overlay_file = "images/frame_badge_confetti.png"
tweet_string = '#IVOTED'
else:
img_overlay_file = "images/frame_badge_voter.png"
tweet_string = 'Voting!'
foreground = Image.open(img_overlay_file)
im.paste(foreground, (0, 0), foreground)
im.save("tmp/%s.png" % filename)
auth = tweepy.OAuthHandler(os.environ.get('TWITTER_API_KEY'),
os.environ.get('TWITTER_API_SECRET'))
auth.set_access_token(os.environ.get('TWITTER_TOKEN'),
os.environ.get('TWITTER_TOKEN_SECRET'))
api = tweepy.API(auth)
r = api.update_with_media(
"tmp/%s.png" % filename,
(u"%s Your "
u"turn\u2014txt VOTE to 384-387 or visit hello.vote for your "
u"voting location & selfie!") % tweet_string)
item = r._json
conn = S3Connection(os.environ.get('AWS_ACCESS_KEY'),
os.environ.get('AWS_SECRET_KEY'),
calling_format=OrdinaryCallingFormat())
bucket = conn.get_bucket(os.environ.get('AWS_S3_BUCKET'))
bucket_url = "https://%s.s3.amazonaws.com" % os.environ.get(
'AWS_S3_BUCKET')
username = item.get('user').get('screen_name')
user_id = item.get('user').get('id')
user_avatar = item.get('user').get('profile_image_url')
photo_id = str(item.get('id'))
photo_url = item.get('entities').get('media')[0].get('media_url')
display_url = item.get('entities').get('media')[0].get('display_url')
user_avatar_s3 = "%s/avatars/%s.jpg" % (bucket_url, user_id)
photo_url_s3 = "%s/photos/%s.png" % (bucket_url, photo_id)
photo = Badge(source='twitter',
source_id=photo_id,
username=username,
user_avatar=user_avatar,
user_avatar_s3=user_avatar_s3,
photo_url=photo_url,
photo_url_s3=photo_url_s3,
photo_display_url=display_url,
original_url='https://twitter.com/%s/status/%s' %
(username, photo_id),
caption=item.get('text'),
visible=False,
priority=0,
random_index=random.randint(0, 2147483647),
permalink_slug=filename,
created=datetime.datetime.now())
photo.save()
# save avatar to s3
avatar = requests.get(user_avatar).content
k = Key(bucket)
k.key = "avatars/%s.jpg" % user_id
k.delete()
k.set_metadata('Content-Type', 'image/jpeg')
k.set_contents_from_string(avatar)
k.set_acl('public-read')
# save photo to s3
k = Key(bucket)
k.key = "photos/%s.png" % photo_id
k.delete()
k.set_metadata('Content-Type', 'image/png')
k.set_contents_from_filename("tmp/%s.png" % filename)
k.set_acl('public-read')
# except:
# return '{"error": true}'
return photo.to_json()
elif player == 'William Lunde':
player_map['players'][player] = 'Bill Lunde'
elif player == 'Carlos Sainz':
player_map['players'][player] = 'Carlos Sainz Jr'
elif player == 'Steve Lewton':
player_map['players'][player] = 'Stephen Lewton'
elif player == 'Wen-Chong Liang':
player_map['players'][player] = 'WC Liang'
elif player == 'Fredrik Jacobson':
player_map['players'][player] = 'Freddie Jacobson'
elif player == 'Rodolfo Cazaubon':
player_map['players'][player] = 'Rodolfo E. Cazaubon'
elif player == 'Yan-wei Liu':
player_map['players'][player] = 'Yan Wei Liu'
elif player == 'Richard Johnson':
player_map['players'][player] = 'Richard S. Johnson'
elif player == 'Rafael Cabrera-Bello':
player_map['players'][player] = 'Rafael Cabrera Bello'
else:
player_map['players'][player] = player
print player_map
# get PGA player list
player_map = json.dumps(player_map)
k3 = Key(b)
k3.key = 'playerData/sportsDataToPgaMapping'
pga_player_list = k3.set_contents_from_string(player_map)
#for index, row in df.iterrows(): # # usernames.append(row['EntryName']) # # player_one.append(lineup_array[0][4:]) # player_two.append(lineup_array[1][4:]) # player_three.append(lineup_array[2][4:]) # player_four.append(lineup_array[3][4:]) # player_five.append(lineup_array[4][4:]) # player_six.append(lineup_array[5][4:]) # # #df = pd.DataFrame(usernames,columns=['usernames']) #df['player_one'] = player_one #df['player_two'] = player_two #df['player_three'] = player_three #df['player_four'] = player_four #df['player_five'] = player_five #df['player_six'] = player_six # #csv = df.to_csv() contest_data = json.dumps(contest_data) k = Key(b) k.key = 'dk/4161803' k.set_contents_from_string(contest_data) k.make_public() print contest_data
sports_data_key = 'd6aw46aafm49s5xyc2bj8vwr'
year = 2015
k.key = 'sportsData/' + str(year) + '/schedule.json'
schedule_string = k.get_contents_as_string()
schedule = json.loads(schedule_string)
# get tournament id
for tournament in schedule['tournaments']:
# uncomment line below to identify the tournament names
# print tournament['name'],tournament['id']
# if tournament['name'] == target_tournament: break
if tournament_name == tournament['name']:
tournament_id = tournament['id']
for rd in [1,2,3,4]:
# use tournament id to get round scores
request_string = 'http://api.sportsdatallc.org/golf-t1/teetimes/pga/' + str(year) + '/tournaments/' + tournament_id + '/rounds/'+ str(rd) + '/teetimes.json?api_key=' + sports_data_key
r = requests.get(request_string)
teetimes = r.json()
teetimes = json.dumps(teetimes)
# save hole_stats to AWS S3
k.key = 'sportsData/' + str(year) + '/' + tournament_name + '/rounds/' + str(rd) + '/teetimes.json'
k.set_contents_from_string(teetimes)
time.sleep(3)
print year, tournament_name, rd
break
def create_directory(self, directory):
self.logger.info("creating directory %s", directory)
k = Key(self.bucket)
k.key = directory[1:] + '/'
k.set_metadata('Content-Type', 'application/x-directory')
k.set_contents_from_string('')
def mock_get_key(keyname):
key = Key(name=keyname)
key.set_contents_from_string('content')
key.set_metadata('metadata_name', 'metadata_val')
return key
def upload_to_s3(self, path: Text, file_data: binary_type,
headers: Optional[Dict[Text, Text]]) -> None:
key = Key(self.bucket)
key.key = path
key.set_contents_from_string(force_str(file_data), headers=headers)
# uploads to S3 must be to a bucket, which must have a unique name
bucket_name = "boto-example-boop-beep-%s" % uuid.uuid4()
print "Check it out, your bucket name will be: " + bucket_name
my_shiny_new_bucket = s3_client.create_bucket(bucket_name)
print "Just made a bucket!..."
# Object (a.k.a. files): Refer to it by its key (name)
from boto.s3.key import Key
key_aka_nameOfObject = Key(my_shiny_new_bucket)
key_aka_nameOfObject.key = 'boto-example-beepBoop-key.txt'
print "Uploading data to " + bucket_name + " with key: " + key_aka_nameOfObject.key
# Put a bit of data into the object a.k.a. file
key_aka_nameOfObject.set_contents_from_string('BLAH BLAH BLAH!!!!!!!!!!!!')
# use generate_url to make an URL
seconds_to_expire = 240 # you have 4 minutes
print "Making a public URL for the uploaded object. Lives for %d seconds." % seconds_to_expire
print
print key_aka_nameOfObject.generate_url(seconds_to_expire)
print
raw_input("Press enter to delete object and bucket...")
# Need to delete object first because bucket has to be empty before it can be deleted
print "Deleting object..."
key_aka_nameOfObject.delete()
from filechunkio import FileChunkIO
dsslib.GLOBAL_DEBUG = 1
'''=========================================================
### API Doc testing
'''
usr_str = "user804"
obj = dsslib.getConnection(dsslib.USER_keystone804)
dsslib.listBucket(obj, usr_str)
b = obj.create_bucket('docsbuck001')
dsslib.listBucket(obj, usr_str)
b.set_acl('public-read-write')
k = Key(b)
k.key = 'obj1'
k.set_contents_from_string('Data of object')
print "Setting ACL on obj"
k.set_acl('public-read')
print "\nObject has ACL============="
print str(k.get_acl())
##====================================================================
'''=========================================================
## AWS user policy prevents user from listing bucket but allows getting objects inside the particular bucket
## Make listing buckets allowed. Generate signed URLs then make listing buckets not allowed in user policy.
## Check if the URLs still work.
obj = dsslib.getConnection(dsslib.USER_aws2)
b = obj.get_bucket('shivanshubucketauto4')
for i in b.list():
print 'New key: ' + str(i)
try:
url = "GOOGLE SPREADSHEET URL" + "&output=csv"
request = urllib2.Request(url)
cookie_handler = urllib2.HTTPCookieProcessor()
redirect_handler = urllib2.HTTPRedirectHandler()
opener = urllib2.build_opener(redirect_handler, cookie_handler)
u = opener.open(request)
print "Getting JSON"
data = csv.DictReader(u)
out = json.dumps([row for row in data])
print "Connecting to S3"
conn = boto.connect_s3()
#Replace 'my-bucket' with your bucket name
bucket = conn.get_bucket('my-bucket')
from boto.s3.key import Key
#Replace 'data.json' with your preferred file name
k = Key(bucket)
k.key = "data.json"
k.set_metadata("Cache-Control", "max-age=180")
k.set_metadata("Content-Type", "application/json")
k.set_contents_from_string(out)
k.set_acl("public-read")
print "Done, JSON is updated"
def upload_to_s3(self, path: str, file_data: bytes,
headers: Optional[Dict[str, str]]) -> None:
key = Key(self.bucket)
key.key = path
key.set_contents_from_string(force_str(file_data), headers=headers)
def upload_string_to_bucket(bucket, key_name, s, public=True):
k = Key(bucket)
k.key = key_name
k.set_contents_from_string(s)
if public:
k.set_acl('public-read')
port=RADOSPORT,
is_secure=False,
calling_format=boto.s3.connection.OrdinaryCallingFormat())
####################################################
################## TEST CASE #######################
print "\nCreating and populating the bucket for user1..."
b1 = conn_u1.create_bucket('bucket_b')
k = Key(b1)
for i in range(1, 11):
print "\tCreating obj %d" % (i)
keyv = 'keynum' + str(i)
valv = 'Contents of object'
k.key = keyv
k.set_contents_from_string(valv)
print "\nSetting ACL..."
b1.set_acl('public-read-write')
b2 = conn_u2.get_bucket(b1.name)
print "\nU2: Name of this bucket is {b2name}".format(b2name=b2.name)
print "\nU2: Listing keys from U1 bucket"
for i in b2.list():
print "U1 bucket key: " + str(i)
print "\nU2 tries to write to this bucket"
m = Key(b2)
for i in range(20, 25):
print "\tCreating obj %d" % (i)
strokes_gained = {}
strokes_gained['tournament'] = tournament_name
strokes_gained['id'] = tournament_id
strokes_gained['round'] = rd
# create courses set
for player in scorecards['round']['players']:
player_name = player['first_name'] + ' ' + player['last_name']
strokes_gained[player_name] = {}
course = player['course']['name']
strokes_gained[player_name]['course'] = course
strokes_gained[player_name]['rd_strokes_gained'] = 0
strokes_gained[player_name]['num_holes'] = 0
for score in player['scores']:
hole_num = score['number']
strokes_gained[player_name][hole_num] = {}
strokes_gained[player_name][hole_num]['strokes'] = score['strokes']
if strokes_gained[player_name][hole_num]['strokes'] == 0: continue
strokes_gained[player_name][hole_num]['average'] = hole_averages[course]['holes'][str(hole_num)]['average']
strokes_gained[player_name][hole_num]['strokes_gained'] = float(hole_averages[course]['holes'][str(hole_num)]['average']) - float(score['strokes'])
strokes_gained[player_name]['rd_strokes_gained'] += float(strokes_gained[player_name][hole_num]['strokes_gained'])
strokes_gained[player_name]['num_holes'] += 1
# save to AWS S3
strokes_gained = json.dumps(strokes_gained)
k2.key = 'sportsData/' + str(year) + '/' + tournament_name + '/rounds/' + str(rd) + '/strokes_gained.json'
k2.set_contents_from_string(strokes_gained)
print year, tournament_name, rd
def set_file_contents(conn, path, s):
bucket, key = get_bucket_key(path)
b = conn.get_bucket(bucket, validate=False)
k = Key(b)
k.key = key
return k.set_contents_from_string(s)
print b.name + "\n"
delBuck = raw_input("Enter the name of bucket to be deleted: ")
conn.delete_bucket(delBuck)
print("Done\n")
elif op == 4:
buck = raw_input("Enter name of bucket: ")
nonexistent = conn.lookup(buck)
if nonexistent is None:
print "No such bucket!"
b = conn.get_bucket(buck)
k = Key(b)
object = raw_input("Enter the key and object sperated by ',': ")
ki, data = object.split(",")
k.key = ki
k.set_contents_from_string(data)
print("Data entered successfully as: ", k.get_contents_as_string())
elif op == 5:
b = raw_input("Enter the name of Bucket: ")
nonexistent = conn.lookup(b)
if nonexistent is None:
print "No such bucket!"
for key in nonexistent.list():
print "{name}\t{size}\t{modified}".format(
name=key.name,
size=key.size,
modified=key.last_modified,
)
elif op == 6:
