def main(stream_url: str, stream_name: str, bucket_name: str, duration: str):
temp_file = 'temp.m4a'
print('beginning rip')
code = subprocess.call(['ffmpeg',
'-i', stream_url,
'-t', duration,
'-acodec', 'copy',
'-absf', 'aac_adtstoasc',
temp_file])
assert code == 0, 'stream rip failed with code ' + str(code)
print('connecting to s3')
conn = S3Connection(is_secure=False) # AWS uses invalid certs
bucket = conn.get_bucket(bucket_name)
print('writing recorded file to s3')
m4a = Key(bucket)
m4a.name = datetime.datetime.utcnow().strftime(stream_name + '--%Y-%m-%d.m4a')
m4a.content_type = MIME_TYPE
m4a.metadata = {'Content-Type': MIME_TYPE}
m4a.storage_class = 'STANDARD_IA'
m4a.set_contents_from_filename(temp_file)
m4a.close()
print('generating new feed.xml from s3 bucket list')
feed_xml = Key(bucket)
feed_xml.name = 'feed.xml'
feed_xml.content_type = 'application/rss+xml'
feed_xml.set_contents_from_string(
rss_xml(stream_name, bucket_name, bucket.list()))
feed_xml.close()
def init_bucket(self):
bucket = self._connection.create_bucket(self._bucket_name)
bucket.set_policy(json.dumps({
"Version": "2012-10-17",
"Statement": [
{
"Sid": "PublicReadGetObject",
"Effect": "Allow",
"Principal": "*",
"Action": ["s3:GetObject"],
"Resource": [
"arn:aws:s3:::{bucket}/*".format(
bucket=self._bucket_name)]
}
]
}))
error_key = Key(bucket, 'error.html')
error_key.content_type = "text/html"
error_key.set_contents_from_string("""
<!doctype html>
<h1>It's all gone wrong!
""")
index_key = Key(bucket, "index.html")
index_key.content_type = "text/html"
index_key.set_contents_from_string("""
<!doctype html>
<h1>Welcome to gifshare
""")
bucket.set_website_configuration(WebsiteConfiguration(
'index.html',
'error.html',
))
def upload_image(folder_name, file_uuid, image_file, type_known=False):
"""
Creates a connection to the s3 service then uploads the file which was
passed
to this function an uses the uuid as the filename.
:param type_known:
:param image_file:
:param folder_name:
:param file_uuid:
:return:
"""
bucket = settings.AWS_STORAGE_BUCKET_NAME
conn = connect_s3(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY)
k = Key(conn.get_bucket(bucket))
if type_known:
key_string = "%s/%s" % (folder_name, file_uuid)
k.content_type = 'image/%s' % file_uuid[file_uuid.find('.') + 1:]
else:
key_string = "%s/%s%s" % (folder_name, file_uuid, ".png")
k.content_type = 'image/png'
k.key = key_string
if not isinstance(image_file, str):
image_file.seek(0)
k.set_contents_from_string(image_file.read())
else:
k.set_contents_from_string(image_file)
k.make_public()
image_uri = k.generate_url(expires_in=0, query_auth=False)
return image_uri
def Upload(self):
import boto
import ssl
import os
import sys
import ssl
from boto.s3.key import Key
s3 = boto.connect_s3()
bucket=s3.get_bucket(self.b, validate=False)
self.lock.acquire()
print "s3Upload",(self.f+"1.csv")," to ",bucket
self.lock.release()
try:
k = Key(bucket)
print(k)
k.key = self.f+"1.csv"
k.content_type = 'text/html'
k.set_contents_from_filename(str(self.f)+"1.csv")
k.set_acl('public-read')
k.key = self.f+"1.json"
k.content_type = 'text/html'
k.set_contents_from_filename(str(self.f)+"1.json")
k.set_acl('public-read')
except IOError as e:
print "I/O error({0}): {1}".format(e.errno, e.strerror)
except SSLError as e:
print "SSL error({0}): {1}".format(e.errno, e.strerror)
except:
print "Unexpected error:", sys.exc_info()[0]
raise
def save_s3(bucket,
filename,
contents,
systemfile,
content_type=None,
acl='public-read',
meta=None,
encode=None):
from boto.dynamodb2.table import Item
key = Key(bucket, filename)
print 'new s3 key:', 'http://s3.amazonaws.com/' + bucket.name + (
key.name if key.name.startswith('/') else '/' + key.name)
if isinstance(meta, Item):
meta = meta._data
if isinstance(meta, dict):
trim_meta = fixed.check_entity_size(meta)
trim_meta = dict([(k, value) for (k, value) in trim_meta.items()
if value is not None and value])
trim_meta = json.loads(json.dumps(trim_meta, cls=fixed.SetEncoder))
print 'meta key length:', len(trim_meta.keys())
key.metadata = trim_meta
if content_type is not None:
print 'set content type:', content_type
key.content_type = content_type
elif systemfile and systemfile.endswith('js.map'):
print 'js map!'
key.content_type = 'application/json'
elif systemfile:
gt = mimetypes.guess_type(systemfile)
key.set_metadata('Content-Type', gt[0])
if encode is not None and encode == 'gzip':
key.set_metadata('Content-Encoding', 'gzip')
gzmem = StringIO.StringIO()
gzip_file = gzip.GzipFile(fileobj=gzmem, mode='w')
if contents is not None:
gzip_file.write(contents)
elif systemfile is not None:
with open(systemfile, 'rb') as outfile:
gzip_file.write(outfile.read())
gzip_file.close()
key.set_contents_from_string(gzmem.getvalue())
print 'gzip!'
elif contents is not None:
print 'from string'
key.set_contents_from_string(contents)
elif systemfile is not None:
io = StringIO.StringIO(open(systemfile, 'r').read()).getvalue()
print 'from disk:', systemfile, 'io:', len(io)
key.set_contents_from_string(io)
if acl is not None:
print 'save acl:', acl
key.set_acl(acl)
print 'save complete:', key.name
return key
def upload():
current_app.logger.info('Upload Image')
login_key = request.form.get('login_key', False)
# app_token = request.form.get('app_token', False)
app_user = request.form.get('u', False)
if login_key:
user = get_user_name(login_key)
else:
user = app_user
if 'file' in request.files:
file = request.files['file']
if is_image(file.filename):
object_id = Picture.create(user, {}, file.filename)
elif is_attachment(file.filename):
object_id = Attachment.create(user, {}, file.filename)
filename = secure_filename(object_id + '-' + file.filename)
c = connect_s3()
bucket_name = get_config_value('BUCKET')
bucket = c.create_bucket(bucket_name)
k = Key(bucket)
k.key = user + '/' + filename
k.set_metadata('owner', user)
extension = os.path.splitext(filename)[1]
k.content_type = file.content_type
current_app.logger.info('Extension: ' + str(extension))
current_app.logger.info('file.content_type: ' + str(file.content_type))
if extension.lower() == '.jpg':
k.content_type = 'image/jpeg'
if extension.lower() == '.png':
k.content_type = 'image/png'
if extension.lower() == '.gif':
k.content_type = 'image/gif'
current_app.logger.info('Extension: ' + str(extension))
current_app.logger.info('file.content_type: ' + str(k.content_type))
k.set_contents_from_string(file.read())
k.make_public()
url = 'http://%s/%s' % (bucket_name, k.key)
current_app.logger.info(
'########## url: ' + str(url) + ' ' + str(bucket)
)
if is_image(file.filename):
Picture.add_url(object_id, url)
elif is_attachment(file.filename):
object_id = Attachment.add_url(object_id, file.filename)
return jsonify({'upload': url})
return jsonify({'upload': 'error'})
def generate_thumbnail(file_id, content_type):
bucketname = 's3.mediasnak.com'
if content_type.startswith('image/'):
botoconn = S3Connection(access_keys.key_id, access_keys.secret, is_secure=False)
bucket = botoconn.create_bucket(bucketname)
# Locate and download image to thumbnail, unless it's too big
k = bucket.get_key('u/'+file_id)
if k.size > 30000000:
return
data = k.get_contents_as_string()
# Resize the image
img = images.Image(image_data=data)
img.resize(width=200, height=150)
thumb = img.execute_transforms(output_encoding=images.JPEG)
# Store the image on S3
tk = Key(bucket, name='t/'+file_id)
tk.content_type = 'image/jpeg'
tk.set_contents_from_string(thumb)
# No error-checking here as our calling function already did it for us
file_entry = MediaFile.objects.get(file_id=file_id)
file_entry.has_thumb = True
file_entry.save()
def upload_to_s3(file_obj, filename, path, content_type=None):
"""
Method to upload a file to s3 and return a link.
This method automatically tries to guess of the file using filename if content_type is not passed.
:param file_obj: File object (this must not be closed)
:param path: Path where file is to be uploaded (this does not include filename)
:param content_type: Content-type of the file.
:return: AWS file url
"""
# Connect to the bucket
conn = boto.connect_s3(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY)
bucket = conn.get_bucket(settings.AWS_STORAGE_BUCKET_NAME)
key_path = "{}/{}".format(path, filename)
k = Key(bucket)
k.key = key_path
if content_type is None:
# content_type passed; Guess the type
content_type = mimetypes.guess_type(filename)[0]
# Now set type only if is not None
if content_type:
k.content_type = content_type
file_obj.seek(0)
k.set_contents_from_string(file_obj.read())
k.make_public()
return "https://{}/{}".format(settings.AWS_S3_CUSTOM_DOMAIN, key_path)
def post(self, *args, **kwargs):
parse = reqparse.RequestParser()
parse.add_argument('file', type=str, required=True)
args = parse.parse_args()
# create a file object of the image
mimetype, file_data = parse_base64_string(args['file'])
archive_file = StringIO(file_data)
print 'yahan pahuncha 1'
# upload the file to s3 with a unique uuid
key_name = str(uuid.uuid4()) + '.zip'
conn = S3Connection(app.config['S3_ACCESS_KEY'], app.config['S3_SECRET'])
bucket = conn.get_bucket(app.config['S3_UPLOAD_SET_ARCHIVES_BUCKET'])
archive_s3 = Key(bucket)
archive_s3.key = key_name
archive_s3.content_type = 'application/zip'
archive_s3.set_contents_from_string(archive_file.getvalue())
archive_s3.set_acl('public-read')
print 'yahan pahuncha 2'
# close the StringIO object
archive_file.close()
print 'yahan pahuncha 3'
# return the name of the S3 key
return {'archive_s3_key': archive_s3.key}
def upload(self, filename, contents_bytes, content_type=None):
file_key = Key(bucket=self.bucket, name=filename)
# if file_key.exists():
# log.info("File already exists on S3: %s", filename)
# return
if content_type:
file_key.content_type = content_type
log.info("Setting content_type of %s as %s", filename, content_type)
file_key.content_type = content_type
log.info("Uploading %s", filename)
file_key.set_contents_from_string(contents_bytes, replace=True)
return True
def upload(self, path, content, invalidate=None):
"""
Set the content, mime type and ACL for a key on S3. Before setting the
check if the object is new or changed.
Arguments:
path: path for key
content: content to set
invalidate: CloudFront path to add to invalidation list. * will be
added to the end to make sure we invalidate the URL
path with a trailing slash and the html itself.
If None the path will be used.
"""
changed = self.file_changed(path, content)
if not changed:
return
key = Key(self.bucket)
key.content_type = guess_mime_type(path)
key.key = path
key.set_contents_from_string(content)
key.set_acl("public-read")
print("uploaded: {0}".format(path))
if invalidate is None:
invalidate = path
self.to_invalidate.append(invalidate + "*")
def rxnorm_crawler():
"""
Crawl RxNorm weekly updates from NIH's webpage and store them in S3 bucket.
"""
# Target webpage
weburls = [
'https://www.nlm.nih.gov/research/umls/rxnorm/docs/rxnormfiles.html',
'https://www.nlm.nih.gov/research/umls/rxnorm/docs/rxnormarchive.html'
]
for weburl in weburls:
# Get contents of webpage
conn = urllib2.urlopen(weburl)
html = conn.read()
# Find urls of all RxNorm files
pattern = '<a\s*href=[\'|"](.*?/kss/rxnorm/RxNorm_full_\d+.zip)[\'|"]>'
rxnorm_urls = re.findall(pattern, html)
for url in rxnorm_urls:
r = requests.get(url)
if r.status_code == 200:
#upload the file
file_name = re.findall('.*?(\d+.zip)', url)[0]
k = Key(bucket)
k.key = 'rxnorm/' + file_name
k.content_type = r.headers['content-type']
k.set_contents_from_string(r.content)
def upload_to_s3(fp, key_name, extension, reduced_redundancy=False):
"""
Upload the contents of file handle 'fp' to the S3 bucket specified by AWS_STORAGE_BUCKET_NAME,
under the given filename. Return the public URL.
"""
# connect to S3 and send the file contents
conn = S3Connection(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY)
bucket = conn.get_bucket(settings.AWS_STORAGE_BUCKET_NAME)
k = Key(bucket)
k.key = key_name
k.content_type = MIME_TYPE_BY_EXTENSION.get(extension,
'application/octet-stream')
# print "uploading: %s" % key_name
k.set_contents_from_file(fp,
reduced_redundancy=reduced_redundancy,
rewind=True)
k.set_acl('public-read')
# construct the resulting URL, which depends on whether we're using a CNAME alias
# on our bucket
if settings.AWS_BOTO_CALLING_FORMAT == 'VHostCallingFormat':
return "http://%s/%s" % (settings.AWS_STORAGE_BUCKET_NAME, key_name)
else:
return "http://%s.s3.amazonaws.com/%s" % (
settings.AWS_STORAGE_BUCKET_NAME, key_name)
def uploadThumbToS3(self):
#setup the bucket
# c = boto.connect_s3(your_s3_key, your_s3_key_secret)
bucket = 'imgcat-dev'
c = boto.connect_s3()
# b = c.get_bucket(bucket, validate=False)
b = c.lookup(bucket)
if (b == None):
logger.error("Bucket not found: " + bucket)
sys.stdout.flush()
keyname = os.path.basename(os.path.normpath(self.localThumbFilename()))
print 'gonna create key: ' , keyname
k = Key(bucket=b, name=keyname)
k.content_type = self.data['mimetype']
k.set_contents_from_filename(self.localThumbFilename())
k.set_acl('public-read')
self.data['thumb_path'] = k.generate_url(expires_in=0, query_auth=False)
logger.info("uploaded thumbnail:" + keyname + " to S3")
sys.stdout.flush()
self.data['load_status'] = 'thumbnailed'
self.persist()
def upload_sjson_to_s3(config, sjson_data):
"""
Upload sjson data to s3.
Arguments:
config (dict): instance configuration
sjson_data (list): transcript data to be uploaded to `s3`
Returns:
transcript name for 'edxval'
"""
s3_conn = boto.connect_s3()
bucket = s3_conn.get_bucket(config['aws_video_transcripts_bucket'])
k = Key(bucket)
k.content_type = 'application/json'
transcript_name_without_instance_prefix, transcript_name_with_instance_prefix = construct_transcript_names(
config)
k.key = '{}.sjson'.format(transcript_name_with_instance_prefix)
k.set_contents_from_string(json.dumps(sjson_data))
k.set_acl('public-read')
# transcript path is stored in edxval without `instance_prefix`
return '{}.sjson'.format(transcript_name_without_instance_prefix)
def clean_photo(self):
if self.instance.pk:
if 'photo' not in self.files:
return self.instance.photo
if self.files['photo'].content_type not in [
'image/gif', 'image/jpeg', 'image/jpg', 'image/png'
]:
raise ValidationError('Invalid Photo')
uuid = u'{prefix:s}.{suffix:s}'.format(
prefix=str(uuid4()),
suffix=self.files['photo'].content_type.split('/')[-1],
)
try:
photo = Key(
S3Connection(
settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY,
).get_bucket(settings.AWS_BUCKET, ))
photo.content_type = self.files['photo'].content_type
photo.key = uuid
photo.set_contents_from_string(self.files['photo'].read())
return u'https://d2k6ktnea3auzx.cloudfront.net/{uuid:s}'.format(
uuid=uuid)
except Exception:
pass
raise ValidationError('Invalid Photo')
def save_file(self, remote_file, local_file_path, content_type=None):
k = Key(self.bucket)
k.name = self.target_path+'/'+remote_file
if content_type:
k.content_type = content_type
k.set_contents_from_filename(local_file_path)
k.set_acl('public-read')
def scrape_data_to_html():
timestamp = datetime.fromtimestamp(time.time()).strftime("%H:%M:%S on %A, %d %B, %Y")
all_drinkers = drinkers_table.scan()
drinkers = []
for drinker in all_drinkers:
if (drinker['code'] == None):
drinker['code'] = "UNKNOWN"
if (drinker['name'] == None):
drinker['name'] = "UNKNOWN"
if (drinker['volume_consumed'] == None):
drinker['volume_consumed'] = 0
if (drinker['number_of_drinks'] == None):
drinker['number_of_drinks'] = 0
d = {}
d['code'] = drinker['code']
d['name'] = drinker['name']
d['volume_consumed'] = drinker['volume_consumed']
d['number_of_drinks'] = drinker['number_of_drinks']
drinkers.append(d)
loader = FileLoader('templates')
template = loader.load_template('drinks.html.template')
webpage = template.render(locals())
bucket = s3.get_bucket('kegerator')
key = Key(bucket)
key.key = 'drinks.html'
key.content_type = 'text/html'
key.set_contents_from_string(webpage)
key.make_public()
def _get_key(self):
self._get_bucket()
key = Key(self.bucket)
key.name = self.filename
key.content_type = 'application/json'
self.key = key
def upload(self,
data,
content_type,
make_public=True,
bucket=app.config['S3_BUCKET_NAME']):
bucket_name = bucket
try:
bucket = self.get_bucket(bucket_name)
except S3ResponseError as e:
if e.error_code == 'NoSuchBucket':
try:
bucket = self.create_bucket(bucket_name)
except S3CreateError as e:
raise e
else:
raise e
s3key = Key(bucket)
s3key.key = str(uuid.uuid4().get_hex().upper()
[0:16]) + '.' + content_type.split('/')[-1]
s3key.content_type = content_type
s3key.set_contents_from_string(data)
if make_public:
s3key.set_acl('public-read')
url = 'https://s3.amazonaws.com/' + bucket_name + '/' + s3key.key
return url
def _save(self, name, content, mimetype=None, public=None):
if not mimetype:
mimetype = guess_type(name)[0] or "application/x-binary"
if hasattr(content, 'public'):
public = content.public
else:
public = self.public if public is None else public
key = Key(self.bucket, name.encode('utf-8'))
key.content_type = mimetype
headers = getattr(content, 'backend_headers', {})
success = False
for i in range(3):
content.seek(0)
retry(key.set_contents_from_file, content, headers=headers)
if public:
try:
retry(key.set_acl, 'public-read')
except S3ResponseError, exc:
if exc.status == 404:
continue
raise
# Make sure file actually exists on S3
# TODO: Maybe only do this if not settings the acl as that should have the same effect?
success = bool(retry(self.bucket.lookup, name.encode('utf-8')))
if success:
break
logging.warning(u"Failed to write to S3: %s" % name)
def write_to_s3(self, prefix, fname, content, ctype='text/plain'):
"""
Write ``content`` into S3 at ``prefix``/``fname``. If ``self.dry_run``,
write to local disk instead. Return the resulting URL, either an S3
URL or a local 'file://' URL.
:param prefix: the prefix to write S3 files under, or local files under
:type prefix: str
:param fname: the file name to create
:type fname: str
:param content: the content to write into the file
:type content: str
:returns: URL to the created file
:rtype: str
"""
path = os.path.join(prefix, fname)
if self.dry_run:
path = os.path.abspath(path)
logger.warning("DRY RUN: Writing s3-bound content to %s", path)
dest_dir = os.path.dirname(path)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
with open(path, 'w') as fh:
fh.write(content)
return 'file://%s' % path
# else write to S3
logger.debug("Creating S3 key: %s (Content-Type: %s)", path, ctype)
k = Key(self.bucket)
k.content_type = ctype
k.key = path
k.set_contents_from_string(content)
url = self.url_for_s3(path)
logger.debug("Data written to %s", url)
return url
def bucket_create(self, key, val, metadata_dict = {}):
'''
Create an object in the bucket, but only if not yet present (save traffic).
Parameters
---------
key : str
val : file-like object
metadata_dict : dict
Returns
-------
Key
'''
s3_key = Key(self.apk_bucket)
s3_key.key = key
# important: set metadata before actual upload
s3_key.metadata = metadata_dict
s3_key.content_type = 'application/vnd.android.package-archive'
# upload
log.debug("uploading %s", s3_key.key)
s3_key.set_contents_from_file(val, replace = False)
return s3_key
def bucket_create(self, key, val, metadata_dict={}):
'''
Create an object in the bucket, but only if not yet present (save traffic).
Parameters
---------
key : str
val : file-like object
metadata_dict : dict
Returns
-------
Key
'''
s3_key = Key(self.apk_bucket)
s3_key.key = key
# important: set metadata before actual upload
s3_key.metadata = metadata_dict
s3_key.content_type = 'application/vnd.android.package-archive'
# upload
log.debug("uploading %s", s3_key.key)
s3_key.set_contents_from_file(val, replace=False)
return s3_key
def acquire_run_id(comment='', force=False):
key = _get_key(RUN_INFO_FILE)
if key:
run_info = json.loads(key.get_contents_as_string())
else:
key = Key(_get_bucket(), RUN_INFO_FILE)
key.content_type = 'text/plain'
run_info = {
'last_run': 0,
'active_run': None,
}
if run_info['active_run'] is not None:
print 'Already an active run:', run_info['active_run']
if not force:
raise Exception('Already an active run')
run_info['active_run'] = '%s on %s started at %s %s' % (
os.environ.get('USER', 'unknown'), socket.gethostname(),
str(datetime.datetime.utcnow()), comment)
key.set_contents_from_string(json.dumps(run_info))
return run_info['last_run'] + 1
def upload(self, path, content, invalidate=None):
"""
Set the content, mime type and ACL for a key on S3. Before setting the
check if the object is new or changed.
Arguments:
path: path for key
content: content to set
invalidate: CloudFront path to add to invalidation list. * will be
added to the end to make sure we invalidate the URL
path with a trailing slash and the html itself.
If None the path will be used.
"""
changed = self.file_changed(path, content)
if not changed:
return
key = Key(self.bucket)
key.content_type = guess_mime_type(path)
key.key = path
key.set_contents_from_string(content)
key.set_acl("public-read")
print("uploaded: {0}".format(path))
if invalidate is None:
invalidate = path
self.to_invalidate.append(invalidate + "*")
def event_from_facebook(args, data):
event = g.Event(
club=args['club'],
facebook_id=data['id'],
name=data['name'],
start_date=arrow.get(data['start_time']).datetime,
)
if 'description' in data.keys():
event.description = data['description'].replace('\n', '<br />')
if 'end_time' in data.keys():
event.end_date = arrow.get(data['end_time']).datetime
else:
event.end_date = arrow.get(data['start_time']).replace(hours=2).datetime
if 'place' in data.keys():
event.place = Place(
name=data['place']['name']
)
if 'location' in data['place'].keys():
event.address = data['place']['location']['street'] + ', ' + data['place']['location']['city'] \
+ ', ' + data['place']['location']['country']
event.save()
bucket = s3conn.get_bucket(current_app.config['AWS_S3_BUCKET'])
key = Key(bucket)
key.key = g.tenant + '/events/' + str(event.id)
key.content_type = 'image/jpeg'
key.set_contents_from_string(requests.get(data['cover']['source']).content)
key.make_public()
event.poster = 'https://' + current_app.config['AWS_S3_BUCKET'] + '.s3.amazonaws.com/' + g.tenant + '/events/' + str(event.id)
event.save()
def upload_from_server(data_file,
upload_file_name,
bucket_name=None,
public=False,
content_type=None):
"""
@param data_file: the file that you want to upload
@param upload_file_name: the file path where to upload, eg: upload_folder/file_name.txt, or file_name.jpg
@param public: visibility of file on S3
@return: the url of the uploaded file
"""
if data_file is None:
raise OwnException(NO_FILE_SPECIFIED)
conn = boto.connect_s3()
# conn = boto.s3.connect_to_region(region_name='the_region')
# conn = S3Connection('aws_key', 'aws_secret')
bucket_name = __get_bucket_name(bucket_name)
bucket = conn.get_bucket(bucket_name)
k = Key(bucket)
k.key = upload_file_name
policy = 'public-read' if public else 'private'
k.content_type = content_type
k.set_contents_from_file(data_file, policy=policy)
# k.set_contents_from_string(data_file, policy=policy)
url = k.generate_url(expires_in=0, query_auth=False)
return url
def acquire_run_id(comment='', force=False):
key = _get_key(RUN_INFO_FILE)
if key:
run_info = json.loads(key.get_contents_as_string())
else:
key = Key(_get_bucket(), RUN_INFO_FILE)
key.content_type = 'text/plain'
run_info = {
'last_run': 0,
'active_run': None,
}
if run_info['active_run'] is not None:
print 'Already an active run:', run_info['active_run']
if not force:
raise Exception('Already an active run')
run_info['active_run'] = '%s on %s started at %s %s' % (
os.environ.get('USER','unknown'),
socket.gethostname(),
str(datetime.datetime.utcnow()),
comment)
key.set_contents_from_string(json.dumps(run_info))
return run_info['last_run'] + 1
def upload_images_to_s3(markup, archive, question_files_bucket):
"""Uploads all the images referenced in the markup to S3. Exracts the
images from the '***_files' directory in the zip archive.
Keyword Arguments:
markup -- the string markup whose images need to be uploaded to s3.
archive -- the archive object as returned by zipfile.
"""
## Create a BS object from the markup
soup = BeautifulSoup(markup, 'html.parser')
## Find all the image objects
imgs = soup.find_all('img')
## Iterate over all the images, get the file path, upload the file to S3 and change the attribute to point to the S3 hosted image
bucket = question_files_bucket
for img in imgs:
path = img.attrs['src']
img_file = archive.open(path)
img_s3 = Key(bucket)
img_s3.key = ''.join([str(uuid4()), '_', os.path.basename(path)])
img_s3.content_type = mimetypes.guess_type(path)[0]
img_s3.set_contents_from_string(img_file.read())
img_s3.set_acl('public-read')
img_url = ''.join(['https://', app.config['S3_QUESTION_FILES_TEMP_BUCKET'], '.s3.amazonaws.com/', img_s3.key])
img.attrs['src'] = img_url
return str(soup)
def s3_upload(acl, bucket, conn, content, content_type, path):
""" Store an object in our an S3 bucket.
:param acl:
S3 ACL for the object
:param bucket:
S3 bucket to upload to
:param content:
a string representation of the object to upload
:param content_type:
a string MIMETYPE of the object that S3 should
be informed of
:param path:
an object specific portion of the S3 key name
to be passed to gen_url to generate the the location
in S3 of the new object
:raise:
IOError on any failure
:return:
S3 generated URL of the uploaded object
"""
# obj is the object that will be uploaded
obj = Key(conn.get_bucket(bucket))
obj.content_type = content_type
obj.key = path
obj.set_contents_from_string(content)
obj.set_acl(acl)
return gen_url(bucket, path)
def get_badges():
initial_data = requests.get('{}.json'.format(CFGOV_BASE)).json()
BADGES['overall'] = initial_data['badge_url']
commit_sha = initial_data['commit_sha']
build_soup = bs(
requests.get('{}/builds/{}'.format(COVERALLS, commit_sha)).text,
'html.parser')
results_div = build_soup.find(
True, {'class': ['show-item', 'show-last-build-detail']})
rows = results_div.findAll('tr')[1:]
for row in rows:
coverage = row.find('div').find('div').text
if 'frontend' in row.find('a').text:
BADGES['js'] = BADGE_URL.format(int(round(float(coverage), 2)))
elif 'backend' in row.find('a').text:
BADGES['python'] = BADGE_URL.format(int(round(float(coverage), 2)))
if not (BADGES['js'] and BADGES['python']):
print('No update -- only one set of tests were recorded')
return
with open('badges.html', 'w') as f:
content = TEMPLATE.format(BADGES['overall'], BADGES['js'],
BADGES['python']).encode('utf-8')
f.write(content)
with open('badges.json', 'w') as f:
f.write(json.dumps(BADGES))
s3 = boto.connect_s3(S3_KEY,
S3_SECRET,
calling_format=OrdinaryCallingFormat())
bucket = s3.get_bucket(S3_BASE)
prep = Key(bucket=bucket, name='build/badges/badges.html')
prep.content_type = 'text/plain'
prep.set_contents_from_filename('badges.html')
print('badges updated {}'.format(datetime.datetime.now()))
return
def store_file(self, **kwargs):
content = kwargs['content']
k = Key(self.bucket)
filename = None
if 'name' in kwargs:
filename, extension = os.path.splitext(kwargs['name'])
filename = filename.replace(" ", "")
extension = extension.replace('.', '')
if 'ext' in kwargs:
extension = kwargs['ext']
typ, ext = settings.ALLOWED_FILE_TYPES[extension]
if "type" in kwargs:
typ = kwargs['type']
if 'random_name' in kwargs:
filename = None
k.key = "%s.%s" % (filename or uuid4().hex, ext)
k.content_type = typ
try:
content = base64.decodestring(content.split('base64,')[1])
except IndexError:
content = base64.decodestring(content)
k.set_contents_from_string(content)
self.bucket.set_acl('public-read', k.key)
return k.key
def upload_image(image_url, image_name):
"""аплоад изображения"""
try:
# соединение с S3 bucket
connection = boto.connect_s3()
bucket = connection.get_bucket(config.AWS_STORAGE_BUCKET_NAME)
key = Key(bucket)
# присвоение имени файла
key.key = str(int(time())) + "-" + image_name + ".png"
# чтение
file_object = urllib2.urlopen(image_url)
file_data = StringIO.StringIO(file_object.read())
# запись
key.content_type = "image/png"
key.set_contents_from_file(file_data)
# права на чтение
key.make_public()
result_url = key.generate_url(0, expires_in_absolute=True, force_http=True, query_auth=False)
return result_url
except Exception, e:
return e
def post(self, news_id):
news = g.News.objects.with_id(news_id)
if not current_user.is_admin() and not current_user.has_any_permission('club', news.club.id, ['admin', 'news']):
return abort(401)
parser = reqparse.RequestParser()
parser.add_argument('media', type=werkzeug.datastructures.FileStorage, location='files')
args = parser.parse_args()
uid = str(uuid4())
bucket = s3conn.get_bucket(current_app.config['AWS_S3_BUCKET'])
key = Key(bucket)
key.key = g.tenant + '/news/' + str(news.id) + '/' + uid
key.content_type = args['media'].mimetype
key.set_contents_from_file(args['media'].stream)
key.make_public()
news.update(add_to_set__medias=Media(
name=uid,
url='https://' + current_app.config['AWS_S3_BUCKET'] + '.s3.amazonaws.com/' + g.tenant + '/news/' + str(news.id) + '/' + uid
))
return g.News.objects.with_id(news_id)
def upload():
if request.method == 'POST':
# Connect to Amazon S3
s3 = boto.connect_s3(access_id, secret_access_key)
# Get a handle to the S3 bucket
bucket_name = 'emotipic'
bucket = s3.get_bucket(bucket_name)
k = Key(bucket)
data_file = request.files.get('file')
file_contents = data_file.read()
k.key = os.path.join('uploads', data_file.filename)
k.content_type = data_file.content_type
print "Uploading some data to " + bucket_name + " with key: " + k.key
k.set_contents_from_string(file_contents)
# call api
url = k.generate_url(expires_in=300, query_auth=False, force_http=True)
face_headers = {
'Content-Type': 'application/json',
'Ocp-Apim-Subscription-Key': '3014f0d696a144d4bd875661e36057c3'
}
emotion_headers = {
'Content-Type': 'application/json',
'Ocp-Apim-Subscription-Key': 'b94b9a266d7546ef91e64be1380960c9'
}
data = {
'url': url
}
print url
data = json.dumps(data)
faces = requests.post('https://api.projectoxford.ai/face/v0/detections?analyzesFaceLandmarks=true', data=data, headers=face_headers)
faceRectangles = ''
faces = faces.json()
# parse faces
for face in faces:
faceRectangle = face['faceRectangle']
faceRectangles += ';%s,%s,%s,%s' % (faceRectangle['left'], faceRectangle['top'], faceRectangle['width'], faceRectangle['height'])
faceRectangles = faceRectangles[1:]
emotions = requests.post('https://api.projectoxford.ai/emotion/v1.0/recognize?faceRectangles=' + faceRectangles, data=data, headers=emotion_headers)
emotions = emotions.json()
return json.dumps({
'faces': faces,
'emotions': emotions
})
def _put_S3(self, msg):
key = '/'.join([self.application, uuid.uuid4().hex])
key_object = Key(self.bucket_conn)
key_object.key = key
key_object.content_type = 'application/json'
key_object.set_contents_from_string(msg)
key_object.close()
return json.dumps({'Bucket': self.bucket_name, 'Key': key})
def set_resource_from_string(self, resource, data, content_type=None):
bucket, s3_key = self.s3_storage_objects(resource)
key = Key(bucket)
key.key = s3_key
if content_type != None:
key.content_type = content_type
key.set_contents_from_string(data)
def set_resource_from_string(self, resource, data, content_type=None):
bucket, s3_key = self.s3_storage_objects(resource)
key = Key(bucket)
key.key = s3_key
if content_type != None:
key.content_type = content_type
key.set_contents_from_string(data)
def test_s3_content_type():
conn = boto.connect_s3()
bucket = conn.create_bucket('source-bucket')
key = Key(bucket, 'test-file')
key.content_type = 'text/plain'
key.set_contents_from_string('Hello World')
assert 'text/plain; charset=utf-8' == S3.get_content_type('source-bucket', 'test-file')
def write(filename, content):
conn = S3Connection(aws_access_key, aws_secret_key)
pb = conn.get_bucket(bucket_name)
k = Key(pb)
k.name = prefix + filename
k.content_type = "text/javascript"
k.set_contents_from_string(content, headers={"Cache-Control": "max-age=0"})
k.set_acl("public-read")
def s3bucket(ec2, env, source):
""" Copy contents of static directory to s3 bucket """
mime_types = {
"eot" : "application/vnd.ms-fontobject",
"ttf" : "font/truetype",
"otf" : "font/opentype",
"woff": "font/woff",
}
s3b = boto.connect_s3(ec2.access_key,ec2.secret_key)
for machine in env:
if 's3bucket' in machine.keys():
print 'Copying static media for %s' % machine['name']
s3bucket = machine['s3bucket']
# Get the expires
time_format = '%a, %d %b %Y %H:%M:%S'
now = datetime.datetime.now().strftime(time_format)
expires = s3bucket.get('expires',datetime.datetime.utcnow().strftime(time_format))
try:
datetime.datetime.strptime(expires,time_format)
except:
error('Improperly formatted datetime: %s' % expires)
# Get or create bucket using the name
name = s3bucket.get('name','s3%s'%machine['name'])
try: b = s3b.get_bucket(name)
except: b = s3b.create_bucket(name)
# Set ACL Public for all items in the bucket
b.set_acl('public-read')
k = Key(b)
static_dir = os.path.join(source,'project','static')
for root, dirs, files in os.walk(static_dir):
if '.svn' in dirs: dirs.remove('.svn')
key_root = root.split('static')[1]
for file in files:
filename = os.path.join(root,file)
# Set the headers
headers = {'Expires':expires}
if '.gz' in file:
headers.update({'Content-Encoding':'gzip'})
if os.path.isfile(filename):
# Set the mime-type
ext = file.split('.')[-1]
if ext in mime_types.keys():
k.content_type = mime_types[ext]
# Send the file
k.key = os.path.join(key_root,file)
print '\nTransfering %s' % filename
k.set_contents_from_filename(filename, headers=headers, cb=s3_percent_cb, num_cb=10)
print '\nTransfer complete'
invalidate_cache(ec2, env, source)
def postToS3(image, originalImage, topCaption, bottomCaption):
#print "POST!"
conn = S3Connection('AKIAJQ5EWIERMNJBOSMQ', 'CxMK3wAZ97lItqozZkNp8AmVoj58/TdrH+ClXNaB')
imgHash = hashlib.md5()
imgHash.update(image)
imgHashStr = imgHash.hexdigest()
origImgHash = hashlib.md5()
origImgHash.update(originalImage)
origImgHashStr = origImgHash.hexdigest()
bucket = conn.create_bucket('memeyourfriends')
k = Key(bucket)
key = 'complete/' + imgHashStr + '.jpg'
k.key = key
k.content_type = 'image/jpeg'
k.set_metadata('top', topCaption)
k.set_metadata('bot', bottomCaption)
k.set_contents_from_string(image)
k2 = Key(bucket)
key2 = 'raw/' + origImgHashStr + '.jpg'
k2.key = key2
k2.content_type = 'image/jpeg'
k2.set_contents_from_string(originalImage)
sdb = boto.connect_sdb('AKIAJQ5EWIERMNJBOSMQ', 'CxMK3wAZ97lItqozZkNp8AmVoj58/TdrH+ClXNaB')
domain = sdb.create_domain('memeyourfriends')
item = domain.new_item(imgHashStr)
item['imgHash'] = imgHashStr
item['imgHashPath'] = key
item['origImgHash'] = origImgHashStr
item['origImgHashPath'] = key2
item['topCaption'] = topCaption
item['bottomCaption'] = bottomCaption
item.save()
return key
def test_s3_object_metadata():
conn = boto.connect_s3()
bucket = conn.create_bucket('source-bucket')
key = Key(bucket, 'test-file')
key.content_type = 'text/plain'
key.set_contents_from_string('Hello World')
s3 = S3()
assert {'Content-Type': 'text/plain; charset=utf-8'} == s3.metadata('source-bucket', 'test-file')
def upload(self, filename, bucket, key_name, metadata=None):
bucket = Bucket(self.conn, bucket)
key = Key(bucket, key_name)
key.content_type = "application/x-gzip"
if metadata:
for k, v in metadata.items():
key.set_metadata(k, str(v))
with open(filename, "rb") as fp:
self.retry(key.set_contents_from_file, fp)
def upload_view():
f = request.files['file']
conn = boto.connect_s3()
bucket = conn.get_bucket('ym-remote-control')
k = Key(bucket)
k.key = str(uuid.uuid4()) + '.' + f.filename.split('.')[-1]
k.content_type = mimetypes.guess_type(f.filename)[0]
k.set_contents_from_file(f, headers={'Content-Type': k.content_type}, policy='public-read')
return jsonify({'url': k.generate_url(300)})
def _save(self, bucket_name, end_point, File, mimetype=None):
k = Key(self.connection.get_bucket(bucket_name))
k.key = end_point
if mimetype:
k.content_type = mimetype
if isinstance(File, str) or isinstance(File, unicode):
k.set_contents_from_string(File)
else:
k.set_contents_from_file(File)
def upload(self, image_in_base64, file_name_with_extension,
file_extension):
k = Key(self.bucket)
k.key = file_name_with_extension
k.content_type = 'image/' + file_extension
k.set_contents_from_string(base64.b64decode(image_in_base64),
policy='public-read')
k.set_acl('public-read')
return self.get_url(file_name_with_extension)
def call(self, phoneTo, tweet, voice):
xml = xmlTemplate % (voice, tweet['user']['name'], tweet['text'])
print(xml)
k = Key(self.bucket)
k.key = 'tweet_%s.xml' % tweet['id']
k.content_type = 'text/xml'
k.set_contents_from_string(xml)
k.set_acl('public-read')
self.client.calls.create(to=phoneTo, from_=phoneFrom, method='GET',
url='https://s3.amazonaws.com/twinty/tweet_%s.xml' % tweet['id'])
def upload_index(self, content):
"""upload the index.html"""
key_path = os.path.join(self.prefix, 'index.html')
start = datetime.datetime.now()
logger.info("Uploading index.html")
k = Key(self.bucket)
k.key = key_path
k.content_type = 'text/html'
k.set_contents_from_string(content)
logger.info("Upload complete in %s", (datetime.datetime.now() - start))
def save_to_s3():
conn = boto.connect_s3(settings.AWSAccessKeyId, settings.AWSSecretKey)
bucket = conn.get_bucket('tt-test-321', validate=True)
k = Key(bucket)
k.key = "what.txt"
k.content_type = 'text/plain'
with open( 'what.txt', 'r') as f:
content = f.read()
k.set_contents_from_string(content)
k.set_acl('public-read')
def createBlob(category, png_chunk):
objectId = category + '/' + str(uuid.uuid4()) + '.png'
k = Key(bucket)
k.key = objectId
k.content_type = 'image/png'
k.set_contents_from_string(png_chunk)
url = 'https://cocparser.s3.amazonaws.com/' + objectId
return url
def upload_photo(username, photo):
data = base64.b64decode(photo)
s3 = boto.connect_s3()
bucket = s3.get_bucket(BUCKET_NAME)
key = Key(bucket)
key.content_type = 'image/jpg'
key.key = 'photos/%s/%s.jpg' % (username, random_hex())
key.set_contents_from_string(data)
key.close()
key.make_public()
return AWS_URL_TEMPLATE % key.key
def push_file(src_path, s3_path, verbose=False, overwrite=False):
key = _get_key(s3_path)
if key is not None:
if not overwrite:
raise Exception('File already at %s' % s3_path)
if verbose:
print 'Overwriting existing %s.' % s3_path
key = Key(_get_bucket(), s3_path)
if s3_path.endswith('.TIF') or s3_path.endswith('.tif'):
key.content_type = 'image/tiff'
if s3_path.endswith('.jpg'):
key.content_type = 'image/jpeg'
if s3_path.endswith('.txt'):
key.content_type = 'text/plain'
bytes_uploaded = key.set_contents_from_filename(src_path)
if verbose:
print 'Uploaded %d bytes from %s to %s.' % (bytes_uploaded, src_path,
s3_path)
def flush(self):
payload = ''
while len(self.buffer) > 0:
record = self.buffer.pop(0)
if record.levelno >= self.level:
payload += self.format(record)
if payload:
conn = boto.connect_s3()
bucket = conn.get_bucket(self.bucket)
key = Key(bucket)
key.key = '/'.join((self.prefix, 'bang-%f' % time.time()))
key.content_type = 'application/json'
key.set_contents_from_string(payload)
