def add(bkt, key, img, form = 'JPEG'):
bucket = conn.get_bucket(bkt)
newKeyObj = Key(bucket)
newKeyObj.key = key
newKeyObj.set_metadata('Content-Type', 'image/jpg')
buf = s.StringIO()
img.save(buf, form)
newKeyObj.set_contents_from_string(buf.getvalue())
newKeyObj.set_acl('public-read')
def upload(self):
for destination, data, content_type, compressed in self.get_files():
key = Key(self.bucket)
key.content_type = content_type
if compressed:
key.set_metadata('content-encoding', 'gzip')
for header, value in self.headers:
key.set_metadata(header, value)
key.key = destination
key.set_contents_from_string(data)
def add_bucket(self,bucket_name,access,zonename,create_date):
try:
bucket_count=len(self.conn.get_all_buckets())
# print 'bucket_count is %s ' %bucket_count
if bucket_count < self.bucket_limit:
self.conn.create_bucket(bucket_name)
b=self.conn.get_bucket(bucket_name)
try:
# k=b.new_key('create_info')
# k.set_contents_from_string("{'bucket_name':'%s','zonename':'%s','access':'%s','create_date':'%s'}" %(bucket_name,zonename,access,create_date))
k1=Key(b)
k1.key='create_info'
#k1.set_metadata('Bucket_Name',bucket_name),注意，在设置元数据的时候key名不能带有下划线，该示例在创建的时候会报错403
k1.set_metadata('BucketName',bucket_name)
k1.set_metadata('ZoneName',zonename)
k1.set_metadata('Access',access)
k1.set_metadata('CreateDate',create_date)
k1.set_contents_from_string('')
except Exception as e:
print r'\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\',e
return True
else:
return False
except Exception as e:
return False
def add_bucket(self, bucket_name, access, zonename, create_date):
try:
bucket_count = len(self.conn.get_all_buckets())
# print 'bucket_count is %s ' %bucket_count
if bucket_count < self.bucket_limit:
self.conn.create_bucket(bucket_name)
b = self.conn.get_bucket(bucket_name)
try:
# k=b.new_key('create_info')
# k.set_contents_from_string("{'bucket_name':'%s','zonename':'%s','access':'%s','create_date':'%s'}" %(bucket_name,zonename,access,create_date))
k1 = Key(b)
k1.key = 'create_info'
#k1.set_metadata('Bucket_Name',bucket_name),注意，在设置元数据的时候key名不能带有下划线，该示例在创建的时候会报错403
k1.set_metadata('BucketName', bucket_name)
k1.set_metadata('ZoneName', zonename)
k1.set_metadata('Access', access)
k1.set_metadata('CreateDate', create_date)
k1.set_contents_from_string('')
except Exception as e:
print r'\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\', e
return True
else:
return False
except Exception as e:
return False
def _upload_file_to_bucket(self, file_path, destination):
file_name = os.path.basename(file_path)
destination_path = os.path.join(destination, file_name)
log_info("Uploading '%s' to s3 bucket '%s' to '%s'" %
(file_path, self.bucket_name, destination))
file_obj = open(file_path)
k = Key(self.bucket)
k.key = destination_path
# set meta data (has to be before setting content in
# order for it to work)
k.set_metadata("Content-Type", "application/x-compressed")
k.set_contents_from_file(file_obj)
log_info("Completed upload '%s' to s3 bucket '%s'!" %
(file_path, self.bucket_name))
def _upload_file_to_bucket(self, file_path, destination):
file_name = os.path.basename(file_path)
destination_path = os.path.join(destination, file_name)
log_info("Uploading '%s' to s3 bucket '%s' to '%s'" %
(file_path, self.bucket_name, destination))
file_obj = open(file_path)
k = Key(self.bucket)
k.key = destination_path
# set meta data (has to be before setting content in
# order for it to work)
k.set_metadata("Content-Type", "application/x-compressed")
k.set_contents_from_file(file_obj)
log_info("Completed upload '%s' to s3 bucket '%s'!" %
(file_path, self.bucket_name))
def __key(cls,k=None):
from boto.s3.connection import Key
from time import gmtime
from calendar import timegm
from random import choice
from string import ascii_lowercase as letters
if k is None:
key = Key(cls.__bucket())
k = ''
for i in range(12):
k += choice(letters)
key.key = k
key.set_metadata('time',timegm(gmtime()))
else:
key = cls.__bucket().get_key(k)
return key
def save_tree(self, target, new_tree):
self.connect()
now_dt = datetime.utcnow()
now_dt_str = now_dt.strftime('%Y%m%dT%H%M')
sio = StringIO()
tree.save_tree(new_tree, sio)
# Save to S3
print("Saving tree to S3")
s3_pf_prefix = self.cp.get('options', 's3-pf-prefix')
k = Key(self.s3_bucket)
k.key = '{}/trees/{}.{}'.format(
s3_pf_prefix,
target,
now_dt_str
)
k.set_metadata('pf:target', target)
k.set_metadata('pf:saved_dt', now_dt_str)
k.set_contents_from_string(sio.getvalue())
def upload_file(conn, full_path):
b = Bucket(conn, BUCKET)
k = Key(b)
k.key = full_path
expires = datetime.utcnow() + timedelta(days=(25 * 365))
expires = expires.strftime("%a, %d %b %Y %H:%M:%S GMT")
k.set_metadata("Content-Type", mimetypes.guess_type(full_path)[0])
k.set_metadata("Expires", expires)
k.set_metadata("Cache-Control", "max-age={0}, public".format(25 * 365 * 36400))
k.set_contents_from_filename(full_path)
k.set_acl('public-read')
print "{} -> http://s3.amazonaws.com/yaluandmike/{}".format(full_path, full_path)
entityInfo = reduce(lambda df, tbl: df.join(tbl, entity, how="outer"),
partTbls)
entityInfo = add_computed_cols(entityInfo)
# COMMAND ----------
print "writing entityInfo to s3: %s" % destEntityInfoPath
entityInfo.repartition(1).write.format('com.databricks.spark.csv') \
.options(header='true', nullValue='') \
.save('/mnt/' + AWS_BUCKET_NAME + '/' + destEntityInfoPath, mode='overwrite')
print "written entityInfo to S3."
# COMMAND ----------
# compute and write out simMatrix
sim = compute_sim_mat(orderInfoDF.groupby(entity, uID).count())
print "sim length: %d" % len(sim)
binary_dump = dumps(sim, -1)
print "writing simMat to s3 Key: %s/%s" % (AWS_BUCKET_NAME, destSimMatKey)
# Open a connection to S3 and write the contents
conn = S3Connection(ACCESS_KEY, SECRET_KEY)
bucket = conn.get_bucket(AWS_BUCKET_NAME)
k = Key(bucket)
k.key = destSimMatKey
k.set_metadata("encoder", "pickle-version-2")
k.set_metadata("generated-by", "mappr-etl")
k.set_metadata("generated-by-source", srcFilePath)
k.set_contents_from_string(binary_dump)
print "written simMat to S3."
sqlContext.clearCache()
def process_file(aws_conn, filepath):
mtime = get_mtime(filepath)
name_200 = add_size_name(filepath, '200')
name_800 = add_size_name(filepath, '800')
mtime_200 = get_mtime(name_200)
mtime_800 = get_mtime(name_800)
im = None
if mtime_200 is None or mtime_200 < mtime:
try:
im = Image.open(filepath)
except:
return None
generate_200(im, name_200)
if mtime_800 is None or mtime_800 < mtime:
if im is None:
try:
im = Image.open(filepath)
except:
return None
generate_800(im, name_800)
names = {
'original': filepath,
'thumbnail': name_200,
'display': name_800,
}
b = Bucket(aws_conn, BUCKET)
image_result = {}
for image_type, name in names.items():
aws_tag_path = add_size_name(name, 's3t') + '.meta'
aws_key_path = name[len(GALLERY_DIR):].strip('/')
image_result[image_type] = {
'url': 'http://s3.amazonaws.com/{}/{}'.format(
BUCKET,
aws_key_path)
}
if not is_newer(name, aws_tag_path):
try:
resolution = load_data(aws_tag_path)
resolution['width']
except:
resolution = get_resolution(name)
save_data(aws_tag_path, resolution)
image_result[image_type].update(resolution)
continue
resolution = get_resolution(name)
image_result.update(resolution)
save_data(aws_tag_path, resolution)
s3key = b.get_key(aws_key_path)
mtime = get_mtime(name)
if s3key and s3key.last_modified:
print datetime.datetime(*parsedate(s3key.last_modified)[:6])
print mtime
if datetime.datetime(*parsedate(s3key.last_modified)[:6]) > mtime:
with open(aws_tag_path, 'a'):
os.utime(aws_tag_path, None)
continue
print 'Sending {} to S3'.format(name)
k = Key(b)
k.key = aws_key_path
expires = datetime.datetime.utcnow() + datetime.timedelta(days=25 * 365)
expires = expires.strftime("%a, %d %b %Y %H:%M:%S GMT")
k.set_metadata("Content-Type", mimetypes.guess_type(name)[0])
k.set_metadata("Expires", expires)
k.set_metadata("Cache-Control", "max-age={0}, public".format(86400 * 365 * 25))
k.set_contents_from_filename(name)
k.set_acl('public-read')
with open(aws_tag_path, 'a'):
os.utime(aws_tag_path, None)
photo_age = get_photo_age(filepath)
image_result['caption'] = get_caption(filepath)
return photo_age, image_result
