def all_cvs(config):
bucket = _get_s3_bucket(config)
prefix = "cvs/"
cvs = bucket.list(prefix)
cvs = reversed(sorted(cvs, key=lambda k: k.last_modified))
person_ids = []
result = []
for key in cvs:
person_id = int(re.match("cvs/([0-9]+)/", key.name).group(1))
if person_id == 7777777:
continue
if person_id in person_ids:
continue
result.append({
'name': key.name,
'url': key.generate_url(expires_in=0, query_auth=False),
'last_modified': boto.utils.parse_ts(key.last_modified),
'content_type': key.content_type,
'person_id': person_id
})
person_ids.append(person_id)
return result
def _hash_by_prefix(config, prefix):
print("warming cache _hash_by_prefix", prefix)
bucket = _get_s3_bucket(config)
cvs = bucket.list(prefix)
cvs = reversed(sorted(cvs, key=lambda k: k.last_modified))
person_ids = []
result = collections.OrderedDict()
for key in cvs:
# we use .jpg thumbnails now (and don't accept images as CVs)
if key.name.endswith(".png"):
continue
key_last_modified = boto.utils.parse_ts(key.last_modified)
person_id = int(re.match(prefix + "([0-9]+)[^0-9]", key.name).group(1))
if person_id not in result:
result[person_id] = {
'name': key.name,
'url': key.generate_url(expires_in=0, query_auth=False),
'last_modified': key_last_modified,
'created': key_last_modified,
'person_id': person_id
}
result[person_id]['created'] = key_last_modified
return result
def _hash_by_prefix(config, prefix):
print("warming cache _hash_by_prefix", prefix)
bucket = _get_s3_bucket(config)
cvs = bucket.list(prefix)
cvs = reversed(sorted(cvs, key=lambda k: k.last_modified))
person_ids = []
result = collections.OrderedDict()
for key in cvs:
# we use .jpg thumbnails now (and don't accept images as CVs)
if key.name.endswith(".png"):
continue
key_last_modified = boto.utils.parse_ts(key.last_modified)
person_id = int(re.match(prefix + "([0-9]+)[^0-9]", key.name).group(1))
if person_id not in result:
result[person_id] = {
'name': key.name,
'url': key.generate_url(expires_in=0, query_auth=False),
'last_modified': key_last_modified,
'created': key_last_modified,
'person_id': person_id
}
result[person_id]['created'] = key_last_modified
return result
def test_create_stack_from_s3_url():
s3_conn = boto.s3.connect_to_region("us-west-1")
bucket = s3_conn.create_bucket("foobar", location="us-west-1")
key = boto.s3.key.Key(bucket)
key.key = "template-key"
key.set_contents_from_string(dummy_template_json)
key_url = key.generate_url(expires_in=0, query_auth=False)
conn = boto.cloudformation.connect_to_region("us-west-1")
conn.create_stack("new-stack", template_url=key_url)
stack = conn.describe_stacks()[0]
stack.stack_name.should.equal("new-stack")
stack.get_template().should.equal(
{
"GetTemplateResponse": {
"GetTemplateResult": {
"TemplateBody": dummy_template_json,
"ResponseMetadata": {
"RequestId": "2d06e36c-ac1d-11e0-a958-f9382b6eb86bEXAMPLE"
},
}
}
}
)
def test_create_stack_from_s3_url():
s3_conn = boto.s3.connect_to_region('us-west-1')
bucket = s3_conn.create_bucket("foobar")
key = boto.s3.key.Key(bucket)
key.key = "template-key"
key.set_contents_from_string(dummy_template_json)
key_url = key.generate_url(expires_in=0, query_auth=False)
conn = boto.cloudformation.connect_to_region('us-west-1')
conn.create_stack('new-stack', template_url=key_url)
stack = conn.describe_stacks()[0]
stack.stack_name.should.equal('new-stack')
stack.get_template().should.equal(
{
'GetTemplateResponse': {
'GetTemplateResult': {
'TemplateBody': dummy_template_json,
'ResponseMetadata': {
'RequestId': '2d06e36c-ac1d-11e0-a958-f9382b6eb86bEXAMPLE'
}
}
}
})
def get_direct_upload_url(self, path, mime_type, requires_cors=True):
self._initialize_cloud_conn()
path = self._init_path(path)
key = self._key_class(self._cloud_bucket, path)
url = key.generate_url(300,
"PUT",
headers={"Content-Type": mime_type},
encrypt_key=True)
return url
def updates_join(config, email, postcode):
email = email.lower().replace("/", "_")
bucket = _get_s3_bucket(config)
key = boto.s3.key.Key(bucket)
key.key = "updates/" + str(email)
key.set_contents_from_string(postcode)
url = key.generate_url(expires_in=0, query_auth=False)
def content_redirect_url(self, path):
path = self._init_path(path)
key = self.makeKey(path)
if not key.exists():
raise IOError('No such key: \'{0}\''.format(path))
return key.generate_url(
expires_in=1200,
method='GET',
query_auth=True)
def updates_join(config, email, postcode):
email = email.lower().replace("/", "_")
bucket = _get_s3_bucket(config)
key = boto.s3.key.Key(bucket)
key.key = "updates/" + str(email)
key.set_contents_from_string(postcode)
url = key.generate_url(expires_in=0, query_auth=False)
def content_redirect_url(self, path):
path = self._init_path(path)
key = self.makeKey(path)
if not key.exists():
raise IOError('No such key: \'{0}\''.format(path))
return key.generate_url(
expires_in=1200,
method='GET',
query_auth=True)
def content_redirect_url(self, path):
path = self._init_path(path)
key = self.makeKey(path)
if not key.exists():
raise IOError('No such key: \'{0}\''.format(path))
# No cloudfront? Sign to the bucket
if not self.signer:
return key.generate_url(expires_in=1200,
method='GET',
query_auth=True)
# Have cloudfront? Sign it
return self.signer(path, expire_time=60)
def test_create_stack_from_s3_url():
s3_conn = boto.s3.connect_to_region("us-west-1")
bucket = s3_conn.create_bucket("foobar")
key = boto.s3.key.Key(bucket)
key.key = "template-key"
key.set_contents_from_string(dummy_template_json)
key_url = key.generate_url(expires_in=0, query_auth=False)
conn = boto.cloudformation.connect_to_region("us-west-1")
conn.create_stack("new-stack", template_url=key_url)
stack = conn.describe_stacks()[0]
stack.stack_name.should.equal("new-stack")
stack.get_template().should.equal(dummy_template)
def test_create_stack_from_s3_url():
s3_conn = boto.s3.connect_to_region('us-west-1')
bucket = s3_conn.create_bucket("foobar")
key = boto.s3.key.Key(bucket)
key.key = "template-key"
key.set_contents_from_string(dummy_template_json)
key_url = key.generate_url(expires_in=0, query_auth=False)
conn = boto.cloudformation.connect_to_region('us-west-1')
conn.create_stack('new-stack', template_url=key_url)
stack = conn.describe_stacks()[0]
stack.stack_name.should.equal('new-stack')
stack.get_template().should.equal(dummy_template)
def content_redirect_url(self, path):
path = self._init_path(path)
key = self.makeKey(path)
if not key.exists():
raise IOError('No such key: \'{0}\''.format(path))
# No cloudfront? Sign to the bucket
if not self.signer:
return key.generate_url(
expires_in=1200,
method='GET',
query_auth=True)
# Have cloudfront? Sign it
return self.signer(path, expire_time=60)
def test_create_stack_from_s3_url():
s3_conn = boto.s3.connect_to_region('us-west-1')
bucket = s3_conn.create_bucket("foobar")
key = boto.s3.key.Key(bucket)
key.key = "template-key"
key.set_contents_from_string(dummy_template_json)
key_url = key.generate_url(expires_in=0, query_auth=False)
cf_conn = boto3.client('cloudformation', region_name='us-west-1')
cf_conn.create_stack(
StackName='stack_from_url',
TemplateURL=key_url,
)
cf_conn.get_template(StackName="stack_from_url")['TemplateBody'].should.equal(dummy_template)
def test_create_stack_from_s3_url():
s3_conn = boto.s3.connect_to_region('us-west-1')
bucket = s3_conn.create_bucket("foobar")
key = boto.s3.key.Key(bucket)
key.key = "template-key"
key.set_contents_from_string(dummy_template_json)
key_url = key.generate_url(expires_in=0, query_auth=False)
cf_conn = boto3.client('cloudformation', region_name='us-west-1')
cf_conn.create_stack(
StackName='stack_from_url',
TemplateURL=key_url,
)
cf_conn.get_template(StackName="stack_from_url")[
'TemplateBody'].should.equal(dummy_template)
def get_cv_list(config, person_id):
bucket = _get_s3_bucket(config)
prefix = "cvs/" + str(person_id) + "/"
cvs = bucket.list(prefix)
cvs = reversed(sorted(cvs, key=lambda k: k.last_modified))
result = []
for key in cvs:
result.append({
'name': key.name,
'url': key.generate_url(expires_in=0, query_auth=False),
'last_modified': boto.utils.parse_ts(key.last_modified),
'content_type': key.content_type,
'person_id': person_id
})
return result
def test_create_stack_from_s3_url():
s3_conn = boto.s3.connect_to_region("us-west-1")
bucket = s3_conn.create_bucket("foobar")
key = boto.s3.key.Key(bucket)
key.key = "template-key"
key.set_contents_from_string(dummy_template_json)
key_url = key.generate_url(expires_in=0, query_auth=False)
conn = boto.cloudformation.connect_to_region("us-west-1")
conn.create_stack("new-stack", template_url=key_url)
stack = conn.describe_stacks()[0]
stack.stack_name.should.equal("new-stack")
stack.get_template().should.equal(
{
"GetTemplateResponse": {
"GetTemplateResult": {
"TemplateBody": dummy_template_json,
"ResponseMetadata": {"RequestId": "2d06e36c-ac1d-11e0-a958-f9382b6eb86bEXAMPLE"},
}
}
}
)
def process_item(self, item):
filepath = item['file']
filename = item['filename']
room_id = item['room_id']
user_id = item['user_id']
username = item['username']
room_token = item['room_token']
print "got this job: %s" % item
im = thumbnail = None
try:
im = Image.open(filepath)
except:
pass
message_type = im and 'image' or 'file'
# Generate thumbnail
if im:
thumbnail = Image.open(filepath)
thumbnail.thumbnail((300, 300), Image.ANTIALIAS)
print im
print thumbnail
# Upload thumbnail if necessary
if thumbnail:
name, ext = os.path.splitext(filename)
thumbname = '/uploads/%s/%s_thumb%s' % (room_id, name, ext)
thumbfile = tempfile.NamedTemporaryFile()
thumbnail.save(thumbfile, im.format)
# Determine file mimetype
if im:
mime_type = 'image/%s' % im.format.lower()
else:
mime_type, _ = mimetypes.guess_type(filename)
# Create keys for file
key = boto.s3.key.Key(self.bucket)
key.key = '/uploads/%s/%s' % (room_id, filename)
if mime_type:
key.set_metadata('Content-Type', mime_type)
file = open(filepath)
filesize = os.path.getsize(filepath)
key.set_contents_from_file(file)
file.close()
os.remove(filepath)
print "Uploaded file"
# Upload thumbnail
if thumbnail:
thumb_key = boto.s3.key.Key(self.bucket)
thumb_key.key = thumbname
if mime_type:
thumb_key.set_metadata('Content-Type', mime_type)
thumb_key.set_contents_from_file(thumbfile.file)
print "Uploaded thumbnail"
# Create a message
content = '%s posted a file' % username
message = {
'room': room_id,
'user_id': user_id,
'user_name': username,
'type': message_type,
'filename': filename,
's3_key': key.key,
'content': content,
'created_at': datetime.datetime.utcnow(),
}
if message_type == 'image':
message['size'] = im.size
message['s3_thumbnail_key'] = thumb_key.key
message['thumb_size'] = thumbnail.size
if mime_type:
message['mime_type'] = mime_type
message['filesize'] = filesize
message_id = self.db.messages.insert(message)
m = {
'channel': room_token,
'message': {
'id': str(message_id),
'content': message['content'],
'user_id': str(message['user_id']),
'user_name': message['user_name'],
'type': message_type,
'url': key.generate_url(3600),
}
}
if message_type == 'image':
m['message']['size'] = message['size']
m['message']['thumb_url'] = thumb_key.generate_url(3600)
self.pubnub.publish(m)
