def all_cvs(config): bucket = _get_s3_bucket(config) prefix = "cvs/" cvs = bucket.list(prefix) cvs = reversed(sorted(cvs, key=lambda k: k.last_modified)) person_ids = [] result = [] for key in cvs: person_id = int(re.match("cvs/([0-9]+)/", key.name).group(1)) if person_id == 7777777: continue if person_id in person_ids: continue result.append({ 'name': key.name, 'url': key.generate_url(expires_in=0, query_auth=False), 'last_modified': boto.utils.parse_ts(key.last_modified), 'content_type': key.content_type, 'person_id': person_id }) person_ids.append(person_id) return result
def _hash_by_prefix(config, prefix): print("warming cache _hash_by_prefix", prefix) bucket = _get_s3_bucket(config) cvs = bucket.list(prefix) cvs = reversed(sorted(cvs, key=lambda k: k.last_modified)) person_ids = [] result = collections.OrderedDict() for key in cvs: # we use .jpg thumbnails now (and don't accept images as CVs) if key.name.endswith(".png"): continue key_last_modified = boto.utils.parse_ts(key.last_modified) person_id = int(re.match(prefix + "([0-9]+)[^0-9]", key.name).group(1)) if person_id not in result: result[person_id] = { 'name': key.name, 'url': key.generate_url(expires_in=0, query_auth=False), 'last_modified': key_last_modified, 'created': key_last_modified, 'person_id': person_id } result[person_id]['created'] = key_last_modified return result
def test_create_stack_from_s3_url(): s3_conn = boto.s3.connect_to_region("us-west-1") bucket = s3_conn.create_bucket("foobar", location="us-west-1") key = boto.s3.key.Key(bucket) key.key = "template-key" key.set_contents_from_string(dummy_template_json) key_url = key.generate_url(expires_in=0, query_auth=False) conn = boto.cloudformation.connect_to_region("us-west-1") conn.create_stack("new-stack", template_url=key_url) stack = conn.describe_stacks()[0] stack.stack_name.should.equal("new-stack") stack.get_template().should.equal( { "GetTemplateResponse": { "GetTemplateResult": { "TemplateBody": dummy_template_json, "ResponseMetadata": { "RequestId": "2d06e36c-ac1d-11e0-a958-f9382b6eb86bEXAMPLE" }, } } } )
def test_create_stack_from_s3_url(): s3_conn = boto.s3.connect_to_region('us-west-1') bucket = s3_conn.create_bucket("foobar") key = boto.s3.key.Key(bucket) key.key = "template-key" key.set_contents_from_string(dummy_template_json) key_url = key.generate_url(expires_in=0, query_auth=False) conn = boto.cloudformation.connect_to_region('us-west-1') conn.create_stack('new-stack', template_url=key_url) stack = conn.describe_stacks()[0] stack.stack_name.should.equal('new-stack') stack.get_template().should.equal( { 'GetTemplateResponse': { 'GetTemplateResult': { 'TemplateBody': dummy_template_json, 'ResponseMetadata': { 'RequestId': '2d06e36c-ac1d-11e0-a958-f9382b6eb86bEXAMPLE' } } } })
def get_direct_upload_url(self, path, mime_type, requires_cors=True): self._initialize_cloud_conn() path = self._init_path(path) key = self._key_class(self._cloud_bucket, path) url = key.generate_url(300, "PUT", headers={"Content-Type": mime_type}, encrypt_key=True) return url
def updates_join(config, email, postcode): email = email.lower().replace("/", "_") bucket = _get_s3_bucket(config) key = boto.s3.key.Key(bucket) key.key = "updates/" + str(email) key.set_contents_from_string(postcode) url = key.generate_url(expires_in=0, query_auth=False)
def content_redirect_url(self, path): path = self._init_path(path) key = self.makeKey(path) if not key.exists(): raise IOError('No such key: \'{0}\''.format(path)) return key.generate_url( expires_in=1200, method='GET', query_auth=True)
def content_redirect_url(self, path): path = self._init_path(path) key = self.makeKey(path) if not key.exists(): raise IOError('No such key: \'{0}\''.format(path)) # No cloudfront? Sign to the bucket if not self.signer: return key.generate_url(expires_in=1200, method='GET', query_auth=True) # Have cloudfront? Sign it return self.signer(path, expire_time=60)
def test_create_stack_from_s3_url(): s3_conn = boto.s3.connect_to_region("us-west-1") bucket = s3_conn.create_bucket("foobar") key = boto.s3.key.Key(bucket) key.key = "template-key" key.set_contents_from_string(dummy_template_json) key_url = key.generate_url(expires_in=0, query_auth=False) conn = boto.cloudformation.connect_to_region("us-west-1") conn.create_stack("new-stack", template_url=key_url) stack = conn.describe_stacks()[0] stack.stack_name.should.equal("new-stack") stack.get_template().should.equal(dummy_template)
def test_create_stack_from_s3_url(): s3_conn = boto.s3.connect_to_region('us-west-1') bucket = s3_conn.create_bucket("foobar") key = boto.s3.key.Key(bucket) key.key = "template-key" key.set_contents_from_string(dummy_template_json) key_url = key.generate_url(expires_in=0, query_auth=False) conn = boto.cloudformation.connect_to_region('us-west-1') conn.create_stack('new-stack', template_url=key_url) stack = conn.describe_stacks()[0] stack.stack_name.should.equal('new-stack') stack.get_template().should.equal(dummy_template)
def content_redirect_url(self, path): path = self._init_path(path) key = self.makeKey(path) if not key.exists(): raise IOError('No such key: \'{0}\''.format(path)) # No cloudfront? Sign to the bucket if not self.signer: return key.generate_url( expires_in=1200, method='GET', query_auth=True) # Have cloudfront? Sign it return self.signer(path, expire_time=60)
def test_create_stack_from_s3_url(): s3_conn = boto.s3.connect_to_region('us-west-1') bucket = s3_conn.create_bucket("foobar") key = boto.s3.key.Key(bucket) key.key = "template-key" key.set_contents_from_string(dummy_template_json) key_url = key.generate_url(expires_in=0, query_auth=False) cf_conn = boto3.client('cloudformation', region_name='us-west-1') cf_conn.create_stack( StackName='stack_from_url', TemplateURL=key_url, ) cf_conn.get_template(StackName="stack_from_url")['TemplateBody'].should.equal(dummy_template)
def test_create_stack_from_s3_url(): s3_conn = boto.s3.connect_to_region('us-west-1') bucket = s3_conn.create_bucket("foobar") key = boto.s3.key.Key(bucket) key.key = "template-key" key.set_contents_from_string(dummy_template_json) key_url = key.generate_url(expires_in=0, query_auth=False) cf_conn = boto3.client('cloudformation', region_name='us-west-1') cf_conn.create_stack( StackName='stack_from_url', TemplateURL=key_url, ) cf_conn.get_template(StackName="stack_from_url")[ 'TemplateBody'].should.equal(dummy_template)
def get_cv_list(config, person_id): bucket = _get_s3_bucket(config) prefix = "cvs/" + str(person_id) + "/" cvs = bucket.list(prefix) cvs = reversed(sorted(cvs, key=lambda k: k.last_modified)) result = [] for key in cvs: result.append({ 'name': key.name, 'url': key.generate_url(expires_in=0, query_auth=False), 'last_modified': boto.utils.parse_ts(key.last_modified), 'content_type': key.content_type, 'person_id': person_id }) return result
def test_create_stack_from_s3_url(): s3_conn = boto.s3.connect_to_region("us-west-1") bucket = s3_conn.create_bucket("foobar") key = boto.s3.key.Key(bucket) key.key = "template-key" key.set_contents_from_string(dummy_template_json) key_url = key.generate_url(expires_in=0, query_auth=False) conn = boto.cloudformation.connect_to_region("us-west-1") conn.create_stack("new-stack", template_url=key_url) stack = conn.describe_stacks()[0] stack.stack_name.should.equal("new-stack") stack.get_template().should.equal( { "GetTemplateResponse": { "GetTemplateResult": { "TemplateBody": dummy_template_json, "ResponseMetadata": {"RequestId": "2d06e36c-ac1d-11e0-a958-f9382b6eb86bEXAMPLE"}, } } } )
def process_item(self, item): filepath = item['file'] filename = item['filename'] room_id = item['room_id'] user_id = item['user_id'] username = item['username'] room_token = item['room_token'] print "got this job: %s" % item im = thumbnail = None try: im = Image.open(filepath) except: pass message_type = im and 'image' or 'file' # Generate thumbnail if im: thumbnail = Image.open(filepath) thumbnail.thumbnail((300, 300), Image.ANTIALIAS) print im print thumbnail # Upload thumbnail if necessary if thumbnail: name, ext = os.path.splitext(filename) thumbname = '/uploads/%s/%s_thumb%s' % (room_id, name, ext) thumbfile = tempfile.NamedTemporaryFile() thumbnail.save(thumbfile, im.format) # Determine file mimetype if im: mime_type = 'image/%s' % im.format.lower() else: mime_type, _ = mimetypes.guess_type(filename) # Create keys for file key = boto.s3.key.Key(self.bucket) key.key = '/uploads/%s/%s' % (room_id, filename) if mime_type: key.set_metadata('Content-Type', mime_type) file = open(filepath) filesize = os.path.getsize(filepath) key.set_contents_from_file(file) file.close() os.remove(filepath) print "Uploaded file" # Upload thumbnail if thumbnail: thumb_key = boto.s3.key.Key(self.bucket) thumb_key.key = thumbname if mime_type: thumb_key.set_metadata('Content-Type', mime_type) thumb_key.set_contents_from_file(thumbfile.file) print "Uploaded thumbnail" # Create a message content = '%s posted a file' % username message = { 'room': room_id, 'user_id': user_id, 'user_name': username, 'type': message_type, 'filename': filename, 's3_key': key.key, 'content': content, 'created_at': datetime.datetime.utcnow(), } if message_type == 'image': message['size'] = im.size message['s3_thumbnail_key'] = thumb_key.key message['thumb_size'] = thumbnail.size if mime_type: message['mime_type'] = mime_type message['filesize'] = filesize message_id = self.db.messages.insert(message) m = { 'channel': room_token, 'message': { 'id': str(message_id), 'content': message['content'], 'user_id': str(message['user_id']), 'user_name': message['user_name'], 'type': message_type, 'url': key.generate_url(3600), } } if message_type == 'image': m['message']['size'] = message['size'] m['message']['thumb_url'] = thumb_key.generate_url(3600) self.pubnub.publish(m)