def get(self):
registry = Registry()
if request.path == '/{0}'.format(self._document['document']['url']):
return redirect('{0}/'.format(request.path))
(_, key_name) = request.path.split('/{0}'.format(
self._document['document']['url']))
if not key_name or '/' == key_name:
key_name = '/index.html' # todo must have a default start page for multipage
tmp_dir = os.path.abspath(
os.path.join('/tmp/multipage', self._document['document']['uuid']))
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
file_path = os.path.abspath(os.path.join(tmp_dir, key_name[1:]))
if os.path.exists(file_path):
with open(file_path, 'r') as content:
contents = content.read()
else:
key_name = '{0}{1}'.format(self._document['document']['uuid'],
key_name)
contents = S3.get_string(
registry.get('files').get('bucket_name'), key_name)
dir_name = os.path.dirname(file_path)
if not os.path.exists(dir_name):
os.makedirs(dir_name)
with open(file_path, 'w') as write:
write.write(contents)
mimetype = mimetypes.guess_type(file_path)[0]
return Response(response=contents, status=200, mimetype=mimetype)
def get(self):
registry = Registry()
if request.path == '/{0}'.format(self._document['document']['url']):
return redirect('{0}/'.format(request.path))
(_, key_name) = request.path.split('/{0}'.format(self._document['document']['url']))
if not key_name or '/' == key_name:
key_name = '/index.html' # todo must have a default start page for multipage
tmp_dir = os.path.abspath(os.path.join('/tmp/multipage', self._document['document']['uuid']))
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
file_path = os.path.abspath(os.path.join(tmp_dir, key_name[1:]))
if os.path.exists(file_path):
with open(file_path, 'r') as content:
contents = content.read()
else:
key_name = '{0}{1}'.format(self._document['document']['uuid'], key_name)
contents = S3.get_string(registry.get('files').get('bucket_name'), key_name)
dir_name = os.path.dirname(file_path)
if not os.path.exists(dir_name):
os.makedirs(dir_name)
with open(file_path, 'w') as write:
write.write(contents)
mimetype = mimetypes.guess_type(file_path)[0]
return Response(response=contents, status=200, mimetype=mimetype)
def _create_config_database(self):
conn = boto.rds2.connect_to_region(self.args.region)
while True:
try:
describe = conn.describe_db_instances(
self.tmpl_args['rds']['instance_id'])
info = describe['DescribeDBInstancesResponse'][
'DescribeDBInstancesResult']['DBInstances'][0]
if info['DBInstanceStatus'] == 'available':
# check if we have it in config already
try:
self.rds_config = json.loads(
S3.get_string(self._format_name('config'),
'database'))
except boto.exception.S3ResponseError:
self.rds_config[
'database'] = 'postgres://{0}:{1}@{2}:{3}/{4}'.format(
self.tmpl_args['rds']['username'],
self.tmpl_args['rds']['password'],
info['Endpoint']['Address'], '5432',
self.tmpl_args['rds']['db_name'])
break
print 'rds', info['DBInstanceStatus']
except DBInstanceNotFound as e:
print 'Cannot find instance', str(e)
time.sleep(10)
S3.upload_string(self._format_name('config'),
'database',
json.dumps(self.rds_config),
partition=False)
def _create_config_database(self):
conn = boto.rds2.connect_to_region(self.args.region)
while True:
try:
describe = conn.describe_db_instances(self.tmpl_args['rds']['instance_id'])
info = describe['DescribeDBInstancesResponse']['DescribeDBInstancesResult']['DBInstances'][0]
if info['DBInstanceStatus'] == 'available':
# check if we have it in config already
try:
self.rds_config = json.loads(S3.get_string(self._format_name('config'), 'database'))
except boto.exception.S3ResponseError:
self.rds_config['database'] = 'postgres://{0}:{1}@{2}:{3}/{4}'.format(
self.tmpl_args['rds']['username'], self.tmpl_args['rds']['password'],
info['Endpoint']['Address'], '5432', self.tmpl_args['rds']['db_name']
)
break
print 'rds', info['DBInstanceStatus']
except DBInstanceNotFound as e:
print 'Cannot find instance', str(e)
time.sleep(10)
S3.upload_string(self._format_name('config'), 'database', json.dumps(self.rds_config), partition=False)
def test_get_string():
conn = boto.connect_s3()
conn.create_bucket('source-bucket')
bucket = conn.get_bucket('source-bucket')
key = Key(bucket=bucket, name='the-file')
key.set_contents_from_string('the string')
assert 'the string' == S3.get_string('source-bucket', 'the-file')
def do_work(self, message=None):
"""
:type message: boto.sqs.message.Message | None
:param message:
:return:
"""
if not message:
return
conn = boto.connect_s3()
bucket = conn.get_bucket(self.registry.get('files').get('bucket_name'))
contents = json.loads(message.get_body())
job_id = str(contents['Message'])
job = JobDB.selectBy(uuid=job_id).getOne(None)
if not job:
log.error('Cannot find job %s', job_id)
raise InvalidJobError('Invalid Job ID: {0}'.format(job_id))
job.set(status='running')
message = job.message
document = Document.selectBy(uuid=job.message['document']).getOne(None)
if not document:
message['reason'] = 'No Document exists'
job.set(status='failed', message=message)
raise FatalJobError('No Document Exists')
record = Document.get_document(document)
fp = StringIO(
S3.get_string(
self.registry.get('storage').get('bucket_name'),
record['file']['key']))
with zipfile.ZipFile(fp, 'r') as zip_handle:
for name in zip_handle.namelist():
if name.endswith('/'):
continue
key_name = '{0}/{1}'.format(document.uuid, name)
key = Key(bucket=bucket, name=key_name)
key.content_type = mimetypes.guess_type(name)[0]
key.set_contents_from_string(zip_handle.read(name))
log.info('Uploaded %s', key_name)
job.set(status='complete')
if job.message.get('on_complete', {}).get('alter'):
document.set(**job.message['on_complete']['alter'])
log.info('Setting job=%s to complete', job_id)
def do_work(self, message=None):
"""
:type message: boto.sqs.message.Message | None
:param message:
:return:
"""
if not message:
return
conn = boto.connect_s3()
bucket = conn.get_bucket(self.registry.get('files').get('bucket_name'))
contents = json.loads(message.get_body())
job_id = str(contents['Message'])
job = JobDB.selectBy(uuid=job_id).getOne(None)
if not job:
log.error('Cannot find job %s', job_id)
raise InvalidJobError('Invalid Job ID: {0}'.format(job_id))
job.set(status='running')
message = job.message
document = Document.selectBy(uuid=job.message['document']).getOne(None)
if not document:
message['reason'] = 'No Document exists'
job.set(status='failed', message=message)
raise FatalJobError('No Document Exists')
record = Document.get_document(document)
fp = StringIO(S3.get_string(self.registry.get('storage').get('bucket_name'), record['file']['key']))
with zipfile.ZipFile(fp, 'r') as zip_handle:
for name in zip_handle.namelist():
if name.endswith('/'):
continue
key_name = '{0}/{1}'.format(document.uuid, name)
key = Key(bucket=bucket, name=key_name)
key.content_type = mimetypes.guess_type(name)[0]
key.set_contents_from_string(zip_handle.read(name))
log.info('Uploaded %s', key_name)
job.set(status='complete')
if job.message.get('on_complete', {}).get('alter'):
document.set(**job.message['on_complete']['alter'])
log.info('Setting job=%s to complete', job_id)
def _build_rds(self):
try:
config = json.loads(S3.get_string(self._format_name('config'), 'database'))
parse = urlparse.urlparse(config['database'])
(username, contents, port) = parse.netloc.split(':')
(password, contents) = contents.split('@')
self.tmpl_args['rds'] = {
'username': username,
'password': password,
'db_name': self._format_name('cmsdb').replace('-', '_'),
'instance_id': self._format_name('cmsdb')
}
except boto.exception.S3ResponseError:
self.tmpl_args['rds'] = {
'username': self._format_name('system_database').replace('-', '_'),
'password': uuid.uuid4(),
'db_name': self._format_name('cmsdb').replace('-', '_'),
'instance_id': self._format_name('cmsdb')
}
def _build_rds(self):
try:
config = json.loads(
S3.get_string(self._format_name('config'), 'database'))
parse = urlparse.urlparse(config['database'])
(username, contents, port) = parse.netloc.split(':')
(password, contents) = contents.split('@')
self.tmpl_args['rds'] = {
'username': username,
'password': password,
'db_name': self._format_name('cmsdb').replace('-', '_'),
'instance_id': self._format_name('cmsdb')
}
except boto.exception.S3ResponseError:
self.tmpl_args['rds'] = {
'username':
self._format_name('system_database').replace('-', '_'),
'password': uuid.uuid4(),
'db_name': self._format_name('cmsdb').replace('-', '_'),
'instance_id': self._format_name('cmsdb')
}
