def post(self):
data = request.json
if not data or not data.get('file'):
return Response(status=400)
data['user_id'] = session['auth_user'].get('id', -1)
name = data.get('name', 'Migration Upload {0}'.format(arrow.now().format('YYYY-MM-DD HH:mm')))
job = Job.save({
'name': name,
'status': 'pending',
'message': data
})
registry = Registry()
topic_arn = registry.get('topics').get('topic').get('migrationupload')
conn = boto.sns.connect_to_region(registry.get('region').get('region'))
try:
conn.publish(topic_arn, str(job.uuid), name)
except BotoServerError as e:
log.error('Cannot publish Job=%s to Topic=%s', job.uuid, topic_arn)
log.exception(e)
return Response(status=500)
except AttributeError as e:
log.error('Cannot publish Job=%s to Topic=%s "%s"', job.uuid, topic_arn, str(e))
return Response(status=500)
return Response(response=json.dumps({
'notify_msg': {
'title': 'Job Added',
'message': 'Migration job has been added. Upload will commence shortly.',
'type': 'success'
}}), content_type='application/json', status=200)
def get(self):
"""
{
"jobs": [
{
"uuid": "",
"name": "",
"status": "",
"created": ""
}
],
"meta": {
"offset": offset,
"limit": limit
}
}
:return:
:rtype: flask.Response
"""
offset = int(request.args.get('offset', 0))
limit = int(request.args.get('limit', 100))
items = []
count = JobDB.select().count()
# pylint: disable=no-member
for job in JobDB.select(orderBy=DESC(JobDB.q.created))[offset:limit]:
items.append({
'uuid': job.uuid,
'name': job.name,
'status': job.status.title(),
'created': job.created.strftime('%Y-%m-%d %H:%M:%S'),
'message': {k: v for k, v in job.message.iteritems() if k in ('download', 'content')},
'reason': job.message.get('reason', '')
})
return Response(response=json.dumps({
'jobs': items,
'meta': {
'offset': offset,
'limit': limit,
'total': count
}
}), status=200, content_type='application/json')
def post(self):
"""
For specific parents
{
"documents": [{
"parent_id": "uuid"
}],
"all_documents": false
}
{
"documents": [],
"all_documents": true
}
:rtype: flask.Response
:return: A flask Response object
"""
data = request.json
if not data:
return Response(status=400)
if not ((data.get('document') and not data.get('all_documents')) or
(not data.get('document') and data.get('all_documents'))):
return Response(status=400)
name = data.get('name', 'Migration Download {0}'.format(arrow.now().format('YYYY-MM-DD HH:mm')))
job = Job.save({
'name': name,
'status': 'pending',
'message': data
})
topic_arn = Registry().get('topics').get('topic').get('migrationdownload')
conn = boto.sns.connect_to_region(Registry().get('region').get('region'))
try:
conn.publish(topic_arn, str(job.uuid), name)
except BotoServerError as e:
log.error('Cannot publish Job=%s to Topic=%s', job.uuid, topic_arn)
log.exception(e)
return Response(status=500)
except AttributeError as e:
log.error('Cannot publish Job=%s to Topic=%s "%s"', job.uuid, topic_arn, str(e))
return Response(status=500)
return Response(response=json.dumps({
'notify_msg': {
'title': 'Job Added',
'message': 'Migration job has been added. Download will be ready shortly.',
'type': 'success'
}}), content_type='application/json', status=200)
def do_work(self, message=None):
"""
:type message: boto.sqs.message.Message | None
:param message:
:return:
"""
if not message:
return
conn = boto.connect_s3()
bucket = conn.get_bucket(self.registry.get('files').get('bucket_name'))
contents = json.loads(message.get_body())
job_id = str(contents['Message'])
job = JobDB.selectBy(uuid=job_id).getOne(None)
if not job:
log.error('Cannot find job %s', job_id)
raise InvalidJobError('Invalid Job ID: {0}'.format(job_id))
job.set(status='running')
message = job.message
document = Document.selectBy(uuid=job.message['document']).getOne(None)
if not document:
message['reason'] = 'No Document exists'
job.set(status='failed', message=message)
raise FatalJobError('No Document Exists')
record = Document.get_document(document)
fp = StringIO(
S3.get_string(
self.registry.get('storage').get('bucket_name'),
record['file']['key']))
with zipfile.ZipFile(fp, 'r') as zip_handle:
for name in zip_handle.namelist():
if name.endswith('/'):
continue
key_name = '{0}/{1}'.format(document.uuid, name)
key = Key(bucket=bucket, name=key_name)
key.content_type = mimetypes.guess_type(name)[0]
key.set_contents_from_string(zip_handle.read(name))
log.info('Uploaded %s', key_name)
job.set(status='complete')
if job.message.get('on_complete', {}).get('alter'):
document.set(**job.message['on_complete']['alter'])
log.info('Setting job=%s to complete', job_id)
def do_work(self, message=None):
"""
:type message: boto.sqs.message.Message | None
:param message:
:return:
"""
if not message:
return
conn = boto.connect_s3()
bucket = conn.get_bucket(self.registry.get('files').get('bucket_name'))
contents = json.loads(message.get_body())
job_id = str(contents['Message'])
job = JobDB.selectBy(uuid=job_id).getOne(None)
if not job:
log.error('Cannot find job %s', job_id)
raise InvalidJobError('Invalid Job ID: {0}'.format(job_id))
job.set(status='running')
message = job.message
document = Document.selectBy(uuid=job.message['document']).getOne(None)
if not document:
message['reason'] = 'No Document exists'
job.set(status='failed', message=message)
raise FatalJobError('No Document Exists')
record = Document.get_document(document)
fp = StringIO(S3.get_string(self.registry.get('storage').get('bucket_name'), record['file']['key']))
with zipfile.ZipFile(fp, 'r') as zip_handle:
for name in zip_handle.namelist():
if name.endswith('/'):
continue
key_name = '{0}/{1}'.format(document.uuid, name)
key = Key(bucket=bucket, name=key_name)
key.content_type = mimetypes.guess_type(name)[0]
key.set_contents_from_string(zip_handle.read(name))
log.info('Uploaded %s', key_name)
job.set(status='complete')
if job.message.get('on_complete', {}).get('alter'):
document.set(**job.message['on_complete']['alter'])
log.info('Setting job=%s to complete', job_id)
def do_work(self):
log = logging.getLogger('hermes_cms.helpers.multipage')
updated_record = {}
alter_record = {}
if self.document.published:
alter_record.update({
'published': True
})
updated_record.update({
'published': False
})
log.debug('Document: %s removing published status', str(self.document.uuid))
# create job
name = '{0} multipage job'.format(self.document.name)[0:254]
job = Job.save({
'name': name,
'status': 'pending',
'message': {
'document': str(self.document.uuid),
'on_complete': {
'alter': alter_record
}
}
})
log.info('Created Job for Document %s as JobID=%s', self.document.uuid, job.uuid)
# push this to sns job topic
topic_arn = Registry().get('topics').get('topic').get('multipage')
conn = boto.sns.connect_to_region(Registry().get('region').get('region'))
try:
conn.publish(topic_arn, str(job.uuid), 'Multipage Subject')
except BotoServerError as e:
log.error('Cannot publish Job=%s to Topic=%s', str(job.uuid), topic_arn)
log.exception(e)
if updated_record:
self.document.set(**updated_record)
def post(self):
data = request.json
if not data or not data.get('file'):
return Response(status=400)
data['user_id'] = session['auth_user'].get('id', -1)
name = data.get(
'name', 'Migration Upload {0}'.format(
arrow.now().format('YYYY-MM-DD HH:mm')))
job = Job.save({'name': name, 'status': 'pending', 'message': data})
registry = Registry()
topic_arn = registry.get('topics').get('topic').get('migrationupload')
conn = boto.sns.connect_to_region(registry.get('region').get('region'))
try:
conn.publish(topic_arn, str(job.uuid), name)
except BotoServerError as e:
log.error('Cannot publish Job=%s to Topic=%s', job.uuid, topic_arn)
log.exception(e)
return Response(status=500)
except AttributeError as e:
log.error('Cannot publish Job=%s to Topic=%s "%s"', job.uuid,
topic_arn, str(e))
return Response(status=500)
return Response(response=json.dumps({
'notify_msg': {
'title': 'Job Added',
'message':
'Migration job has been added. Upload will commence shortly.',
'type': 'success'
}
}),
content_type='application/json',
status=200)
def do_work(self, message=None):
"""
{
"documents": [{
"parent_id": "uuid"
}],
"all_documents": false
}
{
"documents": [],
"all_documents": true
}
uuid as filename
{
"document": {},
"file": {},
}
full key name for file
Manifest file structure
{
'documents': [
{
'uuid': 'some-uuid',
'url': 'some-url',
'parent_url': 'some-parent-url',
'parent_uuid': 'some-parent-uuid'
},
...
],
'full': bool
}
:type message: boto.sqs.message.Message | None
:param message:
:return:
"""
if not message:
return
contents = json.loads(message.get_body())
job_id = str(contents['Message'])
job = JobDB.selectBy(uuid=job_id).getOne(None)
if not job:
log.error('Cannot find job %s', job_id)
raise InvalidJobError('Invalid Job ID: {0}'.format(job_id))
job.set(status='running')
and_ops = [Document.q.archived == False, Document.q.published == True]
if not job.message.get('all_documents'):
and_ops.append(self._get_document_query(job.message.get('document')))
manifest = {
'documents': [],
'full': job.message.get('all_documents', False)
}
zip_contents = StringIO()
zip_handle = zipfile.ZipFile(zip_contents, 'w', compression=zipfile.ZIP_DEFLATED)
for document in Document.query(Document.all(), where=AND(*and_ops)):
parent_document = self._get_document_parent_url(document.parent)
manifest['documents'].append({
'uuid': document.uuid,
'url': document.url,
'parent_url': None if not parent_document else parent_document.url,
'parent_uuid': None if not parent_document else parent_document.uuid
})
self._handle_document(document, zip_handle)
log.info('Adding document uuid=%s to zip archive', str(document.uuid))
zip_handle.writestr('manifest', json.dumps(manifest))
zip_handle.close()
zip_key = Key(self.bucket, job_id)
zip_key.content_type = 'application/zip'
zip_key.set_contents_from_string(zip_contents.getvalue())
log.info("Created ZIP for Job '%s'", str(job_id))
message = job.message
message['download'] = {
'bucket': self.bucket.name,
'key': job_id
}
job.set(status='complete', message=message)
log.info('Setting job=%s to complete', job_id)
# setup application
sqlhub.threadConnection = connectionForURI(LocalConfig.DATABASE)
User.createTable(ifNotExists=True)
if not User.select().count():
User(
email='*****@*****.**',
password='******',
first_name='',
last_name='',
permissions=
'add_document,modify_document,delete_document,restore_deleted_document,restore_version_document,add_user,modify_user,delete_user,restore_user'
)
Document.createTable(ifNotExists=True)
Job.createTable(ifNotExists=True)
sqlhub.threadConnection.close()
# close for application to begin
app = create_app(blueprints=[{
'name': 'hermes_cms.views.main',
'from': 'route'
}, {
'name': 'hermes_cms.views.admin',
'from': 'route'
}])
@app.route('/assets/<path:filename>')
def public_static(filename):
return send_from_directory(os.path.abspath('../hermes_ui/dist'), filename)
def do_work(self, message=None):
"""
ZipFile stored in storage. read from Job Message
{
"archive": "path/in/s3"
}
# documents order matter
Manifest file structure
{
'documents': [
{
'uuid': 'some-uuid',
'url': 'some-url',
'parent_url': 'some-parent-url',
'parent_uuid': 'some-parent-uuid'
},
...
],
'full': bool
}
:param message:
:return:
"""
if not message:
return
contents = json.loads(message.get_body())
job_id = str(contents['Message'])
job = JobDB.selectBy(uuid=job_id).getOne(None)
if not job:
log.error('Cannot find job %s', job_id)
raise InvalidJobError('Invalid Job ID: {0}'.format(job_id))
job.set(status='running')
message = job.message
# get archive file
archive_key = self.bucket.get_key(job.message['file']['key'])
if not archive_key:
message['reason'] = 'Cannot find the archive in the S3 bucket.'
job.set(status='failed', message=message)
raise InvalidJobError('Cannot find archive in S3 bucket.')
fp = StringIO(archive_key.get_contents_as_string())
handle = zipfile.ZipFile(fp,
mode='r',
compression=zipfile.ZIP_DEFLATED)
try:
manifest_content = MigrationUploadJob._get_manifest(handle)
except Exception:
message['reason'] = 'Unable to retrieve manifest in archive'
job.set(status='failed', message=message)
raise InvalidJobError('Unable to retrieve manifest')
if not MigrationUploadJob._validate_manifest(
manifest_content['documents']):
message['reason'] = 'Manifest found is not valid'
job.set(status='failed', message=message)
raise InvalidJobError('Manifest is not valid')
for document in manifest_content['documents']:
contents = MigrationUploadJob._get_document_from_archive(
document['uuid'], handle)
if document.get('parent_uuid') and document.get('parent_url'):
self._update_from_parent(contents, document['parent_url'])
created = self._save_document(job.message['user_id'], contents)
if contents.get('file') and contents['document']['type'] == 'File':
self._upload_file(contents, handle)
elif contents.get(
'file') and contents['document']['type'] == 'MultiPage':
self._upload_multipage(contents, handle)
key_name = '{0}/{1}/{2}/{3}'.format(created.day, created.month,
created.year,
contents['document']['uuid'])
key = Key(self.bucket, key_name)
key.set_contents_from_string(json.dumps(contents))
job.set(status='complete')
log.info('Setting job=%s to complete', job_id)
def do_work(self, message=None):
"""
ZipFile stored in storage. read from Job Message
{
"archive": "path/in/s3"
}
# documents order matter
Manifest file structure
{
'documents': [
{
'uuid': 'some-uuid',
'url': 'some-url',
'parent_url': 'some-parent-url',
'parent_uuid': 'some-parent-uuid'
},
...
],
'full': bool
}
:param message:
:return:
"""
if not message:
return
contents = json.loads(message.get_body())
job_id = str(contents['Message'])
job = JobDB.selectBy(uuid=job_id).getOne(None)
if not job:
log.error('Cannot find job %s', job_id)
raise InvalidJobError('Invalid Job ID: {0}'.format(job_id))
job.set(status='running')
message = job.message
# get archive file
archive_key = self.bucket.get_key(job.message['file']['key'])
if not archive_key:
message['reason'] = 'Cannot find the archive in the S3 bucket.'
job.set(status='failed', message=message)
raise InvalidJobError('Cannot find archive in S3 bucket.')
fp = StringIO(archive_key.get_contents_as_string())
handle = zipfile.ZipFile(fp, mode='r', compression=zipfile.ZIP_DEFLATED)
try:
manifest_content = MigrationUploadJob._get_manifest(handle)
except Exception:
message['reason'] = 'Unable to retrieve manifest in archive'
job.set(status='failed', message=message)
raise InvalidJobError('Unable to retrieve manifest')
if not MigrationUploadJob._validate_manifest(manifest_content['documents']):
message['reason'] = 'Manifest found is not valid'
job.set(status='failed', message=message)
raise InvalidJobError('Manifest is not valid')
for document in manifest_content['documents']:
contents = MigrationUploadJob._get_document_from_archive(document['uuid'], handle)
if document.get('parent_uuid') and document.get('parent_url'):
self._update_from_parent(contents, document['parent_url'])
created = self._save_document(job.message['user_id'], contents)
if contents.get('file') and contents['document']['type'] == 'File':
self._upload_file(contents, handle)
elif contents.get('file') and contents['document']['type'] == 'MultiPage':
self._upload_multipage(contents, handle)
key_name = '{0}/{1}/{2}/{3}'.format(created.day, created.month, created.year, contents['document']['uuid'])
key = Key(self.bucket, key_name)
key.set_contents_from_string(json.dumps(contents))
job.set(status='complete')
log.info('Setting job=%s to complete', job_id)