class Stasher(multiprocessing.Process):
"""
Stashers are the ant from the ant and the grashopper fable. They save
things up for winter in persistent storage.
"""
def __init__(self, qin, store_type, store_name, counter, qmsg, nprocs):
multiprocessing.Process.__init__(self)
self.qin = qin
self.counter = counter
self.qmsg = qmsg
self.nprocs = nprocs
self.store_type = store_type
self.store_name = store_name
self.storage = None
def setup(self):
write("%s" % self.qmsg.get())
self.storage = StorageFactory().new_storage(self.store_type, name=self.store_name)
if not self.storage:
print("Error in storage setup")
return False
return self.storage.open()
def run(self):
proceed = self.setup()
write("Proceeding: %s\n" % (str(proceed)))
nfinished = 0
while proceed:
try:
t_data = self.qin.get()
except Empty:
write('S Empty job queue.\n')
proceed = False
else:
if not t_data:
nfinished += 1
proceed = not nfinished == self.nprocs
write('S Received a finished message (%d of %d)\n' % (nfinished, self.nprocs))
else:
write('S\t#%d\t%s\n' % (self.pid, t_data.get('pdf_md5')))
try:
self.storage.store(t_data)
except Exception as e:
write('S\t#%d ERROR storing\t%s\t%s\n' % (self.pid, t_data.get('pdf_md5'), str(e)))
self.counter.inc()
self.qin.task_done()
self.cleanup()
self.finish()
def cleanup(self):
try:
self.storage.close()
except AttributeError:
pass
self.qmsg.task_done()
def finish(self):
write('Stasher: Storage closed. Exiting.\n')
def setup(self):
write("%s" % self.qmsg.get())
self.storage = StorageFactory().new_storage(self.store_type,
name=self.store_name)
if not self.storage:
print("Error in storage setup")
return False
return self.storage.open()
def get_blogs_with_tag(tag: str) -> Dict[str, Blog]:
s = StorageFactory.create(StorageType.S3)
blogs = {
path[6:-5]: Blog.FromString(s.get_blob(path)) for path in s.list_blobs('blogs/')
}
return {k: blog for k, blog in blogs.items() if tag in blog.tags}
def setup(self):
write("%s" % self.qmsg.get())
self.storage = StorageFactory().new_storage(self.store_type, name=self.store_name)
if not self.storage:
print("Error in storage setup")
return False
return self.storage.open()
def _fetch_blog(name: str) -> Blog:
storage = StorageFactory.create(StorageType.S3)
try:
data = storage.get_blob(f"blogs/{name}.blob")
except KeyNotFoundError:
abort(404)
blog = Blog()
blog.ParseFromString(data)
return blog
def _delete_single_blog(path: str) -> bool:
storage = StorageFactory.create(StorageType.S3)
full_path = f"blogs/{path}.blob"
try:
storage.get_blob(full_path)
except Exception:
return False
storage.delete_blob(full_path)
return True
def update_blog(data: Dict[str, Any]) -> Response:
data_keys = [
'path',
'title',
'header-img',
'header-cap-strong',
'header-cap-rest',
'teaser',
'content',
'tags',
]
if not all(k in data for k in data_keys):
return json_response(
ok=False, err="The request to update blog is missing data.", status=400
)
s = StorageFactory.create(StorageType.S3)
path = f"blogs/{data['path']}.blob"
try:
blob = s.get_blob(path)
except KeyNotFoundError:
return json_response(
ok=False, err=f"The blog {data['path']} was not found", status=404
)
# Make sure all tags already exist
all_tags = [tag.name for tag in tag_utils.list_tags()]
if not all(tag in all_tags for tag in data['tags']):
return json_response(
ok=False,
err=f"Some of the tags provided do not already exist.",
status=400,
)
blog = Blog.FromString(blob)
blog.modification_time = int(time.time())
blog.name = data['title']
blog.header_image.path = data['header-img']
blog.header_image.caption_strong = data['header-cap-strong']
blog.header_image.caption_cont = data['header-cap-rest']
blog.teaser = data['teaser']
blog.markdown_content = data['content']
for _ in range(len(blog.tags)):
blog.tags.pop()
for tag in data['tags']:
blog.tags.append(tag)
s.put_blob(path, blog.SerializeToString())
return json_response(ok=True)
def _fetch_all_blogs() -> Dict[str, Blog]:
storage = StorageFactory.create(StorageType.S3)
try:
keys = storage.list_blobs("blogs")
blog_dict = {
_blog_number_from_key(key): (Blog(), storage.get_blob(key)) for key in keys
}
for key in blog_dict.keys():
blog_dict[key][0].ParseFromString(blog_dict[key][1])
except KeyNotFoundError:
abort(500)
return {key: tup[0] for key, tup in blog_dict.items()}
def _create_blog(
path: str,
title: str,
header_img: str,
header_caption_strong: str,
header_caption_rest: str,
teaser: str,
content: str,
tags: List[str],
) -> Response:
storage = StorageFactory.create(StorageType.S3)
blog_path = f"blogs/{path}.blob"
if blog_path in storage.list_blobs("blogs"):
return json_response(
ok=False, err=f"Blog titled {path} already exists", status=400
)
# Make sure all tags already exist
all_tags = [tag.name for tag in tag_utils.list_tags()]
if not all(tag in all_tags for tag in tags):
return json_response(
ok=False,
err=f"Some of the tags provided do not already exist.",
status=400,
)
ts = int(time.time())
blog = Blog(
name=title,
header_image=HeaderImage(
path=header_img,
caption_strong=header_caption_strong,
caption_cont=header_caption_rest,
),
teaser=teaser,
markdown_content=content,
creation_time=ts,
modification_time=ts,
tags=tags,
)
blob = blog.SerializeToString()
storage.put_blob(blog_path, blob)
return json_response(ok=True)
def get_blog(data: Dict[str, Any]) -> Response:
if 'path' not in data:
return json_response(
ok=False, err=f'Request does not contain blog path', status=400
)
s = StorageFactory.create(StorageType.S3)
try:
blob = s.get_blob(f"blogs/{data['path']}.blob")
except KeyNotFoundError:
return json_response(
ok=False, err=f'Blog with path "{data["path"]}" does not exist', status=404
)
blog = Blog.FromString(blob)
dict_blog = json.loads(proto_to_json(blog))
return json_response(ok=True, path=data["path"], **dict_blog)
def __init__(self, storage_type="DictStorage", *args, **kwargs):
if args is None:
args = []
if kwargs is None:
kwargs = {}
self.storage = StorageFactory.create_storage(storage_type, *args, **kwargs)
def __init__(self):
self.storage_db = StorageFactory.get_storage("groups")
def __init__(self):
self.storage_db = StorageFactory.get_storage('provisioners')
def _get_storage():
return StorageFactory.create(StorageType.S3)
def __init__(self):
self.storage_db = StorageFactory.get_storage('instances')
class Stasher(multiprocessing.Process):
"""
Stashers are the ant from the ant and the grashopper fable. They save
things up for winter in persistent storage.
"""
def __init__(self, qin, store_type, store_name, counter, qmsg, nprocs):
multiprocessing.Process.__init__(self)
self.qin = qin
self.counter = counter
self.qmsg = qmsg
self.nprocs = nprocs
self.store_type = store_type
self.store_name = store_name
self.storage = None
def setup(self):
write("%s" % self.qmsg.get())
self.storage = StorageFactory().new_storage(self.store_type,
name=self.store_name)
if not self.storage:
print("Error in storage setup")
return False
return self.storage.open()
def run(self):
proceed = self.setup()
write("Proceeding: %s\n" % (str(proceed)))
nfinished = 0
while proceed:
try:
t_data = self.qin.get()
except Empty:
write('S Empty job queue.\n')
proceed = False
else:
if not t_data:
nfinished += 1
proceed = not nfinished == self.nprocs
write('S Received a finished message (%d of %d)\n' %
(nfinished, self.nprocs))
else:
write('S\t#%d\t%s\n' % (self.pid, t_data.get('pdf_md5')))
try:
self.storage.store(t_data)
except Exception as e:
write('S\t#%d ERROR storing\t%s\t%s\n' %
(self.pid, t_data.get('pdf_md5'), str(e)))
self.counter.inc()
self.qin.task_done()
self.cleanup()
self.finish()
def cleanup(self):
try:
self.storage.close()
except AttributeError:
pass
self.qmsg.task_done()
def finish(self):
write('Stasher: Storage closed. Exiting.\n')
def __init__(self):
self.storage_db = StorageFactory.get_storage('instances')
def __init__(self):
self.storage_db = StorageFactory.get_storage('enviroments')
def __init__(self):
self.storage_db = StorageFactory.get_storage('partitions')
def __init__(self):
self.storage_db = StorageFactory.get_storage('enviroments')
def __init__(self):
self.storage_db = StorageFactory.get_storage('files')
def __init__(self):
self.storage_db = StorageFactory.get_storage('partitions')
