def separate(self, file_id):
num_chunks = int(len(self.content) / CHUNK_SIZE)
self.name_list = []
if num_chunks > 0:
last_index = 0
for i in range(num_chunks):
data = self.content[i * CHUNK_SIZE:(i + 1) * CHUNK_SIZE]
checksum = get_sha1_hash(data)
chunk_object = Chunk(checksum, file_id)
self.name_list.append(chunk_object.get_filename())
self.hash_list.append(checksum.upper())
# compress chunks process
buf = strIO()
f = GzipFile(mode='wb', fileobj=buf)
try:
f.write(data)
finally:
f.close()
chunk = buf.getvalue()
self.chunks.append(chunk)
last_index = i
data = self.content[(last_index + 1) * CHUNK_SIZE:]
else:
data = self.content
# FIXME: Do not repeat code, handle the last chunk differently
if len(data) > 0:
checksum = get_sha1_hash(data)
chunk_object = Chunk(checksum, file_id)
self.name_list.append(chunk_object.get_filename())
self.hash_list.append(checksum.upper())
buf = strIO()
f = GzipFile(mode='wb', fileobj=buf)
try:
f.write(data)
finally:
f.close()
chunk = buf.getvalue()
self.chunks.append(chunk)
def separate(self, file_id):
num_chunks = int(len(self.content)/CHUNK_SIZE)
self.name_list = []
if num_chunks > 0:
last_index = 0
for i in range(num_chunks):
data = self.content[i*CHUNK_SIZE:(i+1)*CHUNK_SIZE]
checksum = get_sha1_hash(data)
chunk_object = Chunk(checksum, file_id)
self.name_list.append(chunk_object.get_filename())
self.hash_list.append(checksum.upper())
# compress chunks process
buf = strIO()
f = GzipFile(mode='wb', fileobj=buf)
try:
f.write(data)
finally:
f.close()
chunk = buf.getvalue()
self.chunks.append(chunk)
last_index = i
data = self.content[(last_index+1)*CHUNK_SIZE:]
else:
data = self.content
# FIXME: Do not repeat code, handle the last chunk differently
if len(data) > 0:
checksum = get_sha1_hash(data)
chunk_object = Chunk(checksum, file_id)
self.name_list.append(chunk_object.get_filename())
self.hash_list.append(checksum.upper())
buf = strIO()
f = GzipFile(mode='wb', fileobj=buf)
try:
f.write(data)
finally:
f.close()
chunk = buf.getvalue()
self.chunks.append(chunk)
def item(dirname):
if dirname == "all":
items = os.listdir(paths["dumpdir"])
else:
items = dirname.split(",")
if len(items) == 1:
itm = items[0]
archive_path = os.path.join(paths["dumpdir"], itm)
if not os.path.isfile(archive_path):
abort(404)
archive = open(archive_path)
archive_size = os.path.getsize(archive_path)
cd = "attachment; filename={0}".format(itm)
return Response(archive.read(),
content_type="application/octet-stream",
headers={
"Content-length": archive_size,
"Content-Disposition": cd
},
direct_passthrough=True)
# tar multiple files
c = strIO()
tarred = tarfile.open(fileobj=c, mode='w')
for item_ in items:
archive_path = os.path.join(paths["dumpdir"], item_)
if not os.path.isfile(archive_path):
abort(404)
tarred.add(archive_path, arcname=item_)
tarred.close()
cd = "attachment; filename=fafdds-{0}.tar.gz".format(
datetime.datetime.now().isoformat())
return Response(c.getvalue(),
content_type="application/octet-stream",
headers={
"Content-length": c.tell(),
"Content-Disposition": cd
},
direct_passthrough=True)
def new(url_fname=None):
"""
Handle dump dir archive uploads
"""
form = NewDumpDirForm()
if request.method in ["POST", "PUT"]:
try:
if request.method == "POST":
if not form.validate() or form.file.name not in request.files:
raise InvalidUsage("Invalid form data.", 400)
archive_file = request.files[form.file.name]
archive_fname = archive_file.filename
if request.method == "PUT":
archive_file = strIO(request.stream.read())
archive_fname = url_fname
archive_file.seek(0, os.SEEK_END)
archive_size = archive_file.tell()
archive_file.seek(0)
if not archive_size:
raise InvalidUsage("Empty archive received", 400)
if not check_filename(archive_fname):
raise InvalidUsage("Wrong archive file name", 400)
# sanitize input filename just to be sure
archive_fname = secure_filename(archive_fname)
if not os.path.exists(paths["dumpdir"]):
raise InvalidUsage(
"That's embarrassing! We have some troubles"
" with deployment. Please try again later.", 500)
count = 0
try:
count = sum(
1 for x in os.listdir(paths["dumpdir"])
if os.path.isfile(os.path.join(paths["dumpdir"], x)))
except Exception as e:
raise InvalidUsage(
"That's embarrassing! We have some troubles"
" with storage. Please try again later.", 500)
if count >= int(config["dumpdir.cachedirectorycountquota"]):
raise InvalidUsage(
"That's embarrassing! We have reached"
" the limit of uploaded archives."
" Please try again later.", 500)
if archive_size > int(config["dumpdir.maxdumpdirsize"]):
raise InvalidUsage("Dump dir archive is too large", 413)
used_space = 0.0
try:
used_space = sum(
float(os.path.getsize(x))
for x in map(lambda f: os.path.join(paths["dumpdir"], f),
os.listdir(paths["dumpdir"]))
if os.path.isfile(x))
except Exception as e:
raise InvalidUsage(
"That's embarrassing! We have some"
" troubles with disk space."
" Please try again later.", 500)
quota = int(config["dumpdir.cachedirectorysizequota"])
if (quota - archive_size) < used_space:
raise InvalidUsage(
"That's embarrassing! We ran out"
" of disk space."
" Please try again later.", 500)
fpath = os.path.join(paths["dumpdir"], archive_fname)
if os.path.exists(fpath):
raise InvalidUsage("Dump dir archive already exists.", 409)
with open(fpath, 'w') as dest:
dest.write(archive_file.read().decode("utf-8"))
if request_wants_json():
response = jsonify({"ok": "ok"})
response.status_code = 201
return response
flash("Uploaded successfully.")
return render_template("dumpdirs/new.html", form=form)
except InvalidUsage as e:
if e.status_code == 500:
logger.error(e.message)
elif e.status_code >= 400:
logger.warning(e.message)
if request_wants_json():
response = jsonify({"error": e.message})
response.status_code = e.status_code
return response
flash(e.message, "danger")
return render_template("dumpdirs/new.html",
form=form), e.status_code
return render_template("dumpdirs/new.html", form=form)
def parse(source, mapper=dict):
"""
Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a VDF)
to a Python object.
``mapper`` specifies the Python object used after deserializetion. ``dict` is
used by default. Alternatively, ``collections.OrderedDict`` can be used if you
wish to preserve key order. Or any object that acts like a ``dict``.
"""
if not issubclass(mapper, dict):
raise TypeError("Expected mapper to be subclass of dict, got %s", type(mapper))
if hasattr(source, "read"):
fp = source
elif isinstance(source, string_type):
try:
fp = unicodeIO(source)
except TypeError:
fp = strIO(source)
else:
raise TypeError("Expected source to be str or file-like object")
# skip past BOMs
fp.seek(bomlen(fp.read(10)))
# init
stack = [mapper()]
expect_bracket = False
re_keyvalue = re.compile(
r'^("(?P<qkey>(?:\\.|[^\\"])+)"|(?P<key>[a-z0-9\-\_]+))'
r"([ \t]*("
r'"(?P<qval>(?:\\.|[^\\"])*)(?P<vq_end>")?'
r"|(?P<val>[a-z0-9\-\_]+)"
r"))?",
flags=re.I,
)
for line in fp:
line = line.lstrip()
# skip empty and comment lines
if line == "" or line[0] == "/":
continue
# one level deeper
if line[0] == "{":
expect_bracket = False
continue
if expect_bracket:
raise SyntaxError("vdf.parse: expected openning bracket")
# one level back
if line[0] == "}":
if len(stack) > 1:
stack.pop()
continue
raise SyntaxError("vdf.parse: one too many closing parenthasis")
# parse keyvalue pairs
while True:
match = re_keyvalue.match(line)
if not match:
raise SyntaxError("vdf.parse: invalid syntax")
key = match.group("key") if match.group("qkey") is None else match.group("qkey")
val = match.group("val") if match.group("qval") is None else match.group("qval")
# we have a key with value in parenthesis, so we make a new dict obj (level deeper)
if val is None:
stack[-1][key] = mapper()
stack.append(stack[-1][key])
expect_bracket = True
# we've matched a simple keyvalue pair, map it to the last dict obj in the stack
else:
# if the value is line consume one more line and try to match again,
# until we get the KeyValue pair
if match.group("vq_end") is None and match.group("qval") is not None:
line += next(fp)
continue
stack[-1][key] = val
# exit the loop
break
if len(stack) != 1:
raise SyntaxError("vdf.parse: unclosed parenthasis or quotes")
return stack.pop()
def parse(source, mapper=dict):
"""
Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a VDF)
to a Python object.
``mapper`` specifies the Python object used after deserializetion. ``dict` is
used by default. Alternatively, ``collections.OrderedDict`` can be used if you
wish to preserve key order. Or any object that acts like a ``dict``.
"""
if not issubclass(mapper, dict):
raise TypeError("Expected mapper to be subclass of dict, got %s", type(mapper))
if hasattr(source, 'read'):
fp = source
elif isinstance(source, string_type):
try:
fp = unicodeIO(source)
except TypeError:
fp = strIO(source)
else:
raise TypeError("Expected source to be str or file-like object")
# skip past BOMs
fp.seek(bomlen(fp.read(10)))
# init
stack = [mapper()]
expect_bracket = False
re_keyvalue = re.compile(r'^("(?P<qkey>(?:\\.|[^\\"])+)"|(?P<key>[a-z0-9\-\_]+))'
r'([ \t]*('
r'"(?P<qval>(?:\\.|[^\\"])*)(?P<vq_end>")?'
r'|(?P<val>[a-z0-9\-\_]+)'
r'))?',
flags=re.I
)
for line in fp:
line = line.lstrip()
# skip empty and comment lines
if line == "" or line[0] == '/':
continue
# one level deeper
if line[0] == "{":
expect_bracket = False
continue
if expect_bracket:
raise SyntaxError("vdf.parse: expected openning bracket")
# one level back
if line[0] == "}":
if len(stack) > 1:
stack.pop()
continue
raise SyntaxError("vdf.parse: one too many closing parenthasis")
# parse keyvalue pairs
while True:
match = re_keyvalue.match(line)
if not match:
raise SyntaxError("vdf.parse: invalid syntax")
key = match.group('key') if match.group('qkey') is None else match.group('qkey')
val = match.group('val') if match.group('qval') is None else match.group('qval')
# we have a key with value in parenthesis, so we make a new dict obj (level deeper)
if val is None:
stack[-1][key] = mapper()
stack.append(stack[-1][key])
expect_bracket = True
# we've matched a simple keyvalue pair, map it to the last dict obj in the stack
else:
# if the value is line consume one more line and try to match again,
# until we get the KeyValue pair
if match.group('vq_end') is None and match.group('qval') is not None:
line += next(fp)
continue
stack[-1][key] = val
# exit the loop
break
if len(stack) != 1:
raise SyntaxError("vdf.parse: unclosed parenthasis or quotes")
return stack.pop()
