def store(fp):
realFilename = fp.filename
md5sum = _hash_file(fp, hashlib.md5())
sha1sum = _hash_file(fp, hashlib.sha1())
fp.seek(0)
fileData = model.File.query.filter(model.File.MD5Sum == md5sum,
model.File.SHA1Sum == sha1sum).first()
if not fileData:
while True:
newFilename = common.generate_random_string(32)
if not os.path.exists(
os.path.join(app.config['UPLOAD_FULL_DIRECTORY'],
newFilename)):
break
fullPath = os.path.join(app.config['UPLOAD_FULL_DIRECTORY'],
newFilename)
fp.save(fullPath)
fileSize = os.stat(fullPath).st_size
fileData = model.File(
os.path.join(app.config["UPLOAD_DIRECTORY"], newFilename), md5sum,
sha1sum, fileSize)
db.session.add(fileData)
db.session.commit()
return fileData
def post(self):
assert util.development() or oauth.is_current_user_admin()
util.log_upload_data(self.request.path, self.request.get("data"))
files_added = {}
data = StringIO.StringIO(self.request.get("data"))
for line in data:
data = json.loads(line)
# We first load the fileset into the database
# For use later, we also add a list of filenames in the fileset
f = model.FileSet(key_name=data["name"],
display_name=data["name"],
files=data["setfiles"])
f.put()
for filename in data["setfiles"]:
if filename not in files_added:
files_added[filename] = [data["name"]]
else:
files_added[filename].append(data["name"])
# We now update the database with the elements in files_added
for filename in files_added:
# TODO: Is there a better way of assigning display names?
split_index = filename.rfind("_")
model.File(key_name=filename,
display_name=filename[:split_index],
file_sets=files_added[filename]).put()
model.filesets().invalidate()
model.files().invalidate()
def setUp(self):
self.model = model.Model()
fileModel = model.File("", self.model)
self.stream = self.model.streams.addFile(fileModel)
self.stromxStream = self.stream.stromxStream
self.factory = stromx.runtime.Factory()
stromx.register('libstromx_cvsupport.so.0.8.0', self.factory)
def setUp(self):
settings.DB_LOCATION = "sqlite:///:memory:"
self.setUpPyfakefs()
self.session = model.get_db_session()
self.path_1 = "/home"
self.directory_1 = model.Directory(path=self.path_1)
self.session.add(self.directory_1)
self.path_2 = "/var"
self.directory_2 = model.Directory(path=self.path_2)
self.session.add(self.directory_2)
self.session.commit()
self.file_1_name = "file_1"
self.file_1_content = "test"
self.fake_file_1 = self.fs.create_file(
f"{self.path_1}/{self.file_1_name}", contents=self.file_1_content)
self.db_file_1 = model.File(
name=self.file_1_name,
size=len(self.file_1_content),
directory_id=self.directory_1.id,
)
self.session.add(self.db_file_1)
self.file_2_name = "file_2"
self.file_2_content = "test"
self.fake_file_2 = self.fs.create_file(
f"{self.path_1}/{self.file_2_name}")
self.db_file_2 = model.File(
name=self.file_2_name,
size=len(self.file_2_content),
directory_id=self.directory_1.id,
)
self.session.commit()
def post(self):
res = entities.check_token(request.headers["Authorization"])
if res == None:
raise Exception("invalid token")
user, group = res
if group != entities.group_student:
uuid = user
else:
stu_obj = model.Student.query.filter_by(username=user).first()
uuid = stu_obj.project.uuid
parser = reqparse.RequestParser()
parser.add_argument(
"file",
type=werkzeug.datastructures.FileStorage,
location="files",
required=True,
)
parser.add_argument("type", required=True)
args = parser.parse_args()
f = args["file"]
fn = f.filename
if entities.filename_validation(fn) == False:
return err.upload_error
fn = entities.make_unique(fn)
fn = werkzeug.utils.secure_filename(fn)
folder = os.path.join(
os.path.abspath(__file__ + "/.."),
"../..",
entities.config.upload_path,
uuid,
)
if not os.path.exists(folder):
os.mkdir(folder)
path = os.path.join(folder, fn)
f.save(path)
fn = os.path.join(entities.config.url_prefix, f"./{uuid}/", fn)
db_f = None
if args["type"] == "file":
db_f = model.File(location=fn)
elif args["type"] == "image":
db_f = model.Image(path=fn, description="")
else:
raise Exception("Type not define")
model.db.session.add(db_f)
model.db.session.commit()
return {"status": "success", "link": fn}
def store_local(normalizedPath):
normalizedFullPath = os.path.join(app.config["UPLOAD_BASE_DIR"],
normalizedPath)
if not os.path.isfile(normalizedFullPath):
return json.dumps({
"result": False,
"message": "Given path is not a file"
})
# We don't check hashsum of local files since it has the possibility of modifying its content,
# which means hashing is meaningless, and has the possibility of pointing a File object
# that has same hashsum but different File.StoredPath.
fileSize = os.stat(normalizedFullPath).st_size
fileData = model.File(normalizedPath, "1", "1", fileSize)
db.session.add(fileData)
db.session.commit()
return fileData
def process_file(directory: model.Directory, file: pathlib.Path,
session: Session) -> None:
"""
If the file doesn't exist in the db, upload it and add it to the db.
If the file exists in the db, upload any new content and update its size.
"""
db_file = (session.query(model.File).filter_by(directory_id=directory.id,
name=file.name).first())
if db_file is None:
upload_logs(file=file, start_pos=0)
new_db_file = model.File(name=file.name,
size=file.stat().st_size,
directory_id=directory.id)
session.add(new_db_file)
session.commit()
elif file.stat().st_size > db_file.size:
upload_logs(file=file, start_pos=db_file.size)
db_file.size = file.stat().st_size
session.add(db_file)
session.commit()
#!/usr/bin/env python3.6 import os import config import model c = config.Config() if not c.parse_args(): os._exit(1) f = model.File(c.index_path) f.load(c) f.gen(c) os._exit(0)