def get(self):
# ATTENTION: Tree debug is quite spammy and deactivated by default. If you want to debug it, remove the silenced=True parameter
log = Logger('directories.get', True)
directoryStructure = {'name': '', 'dirs': [], 'files': []}
treeBase = request.args.get('base')
log.logDebug('========== GET TREE =========')
log.logDebug('os.getcwd:', os.getcwd())
log.logDebug('base:', treeBase)
log.logDebug('==========')
if (treeBase and not os.path.exists(
PathUtils.sanitizePathList([basePath, treeBase]))):
abort(400, 'given treeBase is not part of the dir structure')
for (dirpath, dirnames, filenames) in os.walk(basePath):
filePath = PathUtils.sanitizePathString(dirpath, True)
path = filePath.split('/')
log.logDebug('starting with', path, dirpath)
if (metaSubPath in dirnames):
log.logDebug(
'removing metaSubPath from dirnames to exclude them from walking'
)
dirnames.remove(metaSubPath) # do not visit metaSubPaths
if (len(path) == 0 or metaSubPath in path):
log.logDebug('skipping path', path)
continue # skip meta paths
if (treeBase and len(path) > 1 and treeBase not in path):
log.logDebug('skipping path since its not part of treeBase',
path)
continue
if len(path) == 1 and path[0] == '':
currDir = directoryStructure
log.logDebug(
'Got tree root, using directory Structure directly')
else:
log.logDebug('walking for', dirpath, path)
currDir = self.walkToDir(directoryStructure, path)
if currDir == None:
log.logWarn('Could not find dir for path, skipping',
dirpath)
continue
log.logDebug('crawling content', currDir, filePath)
self.crawlDirContent(dirnames, filenames, currDir, filePath)
if request.args.get('root_only'):
break
result = directoryStructure
if (treeBase):
result = next(
(x
for x in directoryStructure['dirs'] if x['name'] == treeBase),
result)
return json.dumps(result)
def test_sanitizeFilename(self):
testData = {
"aCompletely_ValidFilename-that-needs.no.substitution":
"aCompletely_ValidFilename-that-needs.no.substitution",
"one_invalid_character": "one?invalid_character",
"_a__lot_of_invalid_characters_": "<a//lot?of*invalid'characters>",
"": ""
}
self.assertEqual(PathUtils.sanitizeFilename(None), '')
for result, testData in testData.items():
self.assertEqual(PathUtils.sanitizeFilename(testData), result)
def addMappingToImageFile(self, image_id, dir_path):
pathParts = [basePath, metaSubPath]
imageMapPath = PathUtils.sanitizePathList(pathParts)
Path(imageMapPath).mkdir(parents=True, exist_ok=True)
imageMapFile = PathUtils.sanitizePathList(
[imageMapPath, imagesFileName])
with open(imageMapFile, 'a+', encoding='utf-8') as file:
newLine = f'{image_id} "{dir_path}"\n'
self.log.logDebug(f'adding new entry to image map: {newLine}')
file.write(newLine)
file.close()
def put(self, project_dir):
pathToLinks = PathUtils.sanitizePathList(
[basePath, project_dir, metaSubPath])
Path(pathToLinks).mkdir(parents=True, exist_ok=True)
filePath = PathUtils.sanitizePathList([pathToLinks, linkFileName])
try:
f = request.files['file']
except:
abort(400, 'No or invalid file given')
f.save(filePath)
fileRead = open(filePath, encoding='utf-8')
return fileRead.read()
def delete(self, dir_name):
if request.args.get('dir_path'):
pathToDelete = PathUtils.sanitizePathList(
[basePath, request.args.get('dir_path')])
else:
pathToDelete = basePath
dirPath = PathUtils.sanitizePathList([pathToDelete, dir_name])
self.log.logDebug('will delete directory', dirPath)
if not os.path.exists(dirPath):
abort(400, 'Dir for deletion was not found')
shutil.rmtree(dirPath)
return {}
def put(self, dir_name):
self.log.logDebug('put called', dir_name)
dir_name = PathUtils.sanitizeFilename(dir_name)
if not request.form or (not request.form['doc_path']
and not request.form['root_dir']):
self.log.logWarn('directories put was called with invalid data.')
abort(400, 'Invalid Parameters')
newPath = PathUtils.sanitizePathList(
[basePath, request.form['doc_path'], dir_name])
self.log.logDebug('saving path', newPath)
Path(newPath).mkdir()
self.log.logDebug(
'mkdir:', {'path': PathUtils.sanitizePathString(newPath, True)})
return {'path': PathUtils.sanitizePathString(newPath, True)}
def put(self, doc_name):
if request.form['doc_path']:
pathToSaveTo = PathUtils.sanitizePathList(
[basePath, request.form['doc_path']])
Path(pathToSaveTo).mkdir(parents=True, exist_ok=True)
else:
pathToSaveTo = basePath
name = PathUtils.sanitizeFilename(doc_name)
filePath = PathUtils.sanitizePathList([pathToSaveTo, name])
try:
f = request.files['file']
except:
abort(400, 'No or invalid file given')
f.save(filePath)
return self.getResponseObject(open(filePath, encoding='utf-8'))
def getLabelValPath(self, label_id, value_id, projectDir):
if label_id == 'definitions':
filename = label_def_filename
else:
filename = label_value_prefix + value_id
return PathUtils.sanitizePathList(
[self.getPathToLabels(projectDir), filename])
def setGitConfig(self):
self.log.logInfo('setting git config ...')
pathToGitConfig = PathUtils.sanitizePathList(['.git/config'])
f = open(pathToGitConfig, 'a')
f.write('[user]\n')
f.write('\tname = Writerey\n')
f.write('\temail = writerey@app\n')
f.close()
def getResponseObject(self, file):
fstats = os.stat(file.name)
return {
'path': PathUtils.sanitizePathString(os.path.dirname(file.name),
True),
'name': os.path.basename(file.name),
'last_edited': fstats[ST_MTIME]
}
def get(self, doc_name):
try:
path = PathUtils.sanitizePathList(
[os.getcwd(), self.getPathForImageId(doc_name)])
self.log.logDebug(f'sending file {doc_name} from {path} ....')
return send_from_directory(path, filename=doc_name)
except FileNotFoundError:
abort(404)
except OSError:
abort(500)
def delete(self, doc_name):
if request.args.get('doc_path'):
pathToDelete = PathUtils.sanitizePathList(
[basePath, request.args.get('doc_path')])
else:
pathToDelete = basePath
filePath = PathUtils.sanitizePathList([pathToDelete, doc_name])
if not os.path.exists(filePath):
abort(400, 'File for deletion was not found')
os.remove(filePath)
meta_path = PathUtils.sanitizePathList(
[pathToDelete, metaSubPath, doc_name])
metaExists = os.path.exists(meta_path)
if (metaExists):
shutil.rmtree(meta_path)
return {}
def put(self, doc_name):
context = request.form['context']
if request.form['doc_path']:
pathParts = [
basePath, request.form['doc_path'], metaSubPath, doc_name
]
else:
pathParts = [basePath, metaSubPath, doc_name]
dirPath = PathUtils.sanitizePathList(pathParts)
Path(dirPath).mkdir(parents=True, exist_ok=True)
pathToSaveTo = PathUtils.sanitizePathList([dirPath, context])
try:
f = request.files['file']
f.save(pathToSaveTo)
savedF = open(pathToSaveTo, 'r', encoding='utf-8')
newContent = savedF.read()
return newContent
except:
self.log.logWarn('put paragraph meta failed')
abort(500)
def test_sanitizePathString(self):
testData = {
"this/is/a/path": "this/is/a/path",
"this/is/a/pathWithDouble/Slashes":
"this/is/a///pathWithDouble//Slashes",
"path/With/LeadingAndEnding/Slash":
"/path/With/LeadingAndEnding/Slash/",
"windowsstyled/path/withBackslashes":
"\\windowsstyled\\path\\withBackslashes",
"": ""
}
for result, testData in testData.items():
self.assertEqual(PathUtils.sanitizePathString(testData), result)
def post(self, doc_name):
imgId = str(uuid.uuid4().hex[:8])
path = request.headers['docpath']
if path:
pathParts = [basePath, path, metaSubPath, doc_name]
else:
pathParts = [basePath, metaSubPath, doc_name]
dirPath = PathUtils.sanitizePathList(pathParts)
Path(dirPath).mkdir(parents=True, exist_ok=True)
f = request.files['upload']
name = PathUtils.sanitizeFilename(f.filename)
name = f'{imgId}_{name}'
filePath = PathUtils.sanitizePathList([dirPath, name])
self.log.logDebug(f'saving new image in {filePath}')
f.save(filePath)
self.addMappingToImageFile(name, dirPath)
return {"url": self.getUrlForImage(doc_name, name, path)}
def get(self, project):
path = PathUtils.sanitizePathList(
[basePath, project, metaSubPath, labelPath])
try:
arr = os.listdir(path)
except:
abort(400, 'No or invalid project given')
# remove label def and strip file prefix from label value ids
arr = list(filter(lambda filename: filename != label_def_filename,
arr))
arr = list(
map(lambda filename: filename.replace(label_value_prefix, ''),
arr))
return arr
def get(self, project_dir):
try:
linkFile = PathUtils.sanitizePathList(
[basePath, project_dir, metaSubPath, linkFileName])
f = open(linkFile, encoding='utf-8')
content = f.read()
return content
except FileNotFoundError:
print('get links file not found')
abort(404)
except OSError as err:
self.log.logError('get links failed with OSError', err)
abort(500)
def test_concatPathParts(self):
testData = {
"this/is/a/path": ["this", "is", "a", "path"],
"this/is//a///pathWithDouble/Slashes":
["this", "is//", "a", "///pathWithDouble/", "/Slashes/"],
"path/With/emptyElement": ["path", "With", "", "emptyElement"],
"incomplete/splitted/path/list":
["incomplete", "splitted/path/list"],
"incomplete/splitted\\windows/path\\list":
["incomplete", "splitted\\windows/path\\list"],
"": [""],
"": []
}
for result, testList in testData.items():
self.assertEqual(PathUtils.concatPathParts(testList), result)
def get(self, doc_name):
path = request.args.get('doc_path')
context = request.args.get('context')
try:
path = PathUtils.sanitizePathList(
[basePath, path, metaSubPath, doc_name, context])
f = open(path, encoding='utf-8')
content = f.read()
return content
except FileNotFoundError:
self.log.logInfo(
'paragraph meta file not found, return empty string', context)
return ''
except OSError as err:
self.log.logError('get paragraph meta failed', err)
abort(500)
def get(self, doc_name):
try:
path = PathUtils.sanitizePathList(
[basePath, request.args.get('doc_path'), doc_name])
f = open(path, encoding='utf-8')
response = self.getResponseObject(f)
if request.args.get('with_content'):
content = f.read()
response['content'] = content
return response
except FileNotFoundError:
self.log.logWarn('document not found', doc_name)
abort(404)
except OSError:
self.log.logWarn('OSError on getting document', doc_name)
abort(500)
def updateImageLinks(oldPath, newPath):
log = Logger('updateImageLinks')
imageMapPath = PathUtils.sanitizePathList(
[basePath, metaSubPath, imagesFileName])
# Get content of image map file
file_handle = open(imageMapPath, 'r', encoding='utf-8')
file_string = file_handle.read()
file_handle.close()
regex = f'(.+? ")({oldPath})(.*"\n)'
log.logDebug('trying to find and replace oldPath in image file', oldPath)
file_string = re.sub(regex, r'\1' + newPath + r'\3', file_string)
# Overwrite image map file with updated content
file_handle = open(imageMapPath, 'w')
file_handle.write(file_string)
file_handle.close()
return True
def getPathForImageId(self, image_id):
try:
imageMapPath = PathUtils.sanitizePathList(
[basePath, metaSubPath, imagesFileName])
file = open(imageMapPath, 'r', encoding='utf-8')
pathForImage = re.search(f'{image_id} "(.*)"\n', file.read())
except FileNotFoundError:
self.log.logWarn(
'Could not find image map file, cannot resolve image links!')
abort(500)
if not pathForImage:
self.log.logWarn(f'could not resolve path for image {image_id}')
abort(410,
'could not resolve image_id to path, no mapping available')
self.log.logDebug(
f'resolved for image {image_id} path {pathForImage.group(1)}')
return pathForImage.group(1)
def moveViaGit(self,
doc_path,
doc_name,
new_doc_path,
new_name,
msg,
projectDir=None):
old_path = PathUtils.sanitizePathList([doc_path, doc_name])
new_path = PathUtils.sanitizePathList([new_doc_path, new_name])
old_meta_path = PathUtils.sanitizePathList(
[doc_path, metaSubPath, doc_name])
new_meta_path = PathUtils.sanitizePathList(
[new_doc_path, metaSubPath, new_name])
metaExists = path.exists(
PathUtils.sanitizePathList([basePath, old_meta_path]))
self.logger.logDebug('prepare moving file ...', old_path, new_path)
self.logger.logDebug('check if moving meta ...', metaExists)
if metaExists:
self.logger.logDebug('prepare moving meta ...', old_meta_path,
new_meta_path)
try:
self.git.run(['git', 'ls-files', '--error-unmatch', old_path])
except:
self.logger.logDebug(
'file is not under version control yet, do a file system move')
shutil.move(PathUtils.sanitizePathList([basePath, old_path]),
PathUtils.sanitizePathList([basePath, new_path]))
if (metaExists and new_name):
self.logger.logDebug(
'meta exists, moving meta via file system move')
shutil.move(
PathUtils.sanitizePathList([basePath, old_meta_path]),
PathUtils.sanitizePathList([basePath, new_meta_path]))
self.logger.logDebug('updated file, updating image links....')
updateImageLinks(
PathUtils.sanitizePathList([basePath, old_meta_path]),
PathUtils.sanitizePathList([basePath, new_meta_path]))
return
if not msg:
msg = 'Rename ' + old_path + ' to ' + new_path
self.git.run(["git", "mv", old_path, new_path])
if metaExists and new_name:
self.git.run(["git", "mv", old_meta_path, new_meta_path])
if projectDir:
# add links file to the commit, since it possibly get changed on a move
linksFilePath = PathUtils.sanitizePathList(
[projectDir, metaSubPath, linksFileName])
if path.exists(linksFilePath):
self.git.run(["git", "add", linksFilePath])
updateImageLinks(PathUtils.sanitizePathList([basePath, old_meta_path]),
PathUtils.sanitizePathList([basePath, new_meta_path]))
# add image file to the commit, since it possibly get changed on a move
try:
imageMapPath = PathUtils.sanitizePathList(
[metaSubPath, imagesFileName])
self.logger.logDebug(
f'adding image map to commit ... {imageMapPath} ')
self.git.run(["git", "add", imageMapPath])
except:
self.logger.logWarn('could not add image map to commit')
pass
self.git.run(["git", "commit", "-m", msg])
def getPathToLabels(self, projectDir=''):
return PathUtils.sanitizePathList(
[basePath, projectDir, metaSubPath, labelPath])