def getShare(self, shareID, r=None, asAdmin=False):
sharePath = h.makePath(self.sharesPath, shareID)
if not os.path.exists(sharePath): return None, None
lh, lf = None, h.makePath(h.LOCKS_FOLDER,
"_sfl_share%s" % h.clean(shareID))
try:
lh = h.getLockShared(lf, 5)
shareJson = h.loadJsonFile(sharePath)
files = shareJson.get("files", None)
if files is None:
files = shareJson.get("file", None)
if files is not None: files = [files]
s = share(shareJson["ID"], shareJson["creation"], files,
shareJson.get("views", []), shareJson.get("password"),
shareJson.get("duration", 0))
if not asAdmin and s.duration > 0 and s.duration + s.creation < h.now(
):
rs, rh = None, "Share has expired"
else:
rs, rh = s, "ok"
h.releaseLock(lh)
lh = None
if rs is None: return rs, rh
if r is not None: rs.tag = h.getURLParams(r.url).get("t", None)
return rs, rh
except:
le, lt = h.getLastExceptionAndTrace()
return None, le
finally:
if lh is not None: h.releaseLock(lh)
def processInput(inputFile, configMap):
print("Processing file: ", inputFile)
outputPath = helper.getPath(configMap["outputFolder"])
ptsFilename = Path(os.path.basename(inputFile)).resolve().stem
ptsOutputFilename = helper.getPath(
outputPath,
ptsFilename + configMap["outputFileSuffix"] + configMap["fileType"])
ptsFeedFilename = helper.getPath(
outputPath,
ptsFilename + configMap["feedFileSuffix"] + configMap["fileType"])
testProcedure = configMap["testProcedure"]
expectedValue = configMap["expectedValue"]
helper.makePath(outputPath)
data = pd.read_excel(inputFile,
sheet_name=configMap["sheetName"],
skiprows=int(configMap["startingRow"]))
df = pd.DataFrame(data, columns=[testProcedure, expectedValue])
pts = df[df[testProcedure].notnull()].reset_index()
testCases = {}
overallPts = []
helper.assembleTestCases(pts, testCases, overallPts, testProcedure,
expectedValue)
helper.createFile(ptsOutputFilename, overallPts)
instructions = []
helper.getInstructions(testCases, instructions, configMap)
helper.createFile(ptsFeedFilename, instructions)
def _saveTrackings(self):
if self.trackingsSaved: return
try:
self.trackingsLock.acquire()
trackingFile = h.makePath(self.basePath, ".tracking")
tmpTrackingFile = h.makePath(self.basePath, ".tracking.tmp")
h.logDebug("Saving trackings saved to file", len(self.trackings),
trackingFile)
headers, datas = [
"path", "authorized", "password", "ip", "date", "protected",
"location"
], []
for t in self.trackings:
datas.append([
t.path, t.authorized, t.password, t.ip, t.date,
t.protected, t.location
])
h.writeToCSV(datas, tmpTrackingFile, headers=headers, append=True)
h.delete(trackingFile)
os.rename(tmpTrackingFile, trackingFile)
if self.user is not None:
h.changeFileOwner(trackingFile, self.user)
self.trackingsSaved = True
h.logDebug("Trackings saved to file", len(self.trackings),
trackingFile)
except:
le, lt = h.getLastExceptionAndTrace()
h.logWarning("Can't save trackings", le)
finally:
self.trackingsLock.release()
def callbackReal(*args, **kwargs):
try:
passed, hint = self._firewall(request)
if not passed:
return Template(filename=h.makePath(
h.ROOT_FOLDER, "templates", "error.mako"),
lookup=self.tplLookup).render(
e=hint,
**self._makeBaseNamspace(),
le=None,
lt=None)
r = callback(*args, **kwargs)
if noCache: r = self._noCache(r)
return r
except Exception as e:
le, lt = h.getLastExceptionAndTrace()
print(le)
print(lt)
h.logWarning("Error processing request",
request.data.decode("utf-8"), request.endpoint,
h.formatException(e))
return Template(filename=h.makePath(h.ROOT_FOLDER, "templates",
"error.mako"),
lookup=self.tplLookup).render(
e=e,
**self._makeBaseNamspace(),
le=le,
lt=lt)
def getPasswords(self, path):
path = h.cleanPath(path)
passwordsFile = h.makePath(self.basePath, path, ".password")
if not os.path.exists(passwordsFile): return set()
passwordsCacheKey = h.makeKeyFromArguments(path)
lh, lf = None, h.makePath(h.LOCKS_FOLDER,
"_sfl_password_%s" % h.clean(path))
try:
self.passwordsLock.acquire()
if passwordsCacheKey in self.passwordsCache:
pc = self.passwordsCache[passwordsCacheKey]
if h.getFileModified(passwordsFile) == pc["date"]:
return pc["passwords"]
lh = h.getLockShared(lf, 5)
passwords = set([
p for p in h.readFromFile(
h.makePath(self.basePath, path, ".password")).split("\n")
if p != ""
])
self.passwordsCache[passwordsCacheKey] = {
"passwords": passwords,
"date": h.getFileModified(passwordsFile)
}
return passwords
finally:
self.passwordsLock.release()
if lh is not None: h.releaseLock(lh)
def addShare(self, shareID, paths, duration, password):
paths = [path.lstrip("/").rstrip("/") for path in paths]
sharePath = h.makePath(self.sharesPath, shareID)
password = "" if password is None else password
lh, lf = None, h.makePath(h.LOCKS_FOLDER,
"_sfl_share%s" % h.clean(shareID))
try:
lh = h.getLockExclusive(lf, 5)
s = share(shareID, h.now(), paths, [], password, duration)
h.writeJsonFile(
sharePath, {
"ID": s.ID,
"files": s.files,
"creation": s.creation,
"views": s.views,
"duration": s.duration,
"password": s.password
})
if self.user is not None: h.changeFileOwner(sharePath, self.user)
return s, "ok"
except:
le, lt = h.getLastExceptionAndTrace()
return None, le
finally:
if lh is not None: h.releaseLock(lh)
def removeShare(self, shareID):
sharePath = h.makePath(self.sharesPath, shareID)
if not os.path.exists(sharePath):
raise Exception("Unknown share", shareID)
lh, lf = None, h.makePath(h.LOCKS_FOLDER,
"_sfl_share%s" % h.clean(shareID))
try:
lh = h.getLockExclusive(lf, 5)
os.remove(sharePath)
return True
finally:
if lh is not None: h.releaseLock(lh)
def __init__(self,
ap: ap.authProvider,
basePath,
maxZipSize=50e6,
tmpFolder=None,
tmpFolderDuratioInDays=7,
user=None,
hiddenItems=None):
if hiddenItems is None: hiddenItems = []
self.hiddenItems = set(hiddenItems)
self.basePath = os.path.abspath(basePath)
self.maxZipSize = maxZipSize
self.tmpFolder = tmpFolder.lstrip("/")
self.tmpFolderDuration = tmpFolderDuratioInDays * 24 * 60 * 60
self.user = h.getUserID(user)
self.ap = ap
self.cleanTmpThread = None
if self.tmpFolder is not None:
tmpPath = h.makePath(self.basePath, self.tmpFolder)
if not os.path.exists(tmpPath):
h.makeDir(tmpPath)
if self.user is not None: h.changeFileOwner(tmpPath, self.user)
ap.setListingForbidden(self.tmpFolder)
ap.setDownloadForbidden(self.tmpFolder)
ap.setShowForbidden(self.tmpFolder)
ap.setShareForbidden(self.tmpFolder)
ap.setEditAllowed(self.tmpFolder)
self.cleanTmpThread = CleanTmp(tmpPath, self.tmpFolderDuration)
self.cleanTmpThread.start()
def __init__(self,
ip: ip.itemsProvider,
ap: ap.authProvider,
sp: sp.sharesProvider,
tp: tp.trackingProvider,
port,
ssl=False,
certificateKeyFile=None,
certificateCrtFile=None,
fullchainCrtFile="",
aliases=None,
maxUploadSize=None):
Thread.__init__(self)
self.app = Flask(__name__)
self.ip = ip
self.ap = ap
self.sp = sp
self.tp = tp
self.port = port
self.ssl = ssl
self.certificateKeyFile = certificateKeyFile
self.certificateCrtFile = certificateCrtFile
self.fullchainCrtFile = fullchainCrtFile
self.httpServer = None
self.aliases = aliases
self.maxUploadSize = maxUploadSize
self.firewallIPs = {}
self.firewallNbHits = 150
self.firewallWindowSize = 10
self.firewallLock = Lock()
if self.maxUploadSize is not None:
self.app.config["MAX_CONTENT_LENGTH"] = self.maxUploadSize
self._addRoute("/_hello", self._routeHello, ["GET"])
self._addRoute("/_infos", self._routeInfos, ["GET"])
self._addRoute("/_ping", self._routePing, ["GET"])
self._addRoute("/_list", self._routeList, ["GET"])
self._addRoute("/_list/<path:path>", self._routeList, ["GET"])
self._addRouteRaw("/", self._routeItems, ["GET", "POST"], "index")
self._addRouteRaw("/<path:path>",
self._routeItems, ["GET", "POST"],
noCache=True)
self._addRouteRaw("/_sf_assets/<path:path>", self._routeAssets,
["GET"])
self._addRouteRaw("/admin", self._routeAdmin, ["GET", "POST"])
self._addRouteRaw("/noadmin", self._routeNoAdmin, ["GET"])
self._addRouteRaw("/tracking", self._routeTrackingAdmin,
["GET", "POST"])
self._addRouteRaw("/share=<path:shareIDAndPath>", self._routeShare,
["GET", "POST"])
self._addRouteRaw("/shares", self._routeShares, ["GET", "POST"])
self._addRouteRaw("/remove-share=<shareID>", self._routeShareRemove,
["GET"])
self._addRouteRaw("/create-share=<paths>", self._routeShareAdd,
["GET", "POST"])
self.tplLookup = TemplateLookup(
directories=[h.makePath(h.ROOT_FOLDER, "templates")])
def addLeaf(self, path, file):
path = h.cleanPath(path)
try:
if not self.doesItemExists(path):
return False, "Container does not exists"
filename = self.getPotentialLeafName(file)
if self.isItemLeaf(h.makePath(path, filename)):
return False, "Item already exists"
if self.isItemContainer(h.makePath(path, filename)):
return False, "Item already exists"
filePath = h.makePath(self.basePath, path, filename)
file.save(filePath)
h.touchFile(filePath)
return True, filename
except:
le, lt = h.getLastExceptionAndTrace()
return False, le
def getLowerProtectedPath(self, path):
path = h.cleanPath(path)
relativePath = path
lowerPath = False
paths = relativePath.split("/")
for i in range(len(paths)):
currentPath = h.makePath(*paths[0:len(paths) - i])
if currentPath == "": currentPath = "/"
currentRealPath = os.path.abspath(
h.makePath(self.basePath, currentPath))
if os.path.isfile(h.makePath(currentRealPath, ".password")):
lowerPath = currentRealPath
break
if os.path.isfile(h.makePath(currentRealPath, ".nopassword")):
lowerPath = False
break
if lowerPath == False: return False
return h.cleanPath(lowerPath.replace(self.basePath, ""))
def addNewPassword(self, path, password):
path = h.cleanPath(path)
if password is None: return False
if self.passwordEditForbidden(path): return False
lh, lf = None, h.makePath(h.LOCKS_FOLDER,
"_sfl_password_%s" % h.clean(path))
try:
passwordFile = h.makePath(self.basePath, path, ".password")
requiredPasswords = list(self.getPasswords(path))
requiredPasswords.append(password)
lh = h.getLockExclusive(lf, 5)
h.writeToFile(passwordFile,
"\n".join(list(set(requiredPasswords))))
return True
except:
print(h.getLastExceptionAndTrace())
return False
finally:
if lh is not None: h.releaseLock(lh)
def saveShare(self, s: share):
lh, lf = None, h.makePath(h.LOCKS_FOLDER,
"_sfl_share%s" % h.clean(s.ID))
try:
lh = h.getLockExclusive(lf, 5)
sharePath = h.makePath(self.sharesPath, s.ID)
h.writeJsonFile(
sharePath, {
"ID": s.ID,
"files": s.files,
"creation": s.creation,
"views": s.views,
"duration": s.duration,
"password": s.password
})
if self.user is not None: h.changeFileOwner(sharePath, self.user)
return True
except:
le, lt = h.getLastExceptionAndTrace()
return False
finally:
if lh is not None: h.releaseLock(lh)
def getItem(self, path, r=None, asAdmin=False) -> item:
path = h.cleanPath(path)
if not self.doesItemExists(path): return None
fullPath = h.makePath(self.basePath, path)
isLeaf = self.isItemLeaf(path)
isContainer = not isLeaf
if r is not None:
protected, requiredPasswords, savedPassword, isAuthorized, lowerProtectedPath, protectedFromParent = self.ap.isAuthorized(
path, r)
else:
isAuthorized, requiredPasswords, protected, savedPassword, lowerProtectedPath, protectedFromParent = True, "", False, "", "", False
if asAdmin: isAuthorized = True
if isLeaf:
extension = os.path.splitext(path)[-1].replace(".", "")
size = os.path.getsize(fullPath)
nbItems = 0
isTmpFolder = False
else:
extension = ""
size = 0
nbItems = len(h.listDirectoryItems(fullPath))
isTmpFolder = path.lstrip("/") == self.tmpFolder
isForbidden = self.ap.isForbidden(path)
listingForbidden = self.ap.listingForbidden(path)
shareForbidden = self.ap.shareForbidden(path)
showForbidden = self.ap.showForbidden(path)
editAllowed = self.ap.isEditAllowed(path)
downloadForbidden = self.ap.downloadForbidden(path)
passwordEditForbidden = self.ap.passwordEditForbidden(path)
expires = h.getFileModified(h.makePath(self.basePath,
path)) + self.tmpFolderDuration
return item(path, os.path.basename(path),
os.stat(fullPath).st_mtime, isAuthorized, protected,
isForbidden, showForbidden, listingForbidden,
downloadForbidden, shareForbidden, passwordEditForbidden,
isTmpFolder, editAllowed, isLeaf, isContainer,
lowerProtectedPath, nbItems, requiredPasswords, expires,
size, extension, savedPassword, protectedFromParent)
def getZipFile(self, path, r):
path = h.cleanPath(path)
addedSize = 0
if not self.doesItemExists(path): return None
fullPath = self.getFullPath(path)
zipFilePath = h.makePath(h.TMP_FOLDER, h.uniqueID())
try:
zipf = zipfile.ZipFile(zipFilePath, "w", zipfile.ZIP_DEFLATED)
for root, dirs, files in os.walk(fullPath):
dirs[:] = [
d for d in dirs
if not self.ap.isForbidden(d.replace(self.basePath, ""))
]
dirs[:] = [
d for d in dirs
if self.ap.isAuthorized(d.replace(self.basePath, ""), r)[3]
]
dirs[:] = [
d for d in dirs if
not self.ap.downloadForbidden(d.replace(self.basePath, ""))
]
for f in files:
fpath = h.makePath(root, f)
frpath = fpath.replace(self.basePath, "")
if self.ap.isForbidden(frpath): continue
addedSize += h.getFileSize(fpath)
if addedSize > self.maxZipSize: break
zipf.write(fpath, arcname=frpath)
if addedSize > self.maxZipSize:
h.logDebug("Max zip file size reached", path,
self.maxZipSize)
break
return zipFilePath
except Exception as e:
os.remove(zipFilePath)
raise (e)
def _loadTrackings(self):
try:
self.trackingsLock.acquire()
trackingFile = h.makePath(self.basePath, ".tracking")
datas = h.readCSV(trackingFile)
for d in datas:
self.trackings.append(
tracking(d[0], d[2],
h.parseBool(d[1], False, trueValue="True"), d[3],
h.parseInt(d[4], 0),
h.parseBool(d[5], False, trueValue="True"), d[6]))
h.logDebug("Trackings loaded from file", trackingFile,
len(self.trackings))
except:
le, lt = h.getLastExceptionAndTrace()
h.logWarning("Can't load trackings", le)
finally:
self.trackingsLock.release()
def _doAddView(self, s: share, subPath, baseFile, address, tag,
isAuthorized):
views = s.views
view = {"date": h.now()}
if address is not None:
view["ip"] = address
view["location"] = self._getLocationFromIP(address)
else:
view["ip"] = "unk"
view["location"] = "unk"
if tag is not None: view["tag"] = tag
if subPath is not None:
view["item"] = h.makePath(baseFile if baseFile is not None else "",
subPath).rstrip("/")
view["authorized"] = isAuthorized
views.append(view)
views = sorted(views, key=lambda v: v["date"])[::-1]
s.views = views
self.saveShare(s)
def callbackReal(*args, **kwargs):
try:
passed, hint = self._firewall(request)
if not passed:
return Template(filename=h.makePath(
h.ROOT_FOLDER, "templates", "error.mako"),
lookup=self.tplLookup).render(
e=hint,
**self._makeBaseNamspace(),
le=None,
lt=None)
r = jsonify(callback(*args, **kwargs))
if noCache: r = self._noCache(r)
return r
except Exception as e:
h.logWarning("Error processing request",
request.data.decode("utf-8"), request.endpoint,
h.formatException(e))
return jsonify({"result": 500, "hint": h.formatException(e)})
def _makeTemplate(self, name, **data):
return Template(
filename=h.makePath(h.ROOT_FOLDER, "templates", "%s.mako" % name),
lookup=self.tplLookup).render(**data, **self._makeBaseNamspace())
def _routeShare(self, shareIDAndPath):
shareID = shareIDAndPath.split("/")[0]
query = str(request.query_string, "utf8")
if not self.sp.shareExists(shareID):
return self._makeTemplate("not-found", path=shareID)
if request.form.get("password-submit", False):
response = make_response()
self.ap.setSharePassword(shareID,
request.form.get("password-share", ""),
request, response)
return self._redirect(h.makeURL("/share=%s" % shareID, query),
response)
isAdmin = self.ap.isAdmin(request)
subPath = h.cleanPath(
os.path.normpath(shareIDAndPath.replace(shareID, ""))).lstrip(".")
s, hint = self.sp.getShare(shareID, asAdmin=isAdmin, r=request)
if s is None:
raise Exception("Can't get Share %s, %s" % (shareID, hint))
baseFilesIndexes = {
s.files[i].split("/")[-1]: i
for i in range(len(s.files))
}
subPathBits = subPath.split("/")
if not isAdmin and len(s.files) == 1 and subPath == "":
return self._redirect(
h.makeURL(
"/share=%s/%s" % (shareID, s.files[0].split("/")[-1]),
query))
elif subPath == "":
baseFile, indexFile = None, -1
else:
baseFile = subPathBits.pop(0)
indexFile = baseFilesIndexes.get(baseFile, -2)
isAuthorized, savedPassword = self.ap.isShareAuthorized(s, request)
if not isAdmin:
self.sp.addView(
s, h.makePath(*subPathBits), baseFile,
request.remote_addr if request is not None else None, s.tag,
isAuthorized)
if indexFile == -2:
return self._makeTemplate("not-found", path="%s" % shareID)
if isAdmin and request.args.get("view") is None:
return self._makeTemplate("share-admin", shareID=shareID, share=s)
if indexFile == -1: path, displayPath, shareBasePath = "", shareID, ""
else:
shareBasePath = s.files[indexFile]
path = h.cleanPath(h.makePath(shareBasePath, *subPathBits))
if baseFile is not None:
shareBasePath = shareBasePath.replace(baseFile, "")
displayPath = shareID
if baseFile is not None:
displayPath = h.makePath(displayPath, baseFile)
if subPath != "":
displayPath = h.makePath(displayPath, *subPathBits)
if h.TRACKING and not isAdmin:
self.tp.track(path, request, ap.isShareProtected(s), isAuthorized,
savedPassword, shareID, s.tag)
if not ip.doesItemExists(path):
return self._makeTemplate("not-found", path=displayPath)
if not isAdmin and not isAuthorized:
return self._makeTemplate("share-password",
displayPath=displayPath,
share=s)
if indexFile == -1:
isLeaf, readme = False, None
containers, leafs = [], []
for file in s.files:
item = ip.getItem(file)
if item is None: continue
item.path = item.path.split("/")[-1]
if ip.isItemContainer(file): containers.append(item)
else: leafs.append(item)
else:
isLeaf = ip.isItemLeaf(path)
if not isLeaf:
containers, leafs = self.ip.getItems(
path, overrideListingForbidden=True, overrideNoShow=True)
readme = ip.getReadme(path)
else:
containers, leafs, readme = None, None, None
if isLeaf: return send_from_directory(h.DATA_FOLDER, path)
else:
return self._makeTemplate("share",
displayPath=displayPath,
shareBasePath=shareBasePath,
subPath=subPath,
share=s,
containers=containers,
leafs=leafs,
alerts=[],
readme=readme,
indexFile=indexFile,
query=query)
def downloadForbidden(self, path):
path = h.cleanPath(path)
if path == "": return True
return h.isfile(h.makePath(self.basePath, path,
".nodownload")) or self.showForbidden(
path) or self.listingForbidden(path)
def _routeAssets(self, path):
return send_from_directory(h.makePath(h.ROOT_FOLDER, "assets"), path)
def setDownloadForbidden(self, path):
path = h.cleanPath(path)
h.writeToFile(h.makePath(self.basePath, path, ".nodownload"), "")
def shareForbidden(self, path):
path = h.cleanPath(path)
return h.isfile(h.makePath(self.basePath, path, ".noshare"))
###################################################################################
def stop(self):
self.httpServer.stop()
###################################################################################
# MAIN
###################################################################################
h.displaySplash()
ap = ap.authProvider(h.DATA_FOLDER, h.CONFIG.get("admin password", ""),
h.FORBIDEN_ITEMS)
h.logInfo("Auth provider built")
sp = sp.sharesProvider(h.makePath(h.DATA_FOLDER, "_sf_shares"),
user=h.USER,
locationEnabled=h.TRACKING_IP_GEOLOC,
locationAPIKey=h.CONFIG.get("ip geoloc api key", ""))
h.logInfo("Shares provider built")
ip = ip.itemsProvider(ap,
h.DATA_FOLDER,
tmpFolder=h.CONFIG.get("tmp folder", None),
tmpFolderDuratioInDays=h.CONFIG.get(
"tmp folder duration in days", 30),
user=h.USER,
hiddenItems=h.HIDDEN_ITEMS)
h.logInfo("Items provider built")
tp = tp.trackingProvider(h.DATA_FOLDER,
def getReadmeAdmin(self, path):
path = h.cleanPath(path)
readmeFile = h.makePath(self.getFullPath(path), "README.admin.md")
if not os.path.exists(readmeFile): return self.getReadme(path)
return markdown2.markdown(h.readFromFile(readmeFile))
def getFullPath(self, path):
path = h.cleanPath(path)
fullPath = os.path.abspath(h.makePath(self.basePath, path))
if not self._isFullPathWithinBaseFolder(fullPath): return self.basePath
return fullPath
def setShareForbidden(self, path):
path = h.cleanPath(path)
h.writeToFile(h.makePath(self.basePath, path, ".noshare"), "")
def shareExists(self, shareID):
return os.path.exists(h.makePath(self.sharesPath, shareID))
def passwordEditForbidden(self, path):
path = h.cleanPath(path)
return h.isfile(h.makePath(self.basePath, path, ".nopasswordedit"))
