def run(cls, targetURL):
instance, tableName, query = cls.__parseURL(targetURL)
nameField, contentField, fileExtension = cls._getTableInfo(tableName)
localDir = os.getcwd()
db = SnowFileDB(localDir)
client = SnowClient(tableName, instance)
records = client.getRecords(query)
for record in records:
recordName = getattr(record, nameField)
SLogger.debug("Downloading %s ... " % recordName)
fileName = "%s.%s" % (
getattr(record, nameField).replace(" ", "-").replace("/", "-").replace("(", "[").replace(")", "]"),
fileExtension,
)
filePath = os.path.abspath(os.path.join(localDir, fileName))
if os.path.exists(filePath):
good = False
suffix = 2
while not good:
filePathNew = filePath + "_" + str(suffix)
if not os.path.exists(filePathNew):
good = True
filePath = filePathNew
else:
suffix = suffix + 1
SLogger.warning("Local file %s already exists, will not overwrite" % filePath)
continue
try:
content = normalizeNewlines(getattr(record, contentField))
f = codecs.open(filePath, "w", "utf-8")
f.write(content)
f.close()
db.addFileInfo(
FileInfo(
fileName, instance, tableName, record.sys_id, nameField, contentField, record.sys_updated_on
)
)
SLogger.debug("Done. Written to %s" % fileName)
except Exception:
SLogger.debug("Problem with this record, will not write")
db.commitAndClose()
def run(cls, target):
if not os.path.exists(target):
sys.stderr.write("Unable to find file %s" % target)
sys.exit(1)
dirName, fileName = os.path.split(target)
dirName = os.path.abspath(os.path.normpath(dirName))
db = SnowFileDB(dirName)
fileInfo = db.getFileInfo(fileName)
if not fileInfo:
sys.stderr.write("File %s not found in %s" % (target, os.path.join(dirName, Config.getDBFilename())))
sys.exit(1)
client = SnowClient(fileInfo.getTableName(), fileInfo.getInstance())
record = client.get(fileInfo.getSysId())
recordName = record.sys_id
content = normalizeNewlines(getattr(record, fileInfo.getContentFieldName()))
if os.path.isfile(os.path.join(dirName, Config.getLockFilename())):
#swatch is watching, we do not want him to re-upload, we write a file for swatch.py to know
ignoreWatchFilePath = target + Config.getIgnoreWatchFilenameSuffix()
SLogger.debug("Creating file %s to avoid swatch to re-upload" % ignoreWatchFilePath)
ignoreWatchFile = open(ignoreWatchFilePath, "w")
ignoreWatchFile.close()
f = codecs.open(target, "w", "utf-8")
f.write(content)
db.setUpdatedOn(fileName, record.sys_updated_on)
db.commitAndClose()
SLogger.debug("Updated record %s to file %s. set updated_on to %s" % (recordName, fileName, record.sys_updated_on))
def run(cls, targetURL, *extraArgs):
instance, tableName, sysId = cls.__parseURL(targetURL)
nameField, contentField, fileExtension = cls._getTableInfo(tableName)
localDir = os.getcwd()
db = SnowFileDB(localDir)
client = SnowClient(tableName, instance)
record = client.get(sysId)
if len(extraArgs) > 0:
fileName = extraArgs[0]
else:
fileName = "%s.%s" % (
getattr(record, nameField).replace(" ", "-").replace("/", "-").replace("(", "[").replace(")", "]"),
fileExtension,
)
filePath = os.path.join(localDir, fileName)
if os.path.exists(filePath):
SLogger.warning("Local file %s already exists, will not overwrite" % filePath)
return
# if os.path.isfile(os.path.join(localDir, Config.getLockFilename())):
# #swatch is watching, we do not want him to re-upload, we write a file for swatch.py to know
# ignoreWatchFilePath = filePath + Config.getIgnoreWatchFilenameSuffix()
# SLogger.debug("Creating file %s to avoid swatch to re-upload" % ignoreWatchFilePath)
# ignoreWatchFile = open(ignoreWatchFilePath, "w")
# ignoreWatchFile.close()
f = codecs.open(filePath, "w", "utf-8")
content = normalizeNewlines(getattr(record, contentField))
f.write(content)
f.close()
db.addFileInfo(
FileInfo(fileName, instance, tableName, record.sys_id, nameField, contentField, record.sys_updated_on)
)
db.commitAndClose()
SLogger.debug("Done. Written to %s" % fileName)
def run(cls, src, dst):
src = os.path.abspath(os.path.normpath(src))
dst = os.path.abspath(os.path.normcase(dst))
if src == dst:
return
if not os.path.exists(src):
sys.stderr.write("File %s does not exist\n" % src)
sys.exit(1)
if os.path.exists(dst):
sys.stderr.write("File %s already exists\n" % dst)
sys.exit(1)
sourceDir, sourceFilename = os.path.split(src)
if not SnowFileDB.existsAtDir(sourceDir):
sys.stderr.write("DB file not found at %s, aborting\n" % os.path.join(sourceDir, Config.getDBFilename()))
sys.exit(1)
destDir, destFilename = os.path.split(dst)
if sourceDir != destDir:
sourceDB = SnowFileDB(sourceDir)
destDB = SnowFileDB(destDir)
fileInfo = sourceDB.getFileInfo(sourceFilename)
if not fileInfo:
sys.stderr.write("File %s not present at DB file %s\n" % (sourceFilename, sourceDB.getDBFilePath()))
sys.exit(1)
sourceDB.deleteFile(sourceFilename)
fileInfo.setFileName(destFilename)
destDB.addFileInfo(fileInfo)
shutil.move(src, dst)
sourceDB.commitAndClose()
destDB.commitAndClose()
else:
db = SnowFileDB(sourceDir)
sucesss = db.renameFile(sourceFilename, destFilename)
if not sucesss:
sys.stderr.write("File %s not present at DB file %s\n" % (sourceFilename, db.getDBFilePath()))
sys.exit(1)
shutil.move(src, dst)
db.commitAndClose()
SLogger.debug("File successfully moved from %s to %s\n" % (src, dst))
def run(cls, targetFiles):
for targetFile in targetFiles:
if not os.path.isfile(targetFile):
SLogger.warning("File %s does not exist, ignoring" % os.path.abspath(targetFile))
continue
SLogger.debug("Reading target file %s" % os.path.abspath(targetFile))
loadedParams = {}
execfile(targetFile, globals(), loadedParams)
instance = loadedParams.get("instance", None)
if not instance:
SLogger.warning("No instance declared in target file %s, aborting" % targetFile)
sys.exit(1)
if not instance.endswith("/"):
instance = instance + "/"
for i, target in enumerate(loadedParams["targets"]):
# we extract the table name
table = target.get("table", None)
if not table:
SLogger.warning("No table in target number %d of file %s, ignoring" % (i, targetFile))
continue
# we extract the name field, content field, and how to rename files
tableInfo = cls.__getTableInfo(table, target)
if not tableInfo:
continue
nameField, contentField, transformName = tableInfo
# we extract the filtering functions
query = target.get("query", "")
recordFilter = target.get("recordFilter", lambda x: True)
# we extract the local target directory
localDir = target.get("localDir", None)
if not localDir:
SLogger.debug(
"No localDir in target number %d of file %s, using current directory" % (i, targetFile)
)
localDir = os.getcwd()
if not os.path.isdir(localDir):
SLogger.warning("Local directory %s does not exist, ignoring target %d" % (localDir, i))
continue
# if everything is well, we start downloading and writing
db = SnowFileDB(localDir)
client = SnowClient(table, instance)
records = client.getRecords(query)
for record in records:
if recordFilter(record):
recordName = getattr(record, nameField)
SLogger.debug("Downloading %s ... " % recordName)
fileName = transformName(recordName)
filePath = os.path.abspath(os.path.join(localDir, fileName))
if os.path.exists(filePath):
good = False
suffix = 2
while not good:
filePathNew = filePath + "_" + str(suffix)
if not os.path.exists(filePathNew):
good = True
filePath = filePathNew
else:
suffix = suffix + 1
try:
content = normalizeNewlines(getattr(record, contentField))
f = codecs.open(filePath, "w", "utf-8")
f.write(content)
f.close()
db.addFileInfo(
FileInfo(
fileName,
instance,
table,
record.sys_id,
nameField,
contentField,
record.sys_updated_on,
)
)
SLogger.debug("Done. Written to %s" % fileName)
except Exception:
SLogger.debug("Problem with this record, will not write")
db.commitAndClose()