def run(cls, target): if not os.path.exists(target): sys.stderr.write("Unable to find file %s" % target) sys.exit(1) dirName, fileName = os.path.split(target) dirName = os.path.abspath(os.path.normpath(dirName)) db = SnowFileDB(dirName) fileInfo = db.getFileInfo(fileName) if not fileInfo: sys.stderr.write("File %s not found in %s" % (target, os.path.join(dirName, Config.getDBFilename()))) sys.exit(1) client = SnowClient(fileInfo.getTableName(), fileInfo.getInstance()) record = client.get(fileInfo.getSysId()) recordName = record.sys_id content = normalizeNewlines(getattr(record, fileInfo.getContentFieldName())) if os.path.isfile(os.path.join(dirName, Config.getLockFilename())): #swatch is watching, we do not want him to re-upload, we write a file for swatch.py to know ignoreWatchFilePath = target + Config.getIgnoreWatchFilenameSuffix() SLogger.debug("Creating file %s to avoid swatch to re-upload" % ignoreWatchFilePath) ignoreWatchFile = open(ignoreWatchFilePath, "w") ignoreWatchFile.close() f = codecs.open(target, "w", "utf-8") f.write(content) db.setUpdatedOn(fileName, record.sys_updated_on) db.commitAndClose() SLogger.debug("Updated record %s to file %s. set updated_on to %s" % (recordName, fileName, record.sys_updated_on))
def run(cls, targetURL): instance, tableName, query = cls.__parseURL(targetURL) nameField, contentField, fileExtension = cls._getTableInfo(tableName) localDir = os.getcwd() db = SnowFileDB(localDir) client = SnowClient(tableName, instance) records = client.getRecords(query) for record in records: recordName = getattr(record, nameField) SLogger.debug("Downloading %s ... " % recordName) fileName = "%s.%s" % ( getattr(record, nameField).replace(" ", "-").replace("/", "-").replace("(", "[").replace(")", "]"), fileExtension, ) filePath = os.path.abspath(os.path.join(localDir, fileName)) if os.path.exists(filePath): good = False suffix = 2 while not good: filePathNew = filePath + "_" + str(suffix) if not os.path.exists(filePathNew): good = True filePath = filePathNew else: suffix = suffix + 1 SLogger.warning("Local file %s already exists, will not overwrite" % filePath) continue try: content = normalizeNewlines(getattr(record, contentField)) f = codecs.open(filePath, "w", "utf-8") f.write(content) f.close() db.addFileInfo( FileInfo( fileName, instance, tableName, record.sys_id, nameField, contentField, record.sys_updated_on ) ) SLogger.debug("Done. Written to %s" % fileName) except Exception: SLogger.debug("Problem with this record, will not write") db.commitAndClose()
def run(cls, targetURL, *extraArgs): instance, tableName, sysId = cls.__parseURL(targetURL) nameField, contentField, fileExtension = cls._getTableInfo(tableName) localDir = os.getcwd() db = SnowFileDB(localDir) client = SnowClient(tableName, instance) record = client.get(sysId) if len(extraArgs) > 0: fileName = extraArgs[0] else: fileName = "%s.%s" % ( getattr(record, nameField).replace(" ", "-").replace("/", "-").replace("(", "[").replace(")", "]"), fileExtension, ) filePath = os.path.join(localDir, fileName) if os.path.exists(filePath): SLogger.warning("Local file %s already exists, will not overwrite" % filePath) return # if os.path.isfile(os.path.join(localDir, Config.getLockFilename())): # #swatch is watching, we do not want him to re-upload, we write a file for swatch.py to know # ignoreWatchFilePath = filePath + Config.getIgnoreWatchFilenameSuffix() # SLogger.debug("Creating file %s to avoid swatch to re-upload" % ignoreWatchFilePath) # ignoreWatchFile = open(ignoreWatchFilePath, "w") # ignoreWatchFile.close() f = codecs.open(filePath, "w", "utf-8") content = normalizeNewlines(getattr(record, contentField)) f.write(content) f.close() db.addFileInfo( FileInfo(fileName, instance, tableName, record.sys_id, nameField, contentField, record.sys_updated_on) ) db.commitAndClose() SLogger.debug("Done. Written to %s" % fileName)
def run(cls, targetFiles): for targetFile in targetFiles: if not os.path.isfile(targetFile): SLogger.warning("File %s does not exist, ignoring" % os.path.abspath(targetFile)) continue SLogger.debug("Reading target file %s" % os.path.abspath(targetFile)) loadedParams = {} execfile(targetFile, globals(), loadedParams) instance = loadedParams.get("instance", None) if not instance: SLogger.warning("No instance declared in target file %s, aborting" % targetFile) sys.exit(1) if not instance.endswith("/"): instance = instance + "/" for i, target in enumerate(loadedParams["targets"]): # we extract the table name table = target.get("table", None) if not table: SLogger.warning("No table in target number %d of file %s, ignoring" % (i, targetFile)) continue # we extract the name field, content field, and how to rename files tableInfo = cls.__getTableInfo(table, target) if not tableInfo: continue nameField, contentField, transformName = tableInfo # we extract the filtering functions query = target.get("query", "") recordFilter = target.get("recordFilter", lambda x: True) # we extract the local target directory localDir = target.get("localDir", None) if not localDir: SLogger.debug( "No localDir in target number %d of file %s, using current directory" % (i, targetFile) ) localDir = os.getcwd() if not os.path.isdir(localDir): SLogger.warning("Local directory %s does not exist, ignoring target %d" % (localDir, i)) continue # if everything is well, we start downloading and writing db = SnowFileDB(localDir) client = SnowClient(table, instance) records = client.getRecords(query) for record in records: if recordFilter(record): recordName = getattr(record, nameField) SLogger.debug("Downloading %s ... " % recordName) fileName = transformName(recordName) filePath = os.path.abspath(os.path.join(localDir, fileName)) if os.path.exists(filePath): good = False suffix = 2 while not good: filePathNew = filePath + "_" + str(suffix) if not os.path.exists(filePathNew): good = True filePath = filePathNew else: suffix = suffix + 1 try: content = normalizeNewlines(getattr(record, contentField)) f = codecs.open(filePath, "w", "utf-8") f.write(content) f.close() db.addFileInfo( FileInfo( fileName, instance, table, record.sys_id, nameField, contentField, record.sys_updated_on, ) ) SLogger.debug("Done. Written to %s" % fileName) except Exception: SLogger.debug("Problem with this record, will not write") db.commitAndClose()