Beispiel #1
0
    def run(cls, target):
        if not os.path.exists(target):
            sys.stderr.write("Unable to find file %s" % target)
            sys.exit(1)

        dirName, fileName = os.path.split(target)
        dirName = os.path.abspath(os.path.normpath(dirName))

        db = SnowFileDB(dirName)
        fileInfo = db.getFileInfo(fileName)

        if not fileInfo:
            sys.stderr.write("File %s not found in %s" % (target, os.path.join(dirName, Config.getDBFilename())))
            sys.exit(1)

        client = SnowClient(fileInfo.getTableName(), fileInfo.getInstance())
        record = client.get(fileInfo.getSysId())
        recordName = record.sys_id
        content = normalizeNewlines(getattr(record, fileInfo.getContentFieldName()))

        if os.path.isfile(os.path.join(dirName, Config.getLockFilename())):
            #swatch is watching, we do not want him to re-upload, we write a file for swatch.py to know
            ignoreWatchFilePath = target + Config.getIgnoreWatchFilenameSuffix()
            SLogger.debug("Creating file %s to avoid swatch to re-upload" % ignoreWatchFilePath)
            ignoreWatchFile = open(ignoreWatchFilePath, "w")
            ignoreWatchFile.close()

        f = codecs.open(target, "w", "utf-8")
        f.write(content)

        db.setUpdatedOn(fileName, record.sys_updated_on)
        db.commitAndClose()
        SLogger.debug("Updated record %s to file %s. set updated_on to %s" % (recordName, fileName, record.sys_updated_on))
Beispiel #2
0
    def loadOptionsFromFile(cls, configFilePath):
        found = True
        if not configFilePath:
            currentDir = os.getcwd()
            found = False
            while not found:
                configFilePath = os.path.join(currentDir, "snowconf.py")
                if os.path.isfile(configFilePath):
                    found = True
                else:
                    if cls.__isRootDir(currentDir):
                        break
                    currentDir = os.path.dirname(currentDir)

        elif not os.path.isfile(configFilePath):
            SLogger.warning("Config file not found at %s" % configFilePath)
            sys.exit(1)

        if found:
            cls.__configFilePath = os.path.normpath(os.path.abspath(configFilePath))
            loadedOptions = {}
            execfile(cls.__configFilePath, globals(), loadedOptions)
            cls.__setLoadedOptions(loadedOptions)
        else:
            cls.__configFilePath = None
Beispiel #3
0
 def getRecords(self, query):
     try:
         query = str(query)
         SLogger.debug("Using encoded query: %s" % (query))
         return self.service.getRecords(__inject={'msg': SnowClient.__getRecordsMessage % (self.__table, query)})
     except URLError, e:
         SnowClient.__processURLError(e)
Beispiel #4
0
    def process(self):
        self.__options, self.__args = self.parse_args()

        self.__verifyNArgs()
        Config.loadOptionsFromFile(self.__options.configFilePath)
        self.__setCommandLineOptions()
        Config.setDefaultsIfOptionNotPresent()
        self.__setLoggerLevel()
        if Config.getConfigFilePath():
            SLogger.debug("Loaded config file from %s" % Config.getConfigFilePath())
        self.__launchCommand()
Beispiel #5
0
 def __init__(self, table, instance):
     self.__table = table
     self.__instance = instance
     username = Config.getUsername(instance)
     password = Config.getPassword(instance)
     SLogger.debug("Connecting to %s to download from table %s" % (instance, table))
     try:
         Client.__init__(self,
                         instance + table + "_list.do?WSDL",
                         username=username,
                         password=password,
                         timeout=Config.getTimeout())
     except URLError, e:
         SnowClient.__processURLError(e)
Beispiel #6
0
 def __scheduleDir(cls, dirPath, rootDirPath):
     if not directoryShouldBeIgnored(dirPath):
         SLogger.debug("Watching dir %s" % os.path.abspath(dirPath))
         lockFilePath = os.path.abspath(os.path.join(dirPath, Config.getLockFilename()))
         if os.path.exists(lockFilePath):
             SLogger.warning("%s already exists. Another swatch might be watching this directory or might not have shut down correctly. Consider using the scleanlockfiles command." % (lockFilePath))
             if dirPath == rootDirPath:
                 Commands.warning("Directory already watched", "Another swatch might be watching or might not have shut down correctly.")
         else:
             lockFile = open(lockFilePath, "w")
             lockFile.close()
             cls.__lockFiles.append(lockFilePath)
         
         cls.__observer.schedule(cls.__handler, path=dirPath)
Beispiel #7
0
    def run(cls, src, dst):

        src = os.path.abspath(os.path.normpath(src))
        dst = os.path.abspath(os.path.normcase(dst))
        if src == dst:
            return

        if not os.path.exists(src):
            sys.stderr.write("File %s does not exist\n" % src)
            sys.exit(1)

        if os.path.exists(dst):
            sys.stderr.write("File %s already exists\n" % dst)
            sys.exit(1)

        sourceDir, sourceFilename = os.path.split(src)
        if not SnowFileDB.existsAtDir(sourceDir):
            sys.stderr.write("DB file not found at %s, aborting\n" % os.path.join(sourceDir, Config.getDBFilename()))
            sys.exit(1)

        destDir, destFilename = os.path.split(dst)

        if sourceDir != destDir:
            sourceDB = SnowFileDB(sourceDir)
            destDB = SnowFileDB(destDir)

            fileInfo = sourceDB.getFileInfo(sourceFilename)
            if not fileInfo:
                sys.stderr.write("File %s not present at DB file %s\n" % (sourceFilename, sourceDB.getDBFilePath()))
                sys.exit(1)
            sourceDB.deleteFile(sourceFilename)
            fileInfo.setFileName(destFilename)
            destDB.addFileInfo(fileInfo)
            shutil.move(src, dst)
            sourceDB.commitAndClose()
            destDB.commitAndClose()

        else:
            db = SnowFileDB(sourceDir)
            sucesss = db.renameFile(sourceFilename, destFilename)
            if not sucesss:
                sys.stderr.write("File %s not present at DB file %s\n" % (sourceFilename, db.getDBFilePath()))
                sys.exit(1)

            shutil.move(src, dst)
            db.commitAndClose()

        SLogger.debug("File successfully moved from %s to %s\n" % (src, dst))
Beispiel #8
0
    def run(cls, targetURL, *extraArgs):
        instance, tableName, sysId = cls.__parseURL(targetURL)
        nameField, contentField, fileExtension = cls._getTableInfo(tableName)

        localDir = os.getcwd()
        db = SnowFileDB(localDir)
        client = SnowClient(tableName, instance)
        record = client.get(sysId)

        if len(extraArgs) > 0:
            fileName = extraArgs[0]
        else:
            fileName = "%s.%s" % (
                getattr(record, nameField).replace(" ", "-").replace("/", "-").replace("(", "[").replace(")", "]"),
                fileExtension,
            )
        filePath = os.path.join(localDir, fileName)
        if os.path.exists(filePath):
            SLogger.warning("Local file %s already exists, will not overwrite" % filePath)
            return

        #         if os.path.isfile(os.path.join(localDir, Config.getLockFilename())):
        #             #swatch is watching, we do not want him to re-upload, we write a file for swatch.py to know
        #             ignoreWatchFilePath = filePath + Config.getIgnoreWatchFilenameSuffix()
        #             SLogger.debug("Creating file %s to avoid swatch to re-upload" % ignoreWatchFilePath)
        #             ignoreWatchFile = open(ignoreWatchFilePath, "w")
        #             ignoreWatchFile.close()

        f = codecs.open(filePath, "w", "utf-8")
        content = normalizeNewlines(getattr(record, contentField))
        f.write(content)
        f.close()
        db.addFileInfo(
            FileInfo(fileName, instance, tableName, record.sys_id, nameField, contentField, record.sys_updated_on)
        )
        db.commitAndClose()
        SLogger.debug("Done. Written to %s" % fileName)
Beispiel #9
0
 def __setLoggerLevel(self):
     if self.__options.verbose:
         SLogger.setLevel(logging.DEBUG)
     else:
         level = Config.getLoggerLevel()
         if level == "warning":
             SLogger.setLevel(logging.WARNING)
         elif level == "debug":
             SLogger.setLevel(logging.DEBUG)
Beispiel #10
0
    def run(cls, targetURL):
        instance, tableName, query = cls.__parseURL(targetURL)
        nameField, contentField, fileExtension = cls._getTableInfo(tableName)

        localDir = os.getcwd()
        db = SnowFileDB(localDir)
        client = SnowClient(tableName, instance)

        records = client.getRecords(query)

        for record in records:
            recordName = getattr(record, nameField)
            SLogger.debug("Downloading %s ... " % recordName)
            fileName = "%s.%s" % (
                getattr(record, nameField).replace(" ", "-").replace("/", "-").replace("(", "[").replace(")", "]"),
                fileExtension,
            )
            filePath = os.path.abspath(os.path.join(localDir, fileName))
            if os.path.exists(filePath):
                good = False
                suffix = 2
                while not good:
                    filePathNew = filePath + "_" + str(suffix)
                    if not os.path.exists(filePathNew):
                        good = True
                        filePath = filePathNew
                    else:
                        suffix = suffix + 1
                        SLogger.warning("Local file %s already exists, will not overwrite" % filePath)
                        continue

            try:
                content = normalizeNewlines(getattr(record, contentField))
                f = codecs.open(filePath, "w", "utf-8")
                f.write(content)
                f.close()
                db.addFileInfo(
                    FileInfo(
                        fileName, instance, tableName, record.sys_id, nameField, contentField, record.sys_updated_on
                    )
                )
                SLogger.debug("Done. Written to %s" % fileName)
            except Exception:
                SLogger.debug("Problem with this record, will not write")

        db.commitAndClose()
Beispiel #11
0
    def run(cls, targetDir):
        cls.__dbWorker = DBWorker()
        cls.__dbWorker.start()

        cls.__handler = EventHandler(cls.__dbWorker)

        cls.__lockFiles = []
        cls.__observer = Observer()

        if Config.isRecursive():
            for root, _, _ in os.walk(targetDir):
                cls.__scheduleDir(root, targetDir)
        else:
            cls.__scheduleDir(targetDir, targetDir)

        cls.__observer.start()

        try:
            while True:
                time.sleep(1)
        except KeyboardInterrupt:
            SLogger.debug("Stopping...")
        finally:
            cls.__shutdown()
Beispiel #12
0
    def __shutdown(cls):
        cls.__observer.stop()
        cls.__dbWorker.stop()

        SLogger.debug("Deleting lock files...")
        for lockFilePath in cls.__lockFiles:
            try:
                os.remove(lockFilePath)
            except OSError:
                SLogger.warning("Lock file %s could not be deleted" % lockFilePath)

        cls.__observer.join()
        cls.__dbWorker.join()

        SLogger.debug("Stopped")
Beispiel #13
0
    def run(cls, targetFiles):

        for targetFile in targetFiles:
            if not os.path.isfile(targetFile):
                SLogger.warning("File %s does not exist, ignoring" % os.path.abspath(targetFile))
                continue

            SLogger.debug("Reading target file %s" % os.path.abspath(targetFile))

            loadedParams = {}
            execfile(targetFile, globals(), loadedParams)

            instance = loadedParams.get("instance", None)
            if not instance:
                SLogger.warning("No instance declared in target file %s, aborting" % targetFile)
                sys.exit(1)

            if not instance.endswith("/"):
                instance = instance + "/"

            for i, target in enumerate(loadedParams["targets"]):

                # we extract the table name
                table = target.get("table", None)
                if not table:
                    SLogger.warning("No table in target number %d of file %s, ignoring" % (i, targetFile))
                    continue

                # we extract the name field, content field, and how to rename files
                tableInfo = cls.__getTableInfo(table, target)
                if not tableInfo:
                    continue
                nameField, contentField, transformName = tableInfo

                # we extract the filtering functions
                query = target.get("query", "")
                recordFilter = target.get("recordFilter", lambda x: True)

                # we extract the local target directory
                localDir = target.get("localDir", None)
                if not localDir:
                    SLogger.debug(
                        "No localDir in target number %d of file %s, using current directory" % (i, targetFile)
                    )
                    localDir = os.getcwd()
                if not os.path.isdir(localDir):
                    SLogger.warning("Local directory %s does not exist, ignoring target %d" % (localDir, i))
                    continue

                # if everything is well, we start downloading and writing
                db = SnowFileDB(localDir)
                client = SnowClient(table, instance)

                records = client.getRecords(query)

                for record in records:
                    if recordFilter(record):
                        recordName = getattr(record, nameField)
                        SLogger.debug("Downloading %s ... " % recordName)
                        fileName = transformName(recordName)
                        filePath = os.path.abspath(os.path.join(localDir, fileName))
                        if os.path.exists(filePath):
                            good = False
                            suffix = 2
                            while not good:
                                filePathNew = filePath + "_" + str(suffix)
                                if not os.path.exists(filePathNew):
                                    good = True
                                    filePath = filePathNew
                                else:
                                    suffix = suffix + 1

                        try:
                            content = normalizeNewlines(getattr(record, contentField))
                            f = codecs.open(filePath, "w", "utf-8")
                            f.write(content)
                            f.close()
                            db.addFileInfo(
                                FileInfo(
                                    fileName,
                                    instance,
                                    table,
                                    record.sys_id,
                                    nameField,
                                    contentField,
                                    record.sys_updated_on,
                                )
                            )
                            SLogger.debug("Done. Written to %s" % fileName)
                        except Exception:
                            SLogger.debug("Problem with this record, will not write")

                db.commitAndClose()
Beispiel #14
0
 def __cleanDir(cls, dirPath):
     lockFilePath = os.path.abspath(os.path.join(dirPath, Config.getLockFilename()))
     if os.path.exists(lockFilePath):
         SLogger.debug("Deleting %s" % os.path.abspath(lockFilePath))
         os.remove(lockFilePath)
Beispiel #15
0
 def __init__(self, path):
     self.__dbFilePath = os.path.join(path, Config.getDBFilename())
     SLogger.debug("Opening DB File: %s" % self.__dbFilePath)
     self.__conn = sqlite3.connect(self.__dbFilePath)
     self.__cursor = self.__conn.cursor()
Beispiel #16
0
    def processFile(self, filePath):
        ignoreWatchFilePath = filePath + Config.getIgnoreWatchFilenameSuffix()
        if os.path.isfile(ignoreWatchFilePath):
            #this file was updated by supdate.py, do not process
            os.remove(ignoreWatchFilePath)
            return

        #Loic Horisberger - 17.10.2014:
        #This is a fix for MacVIM where current files are swap files.        
        #If the file ends with ".swp", remove the extention to get the
        #original filename. Remove the "." at the beginning that makes
        #the file hidden as well.
        if filePath.endswith(".swp"):
            filePath = filePath.replace(".swp","")
            filePath = filePath[::-1].replace("/."[::-1],"/"[::-1],1)[::-1]

        self.__dbWorker.getFileInfo(self, filePath)
        fileInfo = self.__inputQueue.get()

        if fileInfo:
            SLogger.debug("Local modification detected for: %s" % filePath)

            client = clientPool.getClient(fileInfo.getTableName(), fileInfo.getInstance())
            localUpdateDateString = fileInfo.getUpdatedOn()

            canUpdate = True
            if localUpdateDateString:
                SLogger.debug("Checking remote version...")
                remoteRecord = client.get(fileInfo.getSysId())
                remoteUpdatedOnString = remoteRecord.sys_updated_on
                canUpdate = checkCanUpdate(localUpdateDateString, remoteUpdatedOnString)
                if canUpdate:
                    SLogger.debug("Done")
                else:
                    SLogger.warning("Remote file changed by %s on %s. Previous local version is %s. Cannot update" %
                        (remoteRecord.sys_updated_by, snowServerToLocalDate(remoteUpdatedOnString), localUpdateDateString))
                    Commands.cannotUpload(remoteRecord.sys_updated_by, remoteUpdatedOnString, localUpdateDateString)


            if canUpdate:
                SLogger.debug("Updating in SNOW...")
                content = codecs.open(filePath, "r", "utf-8").read()

                success = client.updateContent(fileInfo.getSysId(), fileInfo.getContentFieldName(), content)
                if success:
                    SLogger.debug("Updating local update date...")
                    newRemoteRecord = client.get(fileInfo.getSysId())
                    newRemoteUpdatedOnString = newRemoteRecord.sys_updated_on
                    self.__dbWorker.setUpdatedOn(self, filePath, newRemoteUpdatedOnString)
                    confirmation = self.__inputQueue.get()
                    if confirmation:
                        SLogger.debug("Success updating %s" % fileInfo.getFileName())
                        Commands.success(fileInfo)
                    else:
                        SLogger.warning("Update of local date failed for file %s" % fileInfo)
                        Commands.warning("Upload failed", "Update of local date failed for file %s" % fileInfo)

                else:
                    SLogger.warning("Update of local date failed for file %s" % fileInfo)
                    Commands.warning("Upload failed", "Update of local date failed for file %s" % fileInfo)
Beispiel #17
0
 def __processURLError(cls, e):
     if hasattr(e, "reason") and hasattr(e.reason, "args") and e.reason.args[0] == "The read operation timed out":
         SLogger.warning("Connection timed out. Verify your username.")
         sys.exit(1)
     else:
         raise e
Beispiel #18
0
 def __processTransportError(cls, e):
     if hasattr(e, "httpcode") and e.httpcode == 401:
         SLogger.warning("Authentication error 401. Verify your password")
         sys.exit(1)
     else:
         raise e
Beispiel #19
0
 def __processAttributeError(self, e):
     SLogger.warning("Could not retrieve information from ServiceNow. Possibly your password is wrong.")
     Config.clearPassword(self.__instance)
     clientPool.removeClient(self.__table, self.__instance)
     sys.exit(1)