Example #1
0
 def close(self):
     super(HdfsAtomicWritePipe, self).close()
     try:
         remove(self.path)
     except HDFSCliError:
         pass
     if not all(result['result'] for result in rename(self.tmppath, self.path) or []):
         raise HdfsAtomicWriteError('Atomic write to {} failed'.format(self.path))
Example #2
0
 def close(self):
     super(HdfsAtomicWritePipe, self).close()
     try:
         remove(self.path)
     except HDFSCliError:
         pass
     if not all(result['result']
                for result in rename(self.tmppath, self.path) or []):
         raise HdfsAtomicWriteError('Atomic write to {} failed'.format(
             self.path))
Example #3
0
 def close(self):
     super(HdfsAtomicWritePipe, self).close()
     try:
         if exists(self.path):
             remove(self.path)
     except Exception as ex:
         if isinstance(ex, HDFSCliError) or ex.args[0].contains("FileNotFoundException"):
             pass
         else:
             raise ex
     if not all(result['result'] for result in rename(self.tmppath, self.path) or []):
         raise HdfsAtomicWriteError('Atomic write to {} failed'.format(self.path))
Example #4
0
    def close(self):
        super(HdfsAtomicWriteDirPipe, self).close()
        try:
            remove(self.path)
        except HDFSCliError:
            pass

        # it's unlikely to fail in this way but better safe than sorry
        if not all(result['result'] for result in rename(self.tmppath, self.path) or []):
            raise HdfsAtomicWriteError('Atomic write to {} failed'.format(self.path))

        if os.path.basename(self.tmppath) in map(os.path.basename, listdir(self.path)):
            remove(self.path)
            raise HdfsAtomicWriteError('Atomic write to {} failed'.format(self.path))
Example #5
0
 def close(self):
     super(HdfsAtomicWritePipe, self).close()
     try:
         if exists(self.path):
             remove(self.path)
     except Exception as ex:
         if isinstance(ex, HDFSCliError) or ex.args[0].contains(
                 "FileNotFoundException"):
             pass
         else:
             raise ex
     if not all(result['result']
                for result in rename(self.tmppath, self.path) or []):
         raise HdfsAtomicWriteError('Atomic write to {} failed'.format(
             self.path))
Example #6
0
    def close(self):
        super(HdfsAtomicWriteDirPipe, self).close()
        try:
            remove(self.path)
        except HDFSCliError:
            pass

        # it's unlikely to fail in this way but better safe than sorry
        if not all(result['result']
                   for result in rename(self.tmppath, self.path) or []):
            raise HdfsAtomicWriteError('Atomic write to {} failed'.format(
                self.path))

        if os.path.basename(self.tmppath) in map(os.path.basename,
                                                 listdir(self.path)):
            remove(self.path)
            raise HdfsAtomicWriteError('Atomic write to {} failed'.format(
                self.path))
Example #7
0
    def close(self):
        super(HdfsAtomicWriteDirPipe, self).close()
        try:
            if exists(self.path):
                remove(self.path)
        except Exception as ex:
            if isinstance(ex, HDFSCliError) or ex.args[0].contains("FileNotFoundException"):
                pass
            else:
                raise ex

        # it's unlikely to fail in this way but better safe than sorry
        if not all(result['result'] for result in rename(self.tmppath, self.path) or []):
            raise HdfsAtomicWriteError('Atomic write to {} failed'.format(self.path))

        if os.path.basename(self.tmppath) in map(os.path.basename, listdir(self.path)):
            remove(self.path)
            raise HdfsAtomicWriteError('Atomic write to {} failed'.format(self.path))
Example #8
0
    def close(self):
        super(HdfsAtomicWriteDirPipe, self).close()
        try:
            if exists(self.path):
                remove(self.path)
        except Exception as ex:
            if isinstance(ex, HDFSCliError) or ex.args[0].contains(
                    "FileNotFoundException"):
                pass
            else:
                raise ex

        # it's unlikely to fail in this way but better safe than sorry
        if not all(result['result']
                   for result in rename(self.tmppath, self.path) or []):
            raise HdfsAtomicWriteError('Atomic write to {} failed'.format(
                self.path))

        if os.path.basename(self.tmppath) in map(os.path.basename,
                                                 listdir(self.path)):
            remove(self.path)
            raise HdfsAtomicWriteError('Atomic write to {} failed'.format(
                self.path))
Example #9
0
 def abort(self):
     logger.info("Aborting %s('%s'). Removing temporary dir '%s'",
                 self.__class__.__name__, self.path, self.tmppath)
     super(HdfsAtomicWriteDirPipe, self).abort()
     remove(self.tmppath, skip_trash=True)
Example #10
0
File: format.py Project: 01-/luigi
 def abort(self):
     logger.info("Aborting %s('%s'). Removing temporary dir '%s'",
                 self.__class__.__name__, self.path, self.tmppath)
     super(HdfsAtomicWriteDirPipe, self).abort()
     remove(self.tmppath, skip_trash=True)
Example #11
0
 def abort(self):
     logger.info("Aborting %s('%s'). Removing temporary file '%s'",
                 self.__class__.__name__, self.path, self.tmppath)
     super(HdfsAtomicWritePipe, self).abort()
     remove(self.tmppath)