Example #1
0
 def CheckChanges(self, filecheck, dirpath, remotedir, rowInfo):
     '''
     Given a file (with metadata) check if it has changed. If changes have
     been made, then send them to the server and update database.
     
     @param filecheck: The name of the new file
     @param dirpath: Must be a correct relative path where file is
     @param remotedir: The remote counterpart of dirpath
     @param rowinfo: A database row with metainfo of the file
     '''
     localfile = posixpath.join(dirpath,filecheck)
     modifiedTime = os.stat(localfile).st_mtime
     computedSize = os.stat(localfile).st_size    
     if ( rowInfo['localtime'] != modifiedTime or
              rowInfo['size']  != computedSize ):
         # Here! Some file needs care
         self._logger.info('Detected modified file: %s' , localfile )
         hashes = Hashes.open(posixpath.join(
             self._hashesdir, str(rowInfo['idfile'] )))    
         # Get size
         computedSize = os.stat(localfile).st_size
         
         with open ( localfile , "rb" ) as f:
             # Get checksum
             computedChksum = adler32(f.read())
             # Get delta
             f.seek(0)
             delta = hashes.computeDelta(f)
         self._logger.debug ( "Last revision: %s" , repr(rowInfo['lastrev']) )
         self._logger.debug ( "Checksum: %s" , repr(computedChksum) )
         self._logger.debug ( "Size: %s" , repr(computedSize) )
         ret = self._RemoteCaller.SendDelta ( rowInfo['lastrev'], 
             delta.getXMLRPCBinary(), computedChksum, computedSize )
         self._logger.debug ( "Sent, response: %s" , repr(ret) )
         
         with self._db as c:
             c.execute ( '''update files set 
                 lastrev=?, timestamp=?, localtime=?, chksum=?, size=?
                 where idfile=?''' , (ret[0], ret[1], modifiedTime, 
                     computedChksum, computedSize, rowInfo['idfile'] )
                 )
         # update the hashes
         self._logger.debug ( "Updating hash file %s" , str(rowInfo['idfile']) )
         hash = Hashes.eval(localfile)
         hash.save(posixpath.join(self._hashesdir, str(rowInfo['idfile']) ))
     else:
         self._logger.debug('No changes found for file:%s (path:%s)'
             % (filecheck,dirpath) )
Example #2
0
 def NewFile(self, filecheck, dirpath, remotedir):
     '''
     Function used when a potential new file is found.
     
     @param filecheck: The name of the new file
     @param dirpath: Must be a correct relative path where file is
     @param remotedir: The remote counterpart of dirpath
     '''
     localfile = posixpath.join(dirpath,filecheck)
     modifiedTime = os.stat(localfile).st_mtime
     computedSize = os.stat(localfile).st_size    
     self._logger.info('Detected new file: %s' , localfile )
     with open ( localfile , "rb" ) as f:
         # Get checksum
         computedChksum = adler32(f.read())
         # Get data
         f.seek(0)
         dataToSend = Binary(f.read())
         
     # Checking if the server knows the file
     ret = self._RemoteCaller.CheckFileMetadata (computedSize, computedChksum)
     if ret > 0:
         # Say the server that we want to copy
         self._logger.debug( "Creating a copy of revision %s", str(ret) )
         retcp = self._RemoteCaller.CopyFile ( remotedir, filecheck, ret )
         self._logger.debug ( "Sent, response: %s" , repr(retcp) )
         lastrev, timestamp = retcp
     else:
         # Send everything (new) to server
         self._logger.debug( "Transfering new file to server" )
         ret = self._RemoteCaller.SendNewFile( remotedir, filecheck,
             dataToSend, computedChksum, computedSize)
         self._logger.debug ( "Sent, response: %s" , repr(ret) )
         lastrev,timestamp = ret
     # We use the revision id
     with self._db as c:
         cur = c.execute ( '''insert into files 
             (path, file, lastrev, timestamp, localtime, chksum, size) 
             values (?,?,?,?,?,?,?)''' , 
                 (dirpath, filecheck, 
                 lastrev, timestamp, modifiedTime, 
                 computedChksum, computedSize )
             )
         fileId = cur.lastrowid
     # Save the hashes for later use
     hashes = Hashes.eval(localfile)
     hashes.save(posixpath.join(self._hashesdir, str(fileId) ))
Example #3
0
 def SendNewFile (self, path, newfile, bindata , chksum = None, size = None):
     '''
     Create a *non-existing* file in server
     
     @param path: String of an *existing* valid path
     @param newfile: String of the file to create into path
     @param bindata: Binary contents of file
     @param chksum: Checksum of file
     @param size: Size of file
     @return: Error code or raise if something goes wrong. A pair idrev and
     timestamp of the file if everything is ok.
     '''
     filepath = self._basicNewChecks(path, newfile)
     if isinstance(filepath, int):
         return filepath
     
     self._logger.debug ( "Receiving file %s, saving at %s" , newfile , filepath )
     try:
         with open(filepath, "wb") as f:
             f.write(bindata.data)
     except:
         self._seterrormsg('Internal filesystem error when opening ' 
             + filepath)
         return ERR_FS
     
     # Now we have created the file locally
     # let's check that everything is ok
     
     #first checksum
     with open ( filepath , "rb" ) as f:
         computedChecksum = adler32(f.read())
     if chksum and (chksum != computedChecksum):
         self._seterrormsg('Checksums do not match --rolled back')
         os.remove(filepath)
         return ERR_CHKSUM
     
     #then size
     computedSize = os.stat(filepath).st_size
     if size and (computedSize != size):
         self._seterrormsg('Size do not match. Local size: '+ 
             str(computedSize) + ' --rolled back')
         os.remove(filepath)
         return ERR_SIZE
     
     tsnow = datetime.fromtimestamp(int(time.time()))
     
     with self._conn as c:
         cursor = c.execute('''insert into files 
             (path, file, deleted) values (?,?,0)''', (path, newfile) )
         idfile = cursor.lastrowid
         cursor = c.execute('''insert into revisions 
             (idfile,uid,timestamp,fromrev,typefrom,chksum,size,hardexist)
             values (?,?,?,NULL,?,?,?,1)''' , 
             (idfile,self._getUID(),tsnow,REV_NEWFILE,computedChecksum,computedSize) 
             )                    
         idrev = cursor.lastrowid
         c.execute("update files set lastrev=? where idfile=?" , 
             (idrev, idfile) )
     
     revPath = os.path.join ( self._hardsdir ,str(idrev) )
     self._logger.info ( "Proceeding to link %s and %s" , filepath , revPath ) 
     os.link ( filepath ,  revPath )
     
     # calculate here hashes for rsync algorithm
     hashes = Hashes.eval(filepath)
     savehash = os.path.join(self._hashesdir,str(idrev))
     self._logger.debug ("Saving hash in %s", savehash)
     hashes.save (savehash)
     
     return idrev, tsnow