def getLocations(self, files, locations, forceBundle=False):
"""find local copies of a file.
files: list of file names
locations: list with locations (output)
forceBundle: treat all files as bundle files
"""
if not self.isConnected():
return False
retval = 0
debug("getLocations: " + str(files) + " " + str(locations))
try:
fileSystem = FileSystem(self.config)
fetcher = CacheFetcher(self.config, fileSystem, self.connection)
nFiles = len(files)
for i in range(nFiles):
filename = os.path.realpath(files[i])
debug("locate " + filename)
sendExit = (i == (nFiles - 1))
if forceBundle or self._isBundleFile(filename):
loc, retval = self._findBundleLocations(filename, fileSystem, fetcher, sendExit)
else:
loc, retval = self._findLocations(filename, fileSystem, fetcher, sendExit)
locations += loc
except Exception, e:
error("unknown error: %s" % str(e))
retval = 2
def _fetchBundle(self, filename, conjunct):
filename = os.path.realpath(filename)
debug("fetchBundle: %s" % filename)
fileSystem = FileSystem(self.config)
fetcher = CacheFetcher(self.config, fileSystem, self.connection)
if not os.path.isfile(filename):
error("file not found '%s'" % filename)
if self.single: fetcher.sendExit()
return (filename, 1)
destination = self.getDestination(fileSystem, filename)
debug("destination: " + str(destination))
while (os.path.isfile(destination)):
destination = "%s%s.%0.3f.bundle" % (destination[:-6], socket.gethostname(), float(time.time()))
try:
out = file(destination, "w")
except Exception, e:
error("cannot open destination file %s" % destination)
if self.single: fetcher.sendExit()
return (filename, 1)
def _fetchFile(self,
filename,
fileSystem=None,
fetcher=None,
closeOnError=True):
if not fileSystem:
fileSystem = FileSystem(self.config)
if not fetcher:
fetcher = CacheFetcher(self.config, fileSystem, self.connection)
if not os.path.isfile(filename):
error("file not found '%s'" % filename)
if closeOnError and not self.single: fetcher.sendExit()
return (filename, 1)
fileinfo = fileSystem.getFileInfo(filename)
destination = self.getDestination(fileSystem, filename)
if destination is None:
return (filename, 1)
log("destination: " + str(destination))
if os.path.isfile(destination):
wait = fetcher.isActive(destination)
while wait > 0:
log("file transfer in progress. wait %ds" % wait)
time.sleep(wait)
wait = fetcher.isActive(destination)
fileExists, canCopy, removed = fileSystem.destinationExists(
fileinfo, destination)
if fileExists:
if removed:
log("removed " + destination)
fetcher.sendFileRemoved(fileinfo, destination)
if not canCopy:
error("cannot copy file to %s" % destination)
return (filename, 1)
else:
log("using existing file")
fileSystem.setATime(destination)
fetcher.sendFileLocation(fileinfo, destination)
return (destination, 0)
freeSpace, removed = fileSystem.checkFreeSpace(
int(fileinfo[1]), destination)
if not freeSpace:
log("not enough free space in %s" % fileSystem.cacheDir)
if closeOnError and self.single: fetcher.sendExit()
return (filename, 1)
log("request: " + filename)
fetcher.requestFile(fileinfo, destination)
msg = self.connection.receiveMessage()
resultFile, ok, term = fetcher.handleMessage(fileinfo, destination,
msg)
while not ok:
msg = self.connection.receiveMessage()
resultFile, ok, term = fetcher.handleMessage(
fileinfo, destination, msg)
return (resultFile, 0)
def _fetchFile(self, filename, fileSystem=None, fetcher=None, closeOnError=True):
if not fileSystem:
fileSystem = FileSystem(self.config)
if not fetcher:
fetcher = CacheFetcher(self.config, fileSystem, self.connection)
if not os.path.isfile(filename):
error("file not found '%s'" % filename)
if closeOnError and not self.single: fetcher.sendExit()
return (filename, 1)
fileinfo = fileSystem.getFileInfo(filename)
destination = self.getDestination(fileSystem, filename)
if destination is None:
return (filename, 1)
log("destination: " + str(destination))
if os.path.isfile(destination):
wait = fetcher.isActive(destination)
while wait > 0:
log("file transfer in progress. wait %ds" % wait)
time.sleep(wait)
wait = fetcher.isActive(destination)
fileExists, canCopy, removed = fileSystem.destinationExists(fileinfo, destination)
if fileExists:
if removed:
log("removed " + destination)
fetcher.sendFileRemoved(fileinfo, destination)
if not canCopy:
error("cannot copy file to %s" % destination)
return (filename, 1)
else:
log("using existing file")
fileSystem.setATime(destination)
fetcher.sendFileLocation(fileinfo, destination)
return (destination, 0)
freeSpace, removed = fileSystem.checkFreeSpace(int(fileinfo[1]), destination)
if not freeSpace:
log("not enough free space in %s" % fileSystem.cacheDir)
if closeOnError and self.single: fetcher.sendExit()
return (filename, 1)
log("request: " + filename)
fetcher.requestFile(fileinfo, destination)
msg = self.connection.receiveMessage()
resultFile, ok, term = fetcher.handleMessage(fileinfo, destination, msg)
while not ok:
msg = self.connection.receiveMessage()
resultFile, ok, term = fetcher.handleMessage(fileinfo, destination, msg)
return (resultFile, 0)