def __init__(self,space=False):
LtrNode.__init__(self)
LtrUri.__init__(self)
self.record = False
self.name = False
self.space = False
self.uri=False
self.cwd = False #set drop in loadcookie()
if space:
self.setSpace(space)
def getNode(self,fullvolpath):
fname = fullvolpath.split("/")[-1]
dirpath = fullvolpath[0:-len(fname)].rstrip("/") # trailing /
if dirpath=="":
dirpath = "/"
nodes = list(LtrNode.view(self.space.records,"ltrcrawler/path",key=[dirpath,fname],include_docs=True))
nodes = filter(lambda x: x.boxname == self.id ,nodes)
if 0==len(nodes):
return None
node = nodes[0]
node.connect(self.space)
return node
def fetch(self,srcbox,dryrun=True):
print "ltr: pull ", srcbox.fspath
queue=[]
updates = []
wanted = []
import datetime
time = datetime.datetime.now()
queue.append((srcbox,self))
targetfilter = lambda n: n.boxname == self.id
sourcefilter = lambda n: n.boxname == srcbox.id
while len(queue):
(source,target)= queue.pop(0)
source = dictbyname(filter(sourcefilter,source.children()))
target = dictbyname(filter(targetfilter,target.children()))
srcunique = set(source.keys()) - set(target.keys())
common = set(target.keys()) & set(source.keys())
for filename in common:
#fixme, query DB for matching size,hash tuples
current = source[filename]
updating = target[filename]
if LtrNode.SKIP_FLAG in updating.flags:
continue
diff = updating.diff(current)
if diff != []:
show("M %s %s\n" %(current.getVolPath(),diff))
wanted.append((current,updating))
if updating.ftype == "dir":
queue.append((current,updating))
# nodes absent but wanted
for filename in srcunique:
current = source[filename]
absent = LtrNode().new()
absent.name = current.name
absent.path = current.path
absent.boxname = self.id
absent.addtime = time
show("w %s\n" %(absent.getVolPath()))
if current.ftype == "dir":
queue.append((current,absent))
wanted.append((current,absent))
print "ltr: copying %d files" % len(wanted)
for (current,updating) in wanted:
src = os.path.join(srcbox.fspath,current.getVolPath().strip("/"))
dst = os.path.join(self.fspath,updating.getVolPath().strip("/"))
if current.ftype != "dir":
print "cp %s %s " % (src,dst)
if not dryrun:
shutil.copy2(src,dst)
try:
shutil.copy2(src,dst)
except:
print "error"
else:
print "mkdir %s " % dst
if not dryrun:
os.mkdir(dst)
if not LtrNode.COPY_FLAG in updating.flags:
updating.flags.append(LtrNode.COPY_FLAG)
if LtrNode.SKIP_FLAG in updating.flags:
updating.flags.remove(LtrNode.SKIP_FLAG)
updating.log.append({"dt": time, "etype": "pull", "old":current.id})
updating.updateDrop(current)
updates.append(updating)
if not dryrun:
print "." * len(updates)
self.space.records.update(updates)
if len(updates):
#import pprint
#pprint.pprint(updates)
return True
else:
return False
def getRootNode(self):
return LtrNode.load(self.space.records,self.rootnode).connect(self.space,False)
def commit(self,dryrun=True):
""" update target with data from source """
queue=[]
updates = []
import datetime
time = datetime.datetime.now()
queue.append( (LtrDrop("","/",self.fspath),self) )
sourcefilter = lambda n: True
targetfilter = lambda n: n.boxname == self.id
while len(queue):
(source,target)= queue.pop(0)
source = dictbyname(filter(sourcefilter,source.children()))
target = dictbyname(filter(targetfilter,target.children()))
srcunique = set(source.keys()) - set(target.keys())
absent = set(target.keys()) - set(source.keys())
common = set(target.keys()) & set(source.keys())
for filename in common:
existing = source[filename]
updating = target[filename]
diff = updating.diff(existing)
if diff != []:
show("M %s %s\n" %(existing.volpath,diff))
show("[ updateDrop %s ]" % existing.name )
updating.updateDrop(existing)
updates.append(updating)
if updating.ftype == "dir":
queue.append((existing,updating))
for gone in absent:
gone = target.get(gone)
if gone.ftype == "dir":
queue.append((LtrDrop(gone.name,gone.path,self.fspath),gone))
if LtrNode.COPY_FLAG in gone.flags:
show("D %s\n" %(gone.getVolPath()))
gone.flags.remove(LtrNode.COPY_FLAG)
updates.append(gone)
if not LtrNode.COPY_FLAG in gone.flags \
and not LtrNode.SKIP_FLAG in gone.flags:
show("W %s\n" %(gone.getVolPath()))
for new in map(lambda x:source.get(x),srcunique):
newnode = LtrNode().new()
show("[ updateDrop %s ]" % new.volpath )
newnode.updateDrop(new,dryrun=dryrun)
newnode.boxname = self.id
if not LtrNode.COPY_FLAG in newnode.flags:
newnode.flags.append(LtrNode.COPY_FLAG)
newnode.addtime = time
show("N %s\n" %(new.volpath))
updates.append(newnode)
if new.ftype == "dir":
queue.append((new,newnode))
if not dryrun:
print "." * len(updates)
self.space.records.update(updates)
if len(updates):
#import pprint
#pprint.pprint(updates)
return True
else:
return False
