def merge(self, inpath='', outpath=''): input = eblob.blob(inpath) outb = {} for r in self.routes: if r[1] != self.own: if not r[1] in outb: outb[r[1]] = eblob.blob(outpath + '-' + r[1], data_mode='ab', index_mode='ab') for id in input.iterate(want_removed=False): d = self.destination(elliptics_id(list(bytearray(id)), self.own_group, -1)) if d[1] != self.own: b = outb[d[1]] input.read_data() pos = b.dataf.tell() old_pos = input.position input.position = pos idata, data = input.get_data() b.index.write(idata) b.dataf.write(idata) b.dataf.write(input.data[eblob.blob.index_size:]) input.position = old_pos input.mark_removed() input.update()
def __init__(self, remotes=[], groups=[], log='/dev/stdout', mask=8, path=''): self.log = elliptics_log_file(log, mask) self.n = elliptics_node_python(self.log) self.n.add_groups(groups) for r in remotes: try: self.n.add_remote(r[0], r[1]) except: pass if len(self.n.get_routes()) == 0: raise NameError("Route table for group " + str(group) + " is empty") b = eblob.blob(path) for id in b.iterate(want_removed=True): if b.removed(): for g in groups: eid = elliptics_id(list(bytearray(id)), g, -1) self.n.remove(eid, 0) print "%s: flags: 0x%x, position: %d, data_size: %d" % \ (b.sid(count=64), b.flags, b.position, b.data_size)
def process_blob(blob_dir, matcher): data = [] blob = eblob.blob(blob_dir, data_mode='rb', index_mode='rb') for id in blob.iterate(want_removed=False): blob.read_data() meta = parse_meta(blob.data) if matcher.match(meta["object"]): data.append(meta) return data
def __init__(self, remotes=[], groups=[], log='/dev/stdout', mask=8, path=''): self.log = elliptics_log_file(log, mask) self.n = elliptics_node_python(self.log) self.n.add_groups(groups) self.n.add_remote(remotes) if len(self.n.get_routes()) == 0: raise NameError("Route table for group " + str(group) + " is empty") b = eblob.blob(path) for id in b.iterate(want_removed=True): if b.removed(): for g in groups: eid = elliptics_id(list(bytearray(id)), g, -1) self.n.remove(eid, 0) print "%s: flags: 0x%x, position: %d, data_size: %d" % \ (b.sid(count=64), b.flags, b.position, b.data_size)
#!/usr/bin/env python # -*- coding: utf-8 -*- import sys import eblob if __name__ == '__main__': print >> sys.stderr, "Started" for base in sys.argv[1:]: try: print "Processing: %s" % base b = eblob.blob(base, index_mode='w+b') for eid in b.iterate(want_removed=False, over_data=True): print "%s: flags: %x, position: %d, data_size: %d, disk_size: %d" % \ (b.sid(count=64), b.flags, b.position, b.data_size, b.disk_size) b.write_index() except NameError as e: print "Completed:", e print >> sys.stderr, "Finished"
#!/usr/bin/python # -*- coding: utf-8 -*- import eblob try: b = eblob.blob('/tmp/data.0', index_mode='w+b') for eid in b.iterate(want_removed=False, over_data=True): print "%s: flags: %x, position: %d, data_size: %d, disk_size: %d" % \ (b.sid(count=64), b.flags, b.position, b.data_size, b.disk_size) b.write_index() except NameError as e: print "Completed:", e
#!/usr/bin/env python # -*- coding: utf-8 -*- import sys import eblob if __name__ == '__main__': print >>sys.stderr, "Started" for base in sys.argv[1:]: try: print "Processing: %s" % base b = eblob.blob(base, index_mode='w+b') for eid in b.iterate(want_removed=False, over_data=True): print "%s: flags: %x, position: %d, data_size: %d, disk_size: %d" % \ (b.sid(count=64), b.flags, b.position, b.data_size, b.disk_size) b.write_index() except NameError as e: print "Completed:", e print >>sys.stderr, "Finished"