def merge(self, inpath='', outpath=''):
input = eblob.blob(inpath)
outb = {}
for r in self.routes:
if r[1] != self.own:
if not r[1] in outb:
outb[r[1]] = eblob.blob(outpath + '-' + r[1], data_mode='ab', index_mode='ab')
for id in input.iterate(want_removed=False):
d = self.destination(elliptics_id(list(bytearray(id)), self.own_group, -1))
if d[1] != self.own:
b = outb[d[1]]
input.read_data()
pos = b.dataf.tell()
old_pos = input.position
input.position = pos
idata, data = input.get_data()
b.index.write(idata)
b.dataf.write(idata)
b.dataf.write(input.data[eblob.blob.index_size:])
input.position = old_pos
input.mark_removed()
input.update()
def __init__(self,
remotes=[],
groups=[],
log='/dev/stdout',
mask=8,
path=''):
self.log = elliptics_log_file(log, mask)
self.n = elliptics_node_python(self.log)
self.n.add_groups(groups)
for r in remotes:
try:
self.n.add_remote(r[0], r[1])
except:
pass
if len(self.n.get_routes()) == 0:
raise NameError("Route table for group " + str(group) +
" is empty")
b = eblob.blob(path)
for id in b.iterate(want_removed=True):
if b.removed():
for g in groups:
eid = elliptics_id(list(bytearray(id)), g, -1)
self.n.remove(eid, 0)
print "%s: flags: 0x%x, position: %d, data_size: %d" % \
(b.sid(count=64), b.flags, b.position, b.data_size)
def process_blob(blob_dir, matcher):
data = []
blob = eblob.blob(blob_dir, data_mode='rb', index_mode='rb')
for id in blob.iterate(want_removed=False):
blob.read_data()
meta = parse_meta(blob.data)
if matcher.match(meta["object"]):
data.append(meta)
return data
def __init__(self, remotes=[], groups=[], log='/dev/stdout', mask=8, path=''):
self.log = elliptics_log_file(log, mask)
self.n = elliptics_node_python(self.log)
self.n.add_groups(groups)
self.n.add_remote(remotes)
if len(self.n.get_routes()) == 0:
raise NameError("Route table for group " + str(group) + " is empty")
b = eblob.blob(path)
for id in b.iterate(want_removed=True):
if b.removed():
for g in groups:
eid = elliptics_id(list(bytearray(id)), g, -1)
self.n.remove(eid, 0)
print "%s: flags: 0x%x, position: %d, data_size: %d" % \
(b.sid(count=64), b.flags, b.position, b.data_size)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
import eblob
if __name__ == '__main__':
print >> sys.stderr, "Started"
for base in sys.argv[1:]:
try:
print "Processing: %s" % base
b = eblob.blob(base, index_mode='w+b')
for eid in b.iterate(want_removed=False, over_data=True):
print "%s: flags: %x, position: %d, data_size: %d, disk_size: %d" % \
(b.sid(count=64), b.flags, b.position, b.data_size, b.disk_size)
b.write_index()
except NameError as e:
print "Completed:", e
print >> sys.stderr, "Finished"
#!/usr/bin/python
# -*- coding: utf-8 -*-
import eblob
try:
b = eblob.blob('/tmp/data.0', index_mode='w+b')
for eid in b.iterate(want_removed=False, over_data=True):
print "%s: flags: %x, position: %d, data_size: %d, disk_size: %d" % \
(b.sid(count=64), b.flags, b.position, b.data_size, b.disk_size)
b.write_index()
except NameError as e:
print "Completed:", e
#!/usr/bin/env python # -*- coding: utf-8 -*- import sys import eblob if __name__ == '__main__': print >>sys.stderr, "Started" for base in sys.argv[1:]: try: print "Processing: %s" % base b = eblob.blob(base, index_mode='w+b') for eid in b.iterate(want_removed=False, over_data=True): print "%s: flags: %x, position: %d, data_size: %d, disk_size: %d" % \ (b.sid(count=64), b.flags, b.position, b.data_size, b.disk_size) b.write_index() except NameError as e: print "Completed:", e print >>sys.stderr, "Finished"
