def initialize_annex_keys(session, find_output_filename):
filename = find_output_filename
keys = set()
with file(filename) as infile:
while True:
try:
line = infile.next()
except StopIteration:
break
data = gitannex.parse_json_line(line, convert_to_unicode=True, verbose_warning=True)
keys.add(data["key"])
for k in keys:
dbk = AnnexKey()
dbk.name = k
session.add(dbk)
print "added %d keys" % len(keys)
print datetime.now()
session.commit()
print "successful commit", datetime.now()
def populate_files(self):
proc = self.pmgr.make_find_proc()
count = 0
while proc.returncode is None:
try:
line = proc.stdout.next()
count +=1
except StopIteration:
break
data = gitannex.parse_json_line(
line,
convert_to_unicode=True,
verbose_warning=True)
key = data['key']
dbkey = self.keymgr.get_by_name(key)
if dbkey is None:
dbkey = self.keymgr.add_key(key)
self.add_file(data, dbkey.id)
if proc.returncode:
msg = "find proc returned %d" % proc.returncode
raise RuntimeError, msg
def add_files(session, keylookup, find_output_filename):
with file(find_output_filename) as infile:
count = 0
current = datetime.now()
while True:
try:
line = infile.next()
count += 1
except StopIteration:
break
data = gitannex.parse_json_line(line, convert_to_unicode=True, verbose_warning=True)
commit = False
add_file_to_database(session, keylookup, data)
if not count % 5000:
commit = True
print "Committing %d files" % count
now = datetime.now()
print "Diff", now - current
current = now
session.commit()
print "%d files added." % count
session.commit()
print datetime.now()
