def run(self, force=False):
'''
task.run(force=False)
Performs the task.
Parameters
----------
* force: if True, always run the task (even if it ran before)
'''
assert self.can_run()
assert False or not self.finished
args = [value(dep) for dep in self.dependencies]
kwargs = dict((key,value(dep)) for key,dep in self.kwdependencies.iteritems())
self._result = self.f(*args,**kwargs)
name = self.filename()
store.dump(self._result,name)
self.finished = True
if self.print_result:
print self._result
return self._result
# a cheating waw to say if we are empty...
index_empty = True
for k in GID_index.iterkeys():
index_empty = False
break
t0 = time.time()
if not index_empty:
print 'updating...'
##!!! this takes a substantially longer time initially... (about 30x longer)
GID_index, failed, skipped = gid_index(index, GID_index)
else:
print 'indexing...'
GID_index, failed, skipped = gid_index(index)
store.dump(GID_index, INDEX_PATH, index_depth=2)
t1 = time.time()
print 'done in:', t1 - t0, 'seconds.'
json.dump(failed,
file(os.path.join('test', 'failed-to-categorise.json'), 'w'))
##!!! TODO: archive descriptions to help index/tag items...
# NOTE: each import from an existing archive will be as follows:
# - full listing
# - find new subtrees
# - find modified items (file date diff)
print len(index)
index = list(
split_images(index_by_name(list_files(config['ARCHIVE_ROOT']))))
l = len(index)
index = dict(index)
print l, len(index)
json.dump(index, file(os.path.join('test', 'filelist.json'), 'w'))
root_index = store.dump(index,
os.path.join('test', 'index'),
index_depth=1)
## ##!!! this is not used in anything yet...
## json.dump(root_index, file(os.path.join('test', 'index', 'file_index.json'), 'w'))
store.pack(os.path.join('test', 'index'), keep_files=False)
d = store.load(os.path.join('test', 'index'))
print len(d)
k = d.keys()[0]
i = store.Index(os.path.join('test', 'index'))
# a cheating waw to say if we are empty...
index_empty = True
for k in GID_index.iterkeys():
index_empty = False
break
t0 = time.time()
if not index_empty:
print 'updating...'
##!!! this takes a substantially longer time initially... (about 30x longer)
GID_index, failed, skipped = gid_index(index, GID_index)
else:
print 'indexing...'
GID_index, failed, skipped = gid_index(index)
store.dump(GID_index, INDEX_PATH, index_depth=2)
t1 = time.time()
print 'done in:', t1-t0, 'seconds.'
json.dump(failed, file(os.path.join('test', 'failed-to-categorise.json'), 'w'))
##!!! TODO: archive descriptions to help index/tag items...
# NOTE: each import from an existing archive will be as follows:
# - full listing
# - find new subtrees
# - find modified items (file date diff)
print len(index)
index = list(split_images(index_by_name(list_files(config['ARCHIVE_ROOT']))))
l = len(index)
index = dict(index)
print l, len(index)
json.dump(index, file(os.path.join('test', 'filelist.json'), 'w'))
root_index = store.dump(index, os.path.join('test', 'index'), index_depth=1)
## ##!!! this is not used in anything yet...
## json.dump(root_index, file(os.path.join('test', 'index', 'file_index.json'), 'w'))
store.pack(os.path.join('test', 'index'), keep_files=False)
d = store.load(os.path.join('test', 'index'))
print len(d)
k = d.keys()[0]
i = store.Index(os.path.join('test', 'index'))
