def run(self, force=False): ''' task.run(force=False) Performs the task. Parameters ---------- * force: if True, always run the task (even if it ran before) ''' assert self.can_run() assert False or not self.finished args = [value(dep) for dep in self.dependencies] kwargs = dict((key,value(dep)) for key,dep in self.kwdependencies.iteritems()) self._result = self.f(*args,**kwargs) name = self.filename() store.dump(self._result,name) self.finished = True if self.print_result: print self._result return self._result
# a cheating waw to say if we are empty... index_empty = True for k in GID_index.iterkeys(): index_empty = False break t0 = time.time() if not index_empty: print 'updating...' ##!!! this takes a substantially longer time initially... (about 30x longer) GID_index, failed, skipped = gid_index(index, GID_index) else: print 'indexing...' GID_index, failed, skipped = gid_index(index) store.dump(GID_index, INDEX_PATH, index_depth=2) t1 = time.time() print 'done in:', t1 - t0, 'seconds.' json.dump(failed, file(os.path.join('test', 'failed-to-categorise.json'), 'w')) ##!!! TODO: archive descriptions to help index/tag items... # NOTE: each import from an existing archive will be as follows: # - full listing # - find new subtrees # - find modified items (file date diff)
print len(index) index = list( split_images(index_by_name(list_files(config['ARCHIVE_ROOT'])))) l = len(index) index = dict(index) print l, len(index) json.dump(index, file(os.path.join('test', 'filelist.json'), 'w')) root_index = store.dump(index, os.path.join('test', 'index'), index_depth=1) ## ##!!! this is not used in anything yet... ## json.dump(root_index, file(os.path.join('test', 'index', 'file_index.json'), 'w')) store.pack(os.path.join('test', 'index'), keep_files=False) d = store.load(os.path.join('test', 'index')) print len(d) k = d.keys()[0] i = store.Index(os.path.join('test', 'index'))
# a cheating waw to say if we are empty... index_empty = True for k in GID_index.iterkeys(): index_empty = False break t0 = time.time() if not index_empty: print 'updating...' ##!!! this takes a substantially longer time initially... (about 30x longer) GID_index, failed, skipped = gid_index(index, GID_index) else: print 'indexing...' GID_index, failed, skipped = gid_index(index) store.dump(GID_index, INDEX_PATH, index_depth=2) t1 = time.time() print 'done in:', t1-t0, 'seconds.' json.dump(failed, file(os.path.join('test', 'failed-to-categorise.json'), 'w')) ##!!! TODO: archive descriptions to help index/tag items... # NOTE: each import from an existing archive will be as follows: # - full listing # - find new subtrees # - find modified items (file date diff)
print len(index) index = list(split_images(index_by_name(list_files(config['ARCHIVE_ROOT'])))) l = len(index) index = dict(index) print l, len(index) json.dump(index, file(os.path.join('test', 'filelist.json'), 'w')) root_index = store.dump(index, os.path.join('test', 'index'), index_depth=1) ## ##!!! this is not used in anything yet... ## json.dump(root_index, file(os.path.join('test', 'index', 'file_index.json'), 'w')) store.pack(os.path.join('test', 'index'), keep_files=False) d = store.load(os.path.join('test', 'index')) print len(d) k = d.keys()[0] i = store.Index(os.path.join('test', 'index'))