def calculate(f_to_calc, f_to_overwrite, console, download):
"""Calculate a list of features."""
sys.stdout.write('loading')
sys.stdout.flush()
repos = Repo.load_sample()
seen = 0
total = len(repos)
dl_failures = []
calc_failures = []
last_write = datetime.datetime.now()
if f_to_calc or f_to_overwrite or download:
for repo in repos:
seen += 1
success = True
if download:
success = utils.clone(repo)
if not success:
dl_failures.append(repo)
continue
try:
if f_to_calc:
logging.info("calc: %s", repo)
repo.calculate_features(f_to_calc)
if f_to_overwrite:
logging.info("calc: %s", repo)
repo.calculate_features(f_to_overwrite, overwrite=True)
repo._clear_support_features() # we're done with this repo now
except:
print # from status line
logging.exception("!problem: %s", repo)
calc_failures.append(repo)
print
progress_bar(seen, total)
since_write = datetime.datetime.now() - last_write
if since_write > datetime.timedelta(minutes=5):
sys.stdout.write("\r(writing results)")
sys.stdout.flush()
Repo.write_update(repos)
last_write = datetime.datetime.now()
print # from progress bar line
if dl_failures:
print "%s failed to download:" % len(dl_failures)
for f in dl_failures:
print " %s" % f
print
if calc_failures:
print "%s failed during calc:" % len(calc_failures)
for f in calc_failures:
print " %s" % f
print
if console:
message = ('`repos` contains results;\n'
'use ^d to write out or `exit()` to cancel')
code.interact(message, local=locals())
print 'writing out...'
Repo.write_update(repos)
