def go(self):
db = get_easy_algo_db()
extra = self.options.get_extra()
if len(extra) > 1:
msg = 'Only one extra param accepted.'
raise dtu.DTUserError(msg)
if len(extra) == 1:
query = extra[0]
else:
query = 'all'
persons = db.query_and_instance('person', query)
if self.options.roster:
out_file = dtu.expand_all(self.options.roster)
outd = os.path.dirname(out_file)
roster = create_roster(persons, outd)
dtu.write_data_to_file(roster, out_file)
else:
# instance ll
tags = set()
for p in persons.values():
tags.update(p.get_tags())
print(list(persons))
print('tags: %s' % list(tags))
table = table_people(persons)
print(table)
def check(self):
fn = dtu.expand_all(self.filename)
if not os.path.exists(fn):
msg = 'Dir does not exist: %s' % fn
raise CheckFailed(msg)
if not os.path.isdir(fn):
msg = 'Expect this to be a directory: %s' % fn
raise CheckFailed(msg)
def check(self):
d = dtu.expand_all(self.dirname)
if not os.path.exists(d):
msg = 'The repo does not exist'
l = 'The repo does not exist in directory:\n %s' % d
raise CheckFailed(msg, l)
cwd = '.'
cmd = ['git', '-C', d, 'remote', 'get-url', 'origin']
fail_if_stdout_contains(cwd, cmd, 'https')
def check(self):
d = dtu.expand_all(self.dirname)
assert_is_git_repository(d, CheckError) # error = abort
age = get_repo_age(d)
max_age = self.max_age_hours * 60 * 60
if age > max_age:
d1 = duration_compact(age)
d2 = duration_compact(max_age)
msg = "The repository has not been pulled for %s (maximum tolerated %s). " % (
d1, d2)
raise CheckFailed(msg)
def check(self):
fn = expand_all(self.filename)
short = os.path.basename(self.filename)
if not os.path.exists(fn):
msg = 'Path does not exist: %s' % short
l = 'Complete path:\n %s' % fn
raise CheckFailed(msg, l)
if os.path.isdir(fn):
msg = 'Expect this to be a file, not a directory: %s' % short
l = 'Complete path:\n %s' % fn
raise CheckFailed(msg)
def assert_is_git_repository(dirname, ExceptionName):
""" Returns True if the directory exists and is a git repository. """
assert ExceptionName in [CheckError, CheckFailed]
d = dtu.expand_all(dirname)
if not os.path.exists(d):
msg = 'The repo does not exist'
l = 'The repo does not exist in directory:\n %s' % d
raise ExceptionName(msg, l)
gitd = os.path.join(d, '.git')
if not os.path.exists(gitd):
msg = 'The directory does not appear to be a Git repository.'
l = 'The path %r does not exist' % gitd
raise ExceptionName(msg)
def check(self):
fn = expand_all(self.filename)
if not os.path.exists(fn):
msg = 'File does not exist: %s' % fn
raise CheckError(msg)
contents = open(fn).read()
l = 'Entire contents of %s:\n' % fn + indent(contents, ' > ')
if not self.string in contents:
msg = 'String %r not contained in %s' % (self.string, fn)
raise CheckFailed(msg, l)
def check(self):
fn = expand_all(self.filename)
if not os.path.exists(fn):
msg = 'Cannot check permissions if file or dir does not exist.'
raise CheckError(msg)
fstats = os.stat(fn)
filemode = oct(stat.S_IMODE(fstats.st_mode))
if len(filemode) > 4:
filemode = filemode[-4:]
if filemode != self.expected:
msg = ('Expected mode %r, obtained %r.' %
(self.expected, filemode))
raise CheckFailed(msg)
def check(self):
fn = dtu.expand_all(self.filename)
short = os.path.basename(self.filename)
if not os.path.exists(fn):
msg = 'Path does not exist: %s' % short
l = 'Complete path:\n %s' % fn
raise CheckFailed(msg, l)
if os.path.isdir(fn):
msg = 'Expect this to be a file, not a directory: %s' % short
l = 'Complete path:\n %s' % fn
raise CheckFailed(msg)
data = open(fn).read()
lines = data.split('\n')
maxlen = max(map(len, lines))
if len(lines) < 100 and maxlen < 80:
return data
def anti_instagram_annotations_test(dirname, out_base):
base = dtu.expand_all(dirname)
if not os.path.exists(base):
msg = 'Could not find directory %s' % base
raise Exception(msg)
dirs = dtu.locate_files(base, '*.iids1', alsodirs=True)
directory_results = {}
overall_results = []
if not dirs:
raise ValueError('No IIDS1 directories found')
for d in dirs:
out = os.path.join(out_base, os.path.basename(d) + '.v')
if not os.path.exists(out):
os.makedirs(out)
results = examine_dataset(d, out)
overall_results = merge_comparison_results(results, overall_results)
directory_results[d] = results
db = shelve.open('tests_results', flag='n')
db['directory_results'] = directory_results
db['overall_results'] = overall_results
db.close()
logger.info(
"overall average error: %f" %
(overall_results['total_error'] / overall_results['total_pixels']))
logger.info("overall regions checked: %f" %
(overall_results['total_regions']))
for t in overall_results['v_vals'].keys():
logger.info("region %f: RGB %f,%f,%f, HSV %f,%f,%f" %
(t, np.mean(overall_results['r_vals'][t]),
np.mean(overall_results['g_vals'][t]),
np.mean(overall_results['b_vals'][t]),
np.mean(overall_results['h_vals'][t]),
np.mean(overall_results['s_vals'][t]),
np.mean(overall_results['v_vals'][t])))
def examine_dataset(dirname, out):
logger.info(dirname)
dirname = dtu.expand_all(dirname)
jpgs = dtu.locate_files(dirname, '*.jpg')
mats = dtu.locate_files(dirname, '*.mat')
logger.debug('I found %d JPGs and %d .mat.' % (len(jpgs), len(mats)))
if len(jpgs) == 0:
msg = 'Not JPGs found in %r.' % dirname
raise ValueError(msg)
# if len(mats) == 0:
# msg = 'Not enough mats.'
# raise ValueError(msg)
first_jpg = sorted(jpgs)[0]
logger.debug('Using jpg %r to learn transformation.' % first_jpg)
first_jpg_image = dtu.image_cv_from_jpg_fn(first_jpg)
success, health, parameters = calculate_transform(first_jpg_image)
s = ""
s += 'success: %s\n' % str(success)
s += 'health: %s\n' % str(health)
s += 'parameters: %s\n' % str(parameters)
w = os.path.join(out, 'learned_transform.txt')
with open(w, 'w') as f:
f.write(s)
logger.info(s)
transform = ScaleAndShift(**parameters)
duckietown_package_dir = dtu.get_ros_package_path('duckietown')
config_dir = os.path.join(
duckietown_package_dir,
'config/baseline/line_detector/line_detector_node')
if not os.path.exists(config_dir):
msg = 'Could not find configuration dir %s' % config_dir
raise Exception(msg)
config_dir = dtu.expand_all(config_dir)
configurations = dtu.locate_files(config_dir, '*.yaml')
if not configurations:
msg = 'Could not find any configuration file in %s.' % config_dir
raise Exception(msg)
#logger.info('configurations: %r' % configurations)
for j in jpgs:
summaries = []
shape = (200, 160)
interpolation = cv2.INTER_NEAREST
bn = os.path.splitext(os.path.basename(j))[0]
fn = os.path.join(out, '%s.all.png' % (bn))
if os.path.exists(fn):
logger.debug('Skipping because file exists: %r' % fn)
else:
for c in configurations:
logger.info('Trying %r' % c)
name = os.path.splitext(os.path.basename(c))[0]
if name in ['oreo', 'myrtle', 'bad_lighting', '226-night']:
continue
with open(c) as f:
stuff = yaml.load(f)
if not 'detector' in stuff:
msg = 'Cannot find "detector" section in %r' % c
raise ValueError(msg)
detector = stuff['detector']
logger.info(detector)
if not isinstance(detector, list) and len(detector) == 2:
raise ValueError(detector)
def LineDetectorClass():
return dtu.instantiate(detector[0], detector[1])
s = run_detection(transform,
j,
out,
shape=shape,
interpolation=interpolation,
name=name,
LineDetectorClass=LineDetectorClass)
summaries.append(s)
together = dtu.make_images_grid(summaries, cols=1, pad=10)
cv2.imwrite(fn, dtu.zoom_image(together, 4))
overall_results = []
comparison_results = {}
for m in mats:
logger.debug(m)
jpg = os.path.splitext(m)[0] + '.jpg'
if not os.path.exists(jpg):
msg = 'JPG %r for mat %r does not exist' % (jpg, m)
logger.error(msg)
else:
frame_results = test_pair(transform, jpg, m, out)
comparison_results[m] = frame_results
overall_results = merge_comparison_results(comparison_results[m],
overall_results)
print "comparison_results[m]=frame_results"
print "finished mats: " + dirname
return overall_results
