def op_export(app: AppSession, args):
from labler.environ import env
if len(args) == 0:
raise errors.FatalException('no project name specified')
name = args[0]
directory = app.lambdaenv.directory
if directory is None:
raise errors.FatalException('no output directory specified (--dir/-d)')
if not os.path.exists(directory):
raise errors.FatalException(
f'output directory {directory} does not exist')
if not os.access(directory, os.W_OK):
raise errors.FatalException(
f'no write permission for output directory {directory}')
if not os.access(directory, os.X_OK):
raise errors.FatalException(
f'no execute permission for output directory {directory}')
if app.lambdaenv.pretend:
app.printer.notice(
f'would export labels for project {name} to {directory}')
else:
app.printer.action(
f'exporting labels for project {name} to {directory}')
env.data_handler.export_labels(name, app, directory)
def op_add(app: AppSession, args):
from labler.environ import env
if len(args) == 0:
raise errors.FatalException('no project name specified')
name = args[0]
data_dir = app.lambdaenv.directory
if data_dir is None:
raise FatalException(
'no data directory supplied when suing add command')
if app.lambdaenv.output is None:
app.lambdaenv.output = data_dir
if not os.path.exists(data_dir):
raise errors.FatalException(f'data dir "{data_dir}" does not exist')
if not os.access(data_dir, os.R_OK):
raise errors.FatalException(
f'no read permission on data dir "{data_dir}"')
if not os.access(data_dir, os.X_OK):
raise errors.FatalException(
f'no execute permission on data dir "{data_dir}"')
if app.lambdaenv.pretend:
app.printer.notice(
f'would add directory "{data_dir}" to project called {name}')
else:
app.printer.action(f'adding directly "{data_dir}" to project: {name}')
env.db.add_data_dir(name, app)
env.data_handler.add_data_dir(name, app, data_dir)
def op_examples(app: AppSession, args):
from labler.environ import env
if len(args) == 0:
raise errors.FatalException('no project name specified')
project_name = args[0]
app.printer.action(f'listing examples for project {project_name}')
examples = env.db.get_examples(project_name)
app.printer.blanknotice('')
header = 'width \theight \tfile path \tfile name'.expandtabs(tabsize=20)
app.printer.blanknotice(header)
app.printer.blanknotice('-' * len(header))
for example in examples:
file_path = example.file_path
file_name = example.file_name
width = example.width
height = example.height
app.printer.blanknotice(
f'{width} \t{height} \t{file_path} \t{file_name}'.expandtabs(
tabsize=20))
def op_claims(app: AppSession, args):
from labler.environ import env
if len(args) == 0:
raise errors.FatalException('no project name specified')
name = args[0]
if not env.db.project_exists(name):
raise FatalException(f'no project exist with name {name}')
app.printer.action(f'listing claims for project {name}')
claims = env.db.get_claims(name)
app.printer.blanknotice('')
header = 'file name \tstatus \tclaimed by \tclaimed at'.expandtabs(
tabsize=20)
app.printer.blanknotice(header)
app.printer.blanknotice('-' * len(header))
for claim in claims:
claim_json = claim.to_dict()
claimed_at = claim_json[ClaimFields.CLAIMED_AT]
app.printer.blanknotice(
f'{claim.file_name} \t{claim.status} \t{claim.claimed_by} \t{claimed_at}'
.expandtabs(tabsize=20))
def add_examples(
self,
project_name: str,
app: AppSession,
examples: List[Tuple[str, str, int, int]],
session=None
) -> None:
project = session.query(Projects).filter_by(project_name=project_name).first()
if project is None:
raise errors.FatalException(f'no project exist for name {project_name}')
for base_path, file_name, width, height in examples:
existing_example = session.query(Examples)\
.filter_by(project_name=project_name)\
.filter_by(file_path=base_path)\
.filter_by(file_name=file_name)\
.first()
if existing_example is not None:
continue
if app.lambdaenv.pretend:
continue
example = Examples()
example.project_name = project_name
example.file_path = base_path
example.file_name = file_name
example.width = width
example.height = height
session.add(example)
session.commit()
def update_project(self, name, app: AppSession, session=None) -> None:
project = session.query(Projects).filter_by(project_name=name).first()
if project is None:
raise errors.FatalException(f'no project exist for name {name}')
project.project_name = name
project.classes = app.lambdaenv.classes or project.classes
project.project_type = app.lambdaenv.project_type or project.project_type
project.directory = app.lambdaenv.directory or project.directory
session.add(project)
session.commit()
def read_argv(self, argv):
from labler import errors
try:
opts, argv = getopt.gnu_getopt(argv, 'pVsvohuc:t:d:C:O:', [
'pretend', 'version', 'silent', 'verbose', 'overwrite',
'unique', 'help', 'classes=', 'type=', 'dir=', 'cores=',
'output='
])
return opts, argv
except getopt.GetoptError as e:
raise errors.FatalException(f'unknown option {e.opt}')
def add_data_dir(self, name, app: AppSession, session=None) -> None:
project = session.query(Projects).filter_by(project_name=name).first()
if project is None:
raise errors.FatalException(f'no project exist for name {name}')
lambda_dir = app.lambdaenv.directory
if lambda_dir is None:
return
directory = project.directory or ''
project.directory = ';'.join(set([lambda_dir] + directory.split(';')))
session.add(project)
session.commit()
def op_create(app: AppSession, args):
from labler.environ import env
if len(args) == 0:
raise errors.FatalException('no project name specified')
name = args[0]
if env.db.project_exists(name):
raise FatalException(f'project already exists with name {name}')
if app.lambdaenv.pretend:
app.printer.notice(f'would create new project called {name}')
else:
app.printer.action(f'creating project: {name}')
env.db.create_project(name, app)
def op_sync(app: AppSession, args):
from labler.environ import env
if len(args) == 0:
raise errors.FatalException('no project name specified')
project_name = args[0]
if not env.db.project_exists(project_name):
raise FatalException(
f'project with name "{project_name}" does not exist')
if app.lambdaenv.pretend:
app.printer.notice(f'would sync project {project_name}')
else:
app.printer.action(f'syncing project: {project_name}')
data_dirs = env.db.get_data_dirs(project_name)
for data_dir in data_dirs:
try:
env.data_handler.sync_data_dir(project_name, app, data_dir)
except Exception as e:
print(traceback.format_exc())
raise errors.FatalException(
f'could not sync project dir "{data_dir}": {str(e)}')
def load_image(image_file):
"""
Sometimes skimage will hang when loading an image in gpu-02, but not on gpu-01.
Internally skimage uses PIL to open images, but somewhere it gets stuck. Using
PIL directly seems to work around the issues, so try that first. The only downsize
we have to do the conversion to np arrays manually and for some image formats
this might not work, cause some corner case is not covered, so try to use skimage
when using PIL fails.
"""
try:
image = Image.open(image_file)
return pillow2array(image)
except Exception:
pass
try:
return io.imread(image_file)
except Exception as e:
raise errors.FatalException(f'PIL/skimage could not open image {image_file}, giving up: {str(e)}')
def op_labels(app: AppSession, args):
from labler.environ import env
if len(args) == 0:
raise errors.FatalException('no project name specified')
name = args[0]
if not env.db.project_exists(name):
raise FatalException(f'no project exist with name {name}')
if app.lambdaenv.unique:
app.printer.action(f'listing unque labels for project {name}')
labels = env.db.get_unique_labels(name)
header = 'label'
else:
app.printer.action(f'listing labels for project {name}')
labels = env.db.get_labels(name)
header = 'file name \txmin \txmax \tymin \tymax \tsubmitted by \tsubmitted at'.expandtabs(
tabsize=18)
app.printer.blanknotice('')
app.printer.blanknotice(header)
app.printer.blanknotice('-' * len(header))
if app.lambdaenv.unique:
for label in labels:
app.printer.blanknotice(label)
return
for label in labels:
label_json = label.to_dict()
submitted_at = label_json[LabelFields.SUBMITTED_AT]
file_name = label.file_name
if len(file_name) > 15:
file_name = file_name[:12] + '...'
row_1 = f'{file_name} \t{label.xmin} \t{label.xmax} \t{label.ymin} \t'
row_2 = f'{label.ymax} \t{label.submitted_by} \t{submitted_at}'
full_row = row_1 + row_2
app.printer.blanknotice(full_row.expandtabs(tabsize=18))
def __init__(self, argv):
from labler.config import ProjectTypes
from labler import errors
from labler.cli import printer
self.configured = False
self.args = None
self.printer = printer.AppPrinter(self)
self.lambdaenv = LambdaEnviron(LambdaTemplate)
opts, args = self.read_argv(argv)
for opt, arg in opts:
if opt in ('-p', '--pretend'):
self.lambdaenv.pretend = True
elif opt in ('-V', '--version'):
import pkg_resources
version = pkg_resources.require('labler')[0].version
print(version_str.format(version))
return
elif opt in ('-s', '--silent'):
self.lambdaenv.silent = True
self.lambdaenv.suppressed = {'all'}
elif opt in ('-d', '--dir'):
if not os.path.exists(arg):
raise errors.FatalException(
f'directory "{arg}" does not exist')
self.lambdaenv.directory = arg
elif opt in ('-v', '--verbose'):
self.lambdaenv.verbose = True
self.lambdaenv.suppressed = set()
elif opt in ('-u', '--unique'):
self.lambdaenv.unique = True
elif opt in ('-C', '--cores'):
try:
cores = int(arg)
except ValueError:
raise errors.FatalException(
f'argument "{arg}" not a valid number of cores')
if cores < 1:
cores = -1
self.lambdaenv.cores = cores
elif opt in ('-o', '--overwrite'):
self.lambdaenv.overwrite = True
elif opt in ('-O', '--output'):
if not os.path.exists(arg):
raise errors.FatalException(
f'output directory "{arg}" does not exist')
self.lambdaenv.output = arg
elif opt in ('-c', '--classes'):
try:
self.lambdaenv.classes = int(arg)
except ValueError:
raise errors.FatalException(
'invalid argument for option "classes", need an integer value'
)
if self.lambdaenv.classes < 1:
raise errors.FatalException(
f'need at least one class, got "{arg}"')
elif opt in ('-h', '--help'):
print(usage())
sys.exit(0)
elif opt in ('-t', '--type'):
self.lambdaenv.project_type = arg if len(
arg) > 1 else ProjectTypes.get(arg)
if arg not in ProjectTypes.shorts(
) and arg not in ProjectTypes.longs():
raise errors.FatalException(
'unknown project type "{arg}", must be in [{project_types}] or [{project_types_short}]'
.format(arg=arg,
project_types=', '.join(ProjectTypes.values()),
project_types_short=', '.join(
ProjectTypes.keys())))
else:
self.printer.error(f'unknown option: {opt}')
self.args = args
self.configured = True
def get_data_dirs(self, project_name, session=None) -> Set[str]:
project = session.query(Projects).filter_by(project_name=project_name).first()
if project is None:
raise errors.FatalException(f'no project exist for name {project_name}')
return set([directory.rstrip(os.path.sep) for directory in project.directory.split(';')])
