def import_datasources(path: str, sync: str, recursive: bool) -> None:
"""Import datasources from YAML"""
from superset.utils import dict_import_export
sync_array = sync.split(",")
path_object = Path(path)
files = []
if path_object.is_file():
files.append(path_object)
elif path_object.exists() and not recursive:
files.extend(path_object.glob("*.yaml"))
files.extend(path_object.glob("*.yml"))
elif path_object.exists() and recursive:
files.extend(path_object.rglob("*.yaml"))
files.extend(path_object.rglob("*.yml"))
for file_ in files:
logger.info("Importing datasources from file %s", file_)
try:
with file_.open() as data_stream:
dict_import_export.import_from_dict(
db.session, yaml.safe_load(data_stream), sync=sync_array)
except Exception as ex: # pylint: disable=broad-except
logger.error("Error when importing datasources from file %s",
file_)
logger.error(ex)
def cleanse():
"""Remove old unreferenced media files."""
now = datetime.now()
max_age = timedelta(minutes=7)
p = Path(current_app.config['UPLOAD_MNT'])
click.echo('Looking for unreferenced uploaded files...')
for child in p.rglob('*'):
if not child.is_file():
continue
public_path = str(child.relative_to(p))
mtime = datetime.fromtimestamp(child.stat().st_mtime)
if now - mtime < max_age:
click.secho('* %s is fresh, skipping' % child, fg='green')
else:
media_documents, _ = current_app.datastore.search_documents(
index='media',
doc_type=None,
query={'term': {
'sys_filename.raw': public_path
}})
if len(media_documents) > 0:
click.secho('* %s is in use, skipping' % child, fg='green')
else:
click.secho('* %s is stale and unreferenced, deleting' % child,
fg='yellow')
child.unlink()
click.echo('Removing any empty directories...')
for child in p.rglob('*'):
if not child.is_dir():
continue
try:
child.rmdir()
except OSError:
# expected for all nonempty dirs, faster than checking if they are indeed empty
pass
else:
click.secho('* deleted: %s' % child, fg='yellow')
click.echo('Done.')
def _load_images(
img_folder: Path, mask_folder_list: List[Path], img_extension: str
) -> Tuple[List[Path], Dict[str, List[Path]]]:
img_paths: List[Path] = list(
img_folder.rglob(f'**/*.{img_extension.replace(".", "")}')
)
assert (
len(img_paths) > 0
), f"The length of the image must be higher than 1, given {len(img_paths)}."
mask_paths_dict: Dict[str, List[Path]] = {}
for m in mask_folder_list:
mask_paths_dict[str(m)] = list(
m.rglob(f'**/*.{img_extension.replace(".", "")}')
)
assert (
mask_paths_dict[str(m)].__len__() > 0
), f"The length of the masks {m} must be higher than 1, \
given {mask_paths_dict[str(m)].__len__()}"
if mask_folder_list.__len__() > 0:
assert len(img_paths) == list(mask_paths_dict.values())[0].__len__()
assert len(set(map(len, list(mask_paths_dict.values())))) == 1, len(
set(map(len, list(mask_paths_dict.values())))
)
return img_paths, mask_paths_dict
def main():
doc_path = Path(__file__).absolute().parent.parent / "docs"
# # clean out all the old docs
clean_docs()
cmd = "jupyter nbconvert --ExecutePreprocessor.timeout=1200 --to notebook --execute --inplace "
for note_book_path in doc_path.rglob("*.ipynb"):
# skip any hidden directores, eg .ipynb_checkpoints
if note_book_path.parent.name.startswith('.'):
continue
cmd_to_run = cmd + str(note_book_path)
result = os.system(cmd_to_run)
if result != 0:
raise RuntimeError("failed to run: " + cmd_to_run)
# run auto api-doc
with change_directory(doc_path):
api_doc_command = " sphinx-apidoc ../obsplus -o api"
result = os.system(api_doc_command)
if result != 0:
print('failed to run: ' + api_doc_command)
# make the rest of the docs
doc_cmd = 'make html'
result = os.system(doc_cmd)
if result != 0:
print('failed to run: ' + doc_cmd)
def submission_test(config_file, checkpoint_file, save_path):
model = init_detector(config_file, checkpoint_file)
classes = ('holothurian', 'echinus', 'scallop', 'starfish')
results = ['name,image_id,confidence,xmin,ymin,xmax,ymax\n']
test_path = Path(
'/home/fengyouliang/datasets/underwater/test/test-A-image')
all_test_images = list(test_path.rglob('*.jpg'))
image_bar = tqdm(all_test_images)
for image in image_bar:
result = inference_detector(model, image.as_posix())
if isinstance(result, tuple):
bbox_result, segm_result = result
if isinstance(segm_result, tuple):
segm_result = segm_result[0] # ms rcnn
else:
bbox_result, segm_result = result, None
for class_idx, class_result in enumerate(bbox_result):
for item in class_result:
x1, y1, x2, y2, score = item
csv_line = f"{classes[class_idx]},{image.stem},{score},{round(x1)},{round(y1)},{round(x2)},{round(y2)}\n"
results.append(csv_line)
with open(save_path, 'w', encoding='utf-8') as fid:
fid.writelines(results)
print(f'saved in {save_path}')
def import_datasources(path: str, sync: str, recursive: bool) -> None:
"""Import datasources from YAML"""
from superset.datasets.commands.importers.v0 import ImportDatasetsCommand
sync_array = sync.split(",")
path_object = Path(path)
files: List[Path] = []
if path_object.is_file():
files.append(path_object)
elif path_object.exists() and not recursive:
files.extend(path_object.glob("*.yaml"))
files.extend(path_object.glob("*.yml"))
elif path_object.exists() and recursive:
files.extend(path_object.rglob("*.yaml"))
files.extend(path_object.rglob("*.yml"))
contents = {path.name: open(path).read() for path in files}
try:
ImportDatasetsCommand(contents, sync_array).run()
except Exception: # pylint: disable=broad-except
logger.exception("Error when importing dataset")
def import_datasources(path, sync, recursive):
"""Import datasources from YAML"""
sync_array = sync.split(',')
p = Path(path)
files = []
if p.is_file():
files.append(p)
elif p.exists() and not recursive:
files.extend(p.glob('*.yaml'))
files.extend(p.glob('*.yml'))
elif p.exists() and recursive:
files.extend(p.rglob('*.yaml'))
files.extend(p.rglob('*.yml'))
for f in files:
logging.info('Importing datasources from file %s', f)
try:
with f.open() as data_stream:
dict_import_export.import_from_dict(
db.session, yaml.safe_load(data_stream), sync=sync_array)
except Exception as e:
logging.error('Error when importing datasources from file %s', f)
logging.error(e)
def import_datasources(path, sync, recursive=False):
"""Import datasources from YAML"""
sync_array = sync.split(',')
p = Path(path)
files = []
if p.is_file():
files.append(p)
elif p.exists() and not recursive:
files.extend(p.glob('*.yaml'))
files.extend(p.glob('*.yml'))
elif p.exists() and recursive:
files.extend(p.rglob('*.yaml'))
files.extend(p.rglob('*.yml'))
for f in files:
logging.info('Importing datasources from file %s', f)
try:
with f.open() as data_stream:
dict_import_export_util.import_from_dict(
db.session,
yaml.safe_load(data_stream),
sync=sync_array)
except Exception as e:
logging.error('Error when importing datasources from file %s', f)
logging.error(e)
def main():
doc_path = Path(__file__).absolute().parent.parent / "docs"
# make api documentation
cmd = "jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace"
api_path = doc_path / "api"
# clear API documentation
if api_path.exists():
shutil.rmtree(api_path)
# execute all the notebooks
for note_book_path in doc_path.rglob("*.ipynb"):
cmd_to_run = cmd + " {}".format(str(note_book_path))
result = os.system(cmd_to_run)
if result != 0:
msg = "failed to execute " + str(note_book_path)
raise RuntimeError(msg)
def import_dashboards(path, recursive):
"""Import dashboards from JSON"""
p = Path(path)
files = []
if p.is_file():
files.append(p)
elif p.exists() and not recursive:
files.extend(p.glob("*.json"))
elif p.exists() and recursive:
files.extend(p.rglob("*.json"))
for f in files:
logging.info("Importing dashboard from file %s", f)
try:
with f.open() as data_stream:
dashboard_import_export.import_dashboards(db.session, data_stream)
except Exception as e:
logging.error("Error when importing dashboard from file %s", f)
logging.error(e)
def import_dashboards(path: str, recursive: bool, username: str) -> None:
"""Import dashboards from ZIP file"""
from superset.dashboards.commands.importers.v0 import ImportDashboardsCommand
path_object = Path(path)
files: List[Path] = []
if path_object.is_file():
files.append(path_object)
elif path_object.exists() and not recursive:
files.extend(path_object.glob("*.json"))
elif path_object.exists() and recursive:
files.extend(path_object.rglob("*.json"))
if username is not None:
g.user = security_manager.find_user(username=username)
contents = {path.name: open(path).read() for path in files}
try:
ImportDashboardsCommand(contents).run()
except Exception: # pylint: disable=broad-except
logger.exception("Error when importing dashboard")
def import_dashboards(path, recursive=False):
"""Import dashboards from JSON"""
p = Path(path)
files = []
if p.is_file():
files.append(p)
elif p.exists() and not recursive:
files.extend(p.glob('*.json'))
elif p.exists() and recursive:
files.extend(p.rglob('*.json'))
for f in files:
logging.info('Importing dashboard from file %s', f)
try:
with f.open() as data_stream:
dashboard_import_export.import_dashboards(
db.session, data_stream)
except Exception as e:
logging.error('Error when importing dashboard from file %s', f)
logging.error(e)
def import_dashboards(path, recursive, username):
"""Import dashboards from JSON"""
p = Path(path)
files = []
if p.is_file():
files.append(p)
elif p.exists() and not recursive:
files.extend(p.glob('*.json'))
elif p.exists() and recursive:
files.extend(p.rglob('*.json'))
if username is not None:
g.user = security_manager.find_user(username=username)
for f in files:
logging.info('Importing dashboard from file %s', f)
try:
with f.open() as data_stream:
dashboard_import_export.import_dashboards(
db.session, data_stream)
except Exception as e:
logging.error('Error when importing dashboard from file %s', f)
logging.error(e)
def import_dashboards(path: str, recursive: bool, username: str) -> None:
"""Import dashboards from JSON"""
from superset.utils import dashboard_import_export
path_object = Path(path)
files = []
if path_object.is_file():
files.append(path_object)
elif path_object.exists() and not recursive:
files.extend(path_object.glob("*.json"))
elif path_object.exists() and recursive:
files.extend(path_object.rglob("*.json"))
if username is not None:
g.user = security_manager.find_user(username=username)
for file_ in files:
logger.info("Importing dashboard from file %s", file_)
try:
with file_.open() as data_stream:
dashboard_import_export.import_dashboards(data_stream)
except Exception as ex: # pylint: disable=broad-except
logger.error("Error when importing dashboard from file %s", file_)
logger.error(ex)
def check_source(self, test=False):
path = Path(self.folder_path)
if self.include_subdirectories:
iterator = path.rglob(pattern='*')
else:
iterator = path.glob(pattern='*')
for entry in iterator:
if entry.is_file() or entry.is_symlink():
with entry.open(mode='rb+') as file_object:
try:
fcntl.lockf(file_object, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError as exception:
if exception.errno != errno.EAGAIN:
raise
else:
self.handle_upload(
file_object=file_object,
expand=(
self.uncompress == SOURCE_UNCOMPRESS_CHOICE_Y),
label=entry.name)
if not test:
entry.unlink()
