def get_neox_args(context):
args = AttrMap(context.get_hparams())
exp_config = context.get_experiment_config()
# Gather overrides.
overwrite_values = args.pop("overwrite_values")
# We are going to overwrite certain neox_args with determined config values
# from the experiment config to ensure consistency.
assert ("batches" in exp_config["searcher"]["max_length"]
), "Please specify max_length in batches."
assert ("batches" in exp_config["min_validation_period"]
), "Please specify min_validation_period in batches."
overwrite_values.update({
"train_iters":
exp_config["searcher"]["max_length"]["batches"],
"save_interval":
exp_config["min_validation_period"]["batches"],
"eval_interval":
exp_config["min_validation_period"]["batches"],
"global_num_gpus":
context.distributed.get_size(),
"seed":
context.env.trial_seed,
})
for k, v in overwrite_values.items():
logging.info(f"Setting neox_args.{k} to {v}")
# Build neox args.
neox_args = NeoXArgs.process_parsed_deepy_args(
args, overwrite_values=overwrite_values)
return neox_args
def upward_wrapper(self):
childrens_attrs = {}
for k, v in self.children().items():
attr = getattr(v, func.__name__)
if callable(attr):
childrens_attrs[k] = attr()
else:
childrens_attrs[k] = attr
self_attrs = func(self)
intersection = set(childrens_attrs.keys()).intersection(self_attrs.keys())
if intersection:
raise AttributeError(
f"Found keys {intersection} in both childrens_attrs and self_attrs"
)
attrs = {**childrens_attrs, **self_attrs}
attrs = _remove_empty_dict_leaf(attrs)
# TODO(poweic): It's confusing to have both SpaceDict & AttrMap at the same
# time. Also, return AttrMap in new() method of SpaceDict is weird.
# Consider creating that's both SpaceDict and AttrMap at the same time.
if func.__name__ == "get_states":
attrs = AttrMap(attrs)
else:
attrs = SpaceDict(attrs)
return attrs
def get_stats(root: Any) -> AttrMap:
"""
Выдаем статистику
:return: {stats: [dict(name, score) for user], me: position}
"""
return AttrMap(request('stats', login=root.LOGIN))
def sign_in(root: Any, _locals: dict, login: str, password: str,
password2: str):
"""
Регистрация пользователя
"""
if login == '':
_locals['entry_login'].focus()
root.Alert.show('Введите логин')
elif len(login) > 20:
_locals['entry_login'].focus()
root.Alert.show('Логин не должен быть длиннее 20 символов')
elif password == '':
_locals['entry_password'].focus()
root.Alert.show('Введите пароль')
elif password2 == '':
_locals['entry_password2'].focus()
root.Alert.show('Повторите пароль')
elif password != password2:
root.Alert.show('Пароли не совпадают')
elif (response := request('sing_in', login=login,
password=password))['response']:
if os.path.exists('.auth'):
os.remove('.auth')
with open('.auth', 'w') as data:
data.write(f'{response["login"]}::{password}')
subprocess.check_call(['attrib', '+H', '.auth'])
root.LOGIN = response['login']
root.USER_NAME, root.USER_ID = response['login'].split('#')
root.profile = AttrMap(request(f'profile/{root.USER_ID}'))
root.history_view(_locals=_locals)
def _get_obs(self):
cam_color, cam_depth = self.camera.get_image()
# update objects positions registered with digits
self.digits.update()
colors, depths = self.digits.render()
obj_pose = self.obj.get_base_pose()
return AttrMap({
"camera": {
"color": cam_color,
"depth": cam_depth
},
"digits": [{
"color": color,
"depth": depth
} for color, depth in zip(colors, depths)],
"robot":
self.robot.get_states(),
"object": {
"position": np.array(obj_pose[0]),
"orientation": np.array(obj_pose[1]),
},
})
def load(self, path):
with open(path, "rb") as f:
custom = yaml.safe_load(f)
custom = AttrMap(custom)
for attr in self._config:
if attr in custom:
self._config[attr].update(custom[attr])
def complete_quest(root: Any, quest_data: "interface.QuestProcess"):
"""
Сохраняем результаты тестирования
"""
root.profile = AttrMap(
request(f'profile/{root.USER_ID}/quest/{quest_data.quest.name}',
completed_count=f'{quest_data.completed_count}',
score=f'{quest_data.score}',
answers=f'[{quest_data.answers}]'))
def _main(root: Any, _locals: dict = None):
if LOGIN and PASSWORD and USER_NAME and USER_ID:
if request('auth', login=LOGIN, password=PASSWORD)['response']:
root.profile = AttrMap(request(f'profile/{USER_ID}'))
root.home_view(_locals=_locals)
else:
os.remove('.auth')
root.log_in_view(_locals=_locals)
else:
request('test')
root.log_in_view(_locals=_locals)
def _load_untrusted_json(json_str):
try:
obj = json.loads(json_str, parse_int=int, parse_constant=bool)
if (obj['klass'] == 'Command' and isinstance(obj['type'], str)
and (obj['params'] is None or isinstance(obj['params'], dict))
and isinstance(obj['nth_turn'], int)):
return AttrMap(obj)
except Exception as e:
logger.debug('[S] Failed to decode json.')
logger.debug(str(e))
return None
def __init__(self, context: DeepSpeedTrialContext) -> None:
self.context = context
self.exp_config = self.context.get_experiment_config()
self.args = AttrMap(self.context.get_hparams())
# Initalize and get arguments, timers, and Tensorboard writer.
try:
self.neox_args = get_neox_args(self.context)
except:
traceback.print_exc()
raise InvalidHP("Could not parse neox_args.")
self.wrapped_writer = TorchWriter()
self.neox_args.tensorboard_writer = self.wrapped_writer.writer
self.neox_args.configure_distributed_args()
# The tokenizer needs to be built before model initialization in order to set the
# required padded_vocab_size argument.
self.neox_args.build_tokenizer()
megatron_train.initialize_megatron(neox_args=self.neox_args)
self.timers = megatron_utils.Timers(
use_wandb=False,
tensorboard_writer=self.neox_args.tensorboard_writer)
# Model, optimizer, and learning rate.
self.timers("model and optimizer").start()
(
model,
self.optimizer,
self.lr_scheduler,
) = megatron_train.setup_model_and_optimizer(neox_args=self.neox_args)
self.model = self.context.wrap_model_engine(model)
self.timers("model and optimizer").stop()
# Print setup timing.
megatron_utils.print_rank_0("done with setups ...")
self.timers.log(["model and optimizer"])
megatron_utils.print_rank_0("training ...")
# For tracking.
if not self.args.search_world_size:
self.reducer = self.context.wrap_reducer(LMReducers(
self.neox_args),
for_training=False,
for_validation=True)
self.report_memory_flag = True
self.total_train_loss_dict = {}
self.total_val_loss_dict = {}
self.tflops = 0
self.reported_flops = False
self.overflow_monitor = megatron_utils.OverflowMonitor(self.optimizer)
self.noise_scale_logger = megatron_utils.get_noise_scale_logger(
self.neox_args)
self.timers("interval time").start()
def test_invalid_name():
user_data = {
"_id": 1,
"first name": "John",
"last name": "David",
"email": "*****@*****.**",
}
user = AttrMap(user_data, sequence_type=list)
# 无法用 dict.attr 的风格访问
with pytest.raises(Exception):
user._id
# 但仍可以用 dict[attr] 的风格访问
assert user["_id"] == 1
def test_coco():
# ignore warnings in this test
warnings.simplefilter('ignore')
# args
args = AttrMap({
'epochs': 1,
'steps': 1,
'imagenet_weights': False,
'snapshots': False,
'dataset_type': 'coco',
'coco_path': 'tests/test-data/coco',
})
# run training / evaluation
keras_retinanet.bin.train.main_(args)
def test_csv():
# ignore warnings in this test
warnings.simplefilter('ignore')
args = AttrMap({
'epochs': 1,
'steps': 1,
'imagenet_weights': False,
'snapshots': False,
'dataset_type': 'csv',
'annotations': 'tests/test-data/csv/annotations.csv',
'classes': 'tests/test-data/csv/classes.csv'
})
# run training / evaluation
keras_retinanet.bin.train.main_(args)
def test_vgg():
# ignore warnings in this test
warnings.simplefilter('ignore')
args = AttrMap({
'backbone': 'vgg16',
'epochs': 1,
'steps': 1,
'imagenet_weights': False,
'snapshots': False,
'freeze_backbone': True,
'dataset_type': 'coco',
'coco_path': 'tests/test-data/coco'
})
# run training / evaluation
keras_retinanet.bin.train.main_(args)
def __init__(
self,
message: Optional[int] = None,
*,
peer_id: Optional[int] = None,
random_id: Optional[int] = None,
user_id: Optional[int] = None,
domain: Optional[str] = None,
chat_id: Optional[int] = None,
user_ids: Optional[List[int]] = None,
peer_ids: Optional[List[int]] = None,
lat: Optional[float] = None,
long: Optional[float] = None,
attachment: Optional[List[str]] = None,
reply_to: Optional[int] = None,
forward_messages: Optional[List[int]] = None,
sticker_id: Optional[int] = None,
group_id: Optional[int] = None,
keyboard: Optional[str] = None,
payload: Optional[str] = None,
dont_parse_links: Optional[bool] = None,
disable_mentions: Optional[bool] = None,
intent: Optional[str] = None,
expire_ttl: Optional[int] = None,
silent: Optional[bool] = None,
**kwargs,
):
if random_id is None:
random_id = randint(-2 * 31, 2 * 31)
preload_data = locals().copy()
del preload_data["self"]
kwargs_vals = preload_data.pop("kwargs")
preload_data.update(kwargs_vals)
self._params = AttrMap(
dict(filter(lambda x: x[1] is not None, preload_data.items())))
self._set_path()
self._join_attach()
def test_mutable():
"""
注意: AttrDict中的sequence type会自动转化为tuple, 即非mutable的形式。也就是
对list中的对象无法修改。如果想要允许对list中的对象进行修改, 需要使用AttrMap,
并指定 ``sequence_type = list``。
"""
user_data = {
"id": "EN-0001",
"phone_numbers": [
{"label": "home", "number": "111-222-3333"},
{"label": "work", "number": "444-555-6666"},
{"label": "mobile", "number": "777-888-9999"},
],
"profile": {
"SSN": "123-45-6789",
"drivers_license": {
"state": "DC",
"license_number": "DC-1234-5678",
}
}
}
user = AttrMap(user_data, sequence_type=list)
assert user.id == "EN-0001"
assert user["id"] == "EN-0001"
user.id = "EN-0002"
assert user.id == "EN-0002"
assert user["id"] == "EN-0002"
# nested dict is also attrdict
assert user.phone_numbers[0].number == "111-222-3333"
assert user.phone_numbers[0]["number"] == "111-222-3333"
user.phone_numbers[0].number = "111-111-1111"
assert user.phone_numbers[0].number == "111-111-1111"
assert user.phone_numbers[0]["number"] == "111-111-1111"
def log_in(root: Any, _locals: dict, login: str, password: str):
"""
Авторизация пользователя
"""
if login == '':
_locals['entry_login'].focus()
root.Alert.show('Введите логин')
elif password == '':
_locals['entry_password'].focus()
root.Alert.show('Введите пароль')
elif request('auth', login=login, password=password)['response']:
if os.path.exists('.auth'):
os.remove('.auth')
with open('.auth', 'w') as data:
data.write(f'{login}::{password}')
subprocess.check_call(['attrib', '+H', '.auth'])
root.LOGIN = login
root.USER_NAME, root.USER_ID = login.split('#')
root.profile = AttrMap(request(f'profile/{root.USER_ID}'))
root.history_view(_locals=_locals)
else:
root.Alert.show('Неправильный логин или пароль')
def new(self):
# TODO(poweic): instead of None, use torch.Tensor? (placeholder + strict schema)
return AttrMap({
k: v.new() if isinstance(v, collections.abc.Mapping) else None
for k, v in self.spaces.items()
})
G = AttrMap(
{
'APPEND': [],
'ACTIONS': [],
'BUILD': [],
'BUILDER': ["./build-boot-image.py"],
'BUNDLE_FILE': '',
'BUNDLED': 'bundled',
'DEBUG': 0,
'DEFAULT': 'default',
'DRY_RUN': 0,
'EDIT': '',
'BOOT': False,
'EXCLUDES': [],
'FLAGS': [],
'IMAGE': 'boot-image.raw',
'ISO': 'centos.iso',
'LABEL': '',
'MODE': '',
'MODES': ['default'],
'ONLY': [],
'SUPPORTED_VERSION': '1',
'TOPDIR': 'bundled',
'TAR_CLEANUP': [],
'VARS': [],
'VERBOSE': 0,
'STRICT': 1,
'FORMAT': '',
},
sequence_type=list) #,recursive=False)
def get_children_states(self):
return AttrMap(
_remove_empty_dict_leaf(
{k: v.get_states() for k, v in self.children().items()}
)
)
def test_add_fragment_urls(mock_log_change):
# Ensure shelfmark not existing is properly handled.
command = add_fragment_urls.Command()
row = AttrMap({"shelfmark": "mm", "url": "example.com"})
command.add_fragment_urls(row) # Test would fail if error were raised
assert command.stats["not_found"] == 1
assert not mock_log_change.call_count
# Ensure that the iiif url is not overwritten unless overwrite arg is provided
command = add_fragment_urls.Command()
command.overwrite = None
command.dryrun = None
orig_frag = Fragment.objects.create(
shelfmark="T-S NS 305.66",
iiif_url="https://cudl.lib.cam.ac.uk/iiif/MS-TS-NS-J-00490",
)
row = AttrMap({
"shelfmark": orig_frag.shelfmark,
"url": "https://cudl.lib.cam.ac.uk/view/MS-TS-NS-J-00600",
})
command.add_fragment_urls(row)
fragment = Fragment.objects.get(shelfmark=orig_frag.shelfmark)
assert fragment.url == row["url"]
assert fragment.iiif_url == orig_frag.iiif_url
assert command.stats["url_added"] == 1
assert not command.stats["iiif_added"]
assert not command.stats["iiif_updated"]
assert not command.stats["url_updated"]
mock_log_change.assert_called_with(fragment, "added URL")
command = add_fragment_urls.Command()
command.overwrite = True
command.dryrun = None
orig_frag = Fragment.objects.create(
shelfmark="T-S NS 305.75",
iiif_url="https://cudl.lib.cam.ac.uk/iiif/MS-TS-NS-J-00490",
)
row = AttrMap({
"shelfmark": orig_frag.shelfmark,
"url": "https://cudl.lib.cam.ac.uk/view/MS-TS-NS-J-00600",
})
command.add_fragment_urls(row)
fragment = Fragment.objects.get(shelfmark=orig_frag.shelfmark)
assert fragment.iiif_url != orig_frag.iiif_url
assert fragment.iiif_url == "https://cudl.lib.cam.ac.uk/iiif/MS-TS-NS-J-00600"
assert command.stats["iiif_updated"] == 1
mock_log_change.assert_called_with(fragment,
"added URL and updated IIIF URL")
# test updating url — url matches, should skip
fragment.url = row.url
fragment.save()
command.stats = defaultdict(int)
command.add_fragment_urls(row)
assert not command.stats["url_updated"]
assert not command.stats["url_added"]
assert command.stats["skipped"] == 1
# fragment url is set but does not match, no overwrite
fragment.url = "http://example.com/fragment/view"
fragment.save()
command.overwrite = False
command.stats = defaultdict(int)
command.add_fragment_urls(row)
assert not command.stats["url_updated"]
assert not command.stats["url_added"]
assert command.stats["skipped"] == 1
# url mismatch, overwrite specified
command.overwrite = True
command.stats = defaultdict(int)
command.add_fragment_urls(row)
assert command.stats["url_updated"] == 1
assert not command.stats["url_added"]
assert not command.stats["skipped"]
# Ensure that changes aren't saved if dryrun argument is provided
mock_log_change.reset_mock()
command = add_fragment_urls.Command()
command.overwrite = None
command.dryrun = True
orig_frag = Fragment.objects.create(
shelfmark="T-S NS 305.80",
iiif_url="https://cudl.lib.cam.ac.uk/iiif/MS-TS-NS-J-00490",
)
row = AttrMap({
"shelfmark": orig_frag.shelfmark,
"url": "https://cudl.lib.cam.ac.uk/view/MS-TS-NS-J-00600",
})
command.add_fragment_urls(row)
fragment = Fragment.objects.get(shelfmark=orig_frag.shelfmark)
assert fragment.iiif_url == orig_frag.iiif_url
assert not mock_log_change.call_count
import os
import sys
import yaml
from attrdict import AttrMap
current_dir = os.path.dirname(os.path.abspath(__file__))
with open(current_dir + '/settings.yml', 'r') as stream:
config = yaml.load(stream, Loader=yaml.FullLoader)
sys.modules[__name__] = AttrMap(config)
def __init__(self):
with open(self.DEFAULT, "rb") as f:
default = yaml.safe_load(f)
self._config = AttrMap(default)
log['gen/loss'] = loss_g.item()
log['dis/loss'] = loss_d.item()
logreport(log)
with torch.no_grad():
log_test = test(config, test_data_loader, gen, criterionMSE, epoch)
testreport(log_test)
if epoch % config.snapshot_interval == 0:
checkpoint(config, epoch, gen, dis)
logreport.save_lossgraph()
testreport.save_lossgraph()
if __name__ == '__main__':
with open('config.yml', 'r') as f:
config = yaml.load(f)
config = AttrMap(config)
utils.make_manager()
n_job = utils.job_increment()
config.out_dir = os.path.join(config.out_dir, '{:06}'.format(n_job))
os.makedirs(config.out_dir)
print('Job number: {:04d}'.format(n_job))
shutil.copyfile('config.yml', os.path.join(config.out_dir, 'config.yml'))
train(config)
def process(xmlfile, *xmlfiles, **options):
"""
Main program code.
1. Load XML file.
2. Export structured annotation data from XML to relational database.
3. Save database tables as CSV files.
4. Create polygons from the annotations.
5. Create polygonal selection by adding and subtracting polygons.
6. Save polygonal selection as a binary object.
7. Create binary mask at the specified resolution with the specified
field-of-view (FOV) from the polygonal selection.
8. Export binary mask (values: 0-255) to an 8-bit TIFF file.
+1 (optional): display polygonal selection and binary mask.
"""
p = AttrMap(options)
# Start logging
global logger
temp_logfile = tempfile.mkstemp()
logging.basicConfig(format='[%(asctime)s] %(message)s',
datefmt='%d-%m-%Y %H:%M:%S',
filename=temp_logfile[1],
filemode="w")
logger = logging.getLogger("roi")
logger.setLevel(max(50 - p.verbose, 1))
logger.critical("The program started with the command: {}".format(" ".join(
sys.argv)))
xmlfiles = (xmlfile, ) + xmlfiles
err = 0
for f in xmlfiles:
try:
# Parse data in XML file and create data tables
logger.info("{}".format(f))
if not str(f).lower().endswith(".xml"):
logger.warning("Input does not appear to be an XML file.")
points, regions, annotations = parse_xml(f)
# Create output base name
p.basename = os.path.splitext(os.path.split(f)[-1])[0]
if p.outdir:
if not os.path.isdir(p.outdir):
os.makedirs(p.outdir)
p.outbase = os.path.join(p.outdir, p.basename)
else:
p.outbase = os.path.splitext(f)[0]
# Save the tables of the relational database
if p.csv:
points.to_csv(p.outbase + "_points.csv")
regions.to_csv(p.outbase + "_regions.csv")
annotations.to_csv(p.outbase + "_annotations.csv")
# Retrieve polygons
polygons = create_polygons(points)
# Treat annotation layers separately
for layer in annotations.index.unique():
layerdir = p.outdir or os.path.split(f)[0]
layerdir = os.path.join(
layerdir, "AnnotationLayer_{0:02d}".format(layer))
if not os.path.isdir(layerdir):
os.makedirs(layerdir)
layerbase = os.path.join(layerdir, p.basename)
# Set algebra
selection = create_selection(polygons, regions, layer=layer)
if selection is None:
logger.warning("Annotation layer {} does not have any "
"polygonal selections.".format(layer))
continue
# Display selection
if p.display:
visualise_polygon(selection, show=True, save=False)
# Export the polygonal selection object to a binary file
if p.bin:
with open(layerbase + "_selection.obj", "wb") as fp:
dill.dump(selection, fp)
# Generate binary mask
if p.mask:
if len(p.scale) == 1:
scale_x, scale_y = p.scale * 2 # p.scale is a tuple!
elif len(p.scale) == 2:
scale_x, scale_y = p.scale # p.scale is a tuple!
else:
raise ValueError(
"The number of scaling factors must be 2.")
mask = create_mask(selection,
original_shape=p.original_shape,
target_shape=p.target_shape,
scale_x=scale_x,
dimscale_x=p.dimscale_x,
scale_y=scale_y,
dimscale_y=p.dimscale_y,
tile=p.tile,
fill_value=p.fill_value)
# Display binary mask
if p.display:
plt.imshow(mask, cmap="gray", aspect="equal")
plt.show()
# Save binary mask
Image.fromarray(mask).save(
os.path.join(layerbase + "_mask.tif"))
# Save the corresponding histology image
if p.histo:
histo = create_histo(img=p.image,
dimlevel=p.dimlevel,
original_shape=p.original_shape,
target_shape=p.target_shape,
scale_x=scale_x,
dimscale_x=p.dimscale_x,
scale_y=scale_y,
dimscale_y=p.dimscale_y,
tile=p.tile)
Image.fromarray(histo).save(
os.path.join(layerbase + "_histo.tif"))
except Exception as exc:
logger.critical("FAILURE: {}. Exception: {}".format(
f, exc.args[0]))
err += 1
continue
# Conclude run
if err == 0:
logger.critical("All tasks were successfully completed.")
else:
logger.critical("Tasks were completed with {} error(s).".format(err))
# Save logs
try:
shutil.copy(temp_logfile[1], p.outbase + ".log")
except PermissionError:
pass