def _get_file_and_new_contents(self, key):
filename = self.manifest.get(key)
contents = None
if filename and os.path.exists(filename):
return os.path.join(self.cachedir, filename), None
fileroot = self._key_to_file(key)
filename = os.path.join(self.cachedir, fileroot)
with open(filename, 'wb') as f:
contents = self._get_file_contents(key)
f.write(contents)
self.manifest[key] = fileroot
Yaml.write(self.manifest_file, self.manifest)
return filename, contents
def _get_file_and_new_contents(self, key):
filename = self.manifest.get(key)
contents = None
if filename and os.path.exists(filename):
return os.path.join(self.cachedir, filename), None
fileroot = self._key_to_file(key)
filename = os.path.join(self.cachedir, fileroot)
with open(filename, 'wb') as f:
contents = self._get_file_contents(key)
f.write(contents)
self.manifest[key] = fileroot
Yaml.write(self.manifest_file, self.manifest)
return filename, contents
def _make_elements(score_names, table):
result = {}
for score_file, name in score_names:
resolved_file = CommandFile.resolve('score', score_file)
if not resolved_file:
LOGGER.error('No such score file: "%s".',
CommandFile.base_file('score', score_file))
continue
elements = Yaml.read(resolved_file)
description = {'elements': elements, 'type': 'score'}
parts = resolved_file.split('/')
final_file = '/'.join([parts[1]] + parts[3:])
try:
element = Root.Root(description, final_file)
except Exception:
LOGGER.error("\nError when reading score file %s", score_file, exc_info=1)
continue
name = os.path.splitext(name or score_file)[0]
name = UniqueName.unique_name(name, table)
result[name] = element
element.name = name
return result
def save(self):
saved_files = []
for f, settings in self.file_settings:
if len(settings) > 2 and settings[2]:
saved_files.append(f)
settings[1] = Merge.merge(*settings[1:])
while len(settings) > 2:
settings.pop()
if os.path.exists(f):
with open(f, 'r') as fo:
data = fo.read().split(Yaml.SEPARATOR)[0]
else:
data = ''
parent = os.path.dirname(f)
if not os.path.exists(parent):
from echomesh.util import Log
Log.logger(__name__).info('Creating directory %s.', parent)
os.makedirs(parent)
with open(f, 'wb') as fw:
if data:
fw.write(data)
fw.write(Yaml.SEPARATOR)
fw.write(Yaml.encode_one(settings[1]))
self.arg_settings = Merge.difference_strict(
self.arg_settings, self.changed)
self.recalculate()
return saved_files
def _resolve_extensions(data):
extensions = set()
datas = [data]
while True:
extension = data.get('inherit')
if not extension:
break
extension = Yaml.filename(extension)
if extension in extensions:
raise Exception('Infinite circular extension for %s' % extension)
try:
data = CommandFile.load('score', extension)
except Exception as e:
raise Exception("Couldn't find extension for %s: %s" % (extension, str(e)))
if len(data) > 1:
LOGGER.error("More than one element in extension %s", extension)
data = data[0]
datas.append(data)
extensions.add(extension)
result = {}
for data in reversed(datas):
result.update(data)
if extensions:
del result['inherit']
return result
def __init__(self, filename=''):
super(PersistentDict, self).__init__()
self._filename = filename
if filename:
original = Yaml.read(self._filename)
if original:
super(PersistentDict, self).update(original[0])
def make_root(score_names, table):
result = {}
for score_file, name in score_names:
resolved_file = DataFile.resolve('score', score_file)
if not resolved_file:
LOGGER.error('No such score file: "%s".',
DataFile.base_file('score', score_file))
continue
elements = Yaml.read(resolved_file)
description = {'elements': elements, 'type': 'score'}
parts = resolved_file.split('/')
final_file = '/'.join([parts[1]] + parts[3:])
try:
element = Root(description, final_file)
except Exception:
LOGGER.error("\nError when reading score file %s", score_file)
continue
name = os.path.splitext(name or score_file)[0]
name = UniqueName.unique_name(name, table)
result[name] = element
element.name = name
return result
def save(self):
saved_files = []
for f, settings in self.file_settings:
if len(settings) > 2 and settings[2]:
saved_files.append(f)
settings[1] = Merge.merge(*settings[1:])
while len(settings) > 2:
settings.pop()
if os.path.exists(f):
with open(f, 'r') as fo:
data = fo.read().split(Yaml.SEPARATOR)[0]
else:
data = ''
parent = os.path.dirname(f)
if not os.path.exists(parent):
from echomesh.util import Log
Log.logger(__name__).info('Creating directory %s.', parent)
os.makedirs(parent)
with open(f, 'wb') as fw:
if data:
fw.write(data)
fw.write(Yaml.SEPARATOR)
fw.write(Yaml.encode_one(settings[1]))
self.arg_settings = Merge.difference_strict(self.arg_settings,
self.changed)
self.recalculate()
return saved_files
def resolve(*path):
x = expand(*path)
for f in x:
try:
return Yaml.filename(f)
except:
continue
def __init__(self, filename=''):
super(PersistentDict, self).__init__()
self._filename = filename
if filename:
original = Yaml.read(self._filename)
if original:
super(PersistentDict, self).update(original[0])
def load_resolve(*path):
f = resolve(*path)
if f:
data = Yaml.read(f)
if data:
return f, data
raise Exception("Couldn't read Yaml from file %s" % os.path.join(*path))
def set_config(_, *values):
if values:
for address, value in Config.assign(*values):
LOGGER.info('Set %s=%s', '.'.join(address), value)
elif MergeConfig.LOCAL_CHANGES:
LOGGER.info(Yaml.encode_one(MergeConfig.LOCAL_CHANGES))
else:
LOGGER.info('You have made no changes.')
def load_resolve(*path):
f = resolve(*path)
if f:
data = Yaml.read(f)
if data:
return f, data
raise Exception("Couldn't read Yaml from file %s" % os.path.join(*path))
def merge_config():
config = Merge.merge(*Yaml.read(CommandFile.config_file('default')))
assert config, 'Unable to read default config file'
_set_project_path()
config = _merge_file_config(config)
merge_assignments(config, Args.ARGS)
return config
def set_config(_, *values):
if values:
assignment = Leafs.leafs(Config.assign(values))
for address, value in six.iteritems(assignment):
LOGGER.info('Set %s=%s', '.'.join(address), value)
elif Config.MERGE_CONFIG.has_changes():
LOGGER.info(Yaml.encode_one(dict(Config.MERGE_CONFIG.get_changes())))
else:
LOGGER.info('You have made no changes.')
def _write(self, data):
if self.handler and data:
d = Yaml.encode_one(data)
self.handler.wfile.write(d)
self.handler.wfile.write(Yaml.SEPARATOR)
if LOG_ALL_DATA:
FILE.write(d)
FILE.write(Yaml.SEPARATOR)
FILE.flush()
self.handler.wfile.flush()
def add(self, line):
if line:
if line.startswith(Yaml.SEPARATOR_BASE):
if self.callback and self.lines:
res = ''.join(self.lines)
result = Yaml.decode_one(res)
self.callback(result)
self.lines = []
else:
self.lines.append(line)
def __init__(self, name, suffix):
if suffix.startswith('.'):
self.suffix = suffix
else:
self.suffix = '.' + suffix
name_file = CommandFile.clean('cache', name)
self.cachedir = os.path.abspath(os.path.join(*name_file))
MakeDirs.makedirs(self.cachedir)
self.manifest_file = os.path.join(self.cachedir, MANIFEST_NAME)
self.manifest = Merge.merge(*Yaml.read(self.manifest_file))
def set_settings(_, *values):
if values:
assignment = Leafs.leafs(Settings.assign(values))
for address, value in six.iteritems(assignment):
LOGGER.info('Set %s=%s', '.'.join(address), value)
Settings.update_clients()
elif Settings.MERGE_SETTINGS.has_changes():
LOGGER.info(
Yaml.encode_one(dict(Settings.MERGE_SETTINGS.get_changes())))
else:
LOGGER.info('You have made no changes.')
def __init__(self, name, suffix):
if suffix.startswith('.'):
self.suffix = suffix
else:
self.suffix = '.' + suffix
name_file = DataFile.clean('cache', name)
self.cachedir = os.path.abspath(os.path.join(*name_file))
MakeDirs.makedirs(self.cachedir)
self.manifest_file = os.path.join(self.cachedir, MANIFEST_NAME)
self.manifest = Merge.merge(*Yaml.read(self.manifest_file))
def set_settings(_, *values):
if values:
assignment = Leafs.leafs(Settings.assign(values))
for address, value in six.iteritems(assignment):
LOGGER.info('Set %s=%s', '.'.join(address), value)
Settings.update_clients()
elif Settings.MERGE_SETTINGS.has_changes():
LOGGER.info(
Yaml.encode_one(dict(Settings.MERGE_SETTINGS.get_changes())))
else:
LOGGER.info('You have made no changes.')
def callback(self, data):
data = Yaml.decode_one(data)
event = data['event']
if event == 'start':
self.after_server_starts()
elif event == 'closeButtonPressed':
if Settings.get('execution', 'close_button_quits'):
Quit.request_quit()
elif Settings.get('execution', 'close_button_closes_window'):
Visualizer.set_visible(False)
else:
# print(data)
pass
def _write(self, data):
if self.handler and data:
d = Yaml.encode_one(data)
self.handler.wfile.write(d)
self.handler.wfile.write(Yaml.SEPARATOR)
if self.debug or LOG_ALL_DATA:
_write(d)
_write(Yaml.SEPARATOR, True)
self.handler.wfile.flush()
else:
if not self.queue:
self.queue = queue.Queue(self.max_queue_size)
self.queue.put(data)
def _read_file_settings(self):
self.file_settings = []
base_settings = None
for f in reversed(DataFile.expand_settings()):
settings = Yaml.read(f, 'settings')
for c in settings:
if base_settings:
base_settings = Merge.merge_for_settings(base_settings, c)
else:
base_settings = copy.deepcopy(c)
while len(settings) < 3:
settings.append({})
self.file_settings.append([f, settings])
def _read_file_settings(self):
self.file_settings = []
base_settings = None
for f in reversed(DataFile.expand_settings()):
settings = Yaml.read(f, 'settings')
for c in settings:
if base_settings:
base_settings = Merge.merge_for_settings(base_settings, c)
else:
base_settings = copy.deepcopy(c)
while len(settings) < 3:
settings.append({})
self.file_settings.append([f, settings])
def _read_file_configs(self):
self.file_configs = []
base_config = None
for f in reversed(CommandFile.expand('config.yml')):
configs = Yaml.read(f, 'config')
for c in configs:
if base_config:
base_config = Merge.merge_for_config(base_config, c)
else:
base_config = copy.deepcopy(c)
while len(configs) < 3:
configs.append({})
self.file_configs.append([f, configs])
def _merge_file_config(config):
for f in list(reversed(CommandFile.expand('config.yml')))[1:]:
try:
file_configs = Yaml.read(f)
except Exception as e:
_add_exception_suffix(e, 'while reading config file', f)
raise
for cfg in file_configs:
try:
config = Merge.merge_for_config(config, cfg)
except Exception as e:
_add_exception_suffix(e, ' while merging config file', f)
raise
return config
def start_elements(self, names):
score_names = Split.split_scores(names)
element_names = []
for score_file, name in score_names:
if name:
is_file = True
else:
name = score_file
is_file = Yaml.has_extension(score_file)
if is_file:
name = name[:-4]
elif name not in self.elements:
is_file = True
if is_file:
element_names.extend(self._load_raw_elements([[score_file, name]]))
else:
element_names.append(name)
return self.perform_element('start', element_names)
def save(self):
saved_files = []
for f, configs in self.file_configs:
if len(configs) > 2 and configs[2]:
saved_files.append(f)
configs[1] = Merge.merge(*configs[1:])
while len(configs) > 2:
configs.pop()
with open(f, 'r') as fo:
data = fo.read().split(Yaml.SEPARATOR)[0]
with open(f, 'wb') as fw:
fw.write(data)
fw.write(Yaml.SEPARATOR)
fw.write(Yaml.encode_one(configs[1]))
self.arg_config = Merge.difference_strict(self.arg_config, self.changed)
self.recalculate()
return saved_files
def make_root(score_names, table):
result = {}
for score_file, name in score_names:
resolved_file = DataFile.resolve("score", score_file)
if not resolved_file:
LOGGER.error('No such score file: "%s".', DataFile.base_file("score", score_file))
continue
elements = Yaml.read(resolved_file)
description = {"elements": elements, "type": "score"}
parts = resolved_file.split("/")
final_file = "/".join([parts[1]] + parts[3:])
try:
element = Root(description, final_file)
except Exception:
LOGGER.error("\nError when reading score file %s", score_file)
continue
name = os.path.splitext(name or score_file)[0]
name = UniqueName.unique_name(name, table)
result[name] = element
element.name = name
return result
def read_config(scope='default'): return Yaml.read(config_file(scope))
def split_args(s):
if not isinstance(s, six.string_types):
s = ' '.join(s)
results = []
in_quotes = False
backslashed = False
state = State.BEFORE_ADDRESS
bracket_stack = []
address = []
value = []
for col, ch in enumerate(s):
def error(s):
raise Exception('At column %d: %s.' % (1 + col, s))
perhaps_done = False
if state is State.BEFORE_ADDRESS:
if ch.isalpha():
address.append(ch)
state = State.IN_ADDRESS
elif not ch.isspace():
error('Expected a letter, not "%s"' % ch)
continue
if state is State.IN_ADDRESS:
if ch.isalpha() or ch in '._':
address.append(ch)
elif ch.isspace():
state = State.BEFORE_EQUALS
elif ch == '=':
state = State.BEFORE_VALUE
continue
if state is State.BEFORE_EQUALS:
if ch == '=':
state = State.BEFORE_VALUE
elif not ch.isspace():
error('Expected "=", not "%s"' % ch)
continue
if state is State.BEFORE_VALUE:
if ch.isspace():
continue
state = State.IN_VALUE
if backslashed:
backslashed = False
elif ch == '\\':
backslashed = True
continue
elif ch == '"':
if in_quotes:
in_quotes = False
perhaps_done = True
else:
in_quotes = True
elif in_quotes:
pass
elif ch.isspace():
if not bracket_stack:
perhaps_done = True
elif ch in _LEFT:
bracket_stack.append(ch)
elif ch in _RIGHT_TO_LEFT:
left = _RIGHT_TO_LEFT[ch]
if not bracket_stack:
error('Closing %s without opening %s' % (ch, left))
top = bracket_stack.pop()
if top != left:
error('Got closing %s for opening %s' % (left, top))
perhaps_done = True
value.append(ch)
last_time = col == (len(s) - 1)
if last_time:
if bracket_stack:
error('Missing closing brackets for %s' % ''.join(bracket_stack))
elif in_quotes:
error('unterminated quotation mark')
if perhaps_done or last_time:
if bracket_stack or in_quotes:
continue
if address:
if value:
val = ''.join(value).strip()
results.append([''.join(address).strip().split('.'),
Yaml.decode_one(val)])
address = []
value = []
state = State.BEFORE_ADDRESS
else:
error('empty value for address %s' % address)
elif value:
error('empty address for value %s' % value)
if value:
error('value was incomplete')
elif address:
error('expected to see a value')
return results
def _config(_):
LOGGER.info('\n' + Yaml.encode_one(Config.get_config()))
def add_result(self):
if self.address:
value = ''.join(self.value or 'true')
address = ''.join(self.address)
self.results.append([address, Yaml.decode_one(value)])
self.clear()
def settings(_, scope, cfg):
f = DataFile.settings_file(scope)
settings = Yaml.read(f) + [cfg]
Yaml.write(f, Merge.merge(*settings))
# TODO: needs to propagate!
LOGGER.info('Changing settings for %s', scope)
def config(_, scope, cfg):
f = CommandFile.config_file(scope)
configs = Yaml.read(f) + [cfg]
Yaml.write(f, Merge.merge(*configs))
# TODO: needs to propagate!
LOGGER.info('Changing configuration for %s', scope)
def _write(self): Yaml.write(self._filename, self)
def add_result(self):
if self.address:
value = ''.join(self.value or 'true')
address = ''.join(self.address)
self.results.append([address, Yaml.decode_one(value)])
self.clear()
def settings(_, scope, cfg):
f = DataFile.settings_file(scope)
settings = Yaml.read(f) + [cfg]
Yaml.write(f, Merge.merge(*settings))
# TODO: needs to propagate!
LOGGER.info('Changing settings for %s', scope)
def _format(info, spaces=DEFAULT_INDENT):
s = Yaml.encode_one(info)
LOGGER.info('\n' + DEFAULT_INDENT + s.replace('\n', '\n' + DEFAULT_INDENT))
def _write(self):
Yaml.write(self._filename, self)