def add_connection(self, c):
self.info('Adding connection %r' % str(c))
if not c.dp1 in self.names:
raise_desc(DPSemanticError,
'Invalid connection: %r not found.' % c.dp1,
names=self.names,
c=c)
if not c.dp2 in self.names:
raise_desc(DPSemanticError,
'Invalid connection: %r not found.' % c.dp2,
names=self.names,
c=c)
mcdp_dev_warning('redo this check')
if self.is_new_function(c.dp2):
msg = "Cannot add connection to new function %r." % c.dp2
raise_desc(DPSemanticError, msg, c=c)
if self.is_new_resource(c.dp1):
msg = "Cannot add connection to new resource %r." % c.dp1
raise_desc(DPSemanticError, msg, c=c)
# Find if there is already a connection to c.dp2,c.s2
# for c0 in self.connections:
# if c0.dp2 == c.dp2 and c0.s2 == c.s2:
# msg = 'There is already a connection to function %r of %r.' % (c.s2, c.dp2)
# raise_desc(DPSemanticError, msg)
ndp1 = self.names[c.dp1]
ndp2 = self.names[c.dp2]
rnames = ndp1.get_rnames()
if not c.s1 in rnames:
msg = "Resource %r does not exist (known: %s)" % (
c.s1, format_list(rnames))
raise_desc(DPSemanticError, msg, known=rnames)
fnames = ndp2.get_fnames()
if not c.s2 in fnames:
msg = "Function %r does not exist (known: %s)" % (
c.s2, format_list(fnames))
raise_desc(DPSemanticError, msg, known=fnames)
R1 = ndp1.get_rtype(c.s1)
F2 = ndp2.get_ftype(c.s2)
# print('connecting R1 %s to R2 %s' % (R1, F2))
if not (R1 == F2):
msg = 'Connection between different spaces.'
raise_desc(DPSemanticError,
msg,
c=c,
F2=F2,
R1=R1,
ndp1=ndp1,
ndp2=ndp2)
self.connections.append(c)
def fill_in_none(schema, data):
''' Fills in the fields not passed and that can be None '''
if isinstance(schema, SchemaString):
if isinstance(data, unicode):
return data.encode('utf8')
else:
return data
elif isinstance(schema, SchemaSimple):
schema.validate(data)
return data
elif isinstance(schema, SchemaList):
check_isinstance(data, list)
res = [fill_in_none(schema.prototype, _) for _ in data]
schema.validate(res)
return res
elif isinstance(schema, SchemaHash):
check_isinstance(data, dict)
res = dict([(k, fill_in_none(schema.prototype, v))
for k, v in data.items()])
schema.validate(res)
return res
elif isinstance(schema, SchemaContext):
check_isinstance(data, dict)
# some of the fields might be missing
res = {}
present = set(data)
for k, schema_child in schema.children.items():
if k in data:
# have it
res[k] = fill_in_none(schema_child, data[k])
else:
# dont' have it
if schema_child.can_be_none:
res[k] = None
else:
# no default
msg = 'Expected key "%s".' % k
raise_incorrect_format(
msg,
schema,
data,
)
needed = set(schema.children)
extra = set(present) - needed
if extra:
msg = 'Extra fields: %s.' % format_list(sorted(extra))
msg += '\nPresent: %s' % format_list(present)
msg += '\nNeeded: %s' % format_list(needed)
raise_incorrect_format(msg, schema, data)
schema.validate(res)
return res
else:
assert False, schema
def __getitem__(self, key):
if key in self.mount_points:
return self.mount_points[key]
if not key in self._data:
msg = 'Could not find key "%s"; available keys are %s, mount points %s' % (
key, format_list(self._data), format_list(self.mount_points))
raise_desc(EntryNotFound, msg)
d = self._data[key]
prototype = self._schema.prototype
if is_simple_data(prototype):
self.child(key).check_can_read()
return d
else:
return self._create_view_instance(prototype, d, key)
def _get_file_data(self, basename):
""" returns dict with data, realpath """
for fn in self.file_to_contents:
if fn.lower() == basename.lower():
match = fn
break
else:
msg = 'Could not find file %r.' % basename
ext = os.path.splitext(basename)[1].replace('.', '')
available = sorted(self._list_with_extension(ext),
key=lambda x: x.lower())
if available:
msg += (" Available files with %r extension: %s." %
(ext, format_list(sorted(available))))
else:
msg += " No files with extension %r found." % ext
msg += '\nSearch directories: %s' % self.search_dirs
raise_desc(DPSemanticError, msg)
raise Exception() # XXX
found = self.file_to_contents[match]
return found
def generate_empty(self, **kwargs):
res = {}
for k in kwargs:
if not k in self.children:
msg = 'Extra key %r not in %s.' % (k, format_list(self.children))
raise ValueError(msg)
kwargs = deepcopy(kwargs)
for k, c in self.children.items():
if isinstance(c, SchemaSimple):
if k in kwargs:
res[k] = kwargs.pop(k)
elif c.can_be_none:
res[k] = None
else:
msg = 'Cannot generate empty for child %r: %s.' % (k, c)
raise ValueError(msg)
else:
if k in kwargs:
x = kwargs.pop(k)
if isinstance(x, dict):
res[k] = c.generate_empty(**x)
else:
res[k] = x
else:
res[k] = c.generate_empty()
assert not kwargs
return res
def get_soft_match(x, options):
''' Get a soft match or raise KeyError '''
options = list(options)
res = []
for o in options:
if x.lower() == o.lower():
res.append(o)
if not res:
msg = 'Could not find any soft match for "%s" in %s.' % (
x, format_list(options))
raise KeyError(msg)
if len(res) > 1:
msg = 'Too many matches (%s) for "%s" in %s.' % (format_list(res), x,
format_list(options))
raise KeyError(msg)
return res[0]
def load_library(self, libname, context=None): # @UnusedVariable
check_isinstance(libname, str)
""" hook to pass to MCDPLibrary instances to find their sisters. """
if not libname in self.libraries:
msg = ('Cannot find library "%s". Available: %s.' %
(libname, format_list(sorted(self.libraries))))
raise_desc(DPSemanticError, msg)
l = self.libraries[libname]['library']
return l
def shelf_privilege(repo_name, sname, privilege):
repo = repos[repo_name]
if not sname in repo.shelves:
msg = 'Cannot find shelf "%s" in repo "%s".' % (sname,
repo_name)
msg += '\n available: ' + format_list(repo.shelves)
raise ValueError(msg)
acl = repo.shelves[sname].get_acl()
return acl.allowed2(privilege, e.user)
def disk_events_from_list_remove(disk_map, view, _id, who, name, value):
view_parent = view.get_descendant(name)
data = view_parent._data
for index, v in enumerate(data):
if v == value:
return disk_events_from_list_delete(disk_map, view, _id, who, name,
index)
msg = 'There is no value %s in the list.' % value
msg += '\n values: %s' % format_list(data)
raise InvalidOperation(msg)
def dir_rename(self, name, name2):
if not name in self._directories:
msg = ('Cannot rename directory %r to %r if does not exist in %s.' %
(name, name2, format_list(self._directories)))
raise InvalidDiskOperation(msg)
if name2 in self._directories:
msg = ('Cannot rename directory %r to %r if %r already exists' %
(name, name2, name2))
raise InvalidDiskOperation(msg)
self._directories[name2] = self._directories.pop(name)
def go(self):
options = self.get_options()
if options.config is not None:
logger.info('Reading configuration from %s' % options.config)
logger.warn('Other options from command line will be ignored. ')
parser = RawConfigParser()
parser.read(options.config)
sections = parser.sections()
logger.info('sections: %s' % sections)
s = 'app:main'
if not s in sections:
msg = 'Could not find section "%s": available are %s.' % (
s, format_list(sections))
msg += '\n file %s' % options.config
raise Exception(msg) # XXX
settings = dict((k, parser.get(s, k)) for k in parser.options(s))
prefix = 'mcdp_web.'
mcdp_web_settings = get_only_prefixed(settings,
prefix,
delete=True)
# mcdp_web_settings = {}
# for k,v in list(settings.items()):
# if k.startswith(prefix):
# mcdp_web_settings[k[len(prefix):]] = v
# del settings[k]
options = parse_mcdpweb_params_from_dict(mcdp_web_settings)
logger.debug('Using these options: %s' % options)
else:
logger.info('No configuration .ini specified (use --config).')
settings = {}
wa = WebApp(options, settings=settings)
# Write warning messages now
wa.get_authomatic_config()
msg = """Welcome to PyMCDP!
To access the interface, open your browser at the address
http://localhost:%s/
Use Chrome, Firefox, or Opera - Internet Explorer is not supported.
""" % options.port
logger.info(msg)
if options.delete_cache:
pass # XXX: warning deprecated
# logger.info('Deleting cache...')
# wa._refresh_library(None)
wa.serve(port=options.port)
def event_dict_rename_interpret(view, name, key, key2):
from mcdp_hdb.memdataview import ViewHash0
v = get_view_node(view, name)
check_isinstance(v, ViewHash0)
# permissions
v.check_can_write()
if not key in v._data:
msg = ('Cannot rename key %r to %r if it does not exist in %s.' %
(key, key2, format_list(v._data)))
raise InvalidOperation(msg)
v._data[key2] = v._data.pop(key)
def remove(self, value):
if not value in self._data:
msg = 'The list does not contain %r: available %s' % (
value, format_list(self._data))
raise ValueError(msg)
from .memdata_events import event_list_remove
event = event_list_remove(name=self._prefix,
value=value,
**self._get_event_kwargs())
self._notify(event)
self._data.remove(value)
def load_library(self, library_name, context=None): # @UnusedVariable
# logger.debug('load_library(%r)' % library_name)
repos = self.db_view.repos
all_shelves = set()
all_libraries = set()
for repo_name, repo in repos.items():
all_shelves.update(repo.shelves)
for shelf_name, shelf in repo.shelves.items():
all_libraries.update(shelf.libraries)
for shelf_name in self.subscribed_shelves:
if shelf_name in repo.shelves:
shelf = repo.shelves[shelf_name]
if library_name in shelf.libraries:
return self.make_library(repo_name, shelf_name,
library_name)
msg = 'Could not find library %r.' % library_name
msg += '\n Subscribed shelves: %s.' % format_list(
sorted(self.subscribed_shelves))
msg += '\n All shelves: %s.' % format_list(sorted(all_shelves))
msg += '\n All libraries: %s.' % format_list(sorted(all_libraries))
raise ValueError(msg)
def make_function(self, dp, s):
assert isinstance(dp, str)
if not dp in self.names:
msg = 'Unknown design problem %r.' % dp
raise DPSemanticError(msg)
ndp = self.names[dp]
if not s in ndp.get_fnames():
msg = 'Unknown function %r for design problem %r.' % (s, dp)
msg += ' Known functions: %s.' % format_list(ndp.get_fnames())
raise DPSemanticError(msg)
return CFunction(dp, s)
def check_no_forbidden(s, res, location0):
forbidden = {
'>=': ['≥'],
'<=': ['≤'],
'>>': ['?'] # added by mistake by Atom autocompletion
}
for f in forbidden:
if f in s:
msg = 'Found forbidden sequence %r. This will not end well.' % f
subs = forbidden[f]
msg += ' Try one of these substitutions: %s' % format_list(subs)
c = s.index(f)
where = Where(s, c, c + len(f))
res.note_error(msg, LocationInString(where, location0))
def validate(self, data):
if not isinstance(data, dict):
msg = 'Expected a dictionary object.'
raise_desc(NotValid, msg, data=describe_value(data))
for k, v in self.children.items():
if not k in data:
msg = 'Expecting key "%s" but not found in %s.' % (k, format_list(data))
raise_desc(NotValid, msg, data=describe_value(data), self=str(self))
try:
v.validate(data[k])
except NotValid as e:
msg = 'For child "%s":' % k
raise_wrapped(NotValid, e, msg)
def make_resource(self, dp, s):
if not isinstance(dp, str):
raise DPInternalError((dp, s))
if not dp in self.names:
msg = 'Unknown design problem %r.' % dp
raise DPSemanticError(msg)
ndp = self.names[dp]
if not s in ndp.get_rnames():
msg = 'Unknown resource %r for design problem %r.' % (s, dp)
msg += ' Known functions: %s.' % format_list(ndp.get_rnames())
raise DPSemanticError(msg)
return CResource(dp, s)
def rename(self, key1, key2):
from .memdata_events import event_dict_rename
self.check_can_write()
if not key1 in self._data:
msg = ('Could not rename not existing key "%s"; known: %s.' %
(key1, format_list(self._data)))
raise_desc(InvalidOperation, msg)
event = event_dict_rename(name=self._prefix,
key=key1,
key2=key2,
**self._get_event_kwargs())
self._notify(event)
self._data[key2] = self._data.pop(key1)
def __delitem__(self, key):
from .memdata_events import event_dict_delitem
self.check_can_write()
if not key in self._data:
msg = ('Could not delete not existing key "%s"; known: %s.' %
(key, format_list(self._data)))
raise_desc(InvalidOperation, msg)
event = event_dict_delitem(name=self._prefix,
key=key,
**self._get_event_kwargs())
self._notify(event)
del self._data[key]
def check_no_forbidden(s): # pragma: no cover
if '\t' in s:
i = s.index('\t')
msg = "Tabs bring despair (e.g. Markdown does not recognize them.)"
where = Where(s, i)
raise DPSyntaxError(msg, where=where)
forbidden = {
'>=': ['≥'],
'<=': ['≤'],
'>>': ['?'] # added by mistake by Atom autocompletion
}
for f in forbidden:
if f in s:
msg = 'Found forbidden sequence %r. This will not end well.' % f
subs = forbidden[f]
msg += ' Try one of these substitutions: %s' % format_list(subs)
c = s.index(f)
where = Where(s, c, c + len(f))
raise DPSyntaxError(msg, where=where)
def read_SchemaContext_SER_DIR(self, schema, fh):
if not isinstance(fh, ProxyDirectory):
msg = 'I expected a ProxyDirectory representing dir entries.'
msg += indent(str(schema), 'schema ')
raise_desc(IncorrectFormat, msg, fh=fh)
res = {}
hint = self.get_hint(schema)
for k, schema_child in schema.children.items():
filename = hint.filename_for_key(k)
if filename is None:
# skip directory
res[k] = self.interpret_hierarchy_(schema_child, fh)
else:
if not filename in fh:
if schema_child.can_be_none:
res[k] = None
# logger.debug('Using default for key %s' % k)
else:
msg = 'Expected filename "%s".' % filename
msg += '\n available: %s' % format_list(fh)
raise_incorrect_format(msg, schema, fh.tree())
else:
try:
res[k] = self.interpret_hierarchy_(schema_child,
fh[filename])
except IncorrectFormat as e:
msg = 'While interpreting child "%s", filename "%s":' % (
k, filename)
raise_wrapped(IncorrectFormat,
e,
msg,
compact=True,
exc=sys.exc_info())
schema.validate(res)
return res
def _load_spec_data(self, spec_name, thing_name):
shelf = self.the_context.db_view.repos[self.repo_name].shelves[
self.shelf_name]
library = shelf.libraries[self.library_name]
things = library.things.child(spec_name)
try:
match = get_soft_match(thing_name, list(things))
except KeyError:
msg = 'Soft match failed: Could not find %r in %s.' % (thing_name,
spec_name)
available = sorted(things)
if available:
msg += ("\n Available %s: %s." %
(spec_name, format_list(sorted(available))))
else:
msg += "\n None available."
raise_desc(DPSemanticError, msg)
else:
if match != thing_name:
if MCDPConstants.allow_soft_matching:
logger.warning('Soft matching %r to %r (deprecated)' %
(match, thing_name))
else:
msg = 'Found case in which the user relies on soft matching (%r to refer to %r).' % (
thing_name, match)
raise DPSemanticError(msg)
# TODO: add warning
data = things[match]
spec = specs[spec_name]
basename = match + '.' + spec.extension
realpath = '%s in library %r in shelf %r in repo %r' % (
basename, self.library_name, self.shelf_name, self.repo_name)
return dict(data=data, realpath=realpath)
def dirname_from_data_url_(self, schema, data_url):
if not data_url:
return ()
# We know we will only get Context, List and Hash
check_isinstance(schema, (SchemaHash, SchemaList, SchemaContext))
# things that are serialized using hintdir
hint = self.get_hint(schema)
check_isinstance(hint, HintDir)
first = data_url[0]
rest = data_url[1:]
first_translated = hint.filename_for_key(first)
if isinstance(schema, SchemaHash):
rest_translated = self.dirname_from_data_url_(
schema.prototype, rest)
elif isinstance(schema, SchemaList):
rest_translated = self.dirname_from_data_url_(
schema.prototype, rest)
elif isinstance(schema, SchemaContext):
if not first in schema.children:
msg = (
'Could not translate data url %r because could not find child %r: found %s'
% (data_url, first, format_list(schema.children)))
raise_desc(ValueError, msg, schema=str(schema))
schema_child = schema.children[first]
rest_translated = self.dirname_from_data_url_(schema_child, rest)
else:
raise NotImplementedError(schema)
if first_translated is not None:
res = (first_translated, ) + rest_translated
else:
res = rest_translated
return res
def child(self, name):
if name in self.children_already_provided:
return self.children_already_provided[name]
if name in self.mount_points:
return self.mount_points[name]
if not name in self._data:
msg = 'Cannot get child "%s"; known: %s.' % (
name, format_list(self.keys()))
raise_desc(InvalidOperation, msg)
d = self._data[name]
prototype = self._schema.prototype
v = self._create_view_instance(prototype, d, name)
if isinstance(prototype, SchemaSimple):
def set_callback(value):
self._data[name] = value
v._set_callback = set_callback
self.children_already_provided[name] = v
return v
def disk_events_from_dict_rename(disk_map, view, _id, who, name, key, key2):
view_parent = get_view_node(view, name)
schema_parent = view_parent._schema
check_isinstance(schema_parent, SchemaHash)
prototype = schema_parent.prototype
hint = disk_map.get_hint(schema_parent)
if isinstance(hint, HintDir):
dirname = disk_map.dirname_from_data_url_(view._schema, name)
its_name = hint.filename_for_key(key)
its_name2 = hint.filename_for_key(key2)
# I just need to find out whether it would be ProxyDir or ProxyFile
if not key in view_parent._data:
msg = 'Cannot rename key %r that does not exist in %s.' % (
key, format_list(view_parent._data))
raise InvalidOperation(msg)
value = view_parent._data[key]
d = disk_map.create_hierarchy_(prototype, value)
if isinstance(d, ProxyFile):
disk_event = disk_event_file_rename(_id,
who,
dirname=dirname,
name=its_name,
name2=its_name2)
return [disk_event]
elif isinstance(d, ProxyDirectory):
disk_event = disk_event_dir_rename(_id,
who,
dirname=dirname,
name=its_name,
name2=its_name2)
return [disk_event]
else:
assert False
else:
raise NotImplementedError(hint)
def recompute_available(self):
# all the ones we can discover
repos = self.app.hi.db_view.repos
for repo_name, repo in repos.items():
for shelf_name, shelf in repo.shelves.items():
self.shelves_all[shelf_name] = shelf
# shelf:repo/shelfname
# library:repo/shelf/library
self.shelves_available = OrderedDict()
# the ones that are actually in use
self.shelves_used = OrderedDict()
userstruct = self.get_user_struct()
ui = userstruct.info
for sname, shelf in self.shelves_all.items():
if shelf.get_acl().allowed2(Privileges.DISCOVER, ui):
self.shelves_available[sname] = shelf
else:
#print('hiding shelf %r from %r' % (sname, user))
print shelf.get_acl()
#print('shelves all: %s' % list(self.shelves_all))
#print('shelves available: %s' % list(self.shelves_available))
for sname in ui.get_subscriptions():
if sname in self.shelves_available:
shelf = self.shelves_available[sname]
acl = shelf.get_acl()
if acl.allowed2(Privileges.READ, ui):
self.shelves_used[sname] = self.shelves_available[sname]
else:
msg = 'User %r does not have %r for %r' % (
ui.username, Privileges.READ, sname)
msg += '\n%s' % acl
logger.error(msg)
else:
msg = 'Could not find shelf %r to which user %r is subscribed to.' % (
sname, ui.username)
msg += '\n Available: %s' % list(self.shelves_available)
logger.error(msg)
#print('shelves used: %s' % list(self.shelves_used))
self.librarian = Librarian()
self.libname2shelfname = {}
self.shelfname2reponame = {}
repos = self.app.hi.db_view.repos
for repo_name, repo in repos.items():
for shelf_name, shelf in repo.shelves.items():
if shelf_name in self.shelfname2reponame:
o = self.shelfname2reponame[shelf_name]
msg = ('Two repos with same shelf %r: %r and %r.' %
(shelf_name, repo_name, o))
for r in [o, repo_name]:
msg += '\n Shelves for %r: %s' % (
r, format_list(sorted(self.repos[r].shelves)))
raise ValueError(msg)
self.shelfname2reponame[shelf_name] = repo_name
for sname, shelf in self.shelves_all.items():
for libname in shelf.libraries:
self.libname2shelfname[libname] = sname
for sname, shelf in self.shelves_used.items():
if False: #
logger.error('need to reimplement this part')
for libname, libpath in shelf.get_libraries_path().items():
self.librarian.add_lib_by_path(libpath)
self.libraries = self.librarian.get_libraries()
for _short, data in self.libraries.items():
l = data['library']
path = data['path']
cache_dir = os.path.join(path, '_cached/mcdpweb_cache')
l.use_cache_dir(cache_dir)
def child(self, name):
if not name in self.children:
msg = 'Could not find child %r; available: %s.' % (name, format_list(self.children))
raise ValueError(msg)
child = self.children[name]
return child
def __getitem__(self, name):
if not name in self.children:
msg = 'Could not find %r: available %s.' % (name, format_list(self.children))
raise KeyError(msg)
return self.children[name]
def __init__(self, instance, upstream, root, repo_git, repo_local):
'''
root: where to put our temporary files
repo_git: name -> remote git url
repo_local: name -> local path
As a special case, if name == 'user_db', then it is used as the users database.
If no 'user_db' is passed, then we create an empty one inside root.
'''
self.repos = {}
self.who = {'host': host_name(), 'actor': 'system', 'instance': instance}
for name, dirname in repo_local.items():
if not os.path.exists(dirname):
msg = 'Directory %r for %r does not exist' % (dirname, name)
raise_desc(ValueError, msg)
self.repo_local = repo_local
for repo_name, remote_url in repo_git.items():
where = os.path.join(root, repo_name)
logger.info('Loading %r = %r in dir %s' % (repo_name, remote_url, where))
repo = Repo.init(where)
origin = repo.create_remote('origin', url=remote_url)
assert origin.exists()
for _ in repo.remotes.origin.fetch(): pass
# this is what we want to track: the branch "upstream"
if not upstream in origin.refs:
msg = 'Cannot track remote branch %r because it does not exist.' % upstream
raise Exception(msg)
# Do we already have a branch instance in the remote repo?
if instance in origin.refs:
# if so, check out
logger.info('Checking out remote %r' % instance)
head = repo.create_head(instance, origin.refs[instance])
head.set_tracking_branch(origin.refs[instance])
head.checkout()
else:
# we create it from the upstream branch
logger.info('Creating local %s from remote %r' % (instance, upstream))
head = repo.create_head(instance, origin.refs[upstream])
head.checkout()
logger.info('Pushing local %s' % (instance))
repo.git.push('-u', 'origin', instance)
self.repos[repo_name] = repo
if not 'user_db' in self.repos and not 'user_db' in self.repo_local:
dirname = os.path.join(root, 'user_db_local')
create_empty_dir_from_schema(dirname=dirname, schema=DB.user_db, disk_map=DB.dm)
logger.info('No repository with name "user_db" passed. Creating empty one in %s' % dirname)
self.repo_local['user_db'] = dirname
self.mount()
logger.info('Set up repositories %s.' % format_list(self.db_view.repos))
for repo_name, repo in self.db_view.repos.items():
logger.info('* repo %r has shelves %s ' % (repo_name, format_list(repo.shelves)))
logger.info('Set up users %s.' % format_list(self.db_view.user_db.users))
self.host_cache = HostCache(self.db_view)
