def add_connection(self, c): self.info('Adding connection %r' % str(c)) if not c.dp1 in self.names: raise_desc(DPSemanticError, 'Invalid connection: %r not found.' % c.dp1, names=self.names, c=c) if not c.dp2 in self.names: raise_desc(DPSemanticError, 'Invalid connection: %r not found.' % c.dp2, names=self.names, c=c) mcdp_dev_warning('redo this check') if self.is_new_function(c.dp2): msg = "Cannot add connection to new function %r." % c.dp2 raise_desc(DPSemanticError, msg, c=c) if self.is_new_resource(c.dp1): msg = "Cannot add connection to new resource %r." % c.dp1 raise_desc(DPSemanticError, msg, c=c) # Find if there is already a connection to c.dp2,c.s2 # for c0 in self.connections: # if c0.dp2 == c.dp2 and c0.s2 == c.s2: # msg = 'There is already a connection to function %r of %r.' % (c.s2, c.dp2) # raise_desc(DPSemanticError, msg) ndp1 = self.names[c.dp1] ndp2 = self.names[c.dp2] rnames = ndp1.get_rnames() if not c.s1 in rnames: msg = "Resource %r does not exist (known: %s)" % ( c.s1, format_list(rnames)) raise_desc(DPSemanticError, msg, known=rnames) fnames = ndp2.get_fnames() if not c.s2 in fnames: msg = "Function %r does not exist (known: %s)" % ( c.s2, format_list(fnames)) raise_desc(DPSemanticError, msg, known=fnames) R1 = ndp1.get_rtype(c.s1) F2 = ndp2.get_ftype(c.s2) # print('connecting R1 %s to R2 %s' % (R1, F2)) if not (R1 == F2): msg = 'Connection between different spaces.' raise_desc(DPSemanticError, msg, c=c, F2=F2, R1=R1, ndp1=ndp1, ndp2=ndp2) self.connections.append(c)
def fill_in_none(schema, data): ''' Fills in the fields not passed and that can be None ''' if isinstance(schema, SchemaString): if isinstance(data, unicode): return data.encode('utf8') else: return data elif isinstance(schema, SchemaSimple): schema.validate(data) return data elif isinstance(schema, SchemaList): check_isinstance(data, list) res = [fill_in_none(schema.prototype, _) for _ in data] schema.validate(res) return res elif isinstance(schema, SchemaHash): check_isinstance(data, dict) res = dict([(k, fill_in_none(schema.prototype, v)) for k, v in data.items()]) schema.validate(res) return res elif isinstance(schema, SchemaContext): check_isinstance(data, dict) # some of the fields might be missing res = {} present = set(data) for k, schema_child in schema.children.items(): if k in data: # have it res[k] = fill_in_none(schema_child, data[k]) else: # dont' have it if schema_child.can_be_none: res[k] = None else: # no default msg = 'Expected key "%s".' % k raise_incorrect_format( msg, schema, data, ) needed = set(schema.children) extra = set(present) - needed if extra: msg = 'Extra fields: %s.' % format_list(sorted(extra)) msg += '\nPresent: %s' % format_list(present) msg += '\nNeeded: %s' % format_list(needed) raise_incorrect_format(msg, schema, data) schema.validate(res) return res else: assert False, schema
def __getitem__(self, key): if key in self.mount_points: return self.mount_points[key] if not key in self._data: msg = 'Could not find key "%s"; available keys are %s, mount points %s' % ( key, format_list(self._data), format_list(self.mount_points)) raise_desc(EntryNotFound, msg) d = self._data[key] prototype = self._schema.prototype if is_simple_data(prototype): self.child(key).check_can_read() return d else: return self._create_view_instance(prototype, d, key)
def _get_file_data(self, basename): """ returns dict with data, realpath """ for fn in self.file_to_contents: if fn.lower() == basename.lower(): match = fn break else: msg = 'Could not find file %r.' % basename ext = os.path.splitext(basename)[1].replace('.', '') available = sorted(self._list_with_extension(ext), key=lambda x: x.lower()) if available: msg += (" Available files with %r extension: %s." % (ext, format_list(sorted(available)))) else: msg += " No files with extension %r found." % ext msg += '\nSearch directories: %s' % self.search_dirs raise_desc(DPSemanticError, msg) raise Exception() # XXX found = self.file_to_contents[match] return found
def generate_empty(self, **kwargs): res = {} for k in kwargs: if not k in self.children: msg = 'Extra key %r not in %s.' % (k, format_list(self.children)) raise ValueError(msg) kwargs = deepcopy(kwargs) for k, c in self.children.items(): if isinstance(c, SchemaSimple): if k in kwargs: res[k] = kwargs.pop(k) elif c.can_be_none: res[k] = None else: msg = 'Cannot generate empty for child %r: %s.' % (k, c) raise ValueError(msg) else: if k in kwargs: x = kwargs.pop(k) if isinstance(x, dict): res[k] = c.generate_empty(**x) else: res[k] = x else: res[k] = c.generate_empty() assert not kwargs return res
def get_soft_match(x, options): ''' Get a soft match or raise KeyError ''' options = list(options) res = [] for o in options: if x.lower() == o.lower(): res.append(o) if not res: msg = 'Could not find any soft match for "%s" in %s.' % ( x, format_list(options)) raise KeyError(msg) if len(res) > 1: msg = 'Too many matches (%s) for "%s" in %s.' % (format_list(res), x, format_list(options)) raise KeyError(msg) return res[0]
def load_library(self, libname, context=None): # @UnusedVariable check_isinstance(libname, str) """ hook to pass to MCDPLibrary instances to find their sisters. """ if not libname in self.libraries: msg = ('Cannot find library "%s". Available: %s.' % (libname, format_list(sorted(self.libraries)))) raise_desc(DPSemanticError, msg) l = self.libraries[libname]['library'] return l
def shelf_privilege(repo_name, sname, privilege): repo = repos[repo_name] if not sname in repo.shelves: msg = 'Cannot find shelf "%s" in repo "%s".' % (sname, repo_name) msg += '\n available: ' + format_list(repo.shelves) raise ValueError(msg) acl = repo.shelves[sname].get_acl() return acl.allowed2(privilege, e.user)
def disk_events_from_list_remove(disk_map, view, _id, who, name, value): view_parent = view.get_descendant(name) data = view_parent._data for index, v in enumerate(data): if v == value: return disk_events_from_list_delete(disk_map, view, _id, who, name, index) msg = 'There is no value %s in the list.' % value msg += '\n values: %s' % format_list(data) raise InvalidOperation(msg)
def dir_rename(self, name, name2): if not name in self._directories: msg = ('Cannot rename directory %r to %r if does not exist in %s.' % (name, name2, format_list(self._directories))) raise InvalidDiskOperation(msg) if name2 in self._directories: msg = ('Cannot rename directory %r to %r if %r already exists' % (name, name2, name2)) raise InvalidDiskOperation(msg) self._directories[name2] = self._directories.pop(name)
def go(self): options = self.get_options() if options.config is not None: logger.info('Reading configuration from %s' % options.config) logger.warn('Other options from command line will be ignored. ') parser = RawConfigParser() parser.read(options.config) sections = parser.sections() logger.info('sections: %s' % sections) s = 'app:main' if not s in sections: msg = 'Could not find section "%s": available are %s.' % ( s, format_list(sections)) msg += '\n file %s' % options.config raise Exception(msg) # XXX settings = dict((k, parser.get(s, k)) for k in parser.options(s)) prefix = 'mcdp_web.' mcdp_web_settings = get_only_prefixed(settings, prefix, delete=True) # mcdp_web_settings = {} # for k,v in list(settings.items()): # if k.startswith(prefix): # mcdp_web_settings[k[len(prefix):]] = v # del settings[k] options = parse_mcdpweb_params_from_dict(mcdp_web_settings) logger.debug('Using these options: %s' % options) else: logger.info('No configuration .ini specified (use --config).') settings = {} wa = WebApp(options, settings=settings) # Write warning messages now wa.get_authomatic_config() msg = """Welcome to PyMCDP! To access the interface, open your browser at the address http://localhost:%s/ Use Chrome, Firefox, or Opera - Internet Explorer is not supported. """ % options.port logger.info(msg) if options.delete_cache: pass # XXX: warning deprecated # logger.info('Deleting cache...') # wa._refresh_library(None) wa.serve(port=options.port)
def event_dict_rename_interpret(view, name, key, key2): from mcdp_hdb.memdataview import ViewHash0 v = get_view_node(view, name) check_isinstance(v, ViewHash0) # permissions v.check_can_write() if not key in v._data: msg = ('Cannot rename key %r to %r if it does not exist in %s.' % (key, key2, format_list(v._data))) raise InvalidOperation(msg) v._data[key2] = v._data.pop(key)
def remove(self, value): if not value in self._data: msg = 'The list does not contain %r: available %s' % ( value, format_list(self._data)) raise ValueError(msg) from .memdata_events import event_list_remove event = event_list_remove(name=self._prefix, value=value, **self._get_event_kwargs()) self._notify(event) self._data.remove(value)
def load_library(self, library_name, context=None): # @UnusedVariable # logger.debug('load_library(%r)' % library_name) repos = self.db_view.repos all_shelves = set() all_libraries = set() for repo_name, repo in repos.items(): all_shelves.update(repo.shelves) for shelf_name, shelf in repo.shelves.items(): all_libraries.update(shelf.libraries) for shelf_name in self.subscribed_shelves: if shelf_name in repo.shelves: shelf = repo.shelves[shelf_name] if library_name in shelf.libraries: return self.make_library(repo_name, shelf_name, library_name) msg = 'Could not find library %r.' % library_name msg += '\n Subscribed shelves: %s.' % format_list( sorted(self.subscribed_shelves)) msg += '\n All shelves: %s.' % format_list(sorted(all_shelves)) msg += '\n All libraries: %s.' % format_list(sorted(all_libraries)) raise ValueError(msg)
def make_function(self, dp, s): assert isinstance(dp, str) if not dp in self.names: msg = 'Unknown design problem %r.' % dp raise DPSemanticError(msg) ndp = self.names[dp] if not s in ndp.get_fnames(): msg = 'Unknown function %r for design problem %r.' % (s, dp) msg += ' Known functions: %s.' % format_list(ndp.get_fnames()) raise DPSemanticError(msg) return CFunction(dp, s)
def check_no_forbidden(s, res, location0): forbidden = { '>=': ['≥'], '<=': ['≤'], '>>': ['?'] # added by mistake by Atom autocompletion } for f in forbidden: if f in s: msg = 'Found forbidden sequence %r. This will not end well.' % f subs = forbidden[f] msg += ' Try one of these substitutions: %s' % format_list(subs) c = s.index(f) where = Where(s, c, c + len(f)) res.note_error(msg, LocationInString(where, location0))
def validate(self, data): if not isinstance(data, dict): msg = 'Expected a dictionary object.' raise_desc(NotValid, msg, data=describe_value(data)) for k, v in self.children.items(): if not k in data: msg = 'Expecting key "%s" but not found in %s.' % (k, format_list(data)) raise_desc(NotValid, msg, data=describe_value(data), self=str(self)) try: v.validate(data[k]) except NotValid as e: msg = 'For child "%s":' % k raise_wrapped(NotValid, e, msg)
def make_resource(self, dp, s): if not isinstance(dp, str): raise DPInternalError((dp, s)) if not dp in self.names: msg = 'Unknown design problem %r.' % dp raise DPSemanticError(msg) ndp = self.names[dp] if not s in ndp.get_rnames(): msg = 'Unknown resource %r for design problem %r.' % (s, dp) msg += ' Known functions: %s.' % format_list(ndp.get_rnames()) raise DPSemanticError(msg) return CResource(dp, s)
def rename(self, key1, key2): from .memdata_events import event_dict_rename self.check_can_write() if not key1 in self._data: msg = ('Could not rename not existing key "%s"; known: %s.' % (key1, format_list(self._data))) raise_desc(InvalidOperation, msg) event = event_dict_rename(name=self._prefix, key=key1, key2=key2, **self._get_event_kwargs()) self._notify(event) self._data[key2] = self._data.pop(key1)
def __delitem__(self, key): from .memdata_events import event_dict_delitem self.check_can_write() if not key in self._data: msg = ('Could not delete not existing key "%s"; known: %s.' % (key, format_list(self._data))) raise_desc(InvalidOperation, msg) event = event_dict_delitem(name=self._prefix, key=key, **self._get_event_kwargs()) self._notify(event) del self._data[key]
def check_no_forbidden(s): # pragma: no cover if '\t' in s: i = s.index('\t') msg = "Tabs bring despair (e.g. Markdown does not recognize them.)" where = Where(s, i) raise DPSyntaxError(msg, where=where) forbidden = { '>=': ['≥'], '<=': ['≤'], '>>': ['?'] # added by mistake by Atom autocompletion } for f in forbidden: if f in s: msg = 'Found forbidden sequence %r. This will not end well.' % f subs = forbidden[f] msg += ' Try one of these substitutions: %s' % format_list(subs) c = s.index(f) where = Where(s, c, c + len(f)) raise DPSyntaxError(msg, where=where)
def read_SchemaContext_SER_DIR(self, schema, fh): if not isinstance(fh, ProxyDirectory): msg = 'I expected a ProxyDirectory representing dir entries.' msg += indent(str(schema), 'schema ') raise_desc(IncorrectFormat, msg, fh=fh) res = {} hint = self.get_hint(schema) for k, schema_child in schema.children.items(): filename = hint.filename_for_key(k) if filename is None: # skip directory res[k] = self.interpret_hierarchy_(schema_child, fh) else: if not filename in fh: if schema_child.can_be_none: res[k] = None # logger.debug('Using default for key %s' % k) else: msg = 'Expected filename "%s".' % filename msg += '\n available: %s' % format_list(fh) raise_incorrect_format(msg, schema, fh.tree()) else: try: res[k] = self.interpret_hierarchy_(schema_child, fh[filename]) except IncorrectFormat as e: msg = 'While interpreting child "%s", filename "%s":' % ( k, filename) raise_wrapped(IncorrectFormat, e, msg, compact=True, exc=sys.exc_info()) schema.validate(res) return res
def _load_spec_data(self, spec_name, thing_name): shelf = self.the_context.db_view.repos[self.repo_name].shelves[ self.shelf_name] library = shelf.libraries[self.library_name] things = library.things.child(spec_name) try: match = get_soft_match(thing_name, list(things)) except KeyError: msg = 'Soft match failed: Could not find %r in %s.' % (thing_name, spec_name) available = sorted(things) if available: msg += ("\n Available %s: %s." % (spec_name, format_list(sorted(available)))) else: msg += "\n None available." raise_desc(DPSemanticError, msg) else: if match != thing_name: if MCDPConstants.allow_soft_matching: logger.warning('Soft matching %r to %r (deprecated)' % (match, thing_name)) else: msg = 'Found case in which the user relies on soft matching (%r to refer to %r).' % ( thing_name, match) raise DPSemanticError(msg) # TODO: add warning data = things[match] spec = specs[spec_name] basename = match + '.' + spec.extension realpath = '%s in library %r in shelf %r in repo %r' % ( basename, self.library_name, self.shelf_name, self.repo_name) return dict(data=data, realpath=realpath)
def dirname_from_data_url_(self, schema, data_url): if not data_url: return () # We know we will only get Context, List and Hash check_isinstance(schema, (SchemaHash, SchemaList, SchemaContext)) # things that are serialized using hintdir hint = self.get_hint(schema) check_isinstance(hint, HintDir) first = data_url[0] rest = data_url[1:] first_translated = hint.filename_for_key(first) if isinstance(schema, SchemaHash): rest_translated = self.dirname_from_data_url_( schema.prototype, rest) elif isinstance(schema, SchemaList): rest_translated = self.dirname_from_data_url_( schema.prototype, rest) elif isinstance(schema, SchemaContext): if not first in schema.children: msg = ( 'Could not translate data url %r because could not find child %r: found %s' % (data_url, first, format_list(schema.children))) raise_desc(ValueError, msg, schema=str(schema)) schema_child = schema.children[first] rest_translated = self.dirname_from_data_url_(schema_child, rest) else: raise NotImplementedError(schema) if first_translated is not None: res = (first_translated, ) + rest_translated else: res = rest_translated return res
def child(self, name): if name in self.children_already_provided: return self.children_already_provided[name] if name in self.mount_points: return self.mount_points[name] if not name in self._data: msg = 'Cannot get child "%s"; known: %s.' % ( name, format_list(self.keys())) raise_desc(InvalidOperation, msg) d = self._data[name] prototype = self._schema.prototype v = self._create_view_instance(prototype, d, name) if isinstance(prototype, SchemaSimple): def set_callback(value): self._data[name] = value v._set_callback = set_callback self.children_already_provided[name] = v return v
def disk_events_from_dict_rename(disk_map, view, _id, who, name, key, key2): view_parent = get_view_node(view, name) schema_parent = view_parent._schema check_isinstance(schema_parent, SchemaHash) prototype = schema_parent.prototype hint = disk_map.get_hint(schema_parent) if isinstance(hint, HintDir): dirname = disk_map.dirname_from_data_url_(view._schema, name) its_name = hint.filename_for_key(key) its_name2 = hint.filename_for_key(key2) # I just need to find out whether it would be ProxyDir or ProxyFile if not key in view_parent._data: msg = 'Cannot rename key %r that does not exist in %s.' % ( key, format_list(view_parent._data)) raise InvalidOperation(msg) value = view_parent._data[key] d = disk_map.create_hierarchy_(prototype, value) if isinstance(d, ProxyFile): disk_event = disk_event_file_rename(_id, who, dirname=dirname, name=its_name, name2=its_name2) return [disk_event] elif isinstance(d, ProxyDirectory): disk_event = disk_event_dir_rename(_id, who, dirname=dirname, name=its_name, name2=its_name2) return [disk_event] else: assert False else: raise NotImplementedError(hint)
def recompute_available(self): # all the ones we can discover repos = self.app.hi.db_view.repos for repo_name, repo in repos.items(): for shelf_name, shelf in repo.shelves.items(): self.shelves_all[shelf_name] = shelf # shelf:repo/shelfname # library:repo/shelf/library self.shelves_available = OrderedDict() # the ones that are actually in use self.shelves_used = OrderedDict() userstruct = self.get_user_struct() ui = userstruct.info for sname, shelf in self.shelves_all.items(): if shelf.get_acl().allowed2(Privileges.DISCOVER, ui): self.shelves_available[sname] = shelf else: #print('hiding shelf %r from %r' % (sname, user)) print shelf.get_acl() #print('shelves all: %s' % list(self.shelves_all)) #print('shelves available: %s' % list(self.shelves_available)) for sname in ui.get_subscriptions(): if sname in self.shelves_available: shelf = self.shelves_available[sname] acl = shelf.get_acl() if acl.allowed2(Privileges.READ, ui): self.shelves_used[sname] = self.shelves_available[sname] else: msg = 'User %r does not have %r for %r' % ( ui.username, Privileges.READ, sname) msg += '\n%s' % acl logger.error(msg) else: msg = 'Could not find shelf %r to which user %r is subscribed to.' % ( sname, ui.username) msg += '\n Available: %s' % list(self.shelves_available) logger.error(msg) #print('shelves used: %s' % list(self.shelves_used)) self.librarian = Librarian() self.libname2shelfname = {} self.shelfname2reponame = {} repos = self.app.hi.db_view.repos for repo_name, repo in repos.items(): for shelf_name, shelf in repo.shelves.items(): if shelf_name in self.shelfname2reponame: o = self.shelfname2reponame[shelf_name] msg = ('Two repos with same shelf %r: %r and %r.' % (shelf_name, repo_name, o)) for r in [o, repo_name]: msg += '\n Shelves for %r: %s' % ( r, format_list(sorted(self.repos[r].shelves))) raise ValueError(msg) self.shelfname2reponame[shelf_name] = repo_name for sname, shelf in self.shelves_all.items(): for libname in shelf.libraries: self.libname2shelfname[libname] = sname for sname, shelf in self.shelves_used.items(): if False: # logger.error('need to reimplement this part') for libname, libpath in shelf.get_libraries_path().items(): self.librarian.add_lib_by_path(libpath) self.libraries = self.librarian.get_libraries() for _short, data in self.libraries.items(): l = data['library'] path = data['path'] cache_dir = os.path.join(path, '_cached/mcdpweb_cache') l.use_cache_dir(cache_dir)
def child(self, name): if not name in self.children: msg = 'Could not find child %r; available: %s.' % (name, format_list(self.children)) raise ValueError(msg) child = self.children[name] return child
def __getitem__(self, name): if not name in self.children: msg = 'Could not find %r: available %s.' % (name, format_list(self.children)) raise KeyError(msg) return self.children[name]
def __init__(self, instance, upstream, root, repo_git, repo_local): ''' root: where to put our temporary files repo_git: name -> remote git url repo_local: name -> local path As a special case, if name == 'user_db', then it is used as the users database. If no 'user_db' is passed, then we create an empty one inside root. ''' self.repos = {} self.who = {'host': host_name(), 'actor': 'system', 'instance': instance} for name, dirname in repo_local.items(): if not os.path.exists(dirname): msg = 'Directory %r for %r does not exist' % (dirname, name) raise_desc(ValueError, msg) self.repo_local = repo_local for repo_name, remote_url in repo_git.items(): where = os.path.join(root, repo_name) logger.info('Loading %r = %r in dir %s' % (repo_name, remote_url, where)) repo = Repo.init(where) origin = repo.create_remote('origin', url=remote_url) assert origin.exists() for _ in repo.remotes.origin.fetch(): pass # this is what we want to track: the branch "upstream" if not upstream in origin.refs: msg = 'Cannot track remote branch %r because it does not exist.' % upstream raise Exception(msg) # Do we already have a branch instance in the remote repo? if instance in origin.refs: # if so, check out logger.info('Checking out remote %r' % instance) head = repo.create_head(instance, origin.refs[instance]) head.set_tracking_branch(origin.refs[instance]) head.checkout() else: # we create it from the upstream branch logger.info('Creating local %s from remote %r' % (instance, upstream)) head = repo.create_head(instance, origin.refs[upstream]) head.checkout() logger.info('Pushing local %s' % (instance)) repo.git.push('-u', 'origin', instance) self.repos[repo_name] = repo if not 'user_db' in self.repos and not 'user_db' in self.repo_local: dirname = os.path.join(root, 'user_db_local') create_empty_dir_from_schema(dirname=dirname, schema=DB.user_db, disk_map=DB.dm) logger.info('No repository with name "user_db" passed. Creating empty one in %s' % dirname) self.repo_local['user_db'] = dirname self.mount() logger.info('Set up repositories %s.' % format_list(self.db_view.repos)) for repo_name, repo in self.db_view.repos.items(): logger.info('* repo %r has shelves %s ' % (repo_name, format_list(repo.shelves))) logger.info('Set up users %s.' % format_list(self.db_view.user_db.users)) self.host_cache = HostCache(self.db_view)