def continueObjectEditing(self, user, ref): """ Objects which have been opened but not edited for a certain amount of time are automatically closed by the backend. This command delays this behaviour by increasing the timeout. ================= ========================== Parameter Description ================= ========================== ref UUID / object reference ================= ========================== """ objdsc = self.__get_ref(ref) if not objdsc: raise ValueError(C.make_error("REFERENCE_NOT_FOUND", ref=ref)) if not self.__check_user(ref, user): raise ValueError(C.make_error("NOT_OBJECT_OWNER")) objdsc['last_interaction'] = datetime.datetime.now() with make_session() as session: obj = session.query(OpenObject).filter(OpenObject.ref == ref).one() obj.last_interaction = objdsc['last_interaction'] session.commit() if 'mark_for_deletion' in objdsc: # as this object has been marked for deletion, we have to run the garbage collection # to remove this mark now self.__gc()
def getForemanReleaseName(self, operatingsystem_id): if operatingsystem_id is not None: with make_session() as session: res = session.query(Cache.data).filter(Cache.key == "foreman.operating_system.%s" % operatingsystem_id).one_or_none() if res is not None: return res[0]["release_name"] return None
def checkObjectRef(self, user, session_id, ref): """ Objects which have been opened but not edited for a certain amount of time are automatically closed by the backend. This command delays this behaviour by increasing the timeout. ================= ========================== Parameter Description ================= ========================== ref UUID / object reference ================= ========================== ``Return``: boolean """ objdsc = self.__get_ref(ref) if objdsc and objdsc['user'] == user: # update session-id, which might have changed and is needed to inform the user about object closing if objdsc['session_id'] != session_id: objdsc['session_id'] = session_id with make_session() as session: obj = session.query(OpenObject).filter(OpenObject.ref == ref).one() obj.session_id = session_id session.commit() return True return False
def getContainerTree(self, user, base, object_type=None): table = inspect(ObjectInfoIndex) o2 = aliased(ObjectInfoIndex) base = ObjectProxy.get_adjusted_dn(base, self.env.base) query = and_(getattr(ObjectInfoIndex, "_adjusted_parent_dn") == base, getattr(ObjectInfoIndex, "_type").in_(self.containers)) count = func.count(getattr(o2, "_parent_dn")) parent_join_condition = getattr(o2, "_parent_dn") == getattr(ObjectInfoIndex, "dn") with make_session() as session: query_result = session.query(ObjectInfoIndex, count) \ .outerjoin(o2, and_(getattr(o2, "_invisible").is_(False), parent_join_condition)) \ .filter(query) \ .group_by(*table.c) res = {} factory = ObjectFactory.getInstance() for item, children in query_result: self.update_res(res, item, user, 1) if item.dn in res: res[item.dn]['hasChildren'] = children > 0 res[item.dn]['adjusted_dn'] = ObjectProxy.get_adjusted_dn(item.dn, self.env.base) if object_type is not None: # check if object_type is allowed in this container allowed = factory.getAllowedSubElementsForObject(res[item.dn]['tag'], includeInvisible=False) if "*" in object_type: # all allowed res[item.dn]['allowed_move_target'] = True elif isinstance(object_type, list): res[item.dn]['allowed_move_target'] = len(set(object_type).intersection(allowed)) > 0 else: res[item.dn]['allowed_move_target'] = object_type in allowed return res
def checkObjectRef(self, user, session_id, ref): """ Objects which have been opened but not edited for a certain amount of time are automatically closed by the backend. This command delays this behaviour by increasing the timeout. ================= ========================== Parameter Description ================= ========================== ref UUID / object reference ================= ========================== ``Return``: boolean """ objdsc = self.__get_ref(ref) if objdsc and objdsc['user'] == user: # update session-id, which might have changed and is needed to inform the user about object closing if objdsc['session_id'] != session_id: objdsc['session_id'] = session_id with make_session() as session: obj = session.query(OpenObject).filter( OpenObject.ref == ref).one() obj.session_id = session_id session.commit() return True return False
def __get_ref(self, ref): if ref in self.__stack: return self.__stack[ref] else: # check DB with make_session() as session: obj = session.query(OpenObject).filter(OpenObject.ref == ref).one_or_none() if obj is not None: if obj.backend_uuid != self.env.core_uuid: self.log.debug("open object '%s' is now handled by backend %s" % (obj.uuid, self.env.core_uuid)) with make_session() as session: obj.backend_uuid = self.env.core_uuid session.commit() self.openObject(obj.user, obj.session_id, obj.oid, obj.uuid, db_object=obj) return self.__stack[ref] return None
def __get_user_session(self, sid): if self.env.mode == "proxy": if sid in sessions: return sessions[sid] else: return None else: with make_session() as session: return session.query(UserSession).filter(UserSession.sid == sid).one_or_none()
def __delete_user_session(self, user_session): if user_session is None: return if self.env.mode == "proxy": if user_session.sid in sessions: del sessions[user_session.sid] else: with make_session() as session: session.delete(user_session)
def __gc_sessions(): """ delete sessions that not have been used for 10 hours """ threshold_date = (datetime.datetime.now() - datetime.timedelta(hours=10)) if Environment.getInstance().mode == "proxy": for sid, user_session in sessions.items(): if user_session.last_used < threshold_date: del sessions[sid] else: with make_session() as session: return session.query(UserSession).filter(UserSession.last_used < threshold_date).delete()
def __get_ref(self, ref): if ref in self.__stack: return self.__stack[ref] else: # check DB with make_session() as session: obj = session.query(OpenObject).filter( OpenObject.ref == ref).one_or_none() if obj is not None: if obj.backend_uuid != self.env.core_uuid: self.log.debug( "open object '%s' is now handled by backend %s" % (obj.uuid, self.env.core_uuid)) with make_session() as session: obj.backend_uuid = self.env.core_uuid session.commit() self.openObject(obj.user, obj.session_id, obj.oid, obj.uuid, db_object=obj) return self.__stack[ref] return None
def sync_release_names(self): """ The GOsa proxies need to know the release names of the operating systems, as we do not store them within any object, we save them in the Cache-Database. Proxies can query their replicated database for them """ if self.client: data = self.client.get("operatingsystems") if "results" in data: # clear Database with make_session() as session: session.query(Cache).filter((Cache.key.ilike("foreman.operating_system.%"))).delete(synchronize_session='fetch') for entry in data["results"]: self.sync_release_name(entry, session) session.commit()
def isContainerForObjectType(self, user, container_dn, object_type): with make_session() as session: container_type_query = session.query(getattr(ObjectInfoIndex, "_type")).filter( getattr(ObjectInfoIndex, "dn") == container_dn).one() container_type = container_type_query[0] allowed = ObjectFactory.getInstance().getAllowedSubElementsForObject(container_type) if object_type is None: return True elif "*" in object_type: # all allowed return True elif isinstance(object_type, list): return len(set(object_type).intersection(allowed)) > 0 else: return object_type in allowed
def isContainerForObjectType(self, user, container_dn, object_type): with make_session() as session: container_type_query = session.query( getattr(ObjectInfoIndex, "_type")).filter( getattr(ObjectInfoIndex, "dn") == container_dn).one() container_type = container_type_query[0] allowed = ObjectFactory.getInstance( ).getAllowedSubElementsForObject(container_type) if object_type is None: return True elif "*" in object_type: # all allowed return True elif isinstance(object_type, list): return len(set(object_type).intersection(allowed)) > 0 else: return object_type in allowed
def getPPDURL(self, source_url): """ Downloads the source_url, stores it locally and returns the local URL :param source_url: remote PPD URL :return: local URL to the cached PPD """ source = parse_url(source_url) host = source.host if host is None or host == "localhost": # no host: we assume that the PPD can be found on the current active master backend with make_session() as session: # get any other registered backend master_backend = session.query(RegisteredBackend) \ .filter(RegisteredBackend.uuid != self.env.core_uuid, RegisteredBackend.type == BackendTypes.active_master).first() if master_backend is None: self.log.error(C.make_error("NO_MASTER_BACKEND_FOUND")) return source_url # Try to log in with provided credentials url = parse_url(master_backend.url) host = url.host # check if file exists locally rel_path = source.path[1:] if source.path.startswith("/") else source.path local_path = path.join(self.ppd_dir, host, rel_path) if not path.exists(local_path): # cache locally try: r = requests.get(source_url) if r.ok: local_dir = path.dirname(local_path) if not path.exists(local_dir): makedirs(local_dir) with open(local_path, "w") as f: f.write(r.text) else: self.log.error("requesting PPD from %s failed with status code: %s" % (source_url, r.status_code)) return source_url except requests.exceptions.ConnectionError as e: self.log.error("requesting PPD from %s failed with error: %s" % (source_url, str(e))) return source_url return "%s%s/%s" % (self.base_url, host, rel_path)
def setObjectProperty(self, user, ref, name, value, skip_db_update=False): """ Set a property on an existing stack object. ================= ========================== Parameter Description ================= ========================== ref UUID / object reference name Property name value Property value ================= ========================== """ objdsc = self.__get_ref(ref) if not objdsc: raise ValueError(C.make_error("REFERENCE_NOT_FOUND", ref=ref)) if not name in objdsc['object']['properties']: raise ValueError(C.make_error("PROPERTY_NOT_FOUND", property=name)) details = objdsc['object']['object'].get_attributes(True) if 'auto' in details[name] and details[name]['auto']: raise ValueError(C.make_error("AUTOGENERATED_ATTRIBUTE")) if not self.__check_user(ref, user): raise ValueError(C.make_error("NOT_OBJECT_OWNER")) objdsc['last_interaction'] = datetime.datetime.now() if 'mark_for_deletion' in objdsc: # as this object has been marked for deletion, we have to run the garbage collection # to remove this mark now self.__gc() setattr(objdsc['object']['object'], name, value) if skip_db_update is False: with make_session() as session: obj = session.query(OpenObject).filter( OpenObject.ref == ref).one() obj.last_interaction = objdsc['last_interaction'] if obj.data is None: obj.data = {name: value} else: obj.data[name] = value session.commit()
def getPPDURL(self, source_url): """ Downloads the source_url, stores it locally and returns the local URL :param source_url: remote PPD URL :return: local URL to the cached PPD """ source = parse_url(source_url) host = source.host if host is None or host == "localhost": # no host: we assume that the PPD can be found on the current active master backend with make_session() as session: # get any other registered backend master_backend = session.query(RegisteredBackend) \ .filter(RegisteredBackend.uuid != self.env.core_uuid, RegisteredBackend.type == BackendTypes.active_master).first() if master_backend is None: self.log.error(C.make_error("NO_MASTER_BACKEND_FOUND")) return source_url # Try to log in with provided credentials url = parse_url(master_backend.url) host = url.host # check if file exists locally rel_path = source.path[1:] if source.path.startswith( "/") else source.path local_path = path.join(self.ppd_dir, host, rel_path) if not path.exists(local_path): # cache locally r = requests.get(source_url) if r.ok: local_dir = path.dirname(local_path) if not path.exists(local_dir): makedirs(local_dir) with open(local_path, "w") as f: f.write(r.text) else: self.log.error( "requesting PPD from %s failed with status code: %s" % (source_url, r.status_code)) return source_url return "%s%s/%s" % (self.base_url, host, rel_path)
def setObjectProperty(self, user, ref, name, value, skip_db_update=False): """ Set a property on an existing stack object. ================= ========================== Parameter Description ================= ========================== ref UUID / object reference name Property name value Property value ================= ========================== """ objdsc = self.__get_ref(ref) if not objdsc: raise ValueError(C.make_error("REFERENCE_NOT_FOUND", ref=ref)) if not name in objdsc['object']['properties']: raise ValueError(C.make_error("PROPERTY_NOT_FOUND", property=name)) details = objdsc['object']['object'].get_attributes(True) if 'auto' in details[name] and details[name]['auto']: raise ValueError(C.make_error("AUTOGENERATED_ATTRIBUTE")) if not self.__check_user(ref, user): raise ValueError(C.make_error("NOT_OBJECT_OWNER")) objdsc['last_interaction'] = datetime.datetime.now() if 'mark_for_deletion' in objdsc: # as this object has been marked for deletion, we have to run the garbage collection # to remove this mark now self.__gc() setattr(objdsc['object']['object'], name, value) if skip_db_update is False: with make_session() as session: obj = session.query(OpenObject).filter(OpenObject.ref == ref).one() obj.last_interaction = objdsc['last_interaction'] if obj.data is None: obj.data = {name: value} else: obj.data[name] = value session.commit()
def getForemanDiscoveredHosts(self, user): methods = PluginRegistry.getInstance("RPCMethods") query = and_( ObjectInfoIndex.uuid == ExtensionIndex.uuid, ObjectInfoIndex.uuid == KeyValueIndex.uuid, ObjectInfoIndex._type == "Device", ExtensionIndex.extension == "ForemanHost", KeyValueIndex.key == "status", KeyValueIndex.value == "discovered" ) with make_session() as session: query_result = session.query(ObjectInfoIndex).filter(query) res = {} for item in query_result: methods.update_res(res, item, user, 1) return list(res.values())
def closeObject(self, user, ref): """ Close an object by its reference. This will free the object on the agent side. ================= ========================== Parameter Description ================= ========================== ref UUID / object reference ================= ========================== """ if not self.__get_ref(ref): raise ValueError(C.make_error("REFERENCE_NOT_FOUND", ref=ref)) if not self.__check_user(ref, user): raise ValueError(C.make_error("NOT_OBJECT_OWNER")) with make_session() as session: session.query(OpenObject).filter(OpenObject.ref == ref).delete() del self.__stack[ref]
def getContainerTree(self, user, base, object_type=None): types = [] table = inspect(ObjectInfoIndex) o2 = aliased(ObjectInfoIndex) for container in self.containers: types.append(getattr(ObjectInfoIndex, "_type") == container) query = and_( getattr(ObjectInfoIndex, "_adjusted_parent_dn") == base, or_(*types)) with make_session() as session: query_result = session.query(ObjectInfoIndex, func.count(getattr(o2, "_adjusted_parent_dn"))) \ .outerjoin(o2, and_(getattr(o2, "_invisible").is_(False), getattr(o2, "_adjusted_parent_dn") == getattr(ObjectInfoIndex, "dn"))) \ .filter(query) \ .group_by(*table.c) res = {} factory = ObjectFactory.getInstance() for item, children in query_result: self.update_res(res, item, user, 1) if object_type is not None and item.dn in res: res[item.dn]['hasChildren'] = children > 0 # check if object_type is allowed in this container allowed = factory.getAllowedSubElementsForObject( res[item.dn]['tag'], includeInvisible=False) if "*" in object_type: # all allowed res[item.dn]['allowed_move_target'] = True elif isinstance(object_type, list): res[item.dn]['allowed_move_target'] = len( set(object_type).intersection(allowed)) > 0 else: res[item. dn]['allowed_move_target'] = object_type in allowed return res
async def test_provision_host(self, m_get, m_del, m_put, m_post): """ convert a discovered host to a 'real' host """ self._test_dn = GosaTestCase.create_test_data() container = ObjectProxy(self._test_dn, "IncomingDeviceContainer") container.commit() mocked_foreman = MockForeman() m_get.side_effect = mocked_foreman.get m_del.side_effect = mocked_foreman.delete m_put.side_effect = mocked_foreman.put m_post.side_effect = mocked_foreman.post # create the discovered host + foremanHostgroup d_host = ObjectProxy(container.dn, "Device") d_host.cn = "mac00262df16a2c" d_host.extend("ForemanHost") d_host.status = "discovered" d_host.extend("ieee802Device") d_host.macAddress = "00:26:2d:f1:6a:2c" d_host.extend("IpHost") d_host.ipHostNumber = "192.168.0.1" d_host.commit() hostgroup = ObjectProxy("%s" % self._test_dn, "GroupOfNames") hostgroup.extend("ForemanHostGroup") hostgroup.cn = "Test" hostgroup.foremanGroupId = "4" hostgroup.commit() # add host to group logging.getLogger("test.foreman-integration").info("########### START: Add Host to group ############# %s" % AsyncHTTPTestCase.get_url(self, "/hooks/")) d_host = ObjectProxy("cn=mac00262df16a2c,%s" % container.dn) def check(): logging.getLogger("test.foreman-integration").info("check condition: %s, %s" % (d_host.cn, d_host.status)) return d_host.cn == "mac00262df16a2c" and d_host.status == "discovered" def check2(): logging.getLogger("test.foreman-integration").info("check2 condition: %s" % d_host.cn) return d_host.cn == "Testhost" base_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data") with open(os.path.join(base_dir, "discovered_hosts", "mac00262df16a2c.json")) as f: mocked_foreman.register_conditional_response("http://localhost:8000/api/v2/discovered_hosts/mac00262df16a2c", "get", check, f.read()) with open(os.path.join(base_dir, "conditional", "Testhost.json")) as f: mocked_foreman.register_conditional_response("http://localhost:8000/api/v2/hosts/Testhost", "get", check2, f.read()) def activate(**kwargs): return True mocked_foreman.register_trigger("http://localhost:8000/api/v2/discovered_hosts/mac00262df16a2c", "put", activate, self.execute) with make_session() as session: assert session.query(ObjectInfoIndex.dn)\ .join(ObjectInfoIndex.properties)\ .filter(and_(KeyValueIndex.key == "cn", KeyValueIndex.value == "Testhost"))\ .count() == 0 d_host.cn = "Testhost" d_host.groupMembership = hostgroup.dn d_host.commit() logging.getLogger("test.foreman-integration").info("waiting for 2 seconds") await asyncio.sleep(2) logging.getLogger("test.foreman-integration").info("########### END: Add Host to group #############") # now move the host to the final destination d_host = ObjectProxy("cn=Testhost,ou=incoming,%s" % self._test_dn) assert d_host.status != "discovered" assert d_host.name == "Testhost" assert d_host.hostgroup_id == "4" assert d_host.is_extended_by("RegisteredDevice") is True assert len(d_host.userPassword[0]) > 0 assert d_host.deviceUUID is not None with make_session() as session: assert session.query(ObjectInfoIndex.dn) \ .join(ObjectInfoIndex.properties) \ .filter(and_(KeyValueIndex.key == "cn", KeyValueIndex.value == "Testhost")) \ .count() == 1 logging.getLogger("test.foreman-integration").info("########### START: moving host #############") d_host.move("%s" % self._test_dn) logging.getLogger("test.foreman-integration").info("########### END: moving host #############") # lets check if everything is fine in the database d_host = ObjectProxy("cn=Testhost,ou=devices,%s" % self._test_dn, read_only=True) assert d_host is not None assert d_host.status == "unknown" assert d_host.groupMembership == hostgroup.dn
def search(self, user, base, scope, qstring, fltr=None): """ Performs a query based on a simple search string consisting of keywords. Query the database using the given query string and an optional filter dict - and return the result set. ========== ================== Parameter Description ========== ================== base Query base scope Query scope (SUB, BASE, ONE, CHILDREN) qstring Query string fltr Hash for extra parameters ========== ================== ``Return``: List of dicts """ res = {} keywords = None dn_hook = "_parent_dn" fallback = fltr and "fallback" in fltr and fltr["fallback"] if not base: return [] adjusted_base = base # Set defaults if not fltr: fltr = {} if not 'category' in fltr: fltr['category'] = "all" if not 'secondary' in fltr: fltr['secondary'] = "enabled" if not 'mod-time' in fltr: fltr['mod-time'] = "all" if 'adjusted-dn' in fltr and fltr['adjusted-dn'] is True: dn_hook = "_adjusted_parent_dn" adjusted_base = ObjectProxy.get_adjusted_dn(base, self.env.base) actions = 'actions' in fltr and fltr['actions'] is True # Sanity checks scope = scope.upper() if not scope in ["SUB", "BASE", "ONE", "CHILDREN"]: raise GOsaException(C.make_error("INVALID_SEARCH_SCOPE", scope=scope)) if not fltr['mod-time'] in ["hour", "day", "week", "month", "year", "all"]: raise GOsaException(C.make_error("INVALID_SEARCH_DATE", date=fltr['mod-time'])) # Build query: join attributes and keywords queries = [] # Build query: assemble query = None if scope == "SUB": if queries: query = and_(or_(ObjectInfoIndex._parent_dn == base, ObjectInfoIndex._parent_dn.like("%," + base)), or_(*queries)) else: query = or_(ObjectInfoIndex._parent_dn == base, ObjectInfoIndex._parent_dn.like("%," + base)) elif scope == "ONE": query = and_(or_(ObjectInfoIndex.dn == adjusted_base, getattr(ObjectInfoIndex, dn_hook) == adjusted_base), or_(*queries)) elif scope == "CHILDREN": query = and_(getattr(ObjectInfoIndex, dn_hook) == adjusted_base, or_(*queries)) else: if queries: query = and_(ObjectInfoIndex.dn == base, or_(*queries)) else: query = ObjectInfoIndex.dn == base # Build query: eventually extend with timing information td = None if fltr['mod-time'] != "all": now = datetime.datetime.now() if fltr['mod-time'] == 'hour': td = now - datetime.timedelta(hours=1) elif fltr['mod-time'] == 'day': td = now - datetime.timedelta(days=1) elif fltr['mod-time'] == 'week': td = now - datetime.timedelta(weeks=1) elif fltr['mod-time'] == 'month': td = now - datetime.timedelta(days=31) elif fltr['mod-time'] == 'year': td = now - datetime.timedelta(days=365) query = and_(ObjectInfoIndex._last_modified >= td, query) order_by = None if 'order-by' in fltr: is_desc = 'order' in fltr and fltr['order'] == 'desc' order_by = "_last_changed" if fltr['order-by'] == "last-changed": order_by = "_last_modified" order_by = desc(getattr(ObjectInfoIndex, order_by)) if is_desc else getattr(ObjectInfoIndex, order_by) # Perform primary query and get collect the results squery = [] these = dict([(x, 1) for x in self.__search_aid['used_attrs']]) these.update(dict(dn=1, _type=1, _uuid=1, _last_changed=1)) these = list(these.keys()) with make_session() as session: query_result, ranked = self.finalize_query(query, fltr, session, qstring=qstring, order_by=order_by) # try: # self.log.debug(str(query_result.statement.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}))) # except Exception as e: # self.log.warning(str(e)) # self.log.debug(str(query_result)) # pass # limit only secondary enabled searches, because e.g. the treeitems use this search to resolve and we do not want to limit those results if fltr['secondary'] == "enabled": max_results = self.env.config.get("backend.max-results", default=1000) else: max_results = math.inf counter = 0 total = query_result.count() response = {} if total == 0 and fallback is True and PluginRegistry.getInstance("ObjectIndex").fuzzy is True: # do fuzzy search if qstring: try: keywords = [s.strip("'").strip('"') for s in shlex.split(qstring)] except ValueError: keywords = [s.strip("'").strip('"') for s in qstring.split(" ")] # Make keywords unique keywords = list(set(keywords)) # find most similar words for i, kw in enumerate(keywords): r = session.execute("SELECT word, similarity(word, '{0}') as sim FROM unique_lexeme WHERE length(word) > 2 AND similarity(word, '{0}') > {1} ORDER BY similarity(word, '{0}') DESC LIMIT 3;".format(kw, self.__fuzzy_similarity_threshold)).fetchall() if len(r) > 0 and self.__fuzzy_similarity_threshold < 0.5 <= r[0]['sim']: # we have good results to take only those and ignore the lower similarity threshold r = [x for x in r if x['sim'] >= 0.5] keywords[i] = " or ".join([x['word'] for x in r]) keywords_string = " ".join(keywords) if keywords_string != "": self.log.info("no results found for: '%s' => re-trying with: '%s'" % (qstring, keywords_string)) response['orig'] = qstring if qstring != keywords_string: response['fuzzy'] = keywords_string query_result, ranked = self.finalize_query(query, fltr, session, qstring=" ".join(keywords), order_by=order_by) total = query_result.count() response['primary_total'] = total self.log.debug("Query: %s Keywords: %s, Filter: %s => %s results" % (qstring, keywords, fltr, total)) squery_constraints = {} primary_uuids = [] for tuple in query_result.all(): if ranked is True: item = tuple[0] rank = tuple[1] # make sure that the primary rank is higher that the secondaries rank += 1 else: item = tuple rank = 1 self.update_res(res, item, user, rank, these=these, actions=actions) counter += 1 if counter >= max_results: break primary_uuids.append(item.uuid) # Collect information for secondary search? if fltr['secondary'] != "enabled": continue if item._type in self.__search_aid['resolve']: if len(self.__search_aid['resolve'][item._type]) == 0: continue kv = self.__index_props_to_key_value(item.properties) for r in self.__search_aid['resolve'][item._type]: if r['attribute'] in kv: tag = r['type'] if r['type'] else item._type # If a category was choosen and it does not fit the # desired target tag - skip that one if not (fltr['category'] == "all" or fltr['category'] == tag): continue if hasattr(ObjectInfoIndex, r['filter']): if tag == "*": squery.append(getattr(ObjectInfoIndex, r['filter']).in_(kv[r['attribute']])) else: squery.append(and_(ObjectInfoIndex._type == tag, getattr(ObjectInfoIndex, r['filter']).in_(kv[r['attribute']]))) else: if tag not in squery_constraints: squery_constraints[tag] = {} if r['filter'] not in squery_constraints[tag]: squery_constraints[tag][r['filter']] = [] squery_constraints[tag][r['filter']].extend(kv[r['attribute']]) for type, constraints in squery_constraints.items(): for key, values in constraints.items(): values = list(set(values)) if len(values) > 0: if type == "*": squery.append(KeyValueIndex.key == key, KeyValueIndex.value.in_(values)) else: squery.append(and_(ObjectInfoIndex._type == type, KeyValueIndex.key == key, KeyValueIndex.value.in_(values))) # Perform secondary query and update the result if fltr['secondary'] == "enabled" and squery: query = and_(or_(*squery), ~ObjectInfoIndex.uuid.in_(primary_uuids)) # Add "_last_changed" information to query if fltr['mod-time'] != "all": query = and_(query, ObjectInfoIndex._last_modified >= td) # Execute query and update results sec_result = session.query(ObjectInfoIndex).join(ObjectInfoIndex.properties).options(contains_eager(ObjectInfoIndex.properties)).filter(query) # try: # self.log.debug("Secondary query: %s " % str(sec_result.statement.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}))) # except Exception as e: # self.log.warning(str(e)) # self.log.debug("Secondary query: %s " % str(sec_result)) # pass results = sec_result.all() total += len(results) if counter < max_results: for item in results: self.update_res(res, item, user, self.__make_relevance(item, keywords, fltr, True), secondary=True, these=these, actions=actions) counter += 1 if counter >= max_results: break response['total'] = total response['results'] = list(res.values()) return response
def test_ImageProcessor(self): # read example image with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "test.jpg"), "r+b") as f: byte = f.read() user = mock.MagicMock() with make_session() as session: res = session.query(ObjectInfoIndex.uuid).filter(ObjectInfoIndex.dn == "cn=Frank Reich,ou=people,dc=example,dc=net").one() user.uuid = res[0] user.modifyTimestamp = datetime.datetime.now() test_dict = { "image": { "value": [Binary(byte)] } } image_dir = os.path.join(Environment.getInstance().config.get("user.image-path", "/tmp/images"), user.uuid) tmp_image = mock.MagicMock() with mock.patch("gosa.backend.plugins.user.filters.Base.metadata.create_all") as m_create_all, \ mock.patch("gosa.backend.plugins.user.filters.os.path.exists", return_value=True), \ mock.patch("gosa.backend.plugins.user.filters.os.path.isdir", return_value=True), \ mock.patch("gosa.backend.plugins.user.filters.ImageOps.fit", return_value=tmp_image): filter = ImageProcessor(None) with mock.patch("gosa.backend.plugins.user.filters.make_session") as m: mocked_db_query = m.return_value.__enter__.return_value.query.return_value.filter.return_value.one_or_none mocked_db_query.side_effect = OperationalError(None, None, None) filter.process(user, "image", test_dict, "32", "64") assert m_create_all.called m_create_all.reset_mock() mocked_db_query.side_effect = [None, OperationalError(None, None, None)] filter.process(user, "image", test_dict, "32") assert m_create_all.called assert tmp_image.save.called filter = ImageProcessor(None) with pytest.raises(ElementFilterException): filter.process(None, None, None) with pytest.raises(ElementFilterException), \ mock.patch("gosa.backend.plugins.user.filters.os.path.exists", return_value=True),\ mock.patch("gosa.backend.plugins.user.filters.os.path.isdir", return_value=False), \ mock.patch("gosa.backend.plugins.user.filters.make_session"): filter.process(user, "image", test_dict, "32", "64") with mock.patch("gosa.backend.plugins.user.filters.make_session") as m: m_session = m.return_value.__enter__.return_value m_session.query.return_value.filter.return_value.one_or_none.return_value = None filter.process(user, "image", test_dict, "32", "64") assert m_session.add.called assert m_session.commit.called assert os.path.exists(os.path.join(image_dir, "image", "0", "32.jpg")) assert os.path.exists(os.path.join(image_dir, "image", "0", "64.jpg")) shutil.rmtree(image_dir) found = mock.MagicMock() found.filter.return_value.one_or_none.return_value.modified = user.modifyTimestamp with mock.patch("gosa.backend.plugins.user.filters.make_session") as m: m_session = m.return_value.__enter__.return_value m_session.query.return_value = found filter.process(user, "image", test_dict, "32", "64") assert not m_session.add.called assert not m_session.commit.called assert not os.path.exists(os.path.join(image_dir, "image", "0", "32.jpg")) assert not os.path.exists(os.path.join(image_dir, "image", "0", "64.jpg")) filter.process(user, "image", {'image': {'value': [Binary(b"wrong binary data")]}}, "32", "64") with mock.patch("gosa.backend.plugins.user.filters.make_session") as m: m_session = m.return_value.__enter__.return_value m_session.query.return_value.filter.return_value.one_or_none.return_value = None filter.process(user, "image", {'image': {'value': [Binary(b"wrong binary data")]}}, "32", "64") assert m_session.add.called assert m_session.commit.called assert not os.path.exists(os.path.join(image_dir, "image", "0", "32.jpg")) assert not os.path.exists(os.path.join(image_dir, "image", "0", "64.jpg"))
def check_session(cls, sid, user): with make_session() as session: return session.query(UserSession).filter(UserSession.sid == sid, UserSession.user == user).count() > 0
def __save_user_session(self, user_session): if self.env.mode == "proxy": sessions[user_session.sid] = user_session else: with make_session() as session: session.add(user_session)
def __gc(self): self.env.log.debug("running garbage collector on object store") ten_minutes_ago = datetime.datetime.now() - datetime.timedelta(minutes=self.__inactivity_timeout) e = EventMaker() command = PluginRegistry.getInstance("CommandRegistry") sched = PluginRegistry.getInstance("SchedulerService").getScheduler() for ref, item in list(self.__stack.items()): uuid = item['object']['uuid'] if uuid is None: # new items without uuid do not need to be closed by timeout continue last_interaction_time = item['last_interaction'] if 'last_interaction' in item else item['created'] if last_interaction_time < ten_minutes_ago: if 'mark_for_deletion' in item: if item['mark_for_deletion'] <= datetime.datetime.now(): if 'countdown_job' in item: try: sched.unschedule_job(item['countdown_job']) except KeyError: pass finally: del item['countdown_job'] del self.__stack[ref] with make_session() as session: session.query(OpenObject).filter(OpenObject.ref == ref).delete() event = e.Event( e.ObjectCloseAnnouncement( e.Target(item['user']), e.SessionId(item['session_id']), e.State("closed"), e.UUID(uuid) ) ) command.sendEvent(item['user'], event) else: # notify user to do something otherwise the lock gets removed in 1 minute event = e.Event( e.ObjectCloseAnnouncement( e.Target(item['user']), e.SessionId(item['session_id']), e.State("closing"), e.UUID(uuid), e.Minutes("1") ) ) command.sendEvent(item['user'], event) item['mark_for_deletion'] = datetime.datetime.now() + datetime.timedelta(seconds=59) if 'countdown_job' in item: try: sched.unschedule_job(item['countdown_job']) except KeyError: pass finally: del item['countdown_job'] item['countdown_job'] = sched.add_date_job(self.__gc, datetime.datetime.now() + datetime.timedelta(minutes=1), tag="_internal", jobstore="ram") elif 'mark_for_deletion' in item: # item has been modified -> remove the deletion mark del item['mark_for_deletion'] event = e.Event( e.ObjectCloseAnnouncement( e.Target(item['user']), e.SessionId(item['session_id']), e.State("closing_aborted"), e.UUID(uuid) ) ) command.sendEvent(item['user'], event) if 'countdown_job' in item: try: sched.unschedule_job(item['countdown_job']) except KeyError: pass finally: del item['countdown_job']
def handle_request(self, request_handler): foreman = PluginRegistry.getInstance("Foreman") data = loads(request_handler.request.body) self.log.debug(data) # TODO disable hook logging to file with open("foreman-log.json", "a") as f: f.write("%s,\n" % dumps(data, indent=4, sort_keys=True)) if data["event"] in ForemanHookReceiver.skip_next_event and data["object"] in ForemanHookReceiver.skip_next_event[data["event"]]: ForemanHookReceiver.skip_next_event[data["event"]].remove(data["object"]) self.log.info("skipped '%s' event for object: '%s'" % (data["event"], data["object"])) return data_keys = list(data['data'].keys()) if len(data_keys) == 1: type = data_keys[0] else: # no type given -> skipping this event as other might come with more information self.log.warning("skipping event '%s' for object '%s' as no type information is given in data: '%s'" % (data["event"], data["object"], data["data"])) return # search for real data if len(data['data'][type].keys()) == 1: # something like {data: 'host': {host: {...}}} # or {data: 'discovered_host': {host: {...}}} payload_data = data['data'][type][list(data['data'][type].keys())[0]] else: payload_data = data['data'][type] if type == "operatingsystem": with make_session() as session: foreman.sync_release_name(payload_data, session, event=data['event']) session.commit() return factory = ObjectFactory.getInstance() foreman_type = type if type == "discovered_host": type = "host" object_types = factory.getObjectNamesWithBackendSetting("Foreman", "type", "%ss" % type) object_type = object_types[0] if len(object_types) else None backend_attributes = factory.getObjectBackendProperties(object_type) if object_type is not None else None self.log.debug("Hookevent: '%s' for '%s' (%s)" % (data['event'], data['object'], object_type)) uuid_attribute = None if "Foreman" in backend_attributes: uuid_attribute = backend_attributes["Foreman"]["_uuidSourceAttribute"] \ if '_uuidSourceAttribute' in backend_attributes["Foreman"] else backend_attributes["Foreman"]["_uuidAttribute"] ForemanBackend.modifier = "foreman" update_data = {} if data['event'] in ["update", "create"] and foreman_type == "host": id = payload_data["id"] if "id" in payload_data else None try: foreman.write_parameters(id if id is not None else data['object']) except: foreman.mark_for_parameter_setting(data['object'], { "status": "created", "use_id": id }) if data['event'] == "after_commit" or data['event'] == "update" or data['event'] == "after_create" or data['event'] == "create": host = None if data['event'] == "update" and foreman_type == "host" and "mac" in payload_data and payload_data["mac"] is not None: # check if we have an discovered host for this mac index = PluginRegistry.getInstance("ObjectIndex") res = index.search({ "_type": "Device", "extension": ["ForemanHost", "ieee802Device"], "macAddress": payload_data["mac"], "status": "discovered" }, {"dn": 1}) if len(res): self.log.debug("update received for existing host with dn: %s" % res[0]["dn"]) host = ObjectProxy(res[0]["dn"]) if foreman_type != "discovered_host" and host.is_extended_by("ForemanHost"): host.status = "unknown" foreman_object = foreman.get_object(object_type, payload_data[uuid_attribute], create=host is None) if foreman_object and host: if foreman_object != host: self.log.debug("using known host instead of creating a new one") # host is the formerly discovered host, which might have been changed in GOsa for provisioning # so we want to use this one, foreman_object is the joined one, so copy the credentials from foreman_object to host if not host.is_extended_by("RegisteredDevice"): host.extend("RegisteredDevice") if not host.is_extended_by("simpleSecurityObject"): host.extend("simpleSecurityObject") host.deviceUUID = foreman_object.deviceUUID host.userPassword = foreman_object.userPassword host.otp = foreman_object.otp host.cn = foreman_object.cn # now delete the formerly joined host foreman_object.remove() foreman_object = host elif foreman_object is None and host is not None: foreman_object = host elif foreman_type == "discovered_host": self.log.debug("setting discovered state for %s" % payload_data[uuid_attribute]) if not foreman_object.is_extended_by("ForemanHost"): foreman_object.extend("ForemanHost") foreman_object.status = "discovered" if foreman_type == "host": old_build_state = foreman_object.build foreman.update_type(object_type, foreman_object, payload_data, uuid_attribute, update_data=update_data) if foreman_type == "host" and old_build_state is True and foreman_object.build is False and \ foreman_object.status == "ready": # send notification e = EventMaker() ev = e.Event(e.Notification( e.Title(N_("Host ready")), e.Body(N_("Host '%s' has been successfully build." % foreman_object.cn)), e.Icon("@Ligature/pc"), e.Timeout("10000") )) event_object = objectify.fromstring(etree.tostring(ev, pretty_print=True).decode('utf-8')) SseHandler.notify(event_object) elif data['event'] == "after_destroy": # print("Payload: %s" % payload_data) foreman.remove_type(object_type, payload_data[uuid_attribute]) # because foreman sends the after_commit event after the after_destroy event # we need to skip this event, otherwise the host would be re-created if "after_commit" not in ForemanHookReceiver.skip_next_event: ForemanHookReceiver.skip_next_event["after_commit"] = [data['object']] else: ForemanHookReceiver.skip_next_event["after_commit"].append(data['object']) # add garbage collection for skip sobj = PluginRegistry.getInstance("SchedulerService") sobj.getScheduler().add_date_job(self.cleanup_event_skipper, datetime.datetime.now() + datetime.timedelta(minutes=1), args=("after_commit", data['object']), tag='_internal', jobstore='ram') else: self.log.info("unhandled hook event '%s' received for '%s'" % (data['event'], type)) ForemanBackend.modifier = None
def search(self, user, base, scope, qstring, fltr=None): """ Performs a query based on a simple search string consisting of keywords. Query the database using the given query string and an optional filter dict - and return the result set. ========== ================== Parameter Description ========== ================== base Query base scope Query scope (SUB, BASE, ONE, CHILDREN) qstring Query string fltr Hash for extra parameters ========== ================== ``Return``: List of dicts """ res = {} keywords = None dn_hook = "_parent_dn" fallback = fltr and "fallback" in fltr and fltr["fallback"] if not base: return [] adjusted_base = base # Set defaults if not fltr: fltr = {} if not 'category' in fltr: fltr['category'] = "all" if not 'secondary' in fltr: fltr['secondary'] = "enabled" if not 'mod-time' in fltr: fltr['mod-time'] = "all" if 'adjusted-dn' in fltr and fltr['adjusted-dn'] is True: dn_hook = "_adjusted_parent_dn" adjusted_base = ObjectProxy.get_adjusted_dn(base, self.env.base) actions = 'actions' in fltr and fltr['actions'] is True # Sanity checks scope = scope.upper() if not scope in ["SUB", "BASE", "ONE", "CHILDREN"]: raise GOsaException( C.make_error("INVALID_SEARCH_SCOPE", scope=scope)) if not fltr['mod-time'] in [ "hour", "day", "week", "month", "year", "all" ]: raise GOsaException( C.make_error("INVALID_SEARCH_DATE", date=fltr['mod-time'])) # Build query: join attributes and keywords queries = [] # Build query: assemble query = None if scope == "SUB": if queries: query = and_( or_(ObjectInfoIndex._parent_dn == base, ObjectInfoIndex._parent_dn.like("%," + base)), or_(*queries)) else: query = or_(ObjectInfoIndex._parent_dn == base, ObjectInfoIndex._parent_dn.like("%," + base)) elif scope == "ONE": query = and_( or_(ObjectInfoIndex.dn == adjusted_base, getattr(ObjectInfoIndex, dn_hook) == adjusted_base), or_(*queries)) elif scope == "CHILDREN": query = and_( getattr(ObjectInfoIndex, dn_hook) == adjusted_base, or_(*queries)) else: if queries: query = and_(ObjectInfoIndex.dn == base, or_(*queries)) else: query = ObjectInfoIndex.dn == base # Build query: eventually extend with timing information td = None if fltr['mod-time'] != "all": now = datetime.datetime.now() if fltr['mod-time'] == 'hour': td = now - datetime.timedelta(hours=1) elif fltr['mod-time'] == 'day': td = now - datetime.timedelta(days=1) elif fltr['mod-time'] == 'week': td = now - datetime.timedelta(weeks=1) elif fltr['mod-time'] == 'month': td = now - datetime.timedelta(days=31) elif fltr['mod-time'] == 'year': td = now - datetime.timedelta(days=365) query = and_(ObjectInfoIndex._last_modified >= td, query) order_by = None if 'order-by' in fltr: is_desc = 'order' in fltr and fltr['order'] == 'desc' order_by = "_last_changed" if fltr['order-by'] == "last-changed": order_by = "_last_modified" order_by = desc(getattr(ObjectInfoIndex, order_by)) if is_desc else getattr( ObjectInfoIndex, order_by) # Perform primary query and get collect the results squery = [] these = dict([(x, 1) for x in self.__search_aid['used_attrs']]) these.update(dict(dn=1, _type=1, _uuid=1, _last_changed=1)) these = list(these.keys()) ranked = False with make_session() as session: query_result, ranked = self.finalize_query(query, fltr, session, qstring=qstring, order_by=order_by) # try: # self.log.debug(str(query_result.statement.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}))) # except Exception as e: # self.log.warning(str(e)) # self.log.debug(str(query_result)) # pass # limit only secondary enabled searches, because e.g. the treeitems use this search to resolve and we do not want to limit those results if fltr['secondary'] == "enabled": max_results = self.env.config.get("backend.max-results", default=1000) else: max_results = math.inf counter = 0 total = query_result.count() response = {} if total == 0 and fallback is True and PluginRegistry.getInstance( "ObjectIndex").fuzzy is True: # do fuzzy search if qstring: try: keywords = [ s.strip("'").strip('"') for s in shlex.split(qstring) ] except ValueError: keywords = [ s.strip("'").strip('"') for s in qstring.split(" ") ] # Make keywords unique keywords = list(set(keywords)) # find most similar words for i, kw in enumerate(keywords): r = session.execute( "SELECT word FROM unique_lexeme WHERE length(word) > 2 AND similarity(word, '{0}') > {1} ORDER BY levenshtein(word, '{0}', 2, 2, 1) LIMIT 3;" .format( kw, self.__fuzzy_similarity_threshold)).fetchall() keywords[i] = " or ".join([x['word'] for x in r]) self.log.info( "no results found for: '%s' => re-trying with: '%s'" % (qstring, " ".join(keywords))) response['orig'] = qstring response['fuzzy'] = " ".join(keywords) query_result, ranked = self.finalize_query( query, fltr, session, qstring=" ".join(keywords), order_by=order_by) total = query_result.count() response['primary_total'] = total self.log.debug("Query: %s Keywords: %s, Filter: %s => %s results" % (qstring, keywords, fltr, total)) squery_constraints = {} primary_uuids = [] for tuple in query_result: if ranked is True: item = tuple[0] rank = tuple[1] else: item = tuple rank = 0 self.update_res(res, item, user, rank, these=these, actions=actions) counter += 1 if counter >= max_results: break primary_uuids.append(item.uuid) # Collect information for secondary search? if fltr['secondary'] != "enabled": continue if item._type in self.__search_aid['resolve']: if len(self.__search_aid['resolve'][item._type]) == 0: continue kv = self.__index_props_to_key_value(item.properties) for r in self.__search_aid['resolve'][item._type]: if r['attribute'] in kv: tag = r['type'] if r['type'] else item._type # If a category was choosen and it does not fit the # desired target tag - skip that one if not (fltr['category'] == "all" or fltr['category'] == tag): continue if hasattr(ObjectInfoIndex, r['filter']): if tag == "*": squery.append( getattr(ObjectInfoIndex, r['filter']).in_( kv[r['attribute']])) else: squery.append( and_( ObjectInfoIndex._type == tag, getattr(ObjectInfoIndex, r['filter']).in_( kv[r['attribute']]))) else: if tag not in squery_constraints: squery_constraints[tag] = {} if r['filter'] not in squery_constraints[tag]: squery_constraints[tag][r['filter']] = [] squery_constraints[tag][r['filter']].extend( kv[r['attribute']]) for type, constraints in squery_constraints.items(): for key, values in constraints.items(): values = list(set(values)) if len(values) > 0: if type == "*": squery.append(KeyValueIndex.key == key, KeyValueIndex.value.in_(values)) else: squery.append( and_(ObjectInfoIndex._type == type, KeyValueIndex.key == key, KeyValueIndex.value.in_(values))) # Perform secondary query and update the result if fltr['secondary'] == "enabled" and squery: query = and_(or_(*squery), ~ObjectInfoIndex.uuid.in_(primary_uuids)) # Add "_last_changed" information to query if fltr['mod-time'] != "all": query = and_(query, ObjectInfoIndex._last_modified >= td) # Execute query and update results sec_result = session.query(ObjectInfoIndex).join( ObjectInfoIndex.properties).options( contains_eager( ObjectInfoIndex.properties)).filter(query) # try: # self.log.debug("Secondary query: %s " % str(sec_result.statement.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}))) # except Exception as e: # self.log.warning(str(e)) # self.log.debug("Secondary query: %s " % str(sec_result)) # pass results = sec_result.all() total += len(results) if counter < max_results: for item in results: self.update_res(res, item, user, self.__make_relevance( item, keywords, fltr, True), secondary=True, these=these, actions=actions) counter += 1 if counter >= max_results: break response['total'] = total response['results'] = list(res.values()) return response
def user_sessions_available(cls, user): with make_session() as session: if user is not None: return session.query(UserSession).filter(UserSession.user == user).count() > 0 else: return session.query(UserSession).count() > 0
def process(self, obj, key, valDict, *sizes): # Sanity check if len(sizes) == 0: raise ElementFilterException(C.make_error("USER_IMAGE_SIZE_MISSING")) with make_session() as session: # Do we have an attribute to process? if key in valDict: if valDict[key]['value']: # Check if a cache entry exists... try: entry = session.query(ImageIndex).filter(and_(ImageIndex.uuid == obj.uuid, ImageIndex.attribute == key)).one_or_none() except OperationalError: session.rollback() Base.metadata.create_all(Environment.getInstance().getDatabaseEngine("backend-database"), tables=tables) entry = None if entry: # Nothing to do if it's unmodified if obj.modifyTimestamp == entry.modified: return key, valDict # Create new cache entry else: entry = ImageIndex(uuid=obj.uuid, attribute=key) session.add(entry) # Convert all images to all requested sizes entry.modified = obj.modifyTimestamp for idx in range(0, len(valDict[key]['value'])): image = BytesIO(valDict[key]['value'][idx].get()) try: im = Image.open(image) #@UndefinedVariable except IOError: continue # Check for target directory wd = os.path.join(self.__path, obj.uuid, key, str(idx)) if os.path.exists(wd) and not os.path.isdir(wd): raise ElementFilterException(C.make_error("USER_IMAGE_CACHE_BROKEN")) if not os.path.exists(wd): os.makedirs(wd) for size in sizes: wds = os.path.join(wd, size + ".jpg") s = int(size) tmp = ImageOps.fit(im, (s, s), Image.ANTIALIAS) #@UndefinedVariable tmp.save(wds, "JPEG") # Save size reference if not there yet try: se = session.query(ImageSize.size).filter(and_(ImageSize.uuid == obj.uuid, ImageSize.size == s)).one_or_none() except OperationalError: Base.metadata.create_all(Environment.getInstance().getDatabaseEngine("backend-database"), tables=tables) se = None if not se: se = ImageSize(uuid=obj.uuid, size=s, path=wds) session.add(se) # Flush session.commit() elif 'last_value' in valDict[key] and valDict[key]['last_value']: # Delete from db index try: entry = session.query(ImageIndex).filter(and_(ImageIndex.uuid == obj.uuid, ImageIndex.attribute == key)).one_or_none() if entry is not None: session.delete(entry) except OperationalError: pass # delete from file system for idx in range(0, len(valDict[key]['last_value'])): # Check for target directory wd = os.path.join(self.__path, obj.uuid, key, str(idx)) if os.path.exists(wd) and os.path.isdir(wd): # delete shutil.rmtree(wd) # Flush session.commit() return key, valDict
def getForemanLsbNames(self): res = {} with make_session() as session: for r in session.query(Cache.data).filter(Cache.key.ilike("foreman.operating_system.%")).all(): res[r[0]["release_name"]] = {"value": r[0]["release_name"]} return res
def check_trigger(): with make_session() as session: return session.execute( "select * from pg_trigger WHERE tgname LIKE 'so_index%'" ).rowcount == 0
def _execute(self, transforms, *args, **kwargs): with make_session(skip_context_check=True) as session: with StackContext(SessionContext(session)): super(HSTSRequestHandler, self)._execute(transforms, *args, **kwargs)
def openObject(self, user, session_id, oid, *args, **kwargs): """ Open object on the agent side. This creates an instance on the stack and returns an a JSON description of the object and it's values. ================= ========================== Parameter Description ================= ========================== oid OID of the object to create args/kwargs Arguments to be used when getting an object instance ================= ========================== ``Return``: JSON encoded object description """ # In case of "object" we want to check the lock if oid == 'object': lck = self.__get_lock(args[0]) if lck and (not session_id or lck['user'] != user or lck['session_id'] != session_id): raise Exception(C.make_error("OBJECT_LOCKED", object=args[0], user=lck['user'], when=lck['created'].strftime("%Y-%m-%d (%H:%M:%S)"))) # Use oid to find the object type obj_type = self.__get_object_type(oid) methods, properties = self.__inspect(obj_type) # Load instance, fill with dummy stuff db_object = None if "db_object" in kwargs: db_object = kwargs["db_object"] ref = db_object.ref del kwargs["db_object"] else: ref = str(uuid.uuid1()) # Make object instance and store it kwargs['user'] = user kwargs['session_id'] = session_id obj = obj_type(*args, **kwargs) # Merge in methods that may be available later due to extending more addons methods += obj.get_all_method_names() # Add dynamic information - if available if hasattr(obj, 'get_attributes'): properties = properties + obj.get_attributes() if hasattr(obj, 'get_methods'): methods = methods + obj.get_methods() objdsc = { 'oid': oid, 'dn': obj.dn if hasattr(obj, 'dn') else None, 'uuid': obj.uuid if hasattr(obj, 'uuid') else None, 'object': obj, 'methods': list(set(methods)), 'properties': properties } self.__stack[ref] = { 'user': user, 'session_id': session_id, 'object': objdsc, 'created': datetime.datetime.now() if db_object is None else db_object.created } # Build property dict propvals = {} if properties: propvals = dict([(p, getattr(obj, p)) for p in properties]) propvals['uuid'] = obj.uuid # Build result result = {"__jsonclass__": ["json.JSONObjectFactory", [obj_type.__name__, ref, obj.dn, oid, methods, properties]]} result.update(propvals) if db_object is None: # store in DB with make_session() as session: oo = OpenObject( ref=ref, uuid=obj.uuid, oid=oid, user=user, session_id=session_id, backend_uuid=self.env.core_uuid, created=self.__stack[ref]["created"], last_interaction=self.__stack[ref]["created"] ) session.add(oo) session.commit() elif db_object.data is not None: # apply changes to opened object for prop, value in db_object.data.items(): self.setObjectProperty(user, ref, prop, value, skip_db_update=True) return result
def openObject(self, user, session_id, oid, *args, **kwargs): """ Open object on the agent side. This creates an instance on the stack and returns an a JSON description of the object and it's values. ================= ========================== Parameter Description ================= ========================== oid OID of the object to create args/kwargs Arguments to be used when getting an object instance ================= ========================== ``Return``: JSON encoded object description """ # In case of "object" we want to check the lock if oid == 'object': lck = self.__get_lock(args[0]) if lck and (not session_id or lck['user'] != user or lck['session_id'] != session_id): raise Exception( C.make_error( "OBJECT_LOCKED", object=args[0], user=lck['user'], when=lck['created'].strftime("%Y-%m-%d (%H:%M:%S)"))) # Use oid to find the object type obj_type = self.__get_object_type(oid) methods, properties = self.__inspect(obj_type) # Load instance, fill with dummy stuff db_object = None if "db_object" in kwargs: db_object = kwargs["db_object"] ref = db_object.ref del kwargs["db_object"] else: ref = str(uuid.uuid1()) # Make object instance and store it kwargs['user'] = user kwargs['session_id'] = session_id obj = obj_type(*args, **kwargs) # Merge in methods that may be available later due to extending more addons methods += obj.get_all_method_names() # Add dynamic information - if available if hasattr(obj, 'get_attributes'): properties = properties + obj.get_attributes() if hasattr(obj, 'get_methods'): methods = methods + obj.get_methods() objdsc = { 'oid': oid, 'dn': obj.dn if hasattr(obj, 'dn') else None, 'uuid': obj.uuid if hasattr(obj, 'uuid') else None, 'object': obj, 'methods': list(set(methods)), 'properties': properties } self.__stack[ref] = { 'user': user, 'session_id': session_id, 'object': objdsc, 'created': datetime.datetime.now() if db_object is None else db_object.created } # Build property dict propvals = {} if properties: propvals = dict([(p, getattr(obj, p)) for p in properties]) propvals['uuid'] = obj.uuid # Build result result = { "__jsonclass__": [ "json.JSONObjectFactory", [obj_type.__name__, ref, obj.dn, oid, methods, properties] ] } result.update(propvals) if db_object is None: # store in DB with make_session() as session: oo = OpenObject(ref=ref, uuid=obj.uuid, oid=oid, user=user, session_id=session_id, backend_uuid=self.env.core_uuid, created=self.__stack[ref]["created"], last_interaction=self.__stack[ref]["created"]) session.add(oo) session.commit() elif db_object.data is not None: # apply changes to opened object for prop, value in db_object.data.items(): self.setObjectProperty(user, ref, prop, value, skip_db_update=True) return result
def process(self, obj, key, valDict, *sizes): # Sanity check if len(sizes) == 0: raise ElementFilterException( C.make_error("USER_IMAGE_SIZE_MISSING")) with make_session() as session: # Do we have an attribute to process? if key in valDict: if valDict[key]['value']: # Check if a cache entry exists... try: entry = session.query(ImageIndex).filter( and_(ImageIndex.uuid == obj.uuid, ImageIndex.attribute == key)).one_or_none() except OperationalError: session.rollback() Base.metadata.create_all( Environment.getInstance().getDatabaseEngine( "backend-database")) entry = None if entry: # Nothing to do if it's unmodified if obj.modifyTimestamp == entry.modified: return key, valDict # Create new cache entry else: entry = ImageIndex(uuid=obj.uuid, attribute=key) session.add(entry) # Convert all images to all requested sizes entry.modified = obj.modifyTimestamp for idx in range(0, len(valDict[key]['value'])): image = BytesIO(valDict[key]['value'][idx].get()) try: im = Image.open(image) #@UndefinedVariable except IOError: continue # Check for target directory wd = os.path.join(self.__path, obj.uuid, key, str(idx)) if os.path.exists(wd) and not os.path.isdir(wd): raise ElementFilterException( C.make_error("USER_IMAGE_CACHE_BROKEN")) if not os.path.exists(wd): os.makedirs(wd) for size in sizes: wds = os.path.join(wd, size + ".jpg") s = int(size) tmp = ImageOps.fit( im, (s, s), Image.ANTIALIAS) #@UndefinedVariable tmp.save(wds, "JPEG") # Save size reference if not there yet try: se = session.query(ImageSize.size).filter( and_(ImageSize.uuid == obj.uuid, ImageSize.size == s)).one_or_none() except OperationalError: Base.metadata.create_all( Environment.getInstance( ).getDatabaseEngine("backend-database")) se = None if not se: se = ImageSize(uuid=obj.uuid, size=s, path=wds) session.add(se) # Flush session.commit() elif 'last_value' in valDict[key] and valDict[key][ 'last_value']: # Delete from db index try: entry = session.query(ImageIndex).filter( and_(ImageIndex.uuid == obj.uuid, ImageIndex.attribute == key)).one_or_none() if entry is not None: session.delete(entry) except OperationalError: pass # delete from file system for idx in range(0, len(valDict[key]['last_value'])): # Check for target directory wd = os.path.join(self.__path, obj.uuid, key, str(idx)) if os.path.exists(wd) and os.path.isdir(wd): # delete shutil.rmtree(wd) # Flush session.commit() return key, valDict
def __gc(self): self.env.log.debug("running garbage collector on object store") ten_minutes_ago = datetime.datetime.now() - datetime.timedelta( minutes=self.__inactivity_timeout) e = EventMaker() command = PluginRegistry.getInstance("CommandRegistry") sched = PluginRegistry.getInstance("SchedulerService").getScheduler() for ref, item in list(self.__stack.items()): uuid = item['object']['uuid'] if uuid is None: # new items without uuid do not need to be closed by timeout continue last_interaction_time = item[ 'last_interaction'] if 'last_interaction' in item else item[ 'created'] if last_interaction_time < ten_minutes_ago: if 'mark_for_deletion' in item: if item['mark_for_deletion'] <= datetime.datetime.now(): if 'countdown_job' in item: try: sched.unschedule_job(item['countdown_job']) except KeyError: pass finally: del item['countdown_job'] del self.__stack[ref] with make_session() as session: session.query(OpenObject).filter( OpenObject.ref == ref).delete() event = e.Event( e.ObjectCloseAnnouncement( e.Target(item['user']), e.SessionId(item['session_id']), e.State("closed"), e.UUID(uuid))) command.sendEvent(item['user'], event) else: # notify user to do something otherwise the lock gets removed in 1 minute event = e.Event( e.ObjectCloseAnnouncement( e.Target(item['user']), e.SessionId(item['session_id']), e.State("closing"), e.UUID(uuid), e.Minutes("1"))) command.sendEvent(item['user'], event) item['mark_for_deletion'] = datetime.datetime.now( ) + datetime.timedelta(seconds=59) if 'countdown_job' in item: try: sched.unschedule_job(item['countdown_job']) except KeyError: pass finally: del item['countdown_job'] item['countdown_job'] = sched.add_date_job( self.__gc, datetime.datetime.now() + datetime.timedelta(minutes=1), tag="_internal", jobstore="ram") elif 'mark_for_deletion' in item: # item has been modified -> remove the deletion mark del item['mark_for_deletion'] event = e.Event( e.ObjectCloseAnnouncement(e.Target(item['user']), e.SessionId(item['session_id']), e.State("closing_aborted"), e.UUID(uuid))) command.sendEvent(item['user'], event) if 'countdown_job' in item: try: sched.unschedule_job(item['countdown_job']) except KeyError: pass finally: del item['countdown_job']
def check_trigger(): with make_session() as session: return session.execute("select * from pg_trigger WHERE tgname LIKE 'so_index%'").rowcount == 0
def setUp(self): self.back = ObjectHandler() with make_session() as session: res = session.query(ObjectInfoIndex.uuid).filter(ObjectInfoIndex.dn == "cn=Frank Reich,ou=people,dc=example,dc=net").one() self.user_uuid = res[0]
def process(self, data): """ Process an incoming JSONRPC request and dispatch it thru the *CommandRegistry*. ================= ========================== Parameter Description ================= ========================== data Incoming body data ================= ========================== ``Return``: varries """ try: json = loads(data) except ValueError as e: raise ValueError(C.make_error("INVALID_JSON", data=str(e))) try: method = json['method'] params = json['params'] jid = json['id'] except KeyError as e: raise ValueError(C.make_error("JSON_MISSING_PARAMETER")) if method.startswith('_'): raise tornado.web.HTTPError(403, "Bad method name %s: must not start with _" % method) if not isinstance(params, list) and not isinstance(params, dict): raise ValueError(C.make_error("PARAMETER_LIST_OR_DICT")) # execute command if it is allowed without login if method in no_login_commands: return self.dispatch(method, params, jid) # Check if we're globally locked currently if GlobalLock.exists("scan_index"): raise FilterException(C.make_error('INDEXING', "base")) cls = self.__class__ twofa_manager = PluginRegistry.getInstance("TwoFactorAuthManager") # Create an authentication cookie on login if method == 'login': (user, password) = params # Check password and create session id on success sid = str(uuid.uuid1()) result = { 'state': AUTH_FAILED } lock_result = self.get_lock_result(user) if lock_result is not None: return dict(result=lock_result, error=None, id=jid) dn = self.authenticate(user, password) if dn is not False: # user and password matches so delete the user from observer list if user in cls.__dos_manager: del cls.__dos_manager[user] with make_session() as session: us = UserSession( sid=sid, user=user, dn=dn ) self.set_secure_cookie('REMOTE_USER', user) self.set_secure_cookie('REMOTE_SESSION', sid) factor_method = twofa_manager.get_method_from_user(dn) if factor_method is None: result['state'] = AUTH_SUCCESS self.log.info("login succeeded for user '%s'" % user) elif factor_method == "otp": result['state'] = AUTH_OTP_REQUIRED self.log.info("login succeeded for user '%s', proceeding with OTP two-factor authentication" % user) elif factor_method == "u2f": self.log.info("login succeeded for user '%s', proceeding with U2F two-factor authentication" % user) result['state'] = AUTH_U2F_REQUIRED result['u2f_data'] = twofa_manager.sign(user, dn) us.auth_state = result['state'] session.add(us) else: # Remove current sid if present with make_session() as session: db_session = session.query(UserSession).filter(UserSession.sid == sid).one_or_none() if not self.get_secure_cookie('REMOTE_SESSION') and db_session is not None: session.delete(db_session) self.log.error("login failed for user '%s'" % user) result['state'] = AUTH_FAILED # log login tries if user in cls.__dos_manager: login_stats = cls.__dos_manager[user] # stop counting after 6 tries to avoid "infinity" lock, the user is locked for more than an hour if login_stats['count'] < 6: login_stats['count'] += 1 login_stats['timestamp'] = time.time() cls.__dos_manager[user] = login_stats else: cls.__dos_manager[user] = { 'count': 1, 'timestamp': time.time(), 'ip': self.request.remote_ip } lock_result = self.get_lock_result(user) if lock_result is not None: return dict(result=lock_result, error=None, id=jid) return dict(result=result, error=None, id=jid) # Don't let calls pass beyond this point if we've no valid session ID with make_session() as session: cookie = self.get_secure_cookie('REMOTE_SESSION') if cookie is None: self.log.error("blocked unauthenticated call of method '%s'" % method) raise tornado.web.HTTPError(401, "Please use the login method to authorize yourself.") db_session = session.query(UserSession).filter(UserSession.sid == cookie.decode('ascii')).one_or_none() if db_session is None: self.log.error("blocked unauthenticated call of method '%s'" % method) raise tornado.web.HTTPError(401, "Please use the login method to authorize yourself.") # Remove remote session on logout if method == 'logout': # Remove current sid if present if self.get_secure_cookie('REMOTE_SESSION'): if db_session is not None: session.delete(db_session) # Show logout message if self.get_secure_cookie('REMOTE_USER'): self.log.info("logout for user '%s' succeeded" % self.get_secure_cookie('REMOTE_USER')) self.clear_cookie("REMOTE_USER") self.clear_cookie("REMOTE_SESSION") return dict(result=True, error=None, id=jid) # check two-factor authentication if method == 'verify': (key,) = params if db_session.auth_state == AUTH_OTP_REQUIRED or db_session.auth_state == AUTH_U2F_REQUIRED: if twofa_manager.verify(db_session.user, db_session.dn, key): db_session.auth_state = AUTH_SUCCESS return dict(result={'state': AUTH_SUCCESS}, error=None, id=jid) else: return dict(result={'state': AUTH_FAILED}, error=None, id=jid) if db_session.auth_state != AUTH_SUCCESS: raise tornado.web.HTTPError(401, "Please use the login method to authorize yourself.") return self.dispatch(method, params, jid)