def safe_fetch(item): '''Fetch model or return None if unauthorized''' try: resp = yield item.fetch() defer.returnValue(resp) except Unauthorized: return
def _render_array(self, model, limit, context): tree = list() flattened = list() columns = list() if not context.models or model != context.models[-1]: # this fixes issue with the fact that write() method is passed # the context of the current model instead of the parrent context = context.descend(model) yield self._build_tree(tree, model, limit, context) self._flatten_tree(flattened, columns, dict(), tree[0], limit) headers = [html.tags.th()(x) for x, _ in columns] table = html.tags.table()( html.tags.thead()(*headers)) tbody = html.tags.tbody() table.content.append(tbody) for row in flattened: tr = html.tags.tr() for column in columns: td = html.tags.td() value = row.get(column) if value: item, cur_context = value td.append(self._format_attribute_item(item, cur_context)) tr.append(td) tbody.append(tr) defer.returnValue(table)
def save_document(self, doc): assert IDocument.providedBy(doc) or isinstance(doc, dict), repr(doc) try: self._lock_notifications() serialized = self._serializer.convert(doc) if IDocument.providedBy(doc): following_attachments = dict( (name, attachment) for name, attachment in doc.get_attachments().iteritems() if not attachment.saved) doc_id = doc.doc_id else: following_attachments = dict() doc_id = doc.get('_id') resp = yield self._database.save_doc(serialized, doc_id, following_attachments) self._update_id_and_rev(resp, doc) for attachment in following_attachments.itervalues(): attachment.set_saved() # now process all the documents which have been registered to # be saved together with this document if IDocument.providedBy(doc): while doc.links.to_save: to_link, linker_roles, linkee_roles = ( doc.links.to_save.pop(0)) to_link.links.create(doc=doc, linker_roles=linker_roles, linkee_roles=linkee_roles) yield self.save_document(to_link) defer.returnValue(doc) finally: self._unlock_notifications()
def configure_replicator_database(host, port, username=None, password=None): """ Connects to dabatase, checks the version and creates the design document used by feat (if it doesn't exist). @returns: IDatabaseConnection bound to _replicator database """ database = driver.Database(host, port, '_replicator', username, password) connection = database.get_connection() version = yield database.get_version() if version < (1, 1, 0): database.disconnect() raise ValueError("Found couchdb version %r. " "_replicator database has been introduced in 1.1.0." % (version, )) design_docs = view.DesignDocument.generate_from_views([Replications]) for doc in design_docs: try: doc2 = yield connection.get_document(doc.doc_id) if doc.views != doc2.views or doc.filters != doc2.filters: doc.rev = doc2.rev yield connection.save_document(doc) except NotFoundError: yield connection.save_document(doc) defer.returnValue(connection)
def select_ids(connection, query, skip=0, limit=None, include_responses=False): temp, responses = yield _get_query_response(connection, query) total_count = len(temp) if limit is not None: stop = skip + limit else: stop = None name, direction = query.sorting index = first(v.entries for k, v in responses.iteritems() if k.field == name) if direction == Direction.DESC: index = reversed(index) if query.aggregate: # we have to copy the collection, because _get_sorted_slice() # treats it as a buffer, and modifies the content aggregate_index = set(temp) r = Result(_get_sorted_slice(index, temp, skip, stop)) r.total_count = total_count # count reductions for aggregated fields based on the view index if query.aggregate: r.aggregations = list() for handler, field in query.aggregate: value_index = first(v for k, v in responses.iteritems() if k.field == field) r.aggregations.append(handler(x for x in value_iterator(aggregate_index, value_index))) if include_responses: defer.returnValue((r, responses)) else: defer.returnValue(r)
def _get_agents_at(self, host): agents = [host] hosted_recp = yield host.get_hosted_recipients() for recp in hosted_recp: medium = yield self.driver.find_agent(recp) agents += [medium.get_agent()] defer.returnValue(agents)
def values(connection, query, field, unique=True): if field not in query.fields: raise ValueError("%r doesn't have %s field defined" % (type(query), field)) query.include_value.append(field) query.reset() # ensures the field condition gets included temp, responses = yield _get_query_response(connection, query) index = first(v for k, v in responses.iteritems() if k.field == field) if not index.includes_values: raise ValueError( "The query controller of %s field " "of %s query is not marked to " "keep the value in the cache. You have to enable " "it to make query.value() work." % (field, query.name) ) if unique: resp = set() for x in temp: resp.add(index.get_value(x)) defer.returnValue(list(resp)) else: resp = list() for x in temp: resp.append(index.get_value(x)) defer.returnValue(resp)
def testMisconfiguredPostgresFallbackToSqlite(self): try: import txpostgres except ImportError: raise SkipTest('txpostgres package is missing') postgres = ('postgres://%s:%s@%s/%s' % ('user', 'password', 'localhost', 'name')) tmpfile = self._get_tmp_file() sqlite = 'sqlite://' + tmpfile connstrs = [postgres, sqlite] agency_stub = AgencyStub() jour = journaler.Journaler( on_switch_writer_cb=agency_stub.on_switch_writer) jour.set_connection_strings(connstrs) jour.insert_entry(**self._generate_entry()) @defer.inlineCallbacks def check(): w = jour._writer self.log('writer is %r', w) if isinstance(w, journaler.SqliteWriter): try: num = yield self._get_number_of_entries(jour, 1) self.log('num is %d', num) defer.returnValue(num == 1) except FailTest, e: self.log('assertation failure: %r', e) defer.returnValue(False) defer.returnValue(True) defer.returnValue(False)
def _format_attribute_item(self, item, context): model = yield safe_fetch(item) if not IModel.providedBy(model): defer.returnValue("") result = yield self._format_attribute(model, context.descend(model), context, html_links(item)) defer.returnValue(result)
def assertState(self, _, state): self.assertEqual(state, self.manager._get_medium().state) if state not in (contracts.ContractState.completed, contracts.ContractState.terminated, ): self.assertFailure(self.finished, protocols.ProtocolFailed) yield self.finished defer.returnValue(self.manager)
def _get_query_response(connection, query): cache = connection.get_query_cache() responses = dict() for subquery in query.get_basic_queries(): # subquery -> list of doc ids responses[subquery] = yield cache.query( connection, query.factory, subquery) defer.returnValue((_calculate_query_response(responses, query), responses))
def model_descend(self, model, *path): i = model for part in path: i = yield i.fetch_item(part) if i is None: return i = yield i.fetch() defer.returnValue(i)
def get_replication_status(rconnection, source): database = rconnection.database if not isinstance(database, driver.Database): raise TypeError("This procedure would work only for driver connected" " to the real database. It uses public methods which" " are not the part of IDatabaseDriver interface") version = yield rconnection.database.get_version() if version < (1, 2, 0): raise ValueError("CouchDB 1.2.0 required, found %r" % (version, )) active_tasks = yield database.couchdb_call( database.couchdb.get, '/_active_tasks') # In couchdb version >= 1.2.2 the replication_id is suffixed with # string literal '+continuous'. Here we cut it off for task in active_tasks: if (task.get('type') == 'replication' and task.get('replication_id', '').endswith('+continuous')): task['replication_id'] = task['replication_id'].replace( '+continuous', '') active_tasks = dict((x['replication_id'], x) for x in active_tasks if (x['type'] == 'replication' and 'replication_id' in x)) replications = yield rconnection.query_view(Replications, key=('source', source), include_docs=True) # target -> [(checkpointed_source_seq, continuous, status, replication_id)] result = dict() for replication in replications: target = replication['target'] result.setdefault(target, list()) r_id = replication['_replication_id'] r_state = replication.get('_replication_state') r_continuous = replication.get('continuous', False) if r_state == 'completed': seq = replication['_replication_stats']['checkpointed_source_seq'] result[target].append((seq, False, 'completed', r_id)) elif r_state == 'triggered' and r_continuous: task = active_tasks.get(r_id) if not task: result[target].append((0, True, 'task_missing', r_id)) else: seq = task['checkpointed_source_seq'] result[target].append((seq, True, 'running', r_id)) else: result[target].append((0, r_continuous, r_state, r_id)) # Sort the results so that the first row for the target # is the row with the highest update_seq, aka the most # recent one. Secondary sorting is done to promote continuous # replication over the 1-time replication. for rows in result.itervalues(): rows.sort(key=lambda x: (x[0], x[1]), reverse=True) defer.returnValue(result)
def run_and_configure_db(self): yield self.db_process.restart() c = self.db_process.get_config() db_host, db_port, db_name = c['host'], c['port'], 'test' db = database.Database(db_host, db_port, db_name) self.db = db.get_connection() yield dbtools.create_db(self.db) yield dbtools.push_initial_data(self.db) defer.returnValue((db_host, db_port, db_name, ))
def get_hosted_recipients(self, state): result = list() partners = self.query_partners('all') for partner in partners: agent_id = partner.recipient.key hosted = yield state.medium.check_if_hosted(agent_id) if hosted: result.append(partner.recipient) defer.returnValue(result)
def list_slaves(self): '''Print information about the slave agencies.''' resp = [] for slave_id, slave in self._broker.slaves.iteritems(): resp += ["#### Slave %s ####" % slave_id] table = yield slave.callRemote('list_agents') resp += [table] resp += [] defer.returnValue("\n".join(resp))
def query_partners(self, agent): ''' Generator returning the ShardAgent instances of partners being neighbours of the given ShardAgent. ''' result = [] for p in agent.query_partners('neighbours'): ag = yield self.driver.find_agent(p.recipient.key) result.append(ag.get_agent()) defer.returnValue(result)
def _check_conflict(connection, doc_id): try: raw_doc = yield connection.get_document(doc_id, raw=True, conflicts=True) except NotFoundError: in_conflict = False raw_doc = {'_id': doc_id, '_deleted': True} else: in_conflict = '_conflicts' in raw_doc defer.returnValue((in_conflict, raw_doc))
def select(connection, query, skip=0, limit=None, include_responses=False): res, responses = yield select_ids(connection, query, skip, limit, include_responses=True) temp = yield connection.bulk_get(res) res.update(temp) if query.include_value: yield include_values(res, responses, query) if include_responses: defer.returnValue((res, responses)) else: defer.returnValue(res)
def start_shard(self): a_id = str(uuid.uuid1()) script = format_block(""" agency = spawn_agency(start_host=False) desc = descriptor_factory('shard_agent', shard='%(shard)s') agency.start_agent(desc, run_startup=False) agent = _.get_agent() agent.look_for_neighbours() """) % dict(shard=a_id) yield self.process(script) defer.returnValue(self.get_local('agent'))
def select_ids(connection, query, skip=0, limit=None): temp, responses = yield _get_query_response(connection, query) if query.sorting: temp = sorted(temp, key=_generate_sort_key(responses, query.sorting)) else: temp = list(temp) if limit is not None: stop = skip + limit else: stop = None defer.returnValue(temp[slice(skip, stop)])
def run_and_configure_db(self): yield self.db_process.restart() c = self.db_process.get_config() db_host, db_port, db_name = c['host'], c['port'], 'test' db = database.Database(db_host, db_port, db_name) self.db = db.get_connection() yield tools.create_db(self.db) # disable nagios integration for the purpose of this test from feat.agents.alert import alert_agent alert_config = alert_agent.AlertAgentConfiguration(enabled=False) feat.initial_data(alert_config) yield tools.push_initial_data(self.db) defer.returnValue((db_host, db_port, db_name, ))
def start_host(self, join_shard=True): script = format_block(""" desc = descriptor_factory('host_agent') agency = spawn_agency(start_host=False) medium = agency.start_agent(desc, run_startup=False) agent = medium.get_agent() """) yield self.process(script) agent = self.get_local('agent') if join_shard: yield agent.start_join_shard_manager() yield self.wait_for_idle(20) defer.returnValue(agent)
def wait_for_alert(self, timeout=10): def check(): try: alert = self.browser.switch_to_alert() alert.text return True except exceptions.NoAlertPresentException: return False yield self.wait_for(check, timeout) defer.returnValue(self.browser.switch_to_alert())
def find_agent(self, agent_id): """ Return the medium class of the agent with agent_id if the one is running in simulation. """ try: recp = IRecipient(agent_id) agent_id = recp.key except TypeError: pass agency = self.find_agency(agent_id) if agency: agent = yield agency.find_agent(agent_id) defer.returnValue(agent)
def save_document(self, doc): doc = IDocument(doc) serialized = self._serializer.convert(doc) resp = yield self._database.save_doc(serialized, doc.doc_id) self._update_id_and_rev(resp, doc) for name, attachment in doc.get_attachments().iteritems(): if not attachment.saved: resp = yield self._database.save_attachment( doc.doc_id, doc.rev, attachment) self._update_id_and_rev(resp, doc) attachment.set_saved() defer.returnValue(doc)
def locate_agent(self, recp): '''locate_agent(recp): Return (host, port, should_redirect) tuple. ''' if recipient.IRecipient.providedBy(recp): agent_id = recp.key else: agent_id = recp found = yield self.find_agent(agent_id) if isinstance(found, agency.AgencyAgent): host = self.get_hostname() port = self.gateway_port defer.returnValue((host, port, False, )) elif isinstance(found, broker.AgentReference): host = self.get_hostname() port = yield found.reference.callRemote('get_gateway_port') defer.returnValue((host, port, True, )) else: # None # lazy import not to load descriptor before feat is loaded from feat.utils import locate db = self._database.get_connection() host = yield locate.locate(db, agent_id) port = self.config.gateway.port if host is None: defer.returnValue(None) else: defer.returnValue((host, port, True, ))
def locate_agent(self, recp): """locate_agent(recp): Return (host, port, should_redirect) tuple. """ if recipient.IRecipient.providedBy(recp): agent_id = recp.key else: agent_id = recp found = yield self.find_agent(agent_id) if isinstance(found, agency.AgencyAgent): host = self.get_hostname() port = self.gateway_port defer.returnValue((host, port, False)) elif isinstance(found, broker.AgentReference): host = self.get_hostname() port = yield found.reference.callRemote("get_gateway_port") defer.returnValue((host, port, True)) else: # None # lazy import not to load descriptor before feat is loaded from feat.utils import locate db = self._database.get_connection() host = yield locate.locate(db, agent_id) port = self.config.gateway.port if host is None or (self._broker.is_master() and host == self.get_hostname()): # Second condition reflects the situation when the agent # has its descriptor in the database but is not running. # It breaks the infinite redirect loop. defer.returnValue(None) else: defer.returnValue((host, port, True))
def _get_number_of_entries(self, jour, num): if isinstance(jour, journaler.Journaler): writer = jour._writer else: writer = jour histories = yield writer.get_histories() self.assertIsInstance(histories, list) self.assertTrue(len(histories) > 0) if num > 0: self.assertIsInstance(histories[0], journaler.History) entries = yield writer.get_entries(histories[0]) self.assertIsInstance(entries, list) defer.returnValue(len(entries)) else: defer.returnValue(0)
def locate_agent(self, agent_id): db = self.db() agency = self.source medium = agency.get_agent(agent_id) if medium is not None: defer.returnValue(medium.get_agent()) host = yield locate.locate(db, agent_id) if host is None: return port = self.source.config['gateway']['port'] res = reference.Absolute((host, port), "apps", "dns", "servers", agent_id) defer.returnValue(res)