def check_it(result):
# First determine the contact ID.
key = ['rd.core.content', 'schema_id', 'rd.contact']
result = yield get_doc_model().open_view(key=key, reduce=False,
include_docs=True)
rows = result['rows']
# Should be exactly 1 record with a 'contact' schema.
self.failUnlessEqual(len(rows), 1, str(rows))
key_type, cid = rows[0]['doc']['rd_key']
self.failUnlessEqual(key_type, 'contact')
# should be exact 3 rd.identity.contacts records, each pointing
# at my contact.
key = ['rd.core.content', 'schema_id', 'rd.identity.contacts']
result = yield get_doc_model().open_view(key=key,
reduce=False,
include_docs=True)
rows = result['rows']
self.failUnlessEqual(len(rows), 3, str(rows))
docs = [r['doc'] for r in rows]
for doc in docs:
contacts = doc['contacts']
self.failUnlessEqual(len(contacts), 1, contacts)
this_id, this_rel = contacts[0]
self.failUnlessEqual(this_id, cid)
self.failUnless(this_rel in ['personal', 'public'], this_rel)
def verifyCounts(self, contact_count, identity_count):
# First determine the contact ID.
key = ['schema_id', 'rd.contact']
result = get_doc_model().open_view(key=key, reduce=False)
self.failUnlessEqual(len(result['rows']), contact_count, repr(result))
# each identity should have got 2 schema instances.
keys = [['schema_id', 'rd.identity.exists'],
['schema_id', 'rd.identity.contacts'],
]
result = get_doc_model().open_view(keys=keys, reduce=False)
self.failUnlessEqual(len(result['rows']), identity_count*2, repr(result))
def _prepare_test_doc(self):
doc_model = get_doc_model()
# abuse the schema API to write the outgoing smtp data and the
# 'state' doc in one hit.
body = 'subject: hello\r\n\r\nthe body'
items = {'smtp_from' : '*****@*****.**',
'smtp_to': ['*****@*****.**', '*****@*****.**'],
# The 'state' bit...
'sent_state': None,
'outgoing_state': 'outgoing',
}
sis = [
{ 'rd_key': ['test', 'smtp_test'],
'rd_ext_id': 'testsuite',
'rd_schema_id': 'rd.some_src_schema',
'items': {'outgoing_state': 'outgoing'},
},
{'rd_key': ['test', 'smtp_test'],
'rd_ext_id': 'testsuite',
'rd_schema_id': 'rd.msg.outgoing.smtp',
'items': items,
'attachments': {'smtp_body': {'data': body}},
},
]
doc_model.create_schema_items(sis)
dids = [doc_model.get_doc_id_for_schema_item(si) for si in sis]
return doc_model.open_documents_by_id(dids)
def setup_pipeline(whateva):
global g_pipeline, g_conductor
if g_pipeline is None:
g_pipeline = pipeline.Pipeline(model.get_doc_model(), options)
_ = yield g_pipeline.initialize()
if g_conductor is None:
g_conductor = yield get_conductor(g_pipeline)
def delete_docs(result, parser, options):
"""Delete all documents of a particular type. Use with caution or see
the 'unprocess' command for an alternative.
"""
# NOTE: This is for development only, until we get a way to say
# 'reprocess stuff you've already done' - in the meantime deleting those
# intermediate docs has the same result...
def _del_docs(to_del):
docs = []
for id, rev in to_del:
docs.append({'_id': id, '_rev': rev})
return model.get_doc_model().delete_documents(docs)
def _got_docs(result, dt):
to_del = [(row['id'], row['value']['_rev']) for row in result['rows']]
logger.info("Deleting %d documents of type %r", len(to_del), dt)
return to_del
if not options.schemas:
parser.error("You must specify one or more --schema")
deferreds = []
for st in options.schemas:
key = ['rd.core.content', 'schema_id', st]
d = model.get_doc_model().open_view(key=key, reduce=False
).addCallback(_got_docs, st
).addCallback(_del_docs
)
deferreds.append(d)
return defer.DeferredList(deferreds)
def _prepare_test_doc(self):
doc_model = get_doc_model()
# write a simple outgoing schema
items = {'body' : 'hello there',
'from' : ['email', '*****@*****.**'],
'from_display': 'Sender Name',
'to' : [
['email', '*****@*****.**'],
['email', '*****@*****.**'],
],
'to_display': ['recip 1', 'recip 2'],
'cc' : [
['email', '*****@*****.**'],
],
'cc_display' : ['CC recip 1'],
'subject': "the subject",
# The 'state' bit...
'sent_state': None,
'outgoing_state': 'outgoing',
}
result = yield doc_model.create_schema_items([
{'rd_key': ['test', 'smtp_test'],
'rd_ext_id': 'testsuite',
'rd_schema_id': 'rd.msg.outgoing.simple',
'items': items,
}])
src_doc = yield doc_model.db.openDoc(result[0]['id'])
defer.returnValue(src_doc)
def show_info(parser, options):
"""Print a list of all extensions, loggers etc"""
dm = model.get_doc_model()
print "Database:"
info = dm.db.infoDB()
fmt = " %(doc_count)d docs total, %(doc_del_count)d deleted, " \
"update seq at %(update_seq)d, %(disk_size)d bytes."
print fmt % info
# ouch - this seems a painful way of fetching total unique keys?
results = dm.open_view(
startkey=["key"],
endkey=["key", {}],
group_level=2)
print " %d unique raindrop keys" % len(results['rows'])
print "API groupings:"
from urllib import urlencode
dbconfig = get_config().couches['local']
try:
summaries = _call_api(dbconfig, "_api/inflow/grouping/summary")
print " %d groupings exist" % len(summaries)
for gs in summaries:
title = gs.get('title') or gs['rd_key']
opts = {'limit': 60, 'message_limit': 2,
'keys': json.dumps([gs['rd_key']]),
}
path = "_api/inflow/conversations/in_groups?" + urlencode(opts)
this = _call_api(dbconfig, path)
print " %s: %d conversations" % (title, len(this))
except dm.db.Error, exc:
print "Failed to call the API:", exc
def _call_api(dbconfig, path):
db = model.get_doc_model().db
host = dbconfig['host']
port = dbconfig['port']
dbname = dbconfig['name']
uri = "http://%s:%s/%s/%s" % (host, port, dbname, path)
return db._request("GET", uri)
def _init(req):
global pipeline, conductor
assert pipeline is None # don't call me twice!
opts = options_from_request(req)
pipeline = Pipeline(model.get_doc_model(), opts)
_ = yield pipeline.initialize()
conductor = yield raindrop.sync.get_conductor(pipeline)
def report():
dm = model.get_doc_model()
info = yield dm.db.infoDB()
db_seq = info["update_seq"]
proc_seq = g_pipeline.incoming_processor.current_seq
left = db_seq - proc_seq
print "still waiting for new raindrops (%d items to be processed) ..." % (left)
def show_info(result, parser, options):
"""Print a list of all extensions, loggers etc"""
dm = model.get_doc_model()
print "Database:"
info = yield dm.db.infoDB()
fmt = " %(doc_count)d docs total, %(doc_del_count)d deleted, " "update seq at %(update_seq)d, %(disk_size)d bytes."
print fmt % info
# ouch - this seems a painful way of fetching total unique keys?
results = yield dm.open_view(
startkey=["rd.core.content", "key"], endkey=["rd.core.content", "key", {}], group_level=3
)
print " %d unique raindrop keys" % len(results["rows"])
print "Document counts by schema:"
results = yield dm.open_view(
startkey=["rd.core.content", "schema_id"], endkey=["rd.core.content", "schema_id", {}], group_level=3
)
infos = []
for row in results["rows"]:
sch_id = row["key"][-1]
infos.append((sch_id, row["value"]))
for sch_id, count in sorted(infos):
print " %s: %d" % (sch_id, count)
print
print "Raindrop extensions:"
exts = sorted((yield pipeline.load_extensions(dm)).items()) # sort by ID.
for _, ext in exts:
print " %s: %s" % (ext.id, ext.doc["info"])
print
print "Loggers"
# yuck - reach into impl - and hope all have been initialized by now
# (they should have been as we loaded the extensions above)
for name in logging.Logger.manager.loggerDict:
print " ", name
def test_one_testmsg(self):
# When processing a single test message we end up with 2 identies
# both associated with the same contact
result = self.process_doc()
dm = get_doc_model()
# First determine the contact ID.
key = ['schema_id', 'rd.contact']
result = dm.open_view(key=key, reduce=False, include_docs=True)
rows = result['rows']
# Should be exactly 1 record with a 'contact' schema.
self.failUnlessEqual(len(rows), 1, str(rows))
key_type, cid = rows[0]['doc']['rd_key']
self.failUnlessEqual(key_type, 'contact')
# should be exact 2 rd.identity.contacts records, each pointing
# at my contact.
key = ['schema_id', 'rd.identity.contacts']
result = dm.open_view(key=key, reduce=False, include_docs=True)
rows = result['rows']
self.failUnlessEqual(len(rows), 2, str(rows))
docs = [r['doc'] for r in rows]
for doc in docs:
contacts = doc['contacts']
self.failUnlessEqual(len(contacts), 1, contacts)
this_id, this_rel = contacts[0]
self.failUnlessEqual(this_id, cid)
self.failUnless(this_rel in ['personal', 'public'], this_rel)
def go(_):
try:
_ = yield init_config()
dm = get_doc_model()
_ = yield make_graph(dm)
finally:
print "stopping"
reactor.stop()
def _get_post_client(self, src_doc, raw_doc):
acct = SMTPAccount(get_doc_model(), {})
# we need a factory for error handling...
factory = SMTPClientFactory(None, None, src_doc, raw_doc)
c = SMTPPostingClient(acct, src_doc, raw_doc, 'secret', None, None, None)
c.factory = factory
c.deferred = defer.Deferred()
return c
def test_simple_connection_failed(self):
src_doc = yield self._prepare_test_doc()
server = FakeSMTPServer()
server.connection_made_resp = "452 Out of disk space; try later"
client = self._get_post_client(src_doc, src_doc)
_ = yield self.loopback(server, client)
# now re-open the doc and check the state says 'error'
src_doc = yield get_doc_model().db.openDoc(src_doc['_id'])
self.failUnlessEqual(src_doc['sent_state'], 'error')
def setUp(self):
_ = yield TestCaseWithDB.setUp(self)
raindrop.config.CONFIG = None
self.config = self.make_config()
opts = self.get_options()
self.doc_model = get_doc_model()
self.pipeline = Pipeline(self.doc_model, opts)
_ = yield self.prepare_test_db(self.config)
_ = yield self.pipeline.initialize()
def test_simple(self):
src_doc = yield self._prepare_test_doc()
server = FakeSMTPServer()
client = self._get_post_client(src_doc, src_doc)
_ = yield self.loopback(server, client)
# now re-open the doc and check the state says 'sent'
src_doc = yield get_doc_model().db.openDoc(src_doc['_id'])
self.failUnlessEqual(src_doc['sent_state'], 'sent')
self.failUnless(server.buffer) # must have connected to the test server.
def test_simple_failed(self):
src_doc = yield self._prepare_test_doc()
server = FakeSMTPServer()
client = self._get_post_client(src_doc, src_doc)
client.requireAuthentication = True # this causes failure!
_ = yield self.loopback(server, client)
# now re-open the doc and check the state says 'error'
src_doc = yield get_doc_model().db.openDoc(src_doc['_id'])
self.failUnlessEqual(src_doc['sent_state'], 'error')
def setUp(self):
TestCase.setUp(self)
self._conductor = None
raindrop.config.CONFIG = None
self.config = self.make_config()
opts = self.get_options()
self.doc_model = get_doc_model()
self.pipeline = raindrop.pipeline.Pipeline(self.doc_model, opts)
self.prepare_test_db(self.config)
self.pipeline.initialize()
def test_simple_rejected(self):
src_doc = yield self._prepare_test_doc()
server = FakeSMTPServer()
server.responses["MAIL FROM:"] = "500 sook sook sook"
client = self._get_post_client(src_doc, src_doc)
_ = yield self.loopback(server, client)
# now re-open the doc and check the state says 'error'
src_doc = yield get_doc_model().db.openDoc(src_doc['_id'])
self.failUnlessEqual(src_doc['sent_state'], 'error')
def test_outgoing_twice(self):
doc_model = get_doc_model()
src_doc = yield self._prepare_test_doc()
conductor = yield self.get_conductor()
nc = FakeSMTPServer.num_connections
_ = yield conductor.sync(self.pipeline.options)
self.failUnlessEqual(nc+1, FakeSMTPServer.num_connections)
nc = FakeSMTPServer.num_connections
# sync again - better not make a connection this time!
_ = yield conductor.sync(self.pipeline.options)
self.failUnlessEqual(nc, FakeSMTPServer.num_connections)
def test_simple(self):
src_doc, out_doc = self._prepare_test_doc()
self._send_test_doc(src_doc, out_doc)
# now re-open the doc and check the state says 'sent'
src_doc = get_doc_model().db.openDoc(src_doc['_id'])
self.failUnlessEqual(src_doc['sent_state'], 'sent')
self.failUnlessEqual(self.server.num_connections, 1) # must have connected to the test server.
# check the protocol recorded the success
status = self.acct.status
self.failUnlessEqual(status.get('state'), Rat.GOOD, status)
self.failUnlessEqual(status.get('what'), Rat.EVERYTHING, status)
def test_simple_connection_failed(self):
def filter_log(rec):
return "Out of disk space; try later" in rec.msg
self.log_handler.ok_filters.append(filter_log)
self.server.connection_made_resp = "452 Out of disk space; try later"
src_doc, out_doc = self._prepare_test_doc()
self._send_test_doc(src_doc, out_doc)
# now re-open the doc and check the state says 'error'
src_doc = get_doc_model().db.openDoc(src_doc['_id'])
self.failUnlessEqual(src_doc['sent_state'], 'error')
def test_outgoing_twice(self):
doc_model = get_doc_model()
src_doc = self._prepare_test_doc()
nc = self.server.num_connections
self.ensure_pipeline_complete()
self.failUnlessEqual(nc+1, self.server.num_connections)
nc = self.server.num_connections
# sync again - better not make a connection this time!
# XXX - this isn't testing what it should - it *should* ensure
# the pipeline does see the message again, but the conductor refusing
# to re-send it due to the 'outgoing_state'.
self.ensure_pipeline_complete()
self.failUnlessEqual(nc, self.server.num_connections)
def test_one_testmsg_common(self):
# Here we process 2 test messages which result in both messages
# having an identity in common and one that is unique. When we
# process the second message we should notice the shared identity_id
# is already associated with the contact we created first time round,
# with the end result we still end up with a single contact, but now
# have *three* identities for him
self.test_one_testmsg()
result = self.process_doc()
# First determine the contact ID.
key = ['schema_id', 'rd.contact']
result = get_doc_model().open_view(key=key, reduce=False,
include_docs=True)
rows = result['rows']
# Should be exactly 1 record with a 'contact' schema.
self.failUnlessEqual(len(rows), 1, str(rows))
key_type, cid = rows[0]['doc']['rd_key']
self.failUnlessEqual(key_type, 'contact')
# should be exact 3 rd.identity.contacts records, each pointing
# at my contact.
key = ['schema_id', 'rd.identity.contacts']
result = get_doc_model().open_view(key=key,
reduce=False,
include_docs=True)
rows = result['rows']
self.failUnlessEqual(len(rows), 3, str(rows))
docs = [r['doc'] for r in rows]
for doc in docs:
contacts = doc['contacts']
self.failUnlessEqual(len(contacts), 1, contacts)
this_id, this_rel = contacts[0]
self.failUnlessEqual(this_id, cid)
self.failUnless(this_rel in ['personal', 'public'], this_rel)
self.verifyCounts(1, 3)
def prepare_corpus_environment(self, corpus_name):
raindrop.config.CONFIG = None
cd = self.get_corpus_dir(corpus_name)
self.config = raindrop.config.init_config(os.path.join(cd, "raindrop"))
# hack our couch server in
dbinfo = self.config.couches['local']
dbinfo['name'] = 'raindrop_test_suite'
dbinfo['port'] = 5984
opts = FakeOptions()
self.doc_model = get_doc_model()
self.pipeline = Pipeline(self.doc_model, opts)
return self.prepare_test_db(self.config
).addCallback(lambda _: self.pipeline.initialize()
)
def retry_errors(parser, options):
"""Reprocess all conversions which previously resulted in an error."""
g_pipeline.start_retry_errors()
print "Error retry pipeline has finished..."
# Now reset all outgoing items with a state of 'error' back to 'outgoing'
dm = model.get_doc_model()
result = dm.open_view(viewId="outgoing_by_state", key="error",
include_docs=True)
docs = [r['doc'] for r in result['rows']]
if docs:
for d in docs:
d['outgoing_state'] = 'outgoing'
dm.update_documents(docs)
print "Reset %d outgoing items for retry" % (len(docs),)
def test_one_testmsg_unique(self):
# Here we process 2 test messages but none of the messages emit a
# common identity ID. The end result is we end up with 2 contacts;
# one with 2 identities (from reusing test_one_testmsg), then a second
# contact with only a single identity
self.test_one_testmsg()
result = self.process_doc(False)
# First determine the 2 contact IDs.
key = ['schema_id', 'rd.contact']
result = get_doc_model().open_view(key=key, reduce=False,
include_docs=True)
rows = result['rows']
# Should be exactly 2 records with a 'contact' schema.
self.failUnlessEqual(len(rows), 2, pformat(rows))
key_type, cid1 = rows[0]['doc']['rd_key']
self.failUnlessEqual(key_type, 'contact')
key_type, cid2 = rows[1]['doc']['rd_key']
self.failUnlessEqual(key_type, 'contact')
# should be exact 3 rd.identity.contacts records, each pointing
# at my contact.
key = ['schema_id', 'rd.identity.contacts']
result = get_doc_model().open_view(key=key, reduce=False,
include_docs=True)
rows = result['rows']
self.failUnlessEqual(len(rows), 3, str(rows))
docs = [r['doc'] for r in rows]
for doc in docs:
contacts = doc['contacts']
self.failUnlessEqual(len(contacts), 1, contacts)
this_id, this_rel = contacts[0]
self.failUnless(this_id in [cid1, cid2])
self.failUnless(this_rel in ['personal', 'public'], this_rel)
self.verifyCounts(2, 3)
def add_schemas(result, parser, options, args):
"""Add one or more schema documents to the couch"""
if not args:
parser.error("You must supply filenames containing json for the docs")
dm = model.get_doc_model()
for arg in args:
try:
with open(arg) as f:
try:
vals = json.load(f)
except ValueError, why:
parser.error("file %r has invalid json: %s" % (arg, why))
except IOError:
parser.error("Failed to open json document %r" % arg)
got = yield dm.create_schema_items([vals])
print "Saved doc id %(id)r at rev %(rev)s" % got[0]
def test_simple_rejected(self):
src_doc, out_doc = self._prepare_test_doc()
def filter_log(rec):
return "sook sook sook" in rec.msg
self.log_handler.ok_filters.append(filter_log)
self.server.responses["MAIL FROM:"] = "500 sook sook sook"
self._send_test_doc(src_doc, out_doc)
# now re-open the doc and check the state says 'error'
src_doc = get_doc_model().db.openDoc(src_doc['_id'])
self.failUnlessEqual(src_doc['sent_state'], 'error')
# check the protocol recorded the error.
status = self.acct.status
self.failUnlessEqual(status.get('state'), Rat.BAD, status)
self.failUnlessEqual(status.get('what'), Rat.SERVER, status)
self.failUnless('sook' in status.get('message', ''), status)
