def get_docs(schema_index, previous_export=None, filter=None):
def _filter(results):
if filter is None:
return results
return [doc for doc in results if filter(doc)]
db = Database(settings.COUCH_DATABASE)
if previous_export is not None:
consumer = Consumer(db)
view_results = consumer.fetch(since=previous_export.seq)
if view_results:
try:
include_ids = set([res["id"] for res in view_results["results"]])
possible_ids = set([result['id'] for result in \
db.view("couchexport/schema_index",
key=schema_index).all()])
ids_to_use = include_ids.intersection(possible_ids)
return _filter(res["doc"] for res in \
db.all_docs(keys=list(ids_to_use), include_docs=True).all())
except Exception:
import logging
logging.exception("export failed! results: %s" % view_results)
raise
else:
# sometimes this comes back empty. I think it might be a bug
# in couchdbkit, but it's impossible to consistently reproduce.
# For now, just assume this is fine.
return []
else:
return _filter([result['doc'] for result in \
db.view("couchexport/schema_index",
key=schema_index, include_docs=True).all()])
def get_pats_with_updated_cases(clinic_id, zone, last_seq):
"""
Given a clinic, zone, and last_seq id from couch, get the patients whose
cases have been updated, returning a tuple of updated ids (in a list) and
the last seq number.
"""
CACHE_TIME = 4 * 60 * 60 # cache for 4 hours, in seconds
# first try to get this from the cache
def _build_cache_key(clinic_id, zone, last_seq):
return "sync_patient_list:%s:%s:%s" % (clinic_id, zone, last_seq)
cached_data = cache.get(_build_cache_key(clinic_id, zone, last_seq))
if cached_data:
return cached_data
# first get the patient list of potential matches. use the open case
# list to get patient ids
potential_case_list = get_db().view("case/open_for_chw_for_phone", key=[clinic_id, zone])
possible_pat_ids = []
for row in potential_case_list:
if row["id"] not in possible_pat_ids:
possible_pat_ids.append(row["id"])
# new consumer
consumer = Consumer(get_db())
view_results = consumer.fetch(since=last_seq)
pats_with_open_cases = []
for res in view_results["results"]:
id = res["id"]
if id in possible_pat_ids and not id in pats_with_open_cases:
pats_with_open_cases.append(id)
updated_last_seq = view_results["last_seq"]
ret = (pats_with_open_cases, updated_last_seq)
cache.set(_build_cache_key(clinic_id, zone, last_seq), ret, CACHE_TIME)
return ret
def _ids_by_seq(self, database):
if self.seq == "0" or self.seq is None:
return self.get_all_ids()
consumer = Consumer(database)
view_results = consumer.fetch(since=self.seq)
if view_results:
include_ids = set([res["id"] for res in view_results["results"]])
return include_ids.intersection(self.get_all_ids())
else:
# sometimes this comes back empty. I think it might be a bug
# in couchdbkit, but it's impossible to consistently reproduce.
# For now, just assume this is fine.
return set()
def _ids_by_seq(self, database):
if self.seq == "0" or self.seq is None:
return self.get_all_ids()
consumer = Consumer(database)
view_results = consumer.fetch(since=self.seq)
if view_results:
include_ids = set([res["id"] for res in view_results["results"]])
return include_ids.intersection(self.get_all_ids())
else:
# sometimes this comes back empty. I think it might be a bug
# in couchdbkit, but it's impossible to consistently reproduce.
# For now, just assume this is fine.
return set()
def _potentially_relevant_ids(self):
if self.previous_export is not None:
consumer = Consumer(self.database)
view_results = consumer.fetch(since=self.previous_export.seq)
if view_results:
try:
include_ids = set([res["id"] for res in view_results["results"]])
possible_ids = set(self._all_ids())
return list(include_ids.intersection(possible_ids))
except Exception:
import logging
logging.exception("export failed! results: %s" % view_results)
raise
else:
# sometimes this comes back empty. I think it might be a bug
# in couchdbkit, but it's impossible to consistently reproduce.
# For now, just assume this is fine.
return []
else:
return self._all_ids()
def handle(self, *args, **options):
# this has a verrrrrrrrry similar structure to the conflict resolver.
# we should abstract this out into a base management command class
db = get_db()
c = Consumer(db)
def add_sha1_to_line(line):
try:
xform_id = line["id"]
xform = CXFormInstance.get(xform_id)
add_sha1(None, xform)
except Exception:
logging.exception("problem with sha1 callback")
# Go into receive loop waiting for any conflicting patients to
# come in.
while True:
try:
c.wait(add_sha1_to_line, heartbeat=5000, filter=FILTER_XFORMS)
except Exception, e:
time.sleep(10)
logging.warn("caught exception in sha-1 adder: %s, sleeping and restarting" % e)
def handle(self, *args, **options):
db = get_db()
c = Consumer(db)
# sync design docs to the target db
# lots of source diving to figure out this magic
new_dbs = [(app, global_config.database.uri)
for app, _ in settings.COUCHDB_DATABASES]
couchdbkit_handler = CouchdbkitHandler(new_dbs)
for app, _ in new_dbs:
try:
couchdbkit_handler.sync(models.get_app(app))
except ImproperlyConfigured:
# if django doesn't think this is an app it throws this error
# this is probably fine
pass
# also sync couchapps
sync_design_docs(global_config.database)
def sync_if_necessary(line):
try:
change = Change(line)
# don't bother with deleted or old documents
if change.deleted or not change.is_current(db):
return
# get doc
doc = get_db().get(change.id)
# check if transforms, and if so, save to new domain/db
transforms = global_config.get_transforms(doc)
for transform in transforms:
global_config.save(transform)
# update the checkpoint, somewhat arbitrarily
global domainsync_counter
domainsync_counter = domainsync_counter + 1
if domainsync_counter % CHECKPOINT_FREQUENCY == 0:
Checkpoint.set_checkpoint(CHECKPOINT_ID, change.seq)
except Exception, e:
logging.exception("problem in domain sync for line: %s\n%s" %
(line, e))
def spool():
update_seq = cq.db.info()["update_seq"]
c = Consumer(cq.db)
c.wait(run_item, since=update_seq)
def spool():
update_seq = cq.db.info()['update_seq']
c = Consumer(cq.db)
c.wait(run_item, since=update_seq)