コード例 #1
0
ファイル: schema.py プロジェクト: wbnigeria/couchexport
def get_docs(schema_index, previous_export=None, filter=None):
    
    def _filter(results):
        if filter is None:
            return results
        return [doc for doc in results if filter(doc)]
            
    db = Database(settings.COUCH_DATABASE)
    if previous_export is not None:
        consumer = Consumer(db)
        view_results = consumer.fetch(since=previous_export.seq)
        if view_results:
            try:
                include_ids = set([res["id"] for res in view_results["results"]])
                possible_ids = set([result['id'] for result in \
                                    db.view("couchexport/schema_index", 
                                            key=schema_index).all()])
                ids_to_use = include_ids.intersection(possible_ids)
                return _filter(res["doc"] for res in \
                               db.all_docs(keys=list(ids_to_use), include_docs=True).all())
            except Exception:
                import logging
                logging.exception("export failed! results: %s" % view_results) 
                raise
        else:
            # sometimes this comes back empty. I think it might be a bug
            # in couchdbkit, but it's impossible to consistently reproduce.
            # For now, just assume this is fine.
            return []
    else: 
        return _filter([result['doc'] for result in \
                        db.view("couchexport/schema_index", 
                                key=schema_index, include_docs=True).all()])
コード例 #2
0
ファイル: caselogic.py プロジェクト: dimagi/bhoma
def get_pats_with_updated_cases(clinic_id, zone, last_seq):
    """
    Given a clinic, zone, and last_seq id from couch, get the patients whose
    cases have been updated, returning a tuple of updated ids (in a list) and
    the last seq number.
    """
    CACHE_TIME = 4 * 60 * 60 # cache for 4 hours, in seconds
    
    # first try to get this from the cache
    def _build_cache_key(clinic_id, zone, last_seq):
        return "sync_patient_list:%s:%s:%s" % (clinic_id, zone, last_seq)
    cached_data = cache.get(_build_cache_key(clinic_id, zone, last_seq))
    if cached_data:
        return cached_data
    
    # first get the patient list of potential matches.  use the open case
    # list to get patient ids
    potential_case_list = get_db().view("case/open_for_chw_for_phone", key=[clinic_id, zone])
    possible_pat_ids = []
    for row in potential_case_list:
        if row["id"] not in possible_pat_ids:
            possible_pat_ids.append(row["id"])
    # new consumer
    consumer = Consumer(get_db())
    view_results = consumer.fetch(since=last_seq)
    pats_with_open_cases = []
    for res in view_results["results"]:
        id = res["id"]
        if id in possible_pat_ids and not id in pats_with_open_cases:
            pats_with_open_cases.append(id)
    
    updated_last_seq = view_results["last_seq"]
    ret = (pats_with_open_cases, updated_last_seq)
    cache.set(_build_cache_key(clinic_id, zone, last_seq), ret, CACHE_TIME)
    return ret
コード例 #3
0
ファイル: models.py プロジェクト: kennknowles/couchexport
    def _ids_by_seq(self, database):
        if self.seq == "0" or self.seq is None:
            return self.get_all_ids()

        consumer = Consumer(database)
        view_results = consumer.fetch(since=self.seq)
        if view_results:
            include_ids = set([res["id"] for res in view_results["results"]])
            return include_ids.intersection(self.get_all_ids())
        else:
            # sometimes this comes back empty. I think it might be a bug
            # in couchdbkit, but it's impossible to consistently reproduce.
            # For now, just assume this is fine.
            return set()
コード例 #4
0
    def _ids_by_seq(self, database):
        if self.seq == "0" or self.seq is None:
            return self.get_all_ids()

        consumer = Consumer(database)
        view_results = consumer.fetch(since=self.seq)
        if view_results:
            include_ids = set([res["id"] for res in view_results["results"]])
            return include_ids.intersection(self.get_all_ids())
        else:
            # sometimes this comes back empty. I think it might be a bug
            # in couchdbkit, but it's impossible to consistently reproduce.
            # For now, just assume this is fine.
            return set()
コード例 #5
0
ファイル: export.py プロジェクト: wbnigeria/couchexport
    def _potentially_relevant_ids(self):
        if self.previous_export is not None:
            consumer = Consumer(self.database)
            view_results = consumer.fetch(since=self.previous_export.seq)
            if view_results:
                try:
                    include_ids = set([res["id"] for res in view_results["results"]])
                    possible_ids = set(self._all_ids())
                    return list(include_ids.intersection(possible_ids))
                except Exception:
                    import logging

                    logging.exception("export failed! results: %s" % view_results)
                    raise
            else:
                # sometimes this comes back empty. I think it might be a bug
                # in couchdbkit, but it's impossible to consistently reproduce.
                # For now, just assume this is fine.
                return []
        else:
            return self._all_ids()
コード例 #6
0
ファイル: add_checksums.py プロジェクト: dimagi/bhoma
 def handle(self, *args, **options):
     # this has a verrrrrrrrry similar structure to the conflict resolver.
     # we should abstract this out into a base management command class
     db = get_db()
     c = Consumer(db)
     
     def add_sha1_to_line(line):
         try:
             xform_id = line["id"]
             xform = CXFormInstance.get(xform_id)
             add_sha1(None, xform)
         except Exception:
             logging.exception("problem with sha1 callback")
     
     # Go into receive loop waiting for any conflicting patients to
     # come in.
     while True:
         try:
             c.wait(add_sha1_to_line, heartbeat=5000, filter=FILTER_XFORMS)
         except Exception, e:
             time.sleep(10)
             logging.warn("caught exception in sha-1 adder: %s, sleeping and restarting" % e)
コード例 #7
0
ファイル: sync_domains.py プロジェクト: tsinkala/commcare-hq
    def handle(self, *args, **options):
        db = get_db()
        c = Consumer(db)

        # sync design docs to the target db
        # lots of source diving to figure out this magic
        new_dbs = [(app, global_config.database.uri)
                   for app, _ in settings.COUCHDB_DATABASES]
        couchdbkit_handler = CouchdbkitHandler(new_dbs)
        for app, _ in new_dbs:
            try:
                couchdbkit_handler.sync(models.get_app(app))
            except ImproperlyConfigured:
                # if django doesn't think this is an app it throws this error
                # this is probably fine
                pass

        # also sync couchapps
        sync_design_docs(global_config.database)

        def sync_if_necessary(line):
            try:
                change = Change(line)
                # don't bother with deleted or old documents
                if change.deleted or not change.is_current(db):
                    return

                # get doc
                doc = get_db().get(change.id)

                # check if transforms, and if so, save to new domain/db
                transforms = global_config.get_transforms(doc)
                for transform in transforms:
                    global_config.save(transform)

                # update the checkpoint, somewhat arbitrarily
                global domainsync_counter
                domainsync_counter = domainsync_counter + 1
                if domainsync_counter % CHECKPOINT_FREQUENCY == 0:
                    Checkpoint.set_checkpoint(CHECKPOINT_ID, change.seq)

            except Exception, e:
                logging.exception("problem in domain sync for line: %s\n%s" %
                                  (line, e))
コード例 #8
0
ファイル: spooler.py プロジェクト: zignig/substrate
def spool():
    update_seq = cq.db.info()["update_seq"]
    c = Consumer(cq.db)
    c.wait(run_item, since=update_seq)
コード例 #9
0
ファイル: spooler.py プロジェクト: dicoy-zz/substrate
def spool():
    update_seq = cq.db.info()['update_seq']
    c = Consumer(cq.db)
    c.wait(run_item, since=update_seq)