Beispiel #1
0
    def test_get_list(self):
        """
        Any case in the appropriate domain should be in the list from the API.
        """

        # The actual infrastructure involves saving to CouchDB, having PillowTop
        # read the changes and write it to ElasticSearch.

        #the pillow is set to offline mode - elasticsearch not needed to validate
        pillow = CasePillow(online=False)
        fake_case_es = FakeXFormES()
        v0_4.MOCK_CASE_ES = fake_case_es

        modify_date = datetime.utcnow()

        backend_case = CommCareCase(server_modified_on=modify_date, domain=self.domain.name)
        backend_case.save()

        translated_doc = pillow.change_transform(backend_case.to_json())

        fake_case_es.add_doc(translated_doc['_id'], translated_doc)

        self.client.login(username=self.username, password=self.password)

        response = self.client.get(self.list_endpoint)
        self.assertEqual(response.status_code, 200)

        api_cases = simplejson.loads(response.content)['objects']
        self.assertEqual(len(api_cases), 1)

        api_case = api_cases[0]
        self.assertEqual(dateutil.parser.parse(api_case['server_date_modified']), backend_case.server_modified_on)

        backend_case.delete()
Beispiel #2
0
    def finish_handle(self):
        filepath = os.path.join(settings.FILEPATH, 'corehq', 'pillows',
                                'mappings', 'case_mapping.py')
        case_pillow = CasePillow(create_index=False)

        #check current index
        current_index = case_pillow.es_index

        sys.stderr.write("current index:\n")
        sys.stderr.write('CASE_INDEX="%s"\n' % current_index)

        #regenerate the mapping dict
        mapping = case_mapping.CASE_MAPPING
        case_pillow.default_mapping = mapping
        delattr(case_pillow, '_calc_meta_cache')
        calc_index = "%s_%s" % (case_pillow.es_index_prefix,
                                case_pillow.calc_meta())

        #aliased_indices = case_pillow.check_alias()
        # if calc_index not in aliased_indices and calc_index != current_index:
        #     sys.stderr.write("\n\tWarning, current index %s is not aliased at the moment\n" % current_index)
        #     sys.stderr.write("\tCurrent live aliased index: %s\n\n"  % (','.join(aliased_indices)))

        if calc_index != current_index:
            sys.stderr.write("############# HEADS UP!!! #################\n")
            sys.stderr.write(
                "CASE_INDEX hash has changed, please update \n\t%s\n\tCASE_INDEX property with the line below:\n"
                % filepath)
            sys.stdout.write('CASE_INDEX="%s"\n' % calc_index)
        else:
            sys.stderr.write("CASE_INDEX unchanged\n")
    def finish_handle(self):

        filepath = os.path.join(settings.FILEPATH, 'submodules','core-hq-src','corehq','pillows','mappings','case_mapping.py')
        casepillow = CasePillow(create_index=False)

        #current index
        #check current index
        aliased_indices = casepillow.check_alias()

#        current_index = '%s_%s' % (casepillow.es_index_prefix, casepillow.calc_meta())
        current_index = casepillow.es_index

        #regenerate the mapping dict
        m = DEFAULT_MAPPING_WRAPPER

        m['properties'] = dynamic.set_properties(self.doc_class, custom_types=case_special_types)
        m['_meta']['comment'] = "Autogenerated [%s] mapping from ptop_generate_mapping %s" % (self.doc_class_str, datetime.utcnow().strftime('%m/%d/%Y'))
        casepillow.default_mapping = m
        delattr(casepillow, '_calc_meta_cache')
        output = []
        output.append('CASE_INDEX="%s_%s"' % (casepillow.es_index_prefix, casepillow.calc_meta()))
        output.append('CASE_MAPPING=%s' % pprint.pformat(m))
        newcalc_index = "%s_%s" % (casepillow.es_index_prefix, casepillow.calc_meta())
        print "Writing new case_index and mapping: %s" % output[0]
        with open(filepath, 'w') as outfile:
            outfile.write('\n'.join(output))

        if newcalc_index not in aliased_indices and newcalc_index != current_index:
            sys.stderr.write("\n\tWarning, current index %s is not aliased at the moment\n" % current_index)
            sys.stderr.write("\tCurrent live aliased index: %s\n\n"  % (','.join(aliased_indices)))

        sys.stderr.write("File written to %s\n" % filepath)
    def finish_handle(self):
        filepath = os.path.join(settings.FILEPATH, 'corehq','pillows','mappings','case_mapping.py')
        case_pillow = CasePillow(create_index=False)

        #check current index
        current_index = case_pillow.es_index

        sys.stderr.write("current index:\n")
        sys.stderr.write('CASE_INDEX="%s"\n' % current_index)

        #regenerate the mapping dict
        mapping = case_mapping.CASE_MAPPING
        case_pillow.default_mapping = mapping
        delattr(case_pillow, '_calc_meta_cache')
        calc_index = "%s_%s" % (case_pillow.es_index_prefix, case_pillow.calc_meta())

        #aliased_indices = case_pillow.check_alias()
        # if calc_index not in aliased_indices and calc_index != current_index:
        #     sys.stderr.write("\n\tWarning, current index %s is not aliased at the moment\n" % current_index)
        #     sys.stderr.write("\tCurrent live aliased index: %s\n\n"  % (','.join(aliased_indices)))

        if calc_index != current_index:
            sys.stderr.write("############# HEADS UP!!! #################\n")
            sys.stderr.write("CASE_INDEX hash has changed, please update \n\t%s\n\tCASE_INDEX property with the line below:\n" % filepath)
            sys.stdout.write('CASE_INDEX="%s"\n' % calc_index)
        else:
            sys.stderr.write("CASE_INDEX unchanged\n")
Beispiel #5
0
    def test_get_list(self):
        """
        Any case in the appropriate domain should be in the list from the API.
        """

        # The actual infrastructure involves saving to CouchDB, having PillowTop
        # read the changes and write it to ElasticSearch.

        #the pillow is set to offline mode - elasticsearch not needed to validate
        pillow = CasePillow(online=False)
        fake_case_es = FakeXFormES()
        v0_4.MOCK_CASE_ES = fake_case_es

        modify_date = datetime.utcnow()

        backend_case = CommCareCase(server_modified_on=modify_date, domain=self.domain.name)
        backend_case.type = CC_BIHAR_PREGNANCY
        backend_case.save()

        translated_doc = pillow.change_transform(backend_case.to_json())

        fake_case_es.add_doc(translated_doc['_id'], translated_doc)

        self.client.login(username=self.username, password=self.password)

        response = self.client.get(self.list_endpoint)
        self.assertEqual(response.status_code, 200)

        api_cases = simplejson.loads(response.content)['objects']
        self.assertEqual(len(api_cases), 2)

        api_case = api_cases['mother_lists'][0]
        self.assertEqual(dateutil.parser.parse(api_case['server_date_modified']), backend_case.server_modified_on)

        backend_case.delete()
    def setUpClass(cls):
        case_pillow = CasePillow(online=False)
        group_pillow = GroupPillow(online=False)
        cls.pillows = [case_pillow, group_pillow]

        for pillow in cls.pillows:
            completely_initialize_pillow_index(pillow)

        case = new_case(closed=True)
        case_pillow.send_robust(case.to_json())

        case = new_case(closed=False)
        case_pillow.send_robust(case.to_json())

        case = new_case(closed=True, owner_id="foo")
        case_pillow.send_robust(case.to_json())

        case = new_case(closed=False, owner_id="bar")
        case_pillow.send_robust(case.to_json())

        group = Group(_id=uuid.uuid4().hex, users=["foo", "bar"])
        cls.group_id = group.get_id
        group_pillow.send_robust(group.to_json())

        for pillow in cls.pillows:
            pillow.get_es_new().indices.refresh(pillow.es_index)
    def setUpClass(cls):
        case_pillow = CasePillow(online=False)
        group_pillow = GroupPillow(online=False)
        cls.pillows = [case_pillow, group_pillow]

        with trap_extra_setup(ConnectionError, msg="cannot connect to elasicsearch"):
            for pillow in cls.pillows:
                completely_initialize_pillow_index(pillow)

        case = new_case(closed=True)
        case_pillow.send_robust(case.to_json())

        case = new_case(closed=False)
        case_pillow.send_robust(case.to_json())

        case = new_case(closed=True, owner_id="foo")
        case_pillow.send_robust(case.to_json())

        case = new_case(closed=False, owner_id="bar")
        case_pillow.send_robust(case.to_json())

        group = Group(_id=uuid.uuid4().hex, users=["foo", "bar"])
        cls.group_id = group.get_id
        group_pillow.send_robust(group.to_json())

        for pillow in cls.pillows:
            pillow.get_es_new().indices.refresh(pillow.es_index)
    def testOwnerIDSetOnTransform(self):
        """
        Test that the owner_id gets set to the case when the pillow calls change transform
        """
        case_owner_id = CASE_WITH_OWNER_ID
        case_no_owner_id = CASE_NO_OWNER_ID

        pillow = CasePillow(create_index=False, online=False)
        changed_with_owner_id = pillow.change_transform(case_owner_id)
        changed_with_no_owner_id = pillow.change_transform(case_no_owner_id)

        self.assertEqual(changed_with_owner_id.get("owner_id"), "testuser")
        self.assertEqual(changed_with_no_owner_id.get("owner_id"), "testuser")
    def testOwnerIDSetOnTransform(self):
        """
        Test that the owner_id gets set to the case when the pillow calls change transform
        """
        case_owner_id = CASE_WITH_OWNER_ID
        case_no_owner_id = CASE_NO_OWNER_ID

        pillow = CasePillow(create_index=False, online=False)
        changed_with_owner_id = pillow.change_transform(case_owner_id)
        changed_with_no_owner_id = pillow.change_transform(case_no_owner_id)

        self.assertEqual(changed_with_owner_id.get("owner_id"), "testuser")
        self.assertEqual(changed_with_no_owner_id.get("owner_id"), "testuser")
    def setUpClass(cls):
        form_pillow = XFormPillow(online=False)
        case_pillow = CasePillow(online=False)
        cls.pillows = [form_pillow, case_pillow]

        es = get_es_new()
        with trap_extra_setup(ConnectionError, msg="cannot connect to elasicsearch"):
            for pillow in cls.pillows:
                completely_initialize_pillow_index(pillow)
            initialize_index_and_mapping(es, GROUP_INDEX_INFO)

        case = new_case(closed=True)
        case_pillow.send_robust(case.to_json())

        case = new_case(closed=False)
        case_pillow.send_robust(case.to_json())

        case = new_case(closed=True, owner_id="foo")
        case_pillow.send_robust(case.to_json())

        case = new_case(closed=False, owner_id="bar")
        case_pillow.send_robust(case.to_json())

        group = Group(_id=uuid.uuid4().hex, users=["foo", "bar"])
        cls.group_id = group._id
        send_to_elasticsearch('groups', group.to_json())

        form = new_form(form={"meta": {"userID": None}})
        form_pillow.send_robust(form.to_json())

        form = new_form(form={"meta": {"userID": ""}})
        form_pillow.send_robust(form.to_json())

        form = new_form(form={"meta": {"deviceID": "abc"}})
        form_pillow.send_robust(form.to_json())

        form = new_form(form={"meta": {"userID": uuid.uuid4().hex}})
        form_pillow.send_robust(form.to_json())

        for pillow in cls.pillows:
            pillow.get_es_new().indices.refresh(pillow.es_index)
        es.indices.refresh(GROUP_INDEX_INFO.index)
Beispiel #11
0
    def finish_handle(self):

        filepath = os.path.join(settings.FILEPATH, 'corehq', 'pillows',
                                'mappings', 'case_mapping.py')
        casepillow = CasePillow(create_index=False)

        #current index
        #check current index
        aliased_indices = casepillow.check_alias()

        #        current_index = '%s_%s' % (casepillow.es_index_prefix, casepillow.calc_meta())
        current_index = casepillow.es_index

        #regenerate the mapping dict
        m = DEFAULT_MAPPING_WRAPPER

        m['properties'] = dynamic.set_properties(
            self.doc_class, custom_types=case_special_types)
        m['_meta'][
            'comment'] = "Autogenerated [%s] mapping from ptop_generate_mapping %s" % (
                self.doc_class_str, datetime.utcnow().strftime('%m/%d/%Y'))
        casepillow.default_mapping = m
        delattr(casepillow, '_calc_meta_cache')
        output = []
        output.append('CASE_INDEX="%s_%s"' %
                      (casepillow.es_index_prefix, casepillow.calc_meta()))
        output.append('CASE_MAPPING=%s' % pprint.pformat(m))
        newcalc_index = "%s_%s" % (casepillow.es_index_prefix,
                                   casepillow.calc_meta())
        print "Writing new case_index and mapping: %s" % output[0]
        with open(filepath, 'w') as outfile:
            outfile.write('\n'.join(output))

        if newcalc_index not in aliased_indices and newcalc_index != current_index:
            sys.stderr.write(
                "\n\tWarning, current index %s is not aliased at the moment\n"
                % current_index)
            sys.stderr.write("\tCurrent live aliased index: %s\n\n" %
                             (','.join(aliased_indices)))

        sys.stderr.write("File written to %s\n" % filepath)
Beispiel #12
0
 def setUp(self):
     FormProcessorTestUtils.delete_all_cases()
     self.pillow = CasePillow()
     self.elasticsearch = self.pillow.get_es_new()
     delete_es_index(self.pillow.es_index)
Beispiel #13
0
class CasePillowTest(TestCase):

    domain = 'case-pillowtest-domain'

    def setUp(self):
        FormProcessorTestUtils.delete_all_cases()
        self.pillow = CasePillow()
        self.elasticsearch = self.pillow.get_es_new()
        delete_es_index(self.pillow.es_index)

    def tearDown(self):
        ensure_index_deleted(self.pillow.es_index)

    def test_case_pillow_couch(self):
        # make a case
        case_id = uuid.uuid4().hex
        case_name = 'case-name-{}'.format(uuid.uuid4().hex)
        case = self._make_a_case(case_id, case_name)

        # send to elasticsearch
        self._sync_couch_cases_to_es()

        # verify there
        results = CaseES().run()
        self.assertEqual(1, results.total)
        case_doc = results.hits[0]
        self.assertEqual(self.domain, case_doc['domain'])
        self.assertEqual(case_id, case_doc['_id'])
        self.assertEqual(case_name, case_doc['name'])

        # cleanup
        case.delete()

    def test_case_soft_deletion(self):
        # make a case
        case_id = uuid.uuid4().hex
        case_name = 'case-name-{}'.format(uuid.uuid4().hex)
        case = self._make_a_case(case_id, case_name)

        # send to elasticsearch
        self._sync_couch_cases_to_es()

        # verify there
        results = CaseES().run()
        self.assertEqual(1, results.total)

        seq_before_deletion = self.pillow.get_change_feed().get_latest_change_id()

        # soft delete the case
        case.soft_delete()

        # sync to elasticsearch
        self._sync_couch_cases_to_es(since=seq_before_deletion)

        # ensure not there anymore
        results = CaseES().run()
        self.assertEqual(0, results.total)

        # cleanup
        case.delete()

    @override_settings(TESTS_SHOULD_USE_SQL_BACKEND=True)
    def test_case_pillow_sql(self):
        consumer = get_test_kafka_consumer(topics.CASE_SQL)
        # have to get the seq id before the change is processed
        kafka_seq = consumer.offsets()['fetch'][(topics.CASE_SQL, 0)]

        # make a case
        case_id = uuid.uuid4().hex
        case_name = 'case-name-{}'.format(uuid.uuid4().hex)
        case = self._make_a_case(case_id, case_name)

        # confirm change made it to kafka
        message = consumer.next()
        change_meta = change_meta_from_kafka_message(message.value)
        self.assertEqual(case.case_id, change_meta.document_id)
        self.assertEqual(self.domain, change_meta.domain)

        # send to elasticsearch
        sql_pillow = get_sql_case_to_elasticsearch_pillow()
        sql_pillow.process_changes(since=kafka_seq, forever=False)
        self.elasticsearch.indices.refresh(self.pillow.es_index)

        # confirm change made it to elasticserach
        results = CaseES().run()
        self.assertEqual(1, results.total)
        case_doc = results.hits[0]
        self.assertEqual(self.domain, case_doc['domain'])
        self.assertEqual(case_id, case_doc['_id'])
        self.assertEqual(case_name, case_doc['name'])

    def _make_a_case(self, case_id, case_name):
        # this avoids having to deal with all the reminders code bootstrap
        with drop_connected_signals(case_post_save):
            form, cases = post_case_blocks(
                [
                    CaseBlock(
                        create=True,
                        case_id=case_id,
                        case_name=case_name,
                    ).as_xml()
                ], domain=self.domain
            )
        self.assertEqual(1, len(cases))
        return cases[0]

    def _sync_couch_cases_to_es(self, since=0):
        self.pillow.process_changes(since=since, forever=False)
        self.elasticsearch.indices.refresh(self.pillow.es_index)
Beispiel #14
0
 def setUp(self):
     FormProcessorTestUtils.delete_all_cases()
     with trap_extra_setup(ConnectionError):
         self.pillow = CasePillow()
     self.elasticsearch = self.pillow.get_es_new()
     delete_es_index(self.pillow.es_index)