def setUp(self): self.import_ = Import() self.import_.save() cursor = connection.cursor() for command in open( 'dcdata/scripts/contribution_name_indexes.sql', 'r'): if command.strip() and not command.startswith('--'): cursor.execute(command)
def __init__(self, source, description, imported_by, **kwargs): self.log_func = sys.stderr.write log = kwargs.get('log') if log: self.log_func = log.warn # populate a fieldname/field mapping of model fields self.fields = { } for field in self.model._meta.fields: self.fields[field.name] = field # create a record template with field names from model # remove any fields that are typical ID fields self._new_record = dict([(f, None) for f in self.fields.iterkeys()]) for key in self.ID_FIELDS: if key in self._new_record: del self._new_record[key] # create a new Import instance and save reference self.import_session = Import( source=source, description=description, imported_by=imported_by ) self.import_session.save()
def __init__(self, source, description, imported_by, **kwargs): self.log_func = sys.stderr.write log = kwargs.get('log') if log: self.log_func = log.warn # populate a fieldname/field mapping of model fields self.fields = {} for field in self.model._meta.fields: self.fields[field.name] = field # create a record template with field names from model # remove any fields that are typical ID fields self._new_record = dict([(f, None) for f in self.fields.iterkeys()]) for key in self.ID_FIELDS: if key in self._new_record: del self._new_record[key] # create a new Import instance and save reference self.import_session = Import(source=source, description=description, imported_by=imported_by) self.import_session.save()
class Loader(object): ID_FIELDS = ('id','uuid') model = None field_handlers = { } def __init__(self, source, description, imported_by, **kwargs): self.log_func = sys.stderr.write log = kwargs.get('log') if log: self.log_func = log.warn # populate a fieldname/field mapping of model fields self.fields = { } for field in self.model._meta.fields: self.fields[field.name] = field # create a record template with field names from model # remove any fields that are typical ID fields self._new_record = dict([(f, None) for f in self.fields.iterkeys()]) for key in self.ID_FIELDS: if key in self._new_record: del self._new_record[key] # create a new Import instance and save reference self.import_session = Import( source=source, description=description, imported_by=imported_by ) self.import_session.save() def new_record(self): """ Get an empty copy of record format. """ return self._new_record.copy() # return record template def load_records(self, records): """ Load a list of records. """ for record in records: self.load_record(record) def load_record(self, record): """ Load a dict record into a model and save to the database. """ # a model is required for loading if not self.model: raise ValueError, "model is required" # get a new or existing instance of model obj = self.get_instance(record) if obj is None: return if obj.pk: # object already exists, resolve issue of "conflicting" data self.resolve(record, obj) else: # object is new, copy over all fields self.copy_fields(record, obj) # assign Import reference and save obj.import_reference = self.import_session try: obj.save() except ValueError, e: #import pprint #self.log_func(pprint.pformat(record)) #self.log_func('Error saving record to database: %s' % repr(traceback.format_exc(e))) raise SkipRecordException(e) except:
class Tests(TestCase): def create_entities(self): self.create_contribution(contributor_name='1234') self.create_contribution(organization_name='1234') self.create_contribution(parent_organization_name='5678') self.create_contribution(recipient_name='abcd') self.create_contribution(recipient_name='efgh') self.create_contribution(committee_name='efgh') def assert_num_results(self, expected_num, request): self.assertEqual(expected_num, filter_contributions(request).count()) def create_contribution(self, **kwargs): c = Contribution(**kwargs) if 'cycle' not in kwargs: c.cycle='09' c.transaction_namespace=UNITTEST_TRANSACTION_NAMESPACE c.import_reference=self.import_ c.save() def setUp(self): self.import_ = Import() self.import_.save() cursor = connection.cursor() for command in open( 'dcdata/scripts/contribution_name_indexes.sql', 'r'): if command.strip() and not command.startswith('--'): cursor.execute(command) def test_date(self): self.create_contribution(date=date(1999,12,25)) self.create_contribution(date=date(1999,12,31)) self.create_contribution(date=date(2000,1,1)) self.assert_num_results(0, {'date': ">|2001-01-01"}) self.assert_num_results(1, {'date': "><|1999-12-24|1999-12-26"}) self.assert_num_results(2, {'date': "<|1999-12-31"}) self.assert_num_results(3, {'date': ">|1980-01-01"}) def test_contributor(self): self.create_entities() self.assert_num_results(0, {'contributor_ft': "abcd"}) self.assert_num_results(2, {'contributor_ft': "1234"}) self.assert_num_results(3, {'contributor_ft': "1234|5678"}) def test_recipient(self): self.create_entities() self.assert_num_results(0, {'recipient_ft': "1234"}) self.assert_num_results(1, {'recipient_ft': "abcd"}) self.assert_num_results(2, {'recipient_ft': "abcd|efgh"}) self.assert_num_results(2, {'recipient_ft': "0000|abcd|efgh"}) def test_amount(self): self.create_contribution(amount=100) self.create_contribution(amount=500) self.create_contribution(amount=800) self.assert_num_results(0, {'amount': ">|1000"}) self.assert_num_results(0, {'amount': "<|50"}) self.assert_num_results(1, {'amount': ">|600"}) self.assert_num_results(2, {'amount': "<|500"}) self.assert_num_results(3, {'amount': "><|100|800"}) def test_cycle(self): self.create_contribution(cycle=02) self.create_contribution(cycle=04) self.create_contribution(cycle=04) self.assert_num_results(0, {'cycle': "08"}) self.assert_num_results(1, {'cycle': "02"}) self.assert_num_results(2, {'cycle': "04"}) def test_state(self): self.create_contribution(contributor_state='CA') self.create_contribution(contributor_state='OR') self.create_contribution(contributor_state='OR') self.assert_num_results(0, {'contributor_state': "WA"}) self.assert_num_results(1, {'contributor_state': "CA"}) self.assert_num_results(2, {'contributor_state': "OR"}) def test_conjunctions(self): self.create_contribution(contributor_name='apple', contributor_state="WA", amount=1000) self.create_contribution(contributor_name='apple sauce', contributor_state="WA", amount=100) self.assert_num_results(0, {'contributor_state': "CA", 'amount': ">|500"}) self.assert_num_results(1, {'contributor_state': "WA", 'amount': ">|500"}) self.assert_num_results(1, {'amount': ">|500"}) self.assert_num_results(2, {'contributor_ft': 'apple'}) self.assert_num_results(1, {'contributor_ft': 'apple', 'amount': '>|500'}) self.assert_num_results(0, {'contributor_ft': 'sauce', 'amount': '>|500'}) def test_full_text(self): self.create_contribution(contributor_name="Joe Smith") self.assert_num_results(0, {'contributor_ft': 'Bruce'}) self.assert_num_results(0, {'organization_ft': 'Smith'}) self.assert_num_results(0, {'contributor_ft': 'Bob Smith'}) self.assert_num_results(1, {'contributor_ft': 'Joe'}) self.assert_num_results(1, {'contributor_ft': 'Smith'}) self.assert_num_results(1, {'contributor_ft': 'Joe Smith'}) self.assert_num_results(1, {'contributor_ft': 'Smith Joe'}) self.assert_num_results(1, {'contributor_ft': 'joe smith'}) self.create_contribution(recipient_name="John Adams") self.create_contribution(recipient_name="Barry Adams") self.assert_num_results(2, {'recipient_ft': 'adams'}) self.assert_num_results(1, {'recipient_ft': 'john'}) self.assert_num_results(1, {'recipient_ft': 'barry'}) self.create_contribution(committee_name='Committee to Commit') self.assert_num_results(1, {'committee_ft': 'to'}) self.assert_num_results(1, {'committee_ft': 'commit'}) self.create_contribution(contributor_employer='Meany & Sons') self.create_contribution(organization_name="Jason Q. Meany") self.create_contribution(parent_organization_name="Jason Q. Meany Sr.") self.assert_num_results(3, {'organization_ft': 'Meany'}) self.assert_num_results(1, {'organization_ft': 'Meany Sr.'}) self.assert_num_results(1, {'organization_ft': 'Sons'}) def test_stop_words(self): self.create_contribution(contributor_name='Apple Association Inc') self.assert_num_results(0, {'contributor_ft': 'association'}) self.assert_num_results(0, {'contributor_ft': 'inc'}) self.assert_num_results(0, {'contributor_ft': 'assoc'}) self.assert_num_results(1, {'contributor_ft': 'apple'}) self.assert_num_results(1, {'contributor_ft': 'apple association inc'}) self.assert_num_results(1, {'contributor_ft': 'apple corp'}) def test_contributor_search(self): self.create_contribution(contributor_name='apple') self.create_contribution(contributor_employer='apple') self.create_contribution(organization_name='apple') self.create_contribution(parent_organization_name='apple') self.assert_num_results(4, {'contributor_ft': 'apple'}) self.assert_num_results(3, {'organization_ft': 'apple'}) def test_multiple_ft(self): """ See ticket #196. """ self.create_contribution(contributor_name='Billy Bob') self.create_contribution(contributor_name='Marry Sue') self.assert_num_results(1, {'contributor_ft': 'billy bob'}) self.assert_num_results(1, {'contributor_ft': 'marry sue'}) self.assert_num_results(0, {'contributor_ft': 'billy lee'}) self.assert_num_results(0, {'contributor_ft': 'billy lee|marry sue lee'}) self.assert_num_results(1, {'contributor_ft': 'billy lee|marry sue'}) self.assert_num_results(2, {'contributor_ft': 'billy|marry'}) self.assert_num_results(2, {'contributor_ft': 'billy bob|marry'}) self.assert_num_results(2, {'contributor_ft': 'billy bob|marry sue'}) def test_punctuation_ft(self): self.create_contribution(recipient_name='PG&E Corp') self.create_contribution(recipient_name='PGE Corp') self.create_contribution(recipient_name='PG & E Corp') self.create_contribution(recipient_name='PG and E Corp') self.assert_num_results(3, {'recipient_ft': 'pg&e'}) # in a perfect world this would be found by the searches above, # but Postgres unfortunately treats the ones above as two tokens and this as one. self.assert_num_results(1, {'recipient_ft': 'pge'}) # test parentheses self.create_contribution(recipient_name='John Doe (R)') self.create_contribution(recipient_name='John R James') self.assert_num_results(2, {'recipient_ft': 'John (R)'}) self.assert_num_results(2, {'recipient_ft': 'John R'}) # failing right now: 'and' should be a stop word, but isn't raise SkipTest self.assert_num_results(3, {'recipient_ft': 'pg and e'})
class Loader(object): ID_FIELDS = ('id', 'uuid') model = None field_handlers = {} def __init__(self, source, description, imported_by, **kwargs): self.log_func = sys.stderr.write log = kwargs.get('log') if log: self.log_func = log.warn # populate a fieldname/field mapping of model fields self.fields = {} for field in self.model._meta.fields: self.fields[field.name] = field # create a record template with field names from model # remove any fields that are typical ID fields self._new_record = dict([(f, None) for f in self.fields.iterkeys()]) for key in self.ID_FIELDS: if key in self._new_record: del self._new_record[key] # create a new Import instance and save reference self.import_session = Import(source=source, description=description, imported_by=imported_by) self.import_session.save() def new_record(self): """ Get an empty copy of record format. """ return self._new_record.copy() # return record template def load_records(self, records): """ Load a list of records. """ for record in records: self.load_record(record) def load_record(self, record): """ Load a dict record into a model and save to the database. """ # a model is required for loading if not self.model: raise ValueError, "model is required" # get a new or existing instance of model obj = self.get_instance(record) if obj is None: return if obj.pk: # object already exists, resolve issue of "conflicting" data self.resolve(record, obj) else: # object is new, copy over all fields self.copy_fields(record, obj) # assign Import reference and save obj.import_reference = self.import_session try: obj.save() except ValueError, e: #import pprint #self.log_func(pprint.pformat(record)) #self.log_func('Error saving record to database: %s' % repr(traceback.format_exc(e))) raise SkipRecordException(e) except: