def test_row_stream_query_set(self): """Build a RowStream from a Django query set.""" row_builder = DjangoRowBuilder( [DjangoField('a'), DjangoField('a', rename='b')]) row_stream = from_django(row_builder, MockQuerySet()) self.assertEqual(['a', 'b'], row_stream.fields) self.assertEqual([{'a': 'aa', 'b': 'aa'}], list(row_stream))
def test_django_field_tranform(self): """Transform the value in the Django field.""" field = DjangoField('tags', transform=extract_tags) self.assertEqual([{ 'id': 1, 'name': 'somename', 'hex_color': 'bbccdd' }], [dict(r) for r in field.get_value(MockRecord())])
def test_row_stream_limit_fields(self): row_builder = DjangoRowBuilder( [DjangoField('a'), DjangoField('name'), DjangoField('n')]) row_stream = from_django(row_builder, MockQuerySet(), values=['a', 'name']) self.assertEqual(['a', 'name'], row_stream.fields) self.assertEqual([{'a': 'aa', 'name': 'somename'}], list(row_stream))
def test_row_builder(self): """Build a row from several DjangoFields.""" row_builder = DjangoRowBuilder([ DjangoField('a'), DjangoField('a', rename='b'), DjangoField('tags', transform=extract_tags) ]) self.assertEqual(['a', 'b', 'tags'], row_builder.get_fields()) expected_tag_list = [{ 'id': 1, 'name': 'somename', 'hex_color': 'bbccdd' }] self.assertEqual({ 'a': 'aa', 'b': 'aa', 'tags': expected_tag_list }, row_builder.get_values(MockRecord()))
class PartnersDataSource(DataSource): partner_row_builder = DjangoRowBuilder([ DjangoField('id', rename='partner_id'), DjangoField('data_source'), DjangoField('last_action_time', rename='date'), DjangoField('name'), DjangoField('primary_contact.name', rename='primary_contact'), DjangoField('tags', transform=extract_tags), DjangoField('uri') ]) def run(self, data_type, company, filter_spec, values): return dispatch_run_by_data_type(self, data_type, company, filter_spec, values) def filtered_partners(self, company, filter_spec): qs_filtered = filter_spec.filter_partners(company) qs_distinct = qs_filtered.distinct() return qs_distinct def run_unaggregated(self, company, filter_spec, values): qs_filtered = filter_spec.filter_partners(company) qs_optimized = qs_filtered.prefetch_related('tags', 'primary_contact') partners = from_django(self.partner_row_builder, qs_optimized, values) return list(partners) def run_count_comm_rec_per_month(self, company, filter_spec, values): # Get a list of valid partners (according to our filter). qs_filtered = filter_spec.filter_partners(company) partner_ids = list(qs_filtered.values_list('id', flat=True)) # If there are no partners, we're done. if not partner_ids: return [] sql = count_comm_rec_per_month_per_partner_sql cursor = connection.cursor() cursor.execute(sql, {'partner_list': partner_ids}) count_stream = from_cursor(cursor) # Iterate over counts results. # Find year range and organize counts for later. min_year = None max_year = None db_counts = {} for record in count_stream: partner_id = record['partner_id'] year = record['year'] month = record['month'] comm_rec_count = record['comm_rec_count'] if year < min_year or min_year is None: min_year = year if year > max_year or max_year is None: max_year = year key = (partner_id, year, month) db_counts[key] = comm_rec_count # If there are no communication records, assume current year. if not db_counts: min_year = datetime.now().year max_year = min_year # Create full range of time slots with counts. # DB won't fill in zero's without a complex query. # Do this instead. full_counts = {} for partner_id in partner_ids: count_list = [] full_counts[partner_id] = count_list for year in xrange(min_year, max_year + 1): for month in xrange(1, 12 + 1): key = (partner_id, year, month) if key in db_counts: count = db_counts[key] count_list.append((year, month, count)) else: count_list.append((year, month, 0)) qs_optimized = qs_filtered.prefetch_related('tags', 'primary_contact') partners = from_django(self.partner_row_builder, qs_optimized) # Join the two streams joined_fields = partners.fields + ['year', 'month', 'comm_rec_count'] joined_data = [] for partner_record in partners: count_list = full_counts.get(partner_record['partner_id'], []) for count_tuple in count_list: year, month, count = count_tuple joined_record = dict(partner_record) joined_record.update({ 'year': year, 'month': month, 'comm_rec_count': count, }) joined_data.append(joined_record) joined = from_list(joined_fields, joined_data) return list(joined) def filter_type(self): return PartnersFilter def help(self, company, filter_spec, field, partial): return dispatch_help_by_field_name(self, company, filter_spec, field, partial) def help_city(self, company, filter_spec, partial): """Get help for the city field.""" modified_filter_spec = filter_spec.clone_without_city() partners_qs = modified_filter_spec.filter_partners(company) locations_qs = (Location.objects.filter( contacts__partner__in=partners_qs).filter(city__icontains=partial)) city_qs = locations_qs.values('city').distinct() return [{'value': c['city'], 'display': c['city']} for c in city_qs] def help_state(self, company, filter_spec, partial): """Get help for the state field.""" modified_filter_spec = filter_spec.clone_without_state() partners_qs = modified_filter_spec.filter_partners(company) locations_qs = (Location.objects.filter( contacts__partner__in=partners_qs).filter( state__icontains=partial)) state_qs = locations_qs.values('state').order_by('state').distinct() return [{ 'value': c['state'], 'display': states[c['state']] } for c in state_qs if c['state']] def help_tags(self, company, filter_spec, partial): """Get help for the tags field.""" partners_qs = PartnersFilter().filter_partners(company) tags_qs = (Tag.objects.filter(partner__in=partners_qs).filter( name__icontains=partial).values('name', 'hex_color').distinct()) return [{ 'value': t['name'], 'display': t['name'], 'hexColor': t['hex_color'], } for t in tags_qs] def help_uri(self, company, filter_spec, partial): """Get help for the uri field.""" partners_qs = filter_spec.filter_partners(company) uris_qs = (partners_qs.filter( uri__icontains=partial).values('uri').distinct()) return [{'value': c['uri'], 'display': c['uri']} for c in uris_qs] def help_data_source(self, company, filter_spec, partial): """Get help for the data_source field.""" partners_qs = filter_spec.filter_partners(company) data_sources_qs = (partners_qs.filter( data_source__icontains=partial).values('data_source').distinct()) return [{ 'value': c['data_source'], 'display': c['data_source'] } for c in data_sources_qs] def adorn_filter_items(self, company, found_filter_items): adorned = {} empty = PartnersFilter() if 'tags' in found_filter_items: adorned[u'tags'] = { found['value']: found for found in [result for result in self.help_tags(company, empty, '')] } return adorned def get_default_filter(self, data_type, company): filter_spec = PartnersFilter(date=DateRangeFilter( [datetime(2014, 1, 1), datetime.now()])) adorned = adorn_filter(company, self, filter_spec) return adorned
def test_django_field_tranform(self): """Transform the value in the Django field.""" field = DjangoField('tags', transform=extract_tags) self.assertEqual( [{'id': 1, 'name': 'somename', 'hex_color': 'bbccdd'}], [dict(r) for r in field.get_value(MockRecord())])
def test_django_field_rename(self): """Rename the Django field in the Row.""" field = DjangoField('a', rename='b') self.assertEqual('aa', field.get_value(MockRecord())) self.assertEqual('b', field.get_name())
def test_django_dotted_name_field_none(self): """Recover from None in a dot relation.""" field = DjangoField('contact.n', rename='n') self.assertEqual(None, field.get_value(MockRecord())) self.assertEqual('n', field.get_name())
def test_django_dotted_name_field(self): """Follow a django dot relation.""" field = DjangoField('contact.a', rename='c') self.assertEqual('aa', field.get_value(MockRecord())) self.assertEqual('c', field.get_name())
def test_django_field(self): """Simple field extraction works.""" field = DjangoField('a') self.assertEqual('aa', field.get_value(MockRecord())) self.assertEqual('a', field.get_name())