class Arbiter(object):
"""
"""
def __init__(self, rules):
"""
"""
self.rules = rules
self.cachedcategorytree = CachedCategoryTree()
def apply_rules(self, event, source, external_categories):
"""
This function has no side effects.
It applies all its rules to the event object and returns raw categories
These raw categories may need to be filtered down (using the
concrete_category and abstract_category functions.)
"""
raw_abstracts = []
raw_concretes = []
for rule in self.rules:
concretes, abstracts = rule.classify(event, source, external_categories=external_categories)
# Special handling for event classification rules.
# Could later put this into a special class and abstract out
# the common theme into a BaseChain class.
concretes = [category for category in concretes if category]
if concretes and not raw_concretes:
raw_concretes = concretes
raw_abstracts.extend(abstracts)
return (raw_concretes, raw_abstracts)
def abstract_categories(self, event, source, ext_category_xids=None):
return self.apply_rules(event, source, ext_category_xids)[1]
def concrete_categories(self, event, source, ext_category_xids=None):
categories = self.apply_rules(event, source, ext_category_xids)[0]
return self.cachedcategorytree.deepest_category(categories)
class ImportScrapeData(object):
def __init__(self):
self.x_event_categories = defaultdict(lambda: [])
self.places_by_xid = {}
self.occurrences_by_xid = {}
self.prices_by_xid = {}
self.events_by_xid = {}
self.x_categories_by_xid = {}
self.ctree = CachedCategoryTree()
def run(self):
for x_category in x_models.Category.objects.all():
self.x_categories_by_xid[x_category.id] = x_category
for x_event_category in x_models.EventCategory.objects.all():
self.add_event_category(x_event_category)
for x_loc in x_models.Location.objects.all():
self.add_place(x_loc)
for x_event in x_models.Event.objects.all():
self.add_event(x_event)
for x_occurrence in x_models.Occurrence.objects.all():
self.add_occurrence(x_occurrence)
for x_price in x_models.Price.objects.all():
self.add_price(x_price)
def add_price(self, x_price):
occurrence = self.occurrences_by_xid.get(x_price.occurrence_id)
if occurrence:
price, created = Price.objects.get_or_create(
quantity=x_price.quantity,
units=x_price.units,
remark=x_price.remark or '',
occurrence=occurrence
)
self.prices_by_xid[x_price.id] = price
def add_occurrence(self, x_occurrence):
event = self.events_by_xid.get(x_occurrence.event_id)
if event:
place = self.places_by_xid[x_occurrence.location_id]
occurrence, created = Occurrence.objects.get_or_create(
event=event,
place=place,
# one_off_place= models.CharField(max_length=200, blank=True),
start_date=x_occurrence.start_date,
start_time=x_occurrence.start_time,
end_date= x_occurrence.end_date,
end_time= x_occurrence.end_time,
# is_all_day= models.BooleanField(default=False)
)
self.occurrences_by_xid[x_occurrence.id] = occurrence
def add_event(self, x_event):
x_categories = self.x_event_categories.get(x_event.id) # TODO will be lost
if x_categories:
x_category_xids = map(lambda x_category: x_category.xid, x_categories)
category_ids = ExternalCategory.objects \
.values_list('category_id', flat=True) \
.filter(xid__in=x_category_xids)
concrete_category = self.ctree.deepest_category(
self.ctree.get(id=category_id) for category_id in category_ids
)
event, created = Event.objects.get_or_create(
xid=x_event.guid,
title=x_event.title,
slug=slugify(x_event.title)[:50],
description=x_event.description or '',
# submitted_by = models.ForeignKey(User, blank=True, null=True)
# created = models.DateTimeField(auto_now_add=True)
# modified = models.DateTimeField(auto_now=True)
url=x_event.url or '',
image_url=x_event.image_url or '',
# video_url = models.URLField(verify_exists=False, max_length=200, blank=True)
concrete_category=concrete_category,
# categories = models.ManyToManyField(Category, related_name='events_abstract', verbose_name=_('abstract categories'))
)
self.events_by_xid[x_event.id] = event
def add_place(self, x_location):
city, created = City.objects.get_or_create(
city=x_location.city,
state=x_location.state,
slug=slugify(u'-'.join((x_location.city, x_location.state)))
)
point, created = Point.objects.get_or_create(
latitude=x_location.latitude,
longitude=x_location.longitude,
address=x_location.address,
city=city,
zip=x_location.zipcode,
country='US'
)
place, created = Place.objects.get_or_create(
point=point,
# prefix = '',
title=x_location.title,
slug=slugify(x_location.title)[:50],
# nickname=models.CharField(_('nickname'), blank=True, max_length=100),
# unit=models.CharField(_('unit'), blank=True, max_length=100, help_text='Suite or Apartment #'),
phone=x_location.phone if x_location.phone and (not ':' in x_location.phone) else None or '',
url=x_location.url or x_location.guid, # FIXME source (villagevoice) specific
image_url=x_location.image_url or ''
# email=models.EmailField(_('email'), blank=True),
# description = models.TextField(_('description'), blank=True),
# status = models.IntegerField(_('status'), choices=STATUS_CHOICES, default=1)
# created = models.DateTimeField(auto_now_add=True)
# modified = models.DateTimeField(auto_now=True)
# place_types = models.ManyToManyField(PlaceType, blank=True)
)
self.places_by_xid[x_location.id] = place
def add_event_category(self, x_event_category):
x_category = self.x_categories_by_xid[x_event_category.category_id]
self.x_event_categories[x_event_category.event_id].append(x_category)
class ImportScrapeData(object):
def __init__(self):
self.x_event_categories = defaultdict(lambda: [])
self.places_by_xid = {}
self.occurrences_by_xid = {}
self.prices_by_xid = {}
self.events_by_xid = {}
self.x_categories_by_xid = {}
self.ctree = CachedCategoryTree()
def run(self):
for x_category in x_models.Category.objects.all():
self.x_categories_by_xid[x_category.id] = x_category
for x_event_category in x_models.EventCategory.objects.all():
self.add_event_category(x_event_category)
for x_loc in x_models.Location.objects.all():
self.add_place(x_loc)
for x_event in x_models.Event.objects.all():
self.add_event(x_event)
for x_occurrence in x_models.Occurrence.objects.all():
self.add_occurrence(x_occurrence)
for x_price in x_models.Price.objects.all():
self.add_price(x_price)
def add_price(self, x_price):
occurrence = self.occurrences_by_xid.get(x_price.occurrence_id)
if occurrence:
price, created = Price.objects.get_or_create(
quantity=x_price.quantity,
units=x_price.units,
remark=x_price.remark or '',
occurrence=occurrence)
self.prices_by_xid[x_price.id] = price
def add_occurrence(self, x_occurrence):
event = self.events_by_xid.get(x_occurrence.event_id)
if event:
place = self.places_by_xid[x_occurrence.location_id]
occurrence, created = Occurrence.objects.get_or_create(
event=event,
place=place,
# one_off_place= models.CharField(max_length=200, blank=True),
start_date=x_occurrence.start_date,
start_time=x_occurrence.start_time,
end_date=x_occurrence.end_date,
end_time=x_occurrence.end_time,
# is_all_day= models.BooleanField(default=False)
)
self.occurrences_by_xid[x_occurrence.id] = occurrence
def add_event(self, x_event):
x_categories = self.x_event_categories.get(
x_event.id) # TODO will be lost
if x_categories:
x_category_xids = map(lambda x_category: x_category.xid,
x_categories)
category_ids = ExternalCategory.objects \
.values_list('category_id', flat=True) \
.filter(xid__in=x_category_xids)
concrete_category = self.ctree.deepest_category(
self.ctree.get(id=category_id) for category_id in category_ids)
event, created = Event.objects.get_or_create(
xid=x_event.guid,
title=x_event.title,
slug=slugify(x_event.title)[:50],
description=x_event.description or '',
# submitted_by = models.ForeignKey(User, blank=True, null=True)
# created = models.DateTimeField(auto_now_add=True)
# modified = models.DateTimeField(auto_now=True)
url=x_event.url or '',
image_url=x_event.image_url or '',
# video_url = models.URLField(verify_exists=False, max_length=200, blank=True)
concrete_category=concrete_category,
# categories = models.ManyToManyField(Category, related_name='events_abstract', verbose_name=_('abstract categories'))
)
self.events_by_xid[x_event.id] = event
def add_place(self, x_location):
city, created = City.objects.get_or_create(city=x_location.city,
state=x_location.state,
slug=slugify(u'-'.join(
(x_location.city,
x_location.state))))
point, created = Point.objects.get_or_create(
latitude=x_location.latitude,
longitude=x_location.longitude,
address=x_location.address,
city=city,
zip=x_location.zipcode,
country='US')
place, created = Place.objects.get_or_create(
point=point,
# prefix = '',
title=x_location.title,
slug=slugify(x_location.title)[:50],
# nickname=models.CharField(_('nickname'), blank=True, max_length=100),
# unit=models.CharField(_('unit'), blank=True, max_length=100, help_text='Suite or Apartment #'),
phone=x_location.phone if x_location.phone and
(not ':' in x_location.phone) else None or '',
url=x_location.url
or x_location.guid, # FIXME source (villagevoice) specific
image_url=x_location.image_url or ''
# email=models.EmailField(_('email'), blank=True),
# description = models.TextField(_('description'), blank=True),
# status = models.IntegerField(_('status'), choices=STATUS_CHOICES, default=1)
# created = models.DateTimeField(auto_now_add=True)
# modified = models.DateTimeField(auto_now=True)
# place_types = models.ManyToManyField(PlaceType, blank=True)
)
self.places_by_xid[x_location.id] = place
def add_event_category(self, x_event_category):
x_category = self.x_categories_by_xid[x_event_category.category_id]
self.x_event_categories[x_event_category.event_id].append(x_category)
