Exemplo n.º 1
0
  def _parse_show(self, link):
    LOG.debug("Fetching show: %s" % link)

    event_doc    = html_util.fetch_and_parse(link)

    event_detail = event_doc.get_element_by_id("mainColumn")

    show = Show()
    
    for performer in html_util.get_elements(event_detail, 'h1'):
      name = performer.text_content().strip(' \n\r\t')
      if name:
        show.performers.append(Performer(name))
      
    date_txt = html_util.get_first_element(event_detail, '.date').text_content()
    
    event_match = self.EVENT_URL.match(link)

    show.merge_key = event_match.group('page_id')
    show.venue     = self.venue()
    show.show_time = date_util.parse_date_time(date_txt).replace(hour = 21)
    
    LOG.debug('Date: %s' % show.date)
    
    show.resources.show_url      = link
    show.resources.resource_uris = self.resource_extractor.extract_resources(event_detail)

    for img_tag in event_detail.iter(tag = 'img'):
      if 'main' in img_tag.get('src'):
        show.resources.image_url = img_tag.get('src')
        
        break

    return show
Exemplo n.º 2
0
  def _parse_show(self, link):
    event_doc = html_util.fetch_and_parse(link, parse_500 = True)

    event_detail = html_util.get_first_element(event_doc,    ".event-detail")
    artist_info  = html_util.get_first_element(event_doc,    ".artist-boxes")
    
    date_txt     = html_util.get_first_element(event_detail, ".dates").text_content()
    
    performers = [] 
    
    for el in html_util.get_elements(event_doc, '.headliners'):
      for name in lang_util.parse_performers(el.text_content()):
        performers.append(Performer(name, headliner = True))

    for el in html_util.get_elements(event_doc, '.supports'):
      for name in lang_util.parse_performers(el.text_content()):
        performers.append(Performer(name, headliner = False))

    show = Show()

    show.merge_key  = link
    show.venue      = self.venue()
    show.performers = performers
    show.show_time  = date_util.parse_date_and_time(date_txt, html_util.get_first_element(event_detail, ".times").text_content())

    show.resources.show_url      = link
    show.resources.resource_uris = self.resource_extractor.extract_resources(event_detail, artist_info)

    img = html_util.get_first_element(event_detail, "img", optional = True)
    
    if img is not None:
      show.resources.image_url = img.get('src')

    return show
Exemplo n.º 3
0
  def _month_parser(self, request_date):
    month_url = '%scalendar/%s.html' % (self.BASE_URL, request_date.strftime('%b%y').lower())
    
    logger.debug('Parsing: %s' % month_url)

    doc = html_util.fetch_and_parse(month_url)
    
    main_table = html_util.get_first_element(doc, 'body > table')
    
    trs = main_table.getchildren()
    
    # Remove the "header" row from the table
    trs.pop(0)
    # Remove the "footer" row from the table
    trs.pop()
    
    # The remaining rows look as follows:
    # The Show Details/Time pairing might be repeated
    """
    <tr><td>Monday 4th</td></tr>
    <tr><td>
      <center>
        <p>Show Details</p>
        <p>Show Time</p>
      </center>
    </td></tr>
    """
    
    while trs:
      date_row, show_row = trs.pop(0).getchildren(), trs.pop(0).getchildren()
      
      for i, date_td in enumerate(date_row):
        # At the end of the month, the html doesn't always contain a corresponding table
        # cell in the show_row for each date in the date_row
        if i > len(show_row) - 1:
          break

        date_match = self.DATE_RE.match(date_td.text_content().strip())
        
        if date_match:
          show_date = request_date.replace(day = int(date_match.group('day')))

          show_td = show_row[i]
          
          p_list = show_td.cssselect('center > p')
      
          if len(p_list) % 2 == 0:
            while p_list:
              show_detail, show_time = p_list.pop(0), p_list.pop(0)

              show = self._parse_show(show_date, show_detail, show_time)
              
              if not (show.door_time or show.show_time):
                logger.warning('Unable to determine door or show time for show on %s, discarding' % show_date)
              elif not (show.title or len(show.performers) > 0):
                logger.warning('Unable to determine title or performers for show on %s, discarding' % show_date)
              else:
                yield show
Exemplo n.º 4
0
  def _get_parser(self):
    today = datetime.today().date()

    doc   = html_util.fetch_and_parse(self.CALENDAR_URL)

    main_section = doc.get_element_by_id("col-main")

    for event_detail in main_section.cssselect("#bellhouse"):
      yield self._parse_show(self.CALENDAR_URL, event_detail, today)
Exemplo n.º 5
0
  def _get_parser(self):    
    doc   = html_util.fetch_and_parse(self.BASE_URL)
    
    sidebar = doc.get_element_by_id("sidebar-right-1")
    
    for event_detail in sidebar.cssselect("div.widget.Image"):
      show = self._parse_show(event_detail)

      if show:
        yield show
Exemplo n.º 6
0
 def _month_parser(self, request_date):
   month_url = self.MONTH_URL % {'year': request_date.year, 'month': request_date.month}
   
   LOG.debug('Parsing shows from: %s' % month_url)
   
   doc = html_util.fetch_and_parse(month_url)
   
   for script in doc.iter(tag = 'script'):
     for match in self.EVENT_URL.finditer(script.text_content()):
       yield self._parse_show(match.group(0))
Exemplo n.º 7
0
  def _parse_show(self, link):
    logging.debug('Parsing show from: %s' % link)

    event_doc = html_util.fetch_and_parse(link)

    event        = html_util.get_first_element(event_doc, '.biglisting')
    img          = html_util.get_first_element(event, '.tonightinfo img', optional = True)

    date_el     = html_util.get_first_element(event, '.date')
    
    date_match  = self.DATE_RE.search(date_el.text_content())
    
    if date_match:
      date_txt = date_match.group(0)
    else:
      raise Exception('Unable to determine show date from: %s' % date_el.text_content())
    
    performers = [] 
    first_time = None

    for det in event.cssselect('.showpage-details'):
      header = None
      
      for child in det.getchildren():
        if child.tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'):
          header = child
          
      if header is None:
        logger.error('Unable to determine performer')
      else:
        time_txt = html_util.get_first_element(det, '.time').text_content()
      
        time_match = date_util.TIME_RE.search(time_txt)
      
        if time_match:
          first_time = time_txt = time_match.group('time')
        else:
          time_txt = None

        performers.append(Performer(header.text_content(), start_time = time_txt, headliner = header.tag in ('h1')))

    show = Show()

    show.merge_key  = link
    show.venue      = self.venue()
    show.performers = performers
    show.show_time  = date_util.parse_date_and_time(date_txt, first_time)

    show.resources.show_url      = link
    show.resources.resource_uris = self.resource_extractor.extract_resources(event)
    
    if img is not None:
      show.resources.image_url  = img.get('src')

    return show
Exemplo n.º 8
0
    def _parse_show(self, link):
        event_doc = html_util.fetch_and_parse(link)

        event_detail = event_doc.get_element_by_id("eventDetail")

        title_txt = []

        found_h_el = False

        # Start parsing when we find the first h* element
        # Stop parsing if we found an h* element, but then encounter anything else
        for el in event_detail.getchildren():
            if el.tag in ("h1", "h2"):
                found_h_el = True

                if el.text_content():
                    title_txt.append(el.text_content())
            elif found_h_el:
                break

        """
    <span id="timeDetail">
      Apr 24, 2010<br />
			upstairs<br />
    	Doors @ 7 PM<br/>
    	$15.00 Adv. / $20 at the Door<br />
    	<a href="http://www.deadcellentertainment.tix.com/Schedule.asp?OrganizationNumber=2690" target="_blank">
        <img src="/images/buyticket.png" alt="Purchase Tickets" />
      </a>
    </span>
    """
        time_el = event_detail.get_element_by_id("timeDetail")
        date_txt = time_el.text
        time_txt = time_el.text_content()

        performers = []

        show = Show()

        show.merge_key = link
        show.venue = self.venue()
        show.performers = [Performer(p) for p in lang_util.parse_performers("/".join(title_txt))]
        show.door_time = date_util.parse_show_time(date_txt, time_txt)
        show.show_time = date_util.parse_door_time(date_txt, time_txt)

        show.resources.show_url = link
        show.resources.resource_uris = self.resource_extractor.extract_resources(event_detail)

        img = html_util.get_first_element(event_detail, "img")

        if img is not None:
            show.resources.image_url = img.get("src")

        return show
Exemplo n.º 9
0
  def _month_parser(self, request_date):
    month_url = '%scalendar/%d-%d' % (self.BASE_URL, request_date.year, request_date.month)
    
    logger.debug('Parsing: %s' % month_url)

    doc = html_util.fetch_and_parse(month_url)
    
    main_table = html_util.get_first_element(doc, '.month-view table')
    
    for td in html_util.get_elements(main_table, 'td.has-events'):
      for show in self._parse_shows(request_date, td):
        yield show
Exemplo n.º 10
0
  def _get_parser(self):    
    doc     = html_util.fetch_and_parse(self.BASE_URL)

    events  = html_util.get_first_element(doc, '.defaultText')
    content = html_util.get_displayed_text_content(events).strip()

    for line in content.split('\n'):
      if self.SHOW_START_RE.match(line):
        show = self._parse_show(line)
        
        if show:
          yield show
Exemplo n.º 11
0
  def _get_parser(self):
    today = datetime.today().date()

    doc     = html_util.fetch_and_parse(self.CALENDAR_URL)
    
    by_date = { }
    
    for tr in doc.iter(tag = 'tr'):
      tds = list(tr.iter(tag = 'td'))
      
      date_txt = tds[0].text_content().strip()
      
      logger.debug('Checking if td el is a show: %s' % date_txt)
      
      if self.DATE_RE.match(date_txt):
        show = self._parse_show(tds)
        
        show_date = show[0].date()
        
        if show_date not in by_date:
          by_date[show_date] = []

        by_date[show_date].append(show)
          
    for date, performers in by_date.iteritems():
      performers.sort(lambda x,y: cmp(y[0], x[0]))
      
      show = Show()

      show.venue      = self.venue()
      show.performers = []
      
      all_resources   = []
      
      for time, name, resources in performers:
        all_resources.extend(resources)
        
        if not show.show_time:
          show.show_time = time
        else:
          show.show_time = min(show.show_time, time)

        show.performers.append(Performer(name, start_time = time.strftime('%I:%M')))

      show.resources.resource_uris = list(set(all_resources))
      
      yield show
Exemplo n.º 12
0
  def _parse_show(self, link):
    raw_url   = self.raw_url(link)
    
    match     = self.EVENT_ID.search(link)
    
    if not match:
      raise Exception("Unable to locate event id in: %s" % link)
      
    event_id = match.group(0)
    
    logging.debug('Fetching show info: %s' % link)

    event_doc = html_util.fetch_and_parse(link)
    
    show_el   = html_util.get_first_element(event_doc, '#content')

    header_el = html_util.get_first_element(show_el, 'h1')
    
    header_match = self.HEADER_PARSE.search(header_el.text_content())
    
    if not header_match:
      raise Exception("Unable to parse header: %s" % header_el.text_content())
      
    date_txt = header_match.group('date').strip()
    title    = header_match.group('title').strip()
    
    if date_txt.lower().startswith('tonight'):
      date_txt = datetime.today().date().strftime('%F')

    img   = html_util.get_first_element(show_el, 'img', optional = True)

    show = Show()
    
    show.performers = [Performer(p) for p in lang_util.parse_performers(title)]
    show.show_time  = date_util.parse_date_and_time(date_txt, None)

    show.merge_key = event_id
    show.venue     = self.venue()
  
    show.resources.show_url      = link
    show.resources.resource_uris = self.resource_extractor.extract_resources(show_el)
    
    if img is not None:
      show.resources.image_url = img.get('src')
      
    return show
Exemplo n.º 13
0
  def _get_parser(self):
    doc = html_util.fetch_and_parse(self.CALENDAR_URL)
    
    main_section = None
    
    for tr in doc.iter(tag = 'tr'):
      tds = tr.getchildren()
      
      if len(tds) != 2:
        continue
        
      match = self.DATE_RE.search(tds[0].text_content())
      
      if match:
        show =  self._parse_show(match.group('date'), tds[1])

        if show:
          yield show
Exemplo n.º 14
0
  def _parse_show(self, url, section):
    doc = html_util.fetch_and_parse(url)

    show_el  = html_util.get_first_element(doc, '#detailPage')
    date_txt = html_util.get_first_element(show_el, 'time.dtstart').get('datetime')

    title = html_util.get_first_element(section, 'h4').text_content()

    show = Show()

    show.merge_key  = url
    show.venue      = self.venue()
    show.performers = [Performer(p) for p in lang_util.parse_performers(title)]
    show.show_time  = date_util.parse_date_time(date_txt)

    show.resources.show_url      = url
    show.resources.resource_uris = self.resource_extractor.extract_resources(section, show_el)

    return show
Exemplo n.º 15
0
  def _parse_show(self, link):
    event_doc = html_util.fetch_and_parse(link)

    event        = html_util.get_first_element(event_doc, '#tfly-center-column-wide')
    event_detail = html_util.get_first_element(event,     '#details')
    """
    <div class="info"> 
      Sat, May 22, 2010<br /> 
      Doors: 6:00 PM / Show: 7:00 PM&nbsp;<br /> 
      $5.00<br /> 
    </div> 
    """
    event_info   = html_util.get_first_element(event_detail, ".info")

    date_txt     = event_info.text
    time_txt     = event_info.getchildren()[0].tail

    img          = html_util.get_first_element(event_detail, "img")
    
    performers = [] 

    for tag in ('h1', 'h2', 'h3', 'h4'):
      for h in event_detail.iter(tag = tag):
        performers.extend(self._parse_performers(h))

    show = Show()

    show.merge_key  = link
    show.venue      = self.venue()
    show.performers = performers
    show.show_time  = date_util.parse_show_time(date_txt, time_txt)
    show.door_time  = date_util.parse_door_time(date_txt, time_txt)

    show.resources.show_url      = link
    show.resources.resource_uris = self.resource_extractor.extract_resources(event)
    
    if img is not None:
      show.resources.image_url  = img.get('src')

    return show
Exemplo n.º 16
0
  def _parse_artist(self, a):
    time_match = self.TIME_RE.match(a.text_content())
    
    if time_match:
      start_time = time_match.group('time')
      name       = self.TIME_RE.sub('', a.text_content())
    else:
      start_time = None
      name       = a.text_content()

    link = a.get('href')
    
    if link:
      artist_doc = html_util.fetch_and_parse(link)
      
      artist_el  = html_util.get_first_element(artist_doc, '#bleh')
    else:
      artist_el = None

    logging.debug('Artist (%s) name: %s from (%s)' % (start_time, name, a.text_content()))
      
    return (name, start_time, artist_el)
Exemplo n.º 17
0
  def _parse_show(self, link):
    event_doc    = html_util.fetch_and_parse(link)

    event_detail = event_doc.get_element_by_id("detail")

    show = Show()

    strong_iter = event_detail.iter(tag = 'strong')

    date_tag, title_tag, blank_tag, desc_tag = strong_iter.next(), strong_iter.next(), strong_iter.next(), strong_iter.next()
    
    date_txt = date_tag.text_content()
    
    if desc_tag.getnext().tail:
      time_match = self.TIME_RE.search(desc_tag.getnext().tail)
    else:
      time_match = None
    
    if time_match:
      time_txt = time_match.group('time')
    else:
      time_txt = None

    show.merge_key = link
    show.venue     = self.venue()
    show.title     = title_tag.text_content()
    show.show_time = date_util.parse_date_and_time(date_txt, time_txt)
    
    show.resources.show_url      = link
    show.resources.resource_uris = self.resource_extractor.extract_resources(event_detail)

    for img_tag in event_detail.iter(tag = 'img'):
      if 'main' in img_tag.get('src'):
        show.resources.image_url = img_tag.get('src')
        
        break

    return show
Exemplo n.º 18
0
  def _parse_show(self, link, show_section):
    show_doc    = html_util.fetch_and_parse(link)

    show_detail = html_util.get_first_element(show_doc, "#content .event-detail")

    date_txt    = html_util.get_first_element(show_detail, ".dates").text_content()
    time_txt    = html_util.get_first_element(show_detail, ".times").text_content()
    sold_out    = html_util.get_first_element(show_detail,  '.sold-out', optional = True)
    image       = html_util.get_first_element(show_detail,  'img',       optional = True)
    
    # The image we want is generally the first one, but if the layout changes this may break
    if image is not None:
      image_url = image.get('src')
    else:
      image_url = None
    
    performers = [] 
    
    for tag in ('h1', 'h2', 'h3'):
      for p in show_detail.iter(tag = tag):
        if p.text_content():    
          performers.extend(self._parse_performers(p))

    show = Show()

    show.merge_key               = link
    show.venue                   = self.venue()
    show.performers              = performers
    show.door_time               = date_util.parse_door_time(date_txt, time_txt)
    show.show_time               = date_util.parse_show_time(date_txt, time_txt)
    show.soldout                 = sold_out is not None

    show.resources.show_url      = link
    show.resources.image_url     = image_url
    show.resources.resource_uris = self.resource_extractor.extract_resources(show_detail)

    return show
Exemplo n.º 19
0
  def _parse_show(self, link, show_section):
    show_doc    = html_util.fetch_and_parse(link)

    show_detail = show_doc.get_element_by_id("content")
    
    title       = html_util.get_first_element(show_detail, '.title').text

    date_txt    = html_util.get_first_element(show_section, '.date').text
    image_url   = html_util.get_first_element(show_detail,  '.left-view-header img').get('src')
    
    performers = []
    
    performer_detail = html_util.get_first_element(show_detail, '.performers')
    performer_urls   = []
    
    for anchor in performer_detail.iter(tag = 'a'):
      performers.extend(self._parse_performers(anchor))
      
      if self.IS_ARTIST_URL_RE.match(anchor.get('href')):
        performer_urls.append(anchor.get('href'))
        
    resource_sections = [show_section, show_detail]
    
    for url in performer_urls:
      resource_sections.extend(self.fetch_performer_content(url))
    
    show = Show()

    show.merge_key               = link
    show.venue                   = self.venue()
    show.performers              = performers
    show.show_time               = date_util.parse_date_time(date_txt)
    show.resources.show_url      = link
    show.resources.image_url     = image_url
    show.resources.resource_uris = self.resource_extractor.extract_resources(*resource_sections)

    return show
Exemplo n.º 20
0
    def _parse_show(self, link):
        event_doc = html_util.fetch_and_parse(link)
        match = self.IS_EVENT.match(link)

        event_id = int(match.group("event_id"))
        event_detail = html_util.get_first_element(event_doc, ".tfly-event-id-%d" % event_id)

        date_txt = html_util.get_first_element(event_doc, ".dates").text_content()
        time_txt = html_util.get_first_element(event_doc, ".times").text_content()

        img = html_util.get_first_element(event_detail, "img")

        performers = []

        for p in html_util.get_elements(event_detail, ".headliners"):
            performers.append(Performer(p.text_content(), headliner=True))

        for p in html_util.get_elements(event_detail, ".supports"):
            for pi in lang_util.parse_performers(p.text_content()):
                performers.append(Performer(pi, headliner=False))

        show = Show()

        show.merge_key = link
        show.venue = self.venue()
        show.performers = performers
        show.show_time = date_util.parse_show_time(date_txt, time_txt)
        show.door_time = date_util.parse_door_time(date_txt, time_txt)

        show.resources.show_url = link
        show.resources.resource_uris = self.resource_extractor.extract_resources(event_detail)

        if img is not None:
            show.resources.image_url = img.get("src")

        return show
Exemplo n.º 21
0
 def _get_parser(self):
   doc = html_util.fetch_and_parse(self.CALENDAR_URL)
   
   for el in doc.cssselect('#event-posts .post'):
     yield self._parse_show(el)
Exemplo n.º 22
0
 def fetch_performer_content(self, url):
   doc = html_util.fetch_and_parse(url)
   
   return doc.cssselect('#content')
Exemplo n.º 23
0
 def _get_parser(self):    
   doc   = html_util.fetch_and_parse(self.BASE_URL)
   
   for event_detail in doc.cssselect(".shows .dates"):
     yield self._parse_show(event_detail)