class ProductDetails(demiurge.Item): description = demiurge.TextField( selector='p.card-product-detail-description') location = demiurge.TextField(selector='div.card-product-detail-location') class Meta: selector = 'div.card-product-detail'
class Products(demiurge.Item): title = demiurge.TextField(selector='a.product-info-title') price = demiurge.TextField(selector='span.product-info-price') url = demiurge.AttributeValueField( selector='div.card-product-product-info a.product-info-title', attr='href') class Meta: selector = 'div.card-product'
class SteamCartItem(demiurge.Item): appid = demiurge.AttributeValueField(attr='data-ds-appid', selector='div.cart_row') packageid = demiurge.AttributeValueField(attr='data-ds-packageid', selector='div.cart_row') title = demiurge.TextField(selector='div.cart_item_desc a:first') price = demiurge.TextField(selector='div.cart_item_price div.price:last') remove_button = demiurge.AttributeValueField(attr='href', selector='a.remove_link') class Meta: selector = 'div.cart_row'
class SteamCart(demiurge.Item): count = demiurge.TextField(selector='span#cart_item_count_value') subtotal = demiurge.TextField(selector='div#cart_price_total') balance = demiurge.TextField(selector='a#header_wallet_balance') cart_status_message = demiurge.TextField(selector='div.cart_status_message') cart_checkout_button = demiurge.AttributeValueField(selector='a.continue:eq(0)', attr='href') items = demiurge.RelatedItem(SteamCartItem, selector='div.cart_item_list')
class ItemPriceDetailRow (demiurge.Item): #url = demiurge.AttributeValueField (selector='.pslires .psliimg img', attr='href') seller = demiurge.TextField (selector="td.os-seller-name") rating = demiurge.AttributeValueField (selector="td.os-rating-col span div[aria-label]", attr="aria-label" ) votes = demiurge.TextField (selector="td.os-rating-col a") details = demiurge.TextField (selector="td.os-details-col") price = demiurge.TextField (selector="td.os-price-col span.os-base_price") total = demiurge.TextField (selector="td.os-total-col") class Meta: selector = 'tr.os-row'
class GoogleShoppingItem(demiurge.Item): #url = demiurge.AttributeValueField (selector='td:eq(2) a:eq(1)', attr='href') #name = demiurge.TextField(selector='td:eq(2) a:eq(2)') #size = demiurge.TextField(selector='td:eq(3)') heading = demiurge.TextField(selector='.pslmain h3') image = demiurge.AttributeValueField(selector='.pslimg img', attr='alt') price = demiurge.TextField(selector='.pslline price') main = demiurge.TextField(selector='.pslmain') class Meta: selector = '.pslicont' #'table.maintable:gt(0) tr:gt(0)' base_url = 'https://www.google.com' #'http://www.mininova.org'
class GoogleShoppingItem (demiurge.Item): #url = demiurge.AttributeValueField (selector='td:eq(2) a:eq(1)', attr='href') #name = demiurge.TextField(selector='td:eq(2) a:eq(2)') #size = demiurge.TextField(selector='td:eq(3)') heading = demiurge.TextField(selector='.pslires h3') #'.pslmain h3') image = demiurge.AttributeValueField (selector='.pslires .psliimg img', attr='alt') link = demiurge.AttributeValueField (selector='.pslires h3 a', attr='href') price = demiurge.TextField(selector='.pslires div b:contains("$")') #'.pslline price') # best price, usually 30-40% lower than avg rows = demiurge.RelatedItem (ItemPriceDetailRow, selector=".pslires h3 a", attr="href") #main = demiurge.TextField(selector='.pslmain') class Meta: #selector = '.pslicont' #'table.maintable:gt(0) tr:gt(0)' selector = '.pslires' #'table.maintable:gt(0) tr:gt(0)' base_url = 'https://www.google.com' #'http://www.mininova.org'
class TestItem(demiurge.Item): label = demiurge.TextField(selector='.link') url = demiurge.AttributeValueField(selector='.link', attr='href') class Meta: base_url = 'http://localhost' selector = "p.p_with_link" extra_attribute = 'value'
class StandingsRow(demiurge.Item): team = demiurge.TextField(selector='td.team') pts = demiurge.TextField(selector='td.puntos') p = demiurge.TextField(selector='td.hidden-xs:eq(0)') w = demiurge.TextField(selector='td.hidden-xs:eq(1)') d = demiurge.TextField(selector='td.hidden-xs:eq(2)') l = demiurge.TextField(selector='td.hidden-xs:eq(3)') f = demiurge.TextField(selector='td.hidden-xs:eq(4)') a = demiurge.TextField(selector='td.hidden-xs:eq(5)') gd = demiurge.TextField(selector='td.hidden-xs:eq(6)') class Meta: selector = 'tr.linea' encoding = 'utf-8'
class TestIndexItem(demiurge.Item): title = demiurge.TextField(selector='h1') items_following_link = demiurge.RelatedItem(TestItem, selector='a.link', attr='href') next_page = demiurge.RelatedItem('self', selector='a.next', attr='href') class Meta: base_url = 'http://localhost'
class TestItemWithClean(demiurge.Item): label = demiurge.TextField(selector='.link') def clean_label(self, value): return value.upper() class Meta: base_url = 'http://localhost' selector = 'p'
class SteamGift(demiurge.Item): gift_javascript = demiurge.TextField( selector='div:last-child div.pending_gift_leftcol script') from_link = demiurge.AttributeValueField( selector='div:last-child div.pending_gift_rightcol p:first-child a', attr='href') from_username = demiurge.TextField( selector='div:last-child div.pending_gift_rightcol p:first-child a') accept_button = demiurge.AttributeValueField( selector= 'div:last-child div.pending_gift_rightcol div.gift_controls div.gift_controls_buttons div.btn_medium:first', attr='onclick') class Meta: selector = 'div.pending_gift'
class TestItemWithFieldCoercion(demiurge.Item): label = demiurge.TextField(selector='a.page', coerce=int) url = demiurge.AttributeValueField(selector='a.page', attr='href', coerce=bool) def clean_label(self, value): # remove the Page prefix return value[5:] class Meta: base_url = 'http://localhost' selector = 'div.pagination'
class Round(demiurge.Item): _css_class = demiurge.AttributeValueField(attr='class') title = demiurge.TextField(selector='div.subHeader') matches = demiurge.RelatedItem(Match) class Meta: selector = 'div.fase div.fecha' encoding = 'utf-8' base_url = BASE_URL + 'fixture.html' @property def is_current(self): return 'show' in self._css_class
class RelegationRow(demiurge.Item): team = demiurge.TextField(selector='td.team') average = demiurge.TextField(selector='td.promediodescenso') before1 = demiurge.TextField(selector='td.hidden-xs:eq(0)') before2 = demiurge.TextField(selector='td.hidden-xs:eq(1)') before3 = demiurge.TextField(selector='td.hidden-xs:eq(2)') current = demiurge.TextField(selector='td.puntosactual') pts = demiurge.TextField(selector='td.puntosdescenso') p = demiurge.TextField(selector='td.jugadosdescenso') class Meta: selector = 'tr.linea' encoding = 'utf-8'
class MatchData(demiurge.Item): home = demiurge.TextField(selector='div.local div.equipo') away = demiurge.TextField(selector='div.visitante div.equipo') home_goals = demiurge.TextField(selector='div.local div.resultado') away_goals = demiurge.TextField(selector='div.visitante div.resultado') status = demiurge.TextField(selector='div.detalles div.estado') _date = demiurge.TextField(selector='div.detalles div.dia') _time = demiurge.TextField(selector='div.detalles div.hora') class Meta: selector = 'div.mc-matchContainer' encoding = 'utf-8' base_url = ('http://staticmd1.lavozdelinterior.com.ar/sites/default/' 'files/Datafactory/html/v3/htmlCenter/data/deportes/' 'futbol/primeraa/pages/es/fixture.html') @property def is_finished(self): return self.status.lower() == 'finalizado' @property def is_suspended(self): return self.status.lower() == 'suspendido' @property def in_progress(self): return self.status.lower() == 'en juego' @property def when(self): if self._time is None: return None if self._time.startswith('-'): match_time = "00:00" else: match_time = self._time[:5] date_and_time = "%s %s" % (self._date, match_time) value = datetime.strptime(date_and_time, "%d-%m-%Y %H:%M") return make_aware(value, get_default_timezone())
class Match(demiurge.Item): home = demiurge.TextField(selector='div.local div.equipo') away = demiurge.TextField(selector='div.visitante div.equipo') home_goals = demiurge.TextField(selector='div.local div.resultado') away_goals = demiurge.TextField(selector='div.visitante div.resultado') status = demiurge.TextField(selector='div.detalles div.estado') _date = demiurge.TextField(selector='div.detalles div.dia') _time = demiurge.TextField(selector='div.detalles div.hora') @property def is_finished(self): return self.status.lower() == 'finalizado' @property def in_progress(self): return self.status.lower() in [ '1er tiempo', 'entretiempo', '2do tiempo' ] @property def datetime(self): if self._time is None: return None if self._time.startswith('-'): match_time = "00:00" else: match_time = self._time[:5] date_and_time = "%s %s" % (self._date, match_time) value = datetime.strptime(date_and_time, "%d-%m-%Y %H:%M") return value class Meta: selector = 'div.mc-matchContainer' encoding = 'utf-8'