def __init__(self, base_url = awards_base_url, search_url = ""): Scraper.__init__(self, base_url, search_url) self.file = open('academy_awards.csv', 'wb') self.writer = csv.writer(self.file, delimiter='\t') self.writer.writerow(['Year', 'Category', 'Won', 'FilmName', 'ActorDirectorName']) self.soup = self.connect(base_url) self.next_record = '1'
def __init__(self, geckodriver_path, key_api, keyword_job_title, keyword_job_location): """ Sets up the default URL. """ Scraper.__init__(self, geckodriver_path) self.config = configparser.ConfigParser(interpolation=None) self.config.read('Constants') self.key_api = key_api self.keyword_job_title = keyword_job_title self.keyword_job_location = keyword_job_location self.geckodriver_path = geckodriver_path
def __init__(self): Scraper.__init__(self) api_key = self.config["youtube"]["api_key"] self.url = "https://www.googleapis.com/youtube/v3/search" self.params = { "order": "date", "maxResults": 10, "channelId": "UCH1dpzjCEiGAt8CXkryhkZg", "key": api_key, "type": "upload", "part": "snippet" }
def __init__(self): Scraper.__init__(self) self.url = "https://go.berniesanders.com/page/event/search_results" self.html = HTMLParser() self.params = { 'orderby': 'zip_radius', 'zip_radius[1]': '6000', 'zip_radius[0]': '78218', 'radius_unit': 'mi', 'country': 'US', 'format': 'json' } self.map = { "id": "original_id", "start_dt": "start_time" }
def __init__(self): Scraper.__init__(self) c = self.config["bsd"] self.html = HTMLParser() self.call_path = "/page/api/event/search_events" self.params = { "api_ver": "2", "api_id": c["api_id"], "api_ts": str(int(time.time())) } self.signed_params = self.sign_params(c["api_secret"]) self.url = "".join([ c["endpoint"], self.call_path, "?", self.signed_params ]) self.map = { "event_id": "original_id", "start_dt": "start_time" }
def __init__(self): """ Initially in this function we open up the config file, then get the serverip and port from the config file. """ Scraper.__init__(self) try: with open("../config/server-clients.json") as jsonf: self.cinfo = json.load(jsonf) except: print("Could not find/load server-clients.json config file") exit() try: self.serverip = self.cinfo["serverip"] self.serverport = self.cinfo["serverport"] except: print("server-clients config file not properly formatted") exit() self.socket = self.createSocket()
def __init__(self, *args, **kwargs): Scraper.__init__(self, *args, **kwargs) self.shopify_app_handle = kwargs.get("shopify_app_handle", None) self.shopify_app_url = kwargs.get("shopify_app_url", None) self.tloc = threading.local() self.shop_urls = []
def __init__(self, delay): Scraper.__init__(self, "CZC", "https://www.czc.cz", delay)
def __init__(self): Scraper.__init__(self) self.root = 'http://bitsnoop.com' self.site = "bitsnoop.com"
def __init__(self): Scraper.__init__(self) self.site = "monova.com"
def __init__(self): Scraper.__init__(self) self.url = "https://berniesanders.com/issues/feed/" self.html = HTMLParser()
def __init__(self): Scraper.__init__(self) self.url = "https://berniesanders.com/daily/" self.html = HTMLParser()
def __init__(self, delay): Scraper.__init__(self, "TSBohemia", "https://www.tsbohemia.cz", delay)
def __init__(self): Scraper.__init__(self) self.root = 'http://kat.ph' self.site = "kat.ph"
def __init__(self): Scraper.__init__(self) self.site = "fenopy.se"
def __init__(self): self.username = "******" self.password = "******" self.mySQLactive = [] Scraper.__init__(self)
def __init__(self, *args, **kwargs): # initialize our super classes Scraper.__init__(self) #, prefetch=self.prefetch) # set regex's self.__set_regex()