def __init__(self, base_url = awards_base_url, search_url = ""):
Scraper.__init__(self, base_url, search_url)
self.file = open('academy_awards.csv', 'wb')
self.writer = csv.writer(self.file, delimiter='\t')
self.writer.writerow(['Year', 'Category', 'Won', 'FilmName', 'ActorDirectorName'])
self.soup = self.connect(base_url)
self.next_record = '1'
def __init__(self, geckodriver_path, key_api, keyword_job_title, keyword_job_location):
"""
Sets up the default URL.
"""
Scraper.__init__(self, geckodriver_path)
self.config = configparser.ConfigParser(interpolation=None)
self.config.read('Constants')
self.key_api = key_api
self.keyword_job_title = keyword_job_title
self.keyword_job_location = keyword_job_location
self.geckodriver_path = geckodriver_path
def __init__(self):
Scraper.__init__(self)
api_key = self.config["youtube"]["api_key"]
self.url = "https://www.googleapis.com/youtube/v3/search"
self.params = {
"order": "date",
"maxResults": 10,
"channelId": "UCH1dpzjCEiGAt8CXkryhkZg",
"key": api_key,
"type": "upload",
"part": "snippet"
}
def __init__(self):
Scraper.__init__(self)
self.url = "https://go.berniesanders.com/page/event/search_results"
self.html = HTMLParser()
self.params = {
'orderby': 'zip_radius',
'zip_radius[1]': '6000',
'zip_radius[0]': '78218',
'radius_unit': 'mi',
'country': 'US',
'format': 'json'
}
self.map = {
"id": "original_id",
"start_dt": "start_time"
}
def __init__(self):
Scraper.__init__(self)
c = self.config["bsd"]
self.html = HTMLParser()
self.call_path = "/page/api/event/search_events"
self.params = {
"api_ver": "2",
"api_id": c["api_id"],
"api_ts": str(int(time.time()))
}
self.signed_params = self.sign_params(c["api_secret"])
self.url = "".join([
c["endpoint"],
self.call_path,
"?",
self.signed_params
])
self.map = {
"event_id": "original_id",
"start_dt": "start_time"
}
def __init__(self):
"""
Initially in this function we open up the config file,
then get the serverip and port from the config file.
"""
Scraper.__init__(self)
try:
with open("../config/server-clients.json") as jsonf:
self.cinfo = json.load(jsonf)
except:
print("Could not find/load server-clients.json config file")
exit()
try:
self.serverip = self.cinfo["serverip"]
self.serverport = self.cinfo["serverport"]
except:
print("server-clients config file not properly formatted")
exit()
self.socket = self.createSocket()
def __init__(self, *args, **kwargs):
Scraper.__init__(self, *args, **kwargs)
self.shopify_app_handle = kwargs.get("shopify_app_handle", None)
self.shopify_app_url = kwargs.get("shopify_app_url", None)
self.tloc = threading.local()
self.shop_urls = []
def __init__(self, delay):
Scraper.__init__(self, "CZC", "https://www.czc.cz", delay)
def __init__(self):
Scraper.__init__(self)
self.root = 'http://bitsnoop.com'
self.site = "bitsnoop.com"
def __init__(self):
Scraper.__init__(self)
self.site = "monova.com"
def __init__(self):
Scraper.__init__(self)
self.url = "https://berniesanders.com/issues/feed/"
self.html = HTMLParser()
def __init__(self):
Scraper.__init__(self)
self.url = "https://berniesanders.com/daily/"
self.html = HTMLParser()
def __init__(self, delay):
Scraper.__init__(self, "TSBohemia", "https://www.tsbohemia.cz", delay)
def __init__(self):
Scraper.__init__(self)
self.root = 'http://kat.ph'
self.site = "kat.ph"
def __init__(self):
Scraper.__init__(self)
self.site = "fenopy.se"
def __init__(self):
self.username = "******"
self.password = "******"
self.mySQLactive = []
Scraper.__init__(self)
def __init__(self, *args, **kwargs):
# initialize our super classes
Scraper.__init__(self) #, prefetch=self.prefetch)
# set regex's
self.__set_regex()
