def run(self):
self.log("""
======================================================
====== Starting Victor Spider ======================
""")
if self.isExisting():
self.log("Scrapped for `{0}` already done. Skipping...".format(
self.affiliate))
return True
else:
self.client = UBrowse()
self.client.open_url(
'https://www.betvictoraffiliates.com/en-gb/home/')
time.sleep(5)
if self.login():
self.log("Successfully logged in. Parsing quick stats.")
self.get_quick_stats()
self.select_YTD_option()
self.get_YTD_stats()
self.log("Pulling quick stats reporting...")
self.get_stats_report()
if self.save() == True:
self.log("Pulled data successfully saved!")
else:
self.log("Something went wrong in DB Query.", "error")
else:
self.log("Failed to log in!!", "error")
self.client.close()
def run(self):
self.log("""
======================================================
====== Starting LadBrokes Spider ======================
""")
if self.isExisting() is False:
self.client = UBrowse()
self.client.open_url(
'https://portal.ladbrokespartners.com/portal/#/login')
self.client.set_loginform('//*[@id="userName"]')
self.client.set_passform('//*[@id="password"]')
self.client.set_loginbutton(
'/html/body/div[3]/section/div/form/div[5]/div/input')
if self.client.login('betfyuk', 'WjewEEUV') is True:
self._get_cookies()
self._create_params()
if self.get_data():
self.log("Data stored successfully")
else:
self.log("Failed to write to DB")
else:
self.log("Login Failed", 'error')
self.client.close()
else:
self.log("Already scrapped for `{0}`. Skipping...".format(
self.affiliate))
def run(self):
self.log("""
======================================================
====== Starting StanJames Spider ======================
""")
if self.isExisting():
self.log("Scrapped for `{0}` already done. Skipping...".format(
self.affiliate))
return True
else:
self.client = UBrowse()
if self.login():
self.log("Successfully logged in. Parsing quick stats.")
self.get_quick_stats()
self.select_YTD_option()
self.get_YTD_stats()
self.log("Pulling quick stats reporting...")
self.get_stats_report()
if self.save() == True:
self.log("Pulled data successfully saved!")
return True
else:
return False
else:
self.log("Failed to log in!!", "error")
self.client.close()
def run(self):
self.log("""
======================================================
====== Starting BetFred Spider ======================
""")
if self.isExisting():
self.log("Already scraped for {0} at {1}".format(
self.affiliate, self.get_delta_date()))
else:
self.client = UBrowse()
if self.login():
self.get_quick_stats()
self.select_YTD_option()
self.get_YTD_stats()
self.get_stats_report()
if self.save():
self.log("Successfully saved!")
else:
self.log("Failed to write database")
else:
self.log("Failed to Login.", "error")
self.client.close()
def __init__(self):
self.client = UBrowse()
self.headers = {
'Host': 'program.uffiliates.com',
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0',
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate, br',
'X-Requested-With': 'XMLHttpRequest',
'Referer':
'https://program.uffiliates.com/en/reports/trafficstats',
}
def run(self):
self.log("""
======================================================
====== Starting Bet 10 Spider ======================
""")
app = scheduler.app
with app.app_context():
affiliate = Affiliate.query.filter_by(name = self.affiliate).first()
if self.isExisting():
self.log("Already scraped for {0}".format(self.get_delta_date()))
else:
self.client = UBrowse()
self.log('getting page...')
self.get_page(self.login_url, 5)
self.log('getting pass the gate page...')
self.login()
self.log('getting quick stats table...')
self.parse_stats_tables()
self.log('getting summary page content')
self.get_page(self.report_url, 1)
self.parse_daily_data()
self.log("Saving to Database...")
self.save()
self.close()
def run(self):
self.log("""
======================================================
====== Starting TitanBet Spider ======================
""")
if self.isExisting():
self.log("Scrapped for `{0}` already done. Skipping...".format(
self.affiliate))
return False
self.client = UBrowse()
self.login()
try:
response = json.loads(self.get_data().content)
data = dict()
if response['data']['values']:
values = response['data']['values'][0]['data']
daily_click = int(values[2])
else:
daily_click = 0
created_at = self.get_delta_date()
self.client.close()
app = scheduler.app
with app.app_context():
affiliate = Affiliate.query.filter_by(
name=self.affiliate).first()
if affiliate is None:
affiliate = Affiliate(name=self.affiliate)
db.session.add(affiliate)
db.session.commit()
history = History.query.filter_by(
affiliate_id=affiliate.id, created_at=created_at).first()
if history is None:
history = History(affiliate_id=affiliate.id,
daily_click=daily_click,
created_at=created_at)
db.session.add(history)
db.session.commit()
return True
except Exception as e:
self.log(str(e), "error")
return False
def run(self,
provider='Bet365',
username='******',
password='******'):
self.log("""
======================================================
====== Starting Bet 365 Spider ======================
""")
self.log("Getting data with (" + username + ":" + password + ")")
self.affiliate = provider
if self.isExisting():
self.log("Already scraped for {0} at {1}".format(
provider, self.get_delta_date()))
else:
self.client = UBrowse()
if self.login(username, password):
self.parse_stats()
else:
self.log("Failed to Login with current account.", "error")
self.client.close()
def run(self):
self.log("""
======================================================
====== Starting 888 Spider ======================
""")
if self.isExisting():
self.log("Scrapped for `{0}` already done.".format(self.affiliate))
else:
self.client = UBrowse()
self.log("Logging in....")
self.client.open_url(self.login_url)
time.sleep(1)
if self.login() is True:
self.log("Successfully logged in.")
if self.parse_page() is True:
self.save()
else:
self.log("Parsing Error...")
else:
self.report_error_log("Failed to log in.")
self.close()
def __init__(self):
self.report = SpiderReporter()
self.client = UBrowse()
self.login_url = 'https://affiliatehub.skybet.com/login.asp'
self.report_url = 'https://affiliatehub.skybet.com/reporting/quick_summary_report.asp'
self.username = '******'
self.password = '******'
self.items = []
self.quick_stats_timer = 0
self.YTD_stats_timer = 0
self.report_timer = 0
self.affiliate = "SkyBet"
self.headers = {
'Host': 'affiliatehub.skybet.com',
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0',
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate, br',
'X-Requested-With': 'XMLHttpRequest',
'Referer': 'https://affiliatehub.skybet.com/reporting/quick_summary_report.asp',
}
def run(self):
self.log("""
======================================================
====== Starting William Spider ======================
""")
if self.isExisting():
self.log("Scrapped for `{0}` already done. Skipping...".format(
self.affiliate))
else:
self.client = UBrowse()
if self.login():
if self.get_data():
if self.save():
self.log("Successfully stored to DB.")
else:
self.log("Something went wrong in DB manipulation.")
else:
self.log("Failed to get data")
else:
self.log("Login Failed in william spider", "error")
self.client.close()
class Bet365(object):
"""docstring for bet365Spider"""
def __init__(self):
self.login_url = 'https://www.bet365affiliates.com/ui/pages/affiliates/Affiliates.aspx'
self.stats_url = 'https://www.bet365affiliates.com/members/CMSitePages/Router.aspx?TargetPage=Members%2fStatistics&lng=1'
self.report = SpiderReporter()
self.affiliate = "Bet365"
self.headers = {
'Host':
'www.bet365affiliates.com',
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0',
'Accept':
'application/json, text/javascript, */*; q=0.01',
'Accept-Language':
'en-US,en;q=0.5',
'Accept-Encoding':
'gzip, deflate, br',
'X-Requested-With':
'XMLHttpRequest',
'Referer':
'https://www.bet365affiliates.com/Members/Members/Statistics/',
}
def _create_params(self, from_date, to_date, media=False):
if media:
self.params = (('key', 'value'), )
else:
self.params = (('operatorID', '1'), )
def _get_cookies(self):
self.cookies = dict()
cookies = self.client.driver.get_cookies()
for i in cookies:
self.cookies[i['name']] = i['value']
def get_delta_date(self, delta=DELTA_DAYS):
today = datetime.datetime.today()
diff = datetime.timedelta(days=delta)
return (today - diff).strftime("%Y/%m/%d")
def log(self, message, type='info'):
self.report.write_log("Bet365", message, self.get_delta_date(), type)
def report_error_log(self, message):
self.log(message, "error")
def save(self):
app = scheduler.app
with app.app_context():
try:
monthly_click = int(self.items[2])
monthly_signup = int(self.items[3])
monthly_commission = float(self.items[5])
yearly_click = int(self.items[8])
yearly_signup = int(self.items[9])
yearly_commission = float((self.items[11]).replace(",", ""))
daily_click = int(self.items[13])
daily_signup = int(self.items[14])
daily_commission = float(self.items[16])
paid_signup = int(self.items[4])
created_at = self.get_delta_date()
affiliate = Affiliate.query.filter_by(
name=self.affiliate).first()
if affiliate is None:
affiliate = Affiliate(name=self.affiliate)
db.session.add(affiliate)
db.session.commit()
history = History.query.filter_by(
affiliate_id=affiliate.id, created_at=created_at).first()
if history is None:
history = History(affiliate_id=affiliate.id,
daily_click=daily_click,
daily_signup=daily_signup,
daily_commission=daily_commission,
monthly_click=monthly_click,
monthly_signup=monthly_signup,
monthly_commission=monthly_commission,
yearly_click=yearly_click,
yearly_signup=yearly_signup,
yearly_commission=yearly_commission,
paid_signup=paid_signup,
created_at=created_at)
db.session.add(history)
db.session.commit()
return True
except Exception as e:
self.log(str(e), "error")
return False
def parse_report_table(self, table_name):
app = scheduler.app
with app.app_context():
try:
tblWrapper = self.client.driver.find_element_by_class_name(
'dataTables_scrollBody')
table = tblWrapper.find_element_by_tag_name('table')
row = table.find_elements_by_tag_name('tr')[-1]
pattern = re.compile(r'[\-\d.\d]+')
monthly_click = int(
pattern.search(
row.find_elements_by_tag_name('td')[0].text).group(0))
monthly_signup = int(
pattern.search(
row.find_elements_by_tag_name('td')[1].text).group(0))
paid_signup = int(
pattern.search(
row.find_elements_by_tag_name('td')[9].text).group(0))
monthly_commission = float(
pattern.search(
row.find_elements_by_tag_name('td')[-1].text).group(
0).replace(',', '.'))
created_at = self.get_delta_date()
affiliate = Affiliate.query.filter_by(
name=self.affiliate).first()
if affiliate is None:
affiliate = Affiliate(name=self.affiliate)
db.session.add(affiliate)
db.session.commit()
history = History.query.filter_by(
affiliate_id=affiliate.id, created_at=created_at).first()
if history is None:
history = History(affiliate_id=affiliate.id,
monthly_click=monthly_click,
monthly_signup=monthly_signup,
monthly_commission=monthly_commission,
paid_signup=paid_signup,
created_at=created_at)
db.session.add(history)
db.session.commit()
return True
except Exception as e:
self.log(str(e), "error")
return False
def parse_stats(self, wait_for=10, table_name="bet365s"):
# time.sleep(10)
val = []
try:
self.client.open_url(self.stats_url)
time.sleep(10)
param_date = self.client.get_delta_date(2, "%d/%m/%Y")
report_option = Select(
self.client.driver.find_element_by_xpath(
'//*[@id="m_mainPlaceholder_ReportCriteria"]'))
report_option.select_by_value('dailyReport')
time.sleep(3)
self.client.driver.execute_script(
"document.getElementById('m_mainPlaceholder_FromDate').value = '{0}'"
.format(param_date))
time.sleep(3)
self.client.driver.execute_script(
"document.getElementById('m_mainPlaceholder_ToDate').value = '{0}'"
.format(param_date))
time.sleep(3)
self.client.driver.find_element_by_id(
'm_mainPlaceholder_Refresh').click()
time.sleep(5)
result = self.parse_report_table(table_name)
return result
except Exception as e:
self.report_error_log(str(e))
return False
def login(self, username='******', password='******'):
try:
self.client.open_url(self.login_url)
time.sleep(10)
self.client.driver.find_element_by_css_selector(
"input[name='ctl00$MasterHeaderPlaceHolder$ctl00$userNameTextbox']"
).send_keys(username)
tmp_pass = self.client.driver.find_element_by_xpath(
"//*[@id='ctl00_MasterHeaderPlaceHolder_ctl00_tempPasswordTextbox']"
)
tmp_pass.clear()
self.client.driver.find_element_by_xpath(
"//*[@id='ctl00_MasterHeaderPlaceHolder_ctl00_passwordTextbox']"
).send_keys(password)
self.client.driver.find_element_by_xpath(
"//*[@id='ctl00_MasterHeaderPlaceHolder_ctl00_passwordTextbox']"
).send_keys(Keys.RETURN)
return True
except Exception as e:
self.report_error_log(str(e))
return False
def isExisting(self, date=None):
if date is None:
date = self.get_delta_date()
app = scheduler.app
created_at = self.get_delta_date()
with app.app_context():
affiliate = Affiliate.query.filter_by(name=self.affiliate).first()
if affiliate is None:
return False
history = History.query.filter_by(affiliate_id=affiliate.id,
created_at=created_at).first()
if history is None:
return False
return True
def run(self,
provider='Bet365',
username='******',
password='******'):
self.log("""
======================================================
====== Starting Bet 365 Spider ======================
""")
self.log("Getting data with (" + username + ":" + password + ")")
self.affiliate = provider
if self.isExisting():
self.log("Already scraped for {0} at {1}".format(
provider, self.get_delta_date()))
else:
self.client = UBrowse()
if self.login(username, password):
self.parse_stats()
else:
self.log("Failed to Login with current account.", "error")
self.client.close()
class BetFred(object):
"""docstring for BetFred"""
def __init__(self):
self.login_url = 'https://secure.activewins.com/login.asp'
self.report_url = 'https://secure.activewins.com/reporting/quick_summary_report.asp'
self.username = '******'
self.password = '******'
self.items = []
self.quick_stats_timer = 0
self.YTD_stats_timer = 0
self.report_timer = 0
self.report = SpiderReporter()
self.affiliate = "BetFred"
self.headers = {
'Host':
'secure.activewins.com',
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0',
'Accept':
'application/json, text/javascript, */*; q=0.01',
'Accept-Language':
'en-US,en;q=0.5',
'Accept-Encoding':
'gzip, deflate, br',
'X-Requested-With':
'XMLHttpRequest',
'Referer':
'https://secure.activewins.com/reporting/quick_summary_report.asp',
}
def _create_params(self, from_date, to_date, media=False):
if media:
self.params = (('key', 'value'), )
else:
self.params = (('operatorID', '1'), )
def _get_cookies(self):
self.cookies = dict()
cookies = self.client.driver.get_cookies()
for i in cookies:
self.cookies[i['name']] = i['value']
def get_delta_date(self, delta=DELTA_DAYS, format_string="%Y/%m/%d"):
today = datetime.datetime.today()
diff = datetime.timedelta(days=delta)
return (today - diff).strftime(format_string)
def login(self):
try:
self.client.open_url(self.login_url)
time.sleep(3)
self.client.set_loginform('//*[@id="username"]')
self.client.set_passform('//*[@id="password"]')
self.client.set_loginbutton('//button[@type="submit"]')
if self.client.login(self.username, self.password) is True:
self._get_cookies()
return True
else:
return False
except Exception as e:
self.report_error_log(str(e))
return False
def select_YTD_option(self):
try:
period_select = Select(
self.client.driver.find_element_by_xpath(
'//*[@id="dashboard"]//select[@name="WRQSperiod"]'))
period_select.select_by_value('YTD')
return True
except Exception as e:
self.report_error_log(str(e))
return False
def get_YTD_stats(self):
time.sleep(5)
try:
table = self.client.driver.find_element_by_xpath(
'//*[@id="dashboard_quick_stats"]//tr[@class="row_light_color"]'
)
for td in table.find_elements_by_tag_name('td'):
if td.text == u'':
raise ValueError("Value can't be empty.")
break
self.items.append(td.text)
return True
except:
self.log("Element not found.")
self.YTD_stats_timer += 1
if self.YTD_stats_timer < 10:
return self.get_YTD_stats()
else:
return False
def get_quick_stats(self):
time.sleep(5)
try:
table = self.client.driver.find_element_by_xpath(
'//*[@id="dashboard_quick_stats"]//tr[@class="row_light_color"]'
)
for td in table.find_elements_by_tag_name('td'):
if td.text == u'':
raise ValueError("Value can't be empty.")
break
self.items.append(td.text)
return True
except:
self.log("Element not found.")
self.quick_stats_timer += 1
if self.quick_stats_timer < 6:
return self.get_quick_stats()
else:
return False
def parse_stats_report(self):
time.sleep(3)
param_date = self.get_delta_date()
try:
# tableDiv = Betfred.find_element_by_id("internalreportdata")
table = self.client.driver.find_element_by_xpath(
'//*[@id="internalreportdata"]/table')
# table = tableDiv.find_element_by_tag_name("table")
todayVal = table.find_elements_by_tag_name("tr")
pattern = re.compile(r'[\-\d.\d]+')
impreto = pattern.search(todayVal[1].text).group(0)
self.items.append(impreto)
clito = pattern.search(todayVal[2].text).group(0)
self.items.append(clito)
regto = pattern.search(todayVal[5].text).group(0)
self.items.append(regto)
ndto = pattern.search(todayVal[8].text).group(0)
self.items.append(ndto)
commito = pattern.search(todayVal[-1].text).group(0)
self.items.append(commito)
self.items.append(param_date)
return True
except:
self.log("Element not found.")
self.report_timer += 1
if self.report_timer < 4:
return self.parse_stats_report()
else:
return False
def log(self, message, type='info'):
self.report.write_log("BetFred", message, self.get_delta_date(), type)
def report_error_log(self, message):
self.log(message, "error")
def get_stats_report(self):
try:
self.client.open_url(self.report_url)
time.sleep(10)
merchant = Select(
self.client.driver.find_element_by_xpath(
'//form[@id="FRMReportoptions"]//select[@name="merchantid"]'
))
merchant.select_by_value('0')
param_date = self.get_delta_date()
self.client.driver.execute_script(
"document.getElementById('startdate').value = '{0}'".format(
param_date))
self.client.driver.execute_script(
"document.getElementById('enddate').value = '{0}'".format(
param_date))
self.client.driver.find_element_by_class_name("button").click()
self.parse_stats_report()
return True
except Exception as e:
self.report_error_log(str(e))
return False
def save(self):
try:
app = scheduler.app
monthly_click = int(self.items[2])
monthly_signup = int(self.items[3])
paid_signup = int(self.items[4])
commissionStr = str(self.items[5]).replace(',', '')
pattern = re.compile(r'[\-\d.\d]+')
monthly_commission = float(pattern.search(commissionStr).group(0))
yearly_click = int(self.items[8])
yearly_signup = int(self.items[9])
commiytdStr = str(self.items[11]).replace(',', '')
yearly_commission = float(pattern.search(commiytdStr).group(0))
daily_click = int(self.items[13])
daily_signup = int(self.items[14])
daily_commission = float(self.items[16])
created_at = self.get_delta_date()
with app.app_context():
affiliate = Affiliate.query.filter_by(
name=self.affiliate).first()
if affiliate is None:
affiliate = Affiliate(name=self.affiliate)
db.session.add(affiliate)
db.session.commit()
history = History.query.filter_by(
affiliate_id=affiliate.id, created_at=created_at).first()
if history is None:
history = History(affiliate_id=affiliate.id,
daily_click=daily_click,
daily_signup=daily_signup,
daily_commission=daily_commission,
monthly_click=monthly_click,
monthly_signup=monthly_signup,
monthly_commission=monthly_commission,
yearly_click=yearly_click,
yearly_signup=yearly_signup,
yearly_commission=yearly_commission,
paid_signup=paid_signup,
created_at=created_at)
db.session.add(history)
db.session.commit()
return True
except Exception as e:
self.report_error_log(str(e))
return False
def isExisting(self, date=None):
if date is None:
date = self.get_delta_date()
app = scheduler.app
created_at = self.get_delta_date()
with app.app_context():
affiliate = Affiliate.query.filter_by(name=self.affiliate).first()
if affiliate is None:
return False
history = History.query.filter_by(affiliate_id=affiliate.id,
created_at=created_at).first()
if history is None:
return False
return True
def run(self):
self.log("""
======================================================
====== Starting BetFred Spider ======================
""")
if self.isExisting():
self.log("Already scraped for {0} at {1}".format(
self.affiliate, self.get_delta_date()))
else:
self.client = UBrowse()
if self.login():
self.get_quick_stats()
self.select_YTD_option()
self.get_YTD_stats()
self.get_stats_report()
if self.save():
self.log("Successfully saved!")
else:
self.log("Failed to write database")
else:
self.log("Failed to Login.", "error")
self.client.close()
class LadBrokes(object):
"""docstring for LadBrokes"""
def __init__(self):
self.report = SpiderReporter()
self.report_url = 'https://portal.ladbrokespartners.com/portal/#/statistics'
self.ajax_url = 'https://portal.ladbrokespartners.com/portal/rest/reports/run/atOnce'
self.data = {}
self.affiliate = "LadBrokes"
self.headers = {
'Accept':
'application/json, text/plain, */*',
'Accept-Encoding':
'gzip, deflate, br',
'Accept-Language':
'en-US,en;q=0.5',
'Content-Length':
'9167',
'Content-Type':
'application/json;charset=utf-8',
'Host':
'portal.ladbrokespartners.com',
'Referer':
'https://portal.ladbrokespartners.com/portal/',
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0',
}
def get_delta_date(self, delta=DELTA_DAYS, format_string="%Y/%m/%d"):
today = datetime.datetime.today()
diff = datetime.timedelta(days=delta)
return (today - diff).strftime(format_string)
def _create_params(self, mode='daily'):
delta = datetime.timedelta(days=1)
today = datetime.datetime.now()
yesterday = today - delta
end_date = yesterday.strftime('%Y-%m-%d')
start_date = yesterday.strftime('%Y-%m-%d')
if mode == 'monthly':
delta = dateutil.relativedelta.relativedelta(months=1)
start_date = (yesterday - delta).strftime('%Y-%m-%d')
elif mode == 'yearly':
delta = dateutil.relativedelta.relativedelta(years=1)
start_date = (yesterday - delta).strftime('%Y-%m-%d')
self.ajax_param = '{"report":{"campaignId":"245450","input":[{"name":"startDate","label":"Stat Date","type":"dateRange","required":true,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":"yyyy-MM-dd","jsDateTimeFormat":"yy-MM-dd","precision":3,"value":"custom","values":[],"sublist":[{"name":"startDate_list","label":null,"type":"list","required":true,"collapsed":false,"filter":true,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":null,"value":"today","values":[{"id":"custom","name":null},{"id":"today","name":null},{"id":"yesterday","name":null},{"id":"currweek","name":null},{"id":"prevweek","name":null},{"id":"currmonth","name":null},{"id":"prevmonth","name":null},{"id":"threemonths","name":null},{"id":"sixmonths","name":null},{"id":"curryear","name":null},{"id":"prevyear","name":null}],"sublist":[]},{"name":"startDate_from","label":null,"type":"date","required":true,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":"yyyy-MM-dd","jsDateTimeFormat":"yy-MM-dd","precision":null,"value":"%s","values":[],"sublist":[]},{"name":"startDate_to","label":null,"type":"date","required":true,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":"yyyy-MM-dd","jsDateTimeFormat":"yy-MM-dd","precision":null,"value":"%s","values":[],"sublist":[]}]},{"name":"profile","label":"Profile","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"reportBy1","label":"Report By","type":"list","required":true,"collapsed":false,"filter":true,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":"stat_date","values":[{"id":"stat_date","name":"Date"},{"id":"stat_month","name":"Month"},{"id":"profile","name":"Profile"},{"id":"product","name":"Product"},{"id":"banner","name":"Banner"},{"id":"platform","name":"Platform"},{"id":"country","name":"Country"},{"id":"var1","name":"Var 1"},{"id":"var2","name":"Var 2"},{"id":"var9","name":"Var 9"},{"id":"var10","name":"Var 10"}],"sublist":[]},{"name":"reportBy2","label":"Report By","type":"list","required":false,"collapsed":false,"filter":true,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[{"id":"stat_date","name":"Date"},{"id":"stat_month","name":"Month"},{"id":"profile","name":"Profile"},{"id":"product","name":"Product"},{"id":"banner","name":"Banner"},{"id":"platform","name":"Platform"},{"id":"country","name":"Country"},{"id":"var1","name":"Var 1"},{"id":"var2","name":"Var 2"},{"id":"var9","name":"Var 9"},{"id":"var10","name":"Var 10"}],"sublist":[]},{"name":"reportBy3","label":"Report By","type":"list","required":false,"collapsed":false,"filter":true,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[{"id":"stat_date","name":"Date"},{"id":"stat_month","name":"Month"},{"id":"profile","name":"Profile"},{"id":"product","name":"Product"},{"id":"banner","name":"Banner"},{"id":"platform","name":"Platform"},{"id":"country","name":"Country"},{"id":"var1","name":"Var 1"},{"id":"var2","name":"Var 2"},{"id":"var9","name":"Var 9"},{"id":"var10","name":"Var 10"}],"sublist":[]},{"name":"reportBy4","label":"Report By","type":"list","required":false,"collapsed":false,"filter":true,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[{"id":"stat_date","name":"Date"},{"id":"stat_month","name":"Month"},{"id":"profile","name":"Profile"},{"id":"product","name":"Product"},{"id":"banner","name":"Banner"},{"id":"platform","name":"Platform"},{"id":"country","name":"Country"},{"id":"var1","name":"Var 1"},{"id":"var2","name":"Var 2"},{"id":"var9","name":"Var 9"},{"id":"var10","name":"Var 10"}],"sublist":[]}],"output":[{"name":"stat_date","label":"Date","type":"date","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":"yyyy-MM-dd","jsDateTimeFormat":"yy-MM-dd","precision":3,"value":null,"values":[],"sublist":[]},{"name":"stat_month","label":"Month","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"platform","label":"Platform","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"profile","label":"Profile","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"product","label":"Product","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"banner","label":"Banner","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"country","label":"Country","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"var1","label":"Var 1","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"var2","label":"Var 2","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"var9","label":"Var 9","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"var10","label":"Var 10","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"REAL_IMPS","label":"Unique Impressions","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"RAW_IMPS","label":"Non-Unique Impressions","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"REAL_CLICKS","label":"Unique Clicks","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"RAW_CLICKS","label":"Non-Unique Clicks","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"sports_signups","label":"sports Signups Cnt","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"sports_acquired_count","label":"sports Acquired cnt","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"acquired_count","label":"Acquired cnt","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"WITHDRAWS_CNT","label":"Withdrawal cnt","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"sports_net_gaming_revenue","label":"Sports Net Gaming Revenue","type":"number","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"cpa_commission","label":"CPA Commission","type":"number","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"ng_commission","label":"Net Gaming Commission","type":"number","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"tlr_amount","label":"Top Level Revenue","type":"number","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]}],"language":"EN","name":"trafficStatsNew","id":"1712"},"sc":{"offset":0,"limit":1000,"order":[{"ascending":true,"column":"1"}],"namedQuery":""}}' % (
start_date, end_date)
def _get_auth_token(self):
try:
self.headers['X-Auth-Token'] = self.client.driver.execute_script(
"return localStorage.authToken")
except:
return False
return True
def _get_cookies(self):
self.cookies = dict()
cookies = self.client.driver.get_cookies()
for i in cookies:
self.cookies[i['name']] = i['value']
def log(self, message, type='info'):
self.report.write_log("LadBroke", message, self.get_delta_date(), type)
def get_yearly_data(self):
self._create_params('yearly')
response = requests.post(self.ajax_url,
headers=self.headers,
cookies=self.cookies,
data=self.ajax_param)
content = response.content
response = json.loads(content)
if (response['data']['total']['data']):
values = response['data']['total']['data']
self.data['yearly_click'] = int(values[3])
self.data['yearly_signup'] = int(values[5])
self.data['yearly_commission'] = float(values[10])
return True
else:
self.log("Something went wrong in Ajax for Yearly data", 'error')
return False
def get_monthly_data(self):
self._create_params('monthly')
response = requests.post(self.ajax_url,
headers=self.headers,
cookies=self.cookies,
data=self.ajax_param)
content = response.content
response = json.loads(content)
if (response['data']['total']['data']):
values = response['data']['total']['data']
self.data['monthly_click'] = int(values[3])
self.data['monthly_signup'] = int(values[5])
self.data['monthly_commission'] = float(values[10])
return True
else:
self.log("Something went wrong in Ajax for Monthly data", 'error')
return False
def get_daily_data(self):
self.client.open_url(self.report_url)
time.sleep(2)
response = requests.post(self.ajax_url,
headers=self.headers,
cookies=self.cookies,
data=self.ajax_param)
content = response.content
response = json.loads(content)
if (response['data']['total']['data']):
values = response['data']['total']['data']
self.data['created_at'] = self.get_delta_date()
self.data['daily_click'] = int(values[3])
self.data['daily_signup'] = int(values[5])
self.data['daily_commission'] = float(values[10])
self.data['paid_signup'] = 0
return True
else:
self.log("Something went wrong in Ajax for Daily data", 'error')
return False
def get_data(self):
time.sleep(5)
self._get_auth_token()
self.log("Getting daily data...")
self.get_daily_data()
self.log("Getting monthly data...")
self.get_monthly_data()
self.log("Getting yearly data...")
self.get_yearly_data()
return self.save()
def save(self):
try:
app = scheduler.app
with app.app_context():
affiliate = Affiliate.query.filter_by(
name=self.affiliate).first()
if affiliate is None:
affiliate = Affiliate(name=self.affiliate)
db.session.add(affiliate)
db.session.commit()
print(affiliate)
created_at = self.get_delta_date()
history = History.query.filter_by(
affiliate_id=affiliate.id, created_at=created_at).first()
if history is None:
history = History(
affiliate_id=affiliate.id,
daily_click=self.data['daily_click'],
daily_signup=self.data['daily_signup'],
daily_commission=self.data['daily_commission'],
monthly_click=self.data['monthly_click'],
monthly_signup=self.data['monthly_signup'],
monthly_commission=self.data['monthly_commission'],
yearly_click=self.data['yearly_click'],
yearly_signup=self.data['yearly_signup'],
yearly_commission=self.data['yearly_commission'],
paid_signup=self.data['paid_signup'],
created_at=created_at)
db.session.add(history)
db.session.commit()
return True
except Exception as e:
self.log(str(e), "error")
return False
def isExisting(self, date=None):
if date == None:
date = self.get_delta_date()
app = scheduler.app
with app.app_context():
affiliate = Affiliate.query.filter_by(name=self.affiliate).first()
if affiliate is None:
return False
history = History.query.filter_by(affiliate_id=affiliate.id,
created_at=date).first()
if history is None:
return False
else:
return True
return True
def run(self):
self.log("""
======================================================
====== Starting LadBrokes Spider ======================
""")
if self.isExisting() is False:
self.client = UBrowse()
self.client.open_url(
'https://portal.ladbrokespartners.com/portal/#/login')
self.client.set_loginform('//*[@id="userName"]')
self.client.set_passform('//*[@id="password"]')
self.client.set_loginbutton(
'/html/body/div[3]/section/div/form/div[5]/div/input')
if self.client.login('betfyuk', 'WjewEEUV') is True:
self._get_cookies()
self._create_params()
if self.get_data():
self.log("Data stored successfully")
else:
self.log("Failed to write to DB")
else:
self.log("Login Failed", 'error')
self.client.close()
else:
self.log("Already scrapped for `{0}`. Skipping...".format(
self.affiliate))
def run(self):
print("""
Starting Category Scraper....
""")
if self.client is None:
self.client = UBrowse()
class Eight88(object):
"""docstring for Eight88"""
def __init__(self):
self.report = SpiderReporter()
self.login_url = 'http://affiliates.888.com/'
self.username = '******'
self.password = '******'
self.items = []
self.timer = 0
self.affiliate = "Eight88"
self.headers = {
'Host':
'secure.activewins.com',
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0',
'Accept':
'application/json, text/javascript, */*; q=0.01',
'Accept-Language':
'en-US,en;q=0.5',
'Accept-Encoding':
'gzip, deflate, br',
'X-Requested-With':
'XMLHttpRequest',
'Referer':
'https://secure.activewins.com/reporting/quick_summary_report.asp',
}
def _create_params(self, from_date, to_date, media=False):
if media:
self.params = (('key', 'value'), )
else:
self.params = (('operatorID', '1'), )
def _get_cookies(self):
self.cookies = dict()
cookies = self.client.driver.get_cookies()
for i in cookies:
self.cookies[i['name']] = i['value']
def log(self, message, type='info'):
self.report.write_log("Eight88s", message, self.get_delta_date(), type)
def report_error_log(self, message):
self.log(message, "error")
def get_delta_date(self, delta=DELTA_DAYS, format_string="%Y/%m/%d"):
today = datetime.datetime.today()
diff = datetime.timedelta(days=delta)
return (today - diff).strftime(format_string)
def close(self):
self.client.close()
def login(self, username='******', password='******'):
try:
self.client.driver.find_element_by_class_name(
"hide-under-480").click()
WebDriverWait(self.client.driver, 10).until(
expected_conditions.frame_to_be_available_and_switch_to_it(
self.client.driver.find_element_by_xpath(
'//iframe[contains(@src, "Auth/Login")]')))
self.client.driver.find_element_by_id("userName").send_keys(
"betfyuk")
self.client.driver.find_element_by_id("password").send_keys(
"LALB37hUhs")
self.client.driver.find_element_by_id("btnLogin").click()
time.sleep(1)
self._get_cookies()
return True
except Exception as e:
self.report_error_log(str(e))
return False
def parse_page(self):
try:
self.client.driver.find_element_by_id(
"rbQuickStatID_This Month (1st - Today)").click()
bal = self.client.driver.find_element_by_id("this-month").text
balCents = self.client.driver.find_element_by_id(
"this-month-cents").text
netBal = bal + balCents
prebal = self.client.driver.find_element_by_id("last-month").text
for summarise in self.client.driver.find_elements_by_xpath(
'.//span[@class = "summariseTab"]'):
self.items.append(summarise.text)
self.items.append(netBal)
self.client.driver.find_element_by_id(
"rbQuickStatID_Last 7 Days").click()
time.sleep(2)
for summarise in self.client.driver.find_elements_by_xpath(
'.//span[@class = "summariseTab"]'):
self.items.append(summarise.text)
self.client.driver.find_element_by_id(
"rbQuickStatID_Previous Month").click()
time.sleep(2)
for summarise in self.client.driver.find_elements_by_xpath(
'.//span[@class = "summariseTab"]'):
self.items.append(summarise.text)
self.client.driver.find_element_by_id(
"rbQuickStatID_Today").click()
time.sleep(2)
for summarise in self.client.driver.find_elements_by_xpath(
'.//span[@class = "summariseTab"]'):
self.items.append(summarise.text)
self.items.append(prebal)
return True
except Exception as e:
self.report_error_log(str(e))
return False
def save(self):
app = scheduler.app
try:
with app.app_context():
monthly_click = int(self.items[1])
monthly_signup = int(self.items[2])
monthly_commission = float(self.items[5])
weekly_click = int(self.items[7])
weekly_signup = int(self.items[8])
daily_click = int(self.items[17])
daily_signup = int(self.items[18])
created_at = self.get_delta_date()
affiliate = Affiliate.query.filter_by(
name=self.affiliate).first()
if affiliate is None:
affiliate = Affiliate(name=self.affiliate)
db.session.add(affiliate)
db.session.commit()
history = History.query.filter_by(
affiliate_id=affiliate.id, created_at=created_at).first()
if history is None:
history = History(affiliate_id=affiliate.id,
daily_click=daily_click,
daily_signup=daily_signup,
weekly_click=weekly_click,
weekly_signup=weekly_signup,
monthly_click=monthly_click,
monthly_signup=monthly_signup,
monthly_commission=monthly_commission,
created_at=created_at)
db.session.add(history)
db.session.commit()
return True
except Exception as e:
self.report_error_log(str(e))
return False
def isExisting(self, date=None):
if date is None:
date = self.get_delta_date()
app = scheduler.app
created_at = self.get_delta_date()
with app.app_context():
try:
affiliate = Affiliate.query.filter_by(
name=self.affiliate).first()
if affiliate is None:
return False
history = History.query.filter_by(
affiliate_id=affiliate.id, created_at=created_at).first()
if history is None:
return False
except Exception as e:
self.report_error_log(str(e))
return False
return True
def run(self):
self.log("""
======================================================
====== Starting 888 Spider ======================
""")
if self.isExisting():
self.log("Scrapped for `{0}` already done.".format(self.affiliate))
else:
self.client = UBrowse()
self.log("Logging in....")
self.client.open_url(self.login_url)
time.sleep(1)
if self.login() is True:
self.log("Successfully logged in.")
if self.parse_page() is True:
self.save()
else:
self.log("Parsing Error...")
else:
self.report_error_log("Failed to log in.")
self.close()
class TitanBet(object):
"""docstring for TitanBet"""
def __init__(self):
self.report = SpiderReporter()
self.affiliate = "TitanBet"
self.headers = {
'Accept':
'application/json, text/plain, */*',
'Accept-Encoding':
'gzip, deflate, br',
'Accept-Language':
'en-US,en;q=0.5',
'Content-Length':
'8164',
'Content-Type':
'application/json;charset=utf-8',
'Host':
'portal.europartners.com',
'Referer':
'https://portal.europartners.com/portal/',
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0',
}
def _create_params(self):
one_day = datetime.timedelta(days=1)
day_now = datetime.datetime.now()
yesterday = day_now - one_day
date = yesterday.strftime('%Y-%B-%d')
# date = yesterday.strftime('%Y-%B-02')
self.data = '{"report":{"campaignId":"101341","input":[{"name":"startDate","label":"Start Date","type":"date","required":true,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":"yyyy-MM-dd","jsDateTimeFormat":"yy-MM-dd","precision":3,"value":"%s","values":[],"sublist":[]},{"name":"endDate","label":"End Date","type":"date","required":true,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":"yyyy-MM-dd","jsDateTimeFormat":"yy-MM-dd","precision":3,"value":"%s","values":[],"sublist":[]},{"name":"products","label":"Product:","type":"list","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[{"id":"13523839","name":"Casino Tropez"},{"id":"14275502","name":"Luck"},{"id":"13523860","name":"Europa Casino"},{"id":"13523861","name":"Titan Casino"},{"id":"13523863","name":"Titan Poker"},{"id":"13523864","name":"Titan Bet"},{"id":"13523865","name":"Titan Bet UK Casino"},{"id":"13523866","name":"Titan Bet UK Poker"},{"id":"13523867","name":"Titan Bet UK Sport"},{"id":"13523868","name":"Europaplay Casino"},{"id":"13962520","name":"Casino Tropez Ru"},{"id":"13962521","name":"Europa Casino RU"},{"id":"13962522","name":"Titan Casino RU"},{"id":"13962524","name":"Titan Poker RU"},{"id":"13962525","name":"Titan Bet RU"},{"id":"13965943","name":"Redluck RU"}],"sublist":[]},{"name":"producttypes","label":"Product type :","type":"list","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[{"id":"13523838","name":"Casino"},{"id":"13523858","name":"Poker"},{"id":"13523859","name":"Sport"},{"id":"13966461","name":"bingo"}],"sublist":[]},{"name":"profilename","label":"Profile","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"reportBy1","label":"Report By","type":"list","required":true,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":"date","values":[{"id":"date","name":"Stat Date"},{"id":"month","name":"Stat Month"},{"id":"country","name":"Country"},{"id":"platform","name":"Platform"},{"id":"product","name":"Product"},{"id":"producttype","name":"Product type"},{"id":"profile","name":"Profile"},{"id":"var1","name":"Var1"},{"id":"var2","name":"Var2"},{"id":"var3","name":"Var3"},{"id":"var4","name":"Var4"},{"id":"var5","name":"Var5"},{"id":"var6","name":"Var6"},{"id":"var7","name":"Var7"},{"id":"var8","name":"Var8"},{"id":"var9","name":"Var9"},{"id":"var10","name":"Var10"},{"id":"player","name":"Player"}],"sublist":[]},{"name":"reportBy2","label":"Report By","type":"list","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[{"id":"date","name":"Stat Date"},{"id":"month","name":"Stat Month"},{"id":"country","name":"Country"},{"id":"platform","name":"Platform"},{"id":"product","name":"Product"},{"id":"producttype","name":"Product type"},{"id":"profile","name":"Profile"},{"id":"var1","name":"Var1"},{"id":"var2","name":"Var2"},{"id":"var3","name":"Var3"},{"id":"var4","name":"Var4"},{"id":"var5","name":"Var5"},{"id":"var6","name":"Var6"},{"id":"var7","name":"Var7"},{"id":"var8","name":"Var8"},{"id":"var9","name":"Var9"},{"id":"var10","name":"Var10"},{"id":"player","name":"Player"}],"sublist":[]},{"name":"reportBy3","label":"Report By","type":"list","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[{"id":"date","name":"Stat Date"},{"id":"month","name":"Stat Month"},{"id":"country","name":"Country"},{"id":"platform","name":"Platform"},{"id":"product","name":"Product"},{"id":"producttype","name":"Product type"},{"id":"profile","name":"Profile"},{"id":"var1","name":"Var1"},{"id":"var2","name":"Var2"},{"id":"var3","name":"Var3"},{"id":"var4","name":"Var4"},{"id":"var5","name":"Var5"},{"id":"var6","name":"Var6"},{"id":"var7","name":"Var7"},{"id":"var8","name":"Var8"},{"id":"var9","name":"Var9"},{"id":"var10","name":"Var10"},{"id":"player","name":"Player"}],"sublist":[]}],"output":[{"name":"stat_date","label":"Date","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"stat_month","label":"Stat Month","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"platform","label":"Platform","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"product_name","label":"Product","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"profilename","label":"Profile","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"producttype","label":"Product type","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"tlr_amount","label":"Top Level Revenue","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"REAL_CLICKS","label":"Real Clicks","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"CASINO_U_REAL_COUNT","label":"Casino Real Players","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"casino_d_rf_count","label":"Casino RFD Cnt","type":"number","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"casino_net_gaming","label":"Casino Net Gaming","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"POKER_U_REAL_COUNT","label":"Poker Real Players","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"poker_d_rf_count","label":"Poker RFD Cnt","type":"number","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"poker_net_gaming","label":"Poker Net Gaming","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"SPORT_U_REAL_COUNT","label":"Sport Real Players","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"sport_d_rf_count","label":"Sport RFD Cnt","type":"number","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]}],"language":"EN","name":"Advance Advertiser Stats","id":"1300"},"sc":{"offset":0,"limit":1000,"order":[{"ascending":true,"column":"1"}],"namedQuery":""}}' % (
date, date)
def _get_auth_token(self):
try:
self.headers['X-Auth-Token'] = self.client.driver.execute_script(
"return localStorage.authToken")
except:
self.log("Failed to get Auth Token")
return False
return True
def _get_cookies(self):
self.cookies = dict()
cookies = self.client.driver.get_cookies()
for i in cookies:
self.cookies[i['name']] = i['value']
def log(self, message, type='info'):
self.report.write_log("TitanBet", message, self.get_delta_date(), type)
def get_delta_date(self, delta=DELTA_DAYS, format_string="%Y/%m/%d"):
today = datetime.datetime.today()
diff = datetime.timedelta(days=delta)
return (today - diff).strftime(format_string)
def get_data(self):
self._get_auth_token()
url = 'https://portal.europartners.com/portal/rest/reports/run/atOnce'
response = requests.post(url,
headers=self.headers,
cookies=self.cookies,
data=self.data)
return response
def login(self):
self.client.open_url('https://portal.europartners.com/portal/#/login')
self.client.set_loginform('//*[@id="userName"]')
self.client.set_passform('//*[@id="password"]')
self.client.set_loginbutton(
'/html/body/div[3]/section/div/form/div[5]/div/input')
if self.client.login('betfyuk', 'qwerty123') is True:
time.sleep(3)
self._get_cookies()
self._create_params()
else:
return False
return True
def isExisting(self, date=None):
if date is None:
date = self.get_delta_date()
app = scheduler.app
with app.app_context():
affiliate = Affiliate.query.filter_by(name=self.affiliate).first()
if affiliate is None:
return False
history = History.query.filter_by(affiliate_id=affiliate.id,
created_at=date).first()
if history is None:
return False
else:
return True
return True
def run(self):
self.log("""
======================================================
====== Starting TitanBet Spider ======================
""")
if self.isExisting():
self.log("Scrapped for `{0}` already done. Skipping...".format(
self.affiliate))
return False
self.client = UBrowse()
self.login()
try:
response = json.loads(self.get_data().content)
data = dict()
if response['data']['values']:
values = response['data']['values'][0]['data']
daily_click = int(values[2])
else:
daily_click = 0
created_at = self.get_delta_date()
self.client.close()
app = scheduler.app
with app.app_context():
affiliate = Affiliate.query.filter_by(
name=self.affiliate).first()
if affiliate is None:
affiliate = Affiliate(name=self.affiliate)
db.session.add(affiliate)
db.session.commit()
history = History.query.filter_by(
affiliate_id=affiliate.id, created_at=created_at).first()
if history is None:
history = History(affiliate_id=affiliate.id,
daily_click=daily_click,
created_at=created_at)
db.session.add(history)
db.session.commit()
return True
except Exception as e:
self.log(str(e), "error")
return False
def run(self):
self.log("""
======================================================
====== Starting Paddy Spider ======================
""")
if self.isExisting():
self.log("Scrapped for `{0}` already done. Skipping...".format(self.affiliate))
return True
else:
self.client = UBrowse()
if self.login():
try:
response = json.loads(self.get_data().content)
data = response['data'][0]
one_day = datetime.timedelta(days = 1)
yesterday = datetime.datetime.now() - one_day
date = yesterday.strftime('%Y-%m-%d')
views = data[1]['Value']
uniqueviews = data[2]['Value']
clicks = data[3]['Value']
uniqueclicks = data[4]['Value']
signups = data[5]['Value']
depositingcustomers = data[6]['Value']
activecustomers = data[7]['Value']
newdepositingcustomers = data[8]['Value']
newactivecustomers = data[9]['Value']
firsttimedepositingcustomers = data[10]['Value']
firsttimeactivecustomers = data[11]['Value']
netrevenue = data[12]['Value']
app = scheduler.app
with app.app_context():
affiliate = Affiliate.query.filter_by(name = self.affiliate).first()
if affiliate is None:
affiliate = Affiliate(name = self.affiliate)
db.session.add(affiliate)
db.session.commit()
created_at = self.get_delta_date()
history = History.query.filter_by(affiliate_id = affiliate.id, created_at = created_at).first()
if history is None:
history = History(
affiliate_id = affiliate.id,
daily_click = clicks,
daily_signup = signups,
daily_commission = netrevenue,
paid_signup = newdepositingcustomers,
created_at = created_at
)
db.session.add(history)
db.session.commit()
return False
except Exception as e:
self.log(str(e), "error")
return False
else:
self.log("Failed to login", "error")
return False
self.client.close()
class Paddy(object):
"""docstring for Paddy"""
def __init__(self):
self.report = SpiderReporter()
self.affiliate = "Paddy"
self.headers = {
'Host': 'affiliates.paddypartners.com',
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0',
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate, br',
'Content-Type': 'application/json; charset=utf-8',
'X-Requested-With': 'XMLHttpRequest',
'x-ms-request-root-id': 'cZ8hr',
'x-ms-request-id': 'ZK2p+',
'Referer': 'https://affiliates.paddypartners.com/affiliates/Reports/DailyFigures',
}
def _create_params(self, date = None):
if date is None:
date = self.get_delta_date(2, '%d-%m-%Y')
self.params = (
('dateFilterFrom', [date, date]),
('dateFilterTo', [date, date]),
)
def _get_cookies(self):
self.cookies = dict()
cookies = self.client.driver.get_cookies()
for i in cookies:
self.cookies[i['name']] = i['value']
def get_delta_date(self, delta = DELTA_DAYS, format_string = "%Y/%m/%d"):
today = datetime.datetime.today()
diff = datetime.timedelta(days = delta)
return (today - diff).strftime(format_string)
def log(self, message, type = "info"):
self.report.write_log("Paddy", message, self.get_delta_date(), type)
def get_data(self):
url = 'https://affiliates.paddypartners.com/affiliates/Reports/dailyFiguresReport'
response = requests.get(url, headers=self.headers, params=self.params, cookies=self.cookies)
return response
def login(self):
try:
self.client.open_url('https://affiliates.paddypartners.com/affiliates/Account/Login')
self.client.set_loginform('//*[@id="txtUsername"]')
self.client.set_passform('//*[@id="txtPassword"]')
self.client.set_loginbutton('//*[@id="btnLogin"]')
if self.client.login('betfyuk', 'dontfuckwithme') is True:
self._get_cookies()
self._create_params()
else:
return False
return True
except Exception as e:
self.log(str(e), "error")
return False
def isExisting(self, date = None):
try:
if date is None:
date = self.get_delta_date()
app = scheduler.app
with app.app_context():
affiliate = Affiliate.query.filter_by(name = self.affiliate).first()
if affiliate is None:
return False
history = History.query.filter_by(affiliate_id = affiliate.id, created_at = date).first()
if history is None:
return False
else:
return True
except Exception as e:
self.log(str(e), "error")
return False
def run(self):
self.log("""
======================================================
====== Starting Paddy Spider ======================
""")
if self.isExisting():
self.log("Scrapped for `{0}` already done. Skipping...".format(self.affiliate))
return True
else:
self.client = UBrowse()
if self.login():
try:
response = json.loads(self.get_data().content)
data = response['data'][0]
one_day = datetime.timedelta(days = 1)
yesterday = datetime.datetime.now() - one_day
date = yesterday.strftime('%Y-%m-%d')
views = data[1]['Value']
uniqueviews = data[2]['Value']
clicks = data[3]['Value']
uniqueclicks = data[4]['Value']
signups = data[5]['Value']
depositingcustomers = data[6]['Value']
activecustomers = data[7]['Value']
newdepositingcustomers = data[8]['Value']
newactivecustomers = data[9]['Value']
firsttimedepositingcustomers = data[10]['Value']
firsttimeactivecustomers = data[11]['Value']
netrevenue = data[12]['Value']
app = scheduler.app
with app.app_context():
affiliate = Affiliate.query.filter_by(name = self.affiliate).first()
if affiliate is None:
affiliate = Affiliate(name = self.affiliate)
db.session.add(affiliate)
db.session.commit()
created_at = self.get_delta_date()
history = History.query.filter_by(affiliate_id = affiliate.id, created_at = created_at).first()
if history is None:
history = History(
affiliate_id = affiliate.id,
daily_click = clicks,
daily_signup = signups,
daily_commission = netrevenue,
paid_signup = newdepositingcustomers,
created_at = created_at
)
db.session.add(history)
db.session.commit()
return False
except Exception as e:
self.log(str(e), "error")
return False
else:
self.log("Failed to login", "error")
return False
self.client.close()
class William(object):
"""docstring for William"""
def __init__(self):
self.ajax_url = 'https://account.affiliates.williamhill.com/affiliates/Reports/dailyFiguresReport'
self.report = SpiderReporter()
self.data = {}
self.affiliate = "William"
self.headers = {
'Host':
'account.affiliates.williamhill.com',
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0',
'Accept':
'application/json, text/javascript, */*; q=0.01',
'Accept-Language':
'en-US,en;q=0.5',
'Accept-Encoding':
'gzip, deflate, br',
'Content-Type':
'application/json; charset=utf-8',
'X-Requested-With':
'XMLHttpRequest',
'x-ms-request-root-id':
'7x9tg',
'x-ms-request-id':
'8rW9l',
'Referer':
'https://account.affiliates.williamhill.com/affiliates/Reports/DailyFigures',
}
def _create_params(self, mode='daily', date_format='%d-%m-%Y'):
one_day = datetime.timedelta(days=1)
today = datetime.datetime.now()
yesterday = today - one_day
end_date = yesterday.strftime(date_format)
if mode == 'daily':
start_date = end_date
elif mode == 'monthly':
delta = dateutil.relativedelta.relativedelta(months=1)
start_date = (yesterday - delta).strftime(date_format)
else:
delta = dateutil.relativedelta.relativedelta(years=1)
start_date = (yesterday - delta).strftime(date_format)
self.params = (
('dateFilterFrom', [start_date, start_date]),
('dateFilterTo', [end_date, end_date]),
)
def _get_cookies(self):
self.cookies = dict()
cookies = self.client.driver.get_cookies()
for i in cookies:
self.cookies[i['name']] = i['value']
def get_delta_date(self, delta=DELTA_DAYS):
today = datetime.datetime.today()
diff = datetime.timedelta(days=delta)
return (today - diff).strftime("%Y/%m/%d")
def save(self):
try:
app = scheduler.app
with app.app_context():
affiliate = Affiliate.query.filter_by(
name=self.affiliate).first()
if affiliate is None:
affiliate = Affiliate(name=self.affiliate)
db.session.add(affiliate)
db.session.commit()
history = History.query.filter_by(
affiliate_id=affiliate.id,
created_at=self.data['created_at']).first()
if history is None:
history = History(
affiliate_id=affiliate.id,
daily_click=self.data['daily_click'],
daily_signup=self.data['daily_signup'],
daily_commission=self.data['daily_commission'],
monthly_click=self.data['monthly_click'],
monthly_signup=self.data['monthly_signup'],
monthly_commission=self.data['monthly_commission'],
yearly_click=self.data['yearly_click'],
yearly_signup=self.data['yearly_signup'],
yearly_commission=self.data['yearly_commission'],
paid_signup=self.data['paid_signup'],
created_at=self.data['created_at'])
db.session.add(history)
db.session.commit()
return True
except Exception as e:
self.log(str(e), "error")
return False
def get_ajax_data(self, mode='daily'):
self._create_params(mode)
response = requests.get(self.ajax_url,
headers=self.headers,
params=self.params,
cookies=self.cookies)
return response
def log(self, message, type='info'):
self.report.write_log("William", message, self.get_delta_date(), type)
def get_daily_data(self):
response = self.get_ajax_data()
try:
response = json.loads(response.content)
data = response['data'][0]
self.data['daily_click'] = int(data[3]['Value'])
self.data['daily_signup'] = int(data[5]['Value'])
self.data['daily_commission'] = float(data[13]['Value'])
self.data['paid_signup'] = int(data[10]['Value'])
self.data['created_at'] = self.get_delta_date()
return True
except Exception as e:
self.log(str(e), "error")
return False
def get_monthly_data(self):
response = self.get_ajax_data('monthly')
try:
response = json.loads(response.content)
rows = response['data']
click = 0
signup = 0
commission = 0.0
for row in rows:
click += int(row[3]['Value'])
signup += int(row[5]['Value'])
commission += float(row[13]['Value'])
self.data['monthly_click'] = click
self.data['monthly_signup'] = signup
self.data['monthly_commission'] = commission
return True
except Exception as e:
self.log(str(e), "error")
return False
def get_yearly_data(self):
response = self.get_ajax_data('yearly')
try:
response = json.loads(response.content)
rows = response['data']
click = 0
signup = 0
commission = 0.0
for row in rows:
click += int(row[3]['Value'])
signup += int(row[5]['Value'])
commission += float(row[13]['Value'])
self.data['yearly_click'] = click
self.data['yearly_signup'] = signup
self.data['yearly_commission'] = commission
return True
except Exception as e:
self.log(str(e), "error")
return False
def get_data(self):
return self.get_daily_data() and self.get_monthly_data(
) and self.get_yearly_data()
def login(self):
try:
self.client.open_url(
'https://account.affiliates.williamhill.com/affiliates/Account/Login'
)
self.client.set_loginform('//*[@id="txtUsername"]')
self.client.set_passform('//*[@id="txtPassword"]')
self.client.set_loginbutton('//*[@id="btnLogin"]')
if self.client.login('betfy.co.uk', 'dontfuckwithme') is True:
self._get_cookies()
return True
else:
return False
except Exception as e:
self.log(str(e), "error")
return False
def isExisting(self, date=None):
if date is None:
date = self.get_delta_date()
app = scheduler.app
with app.app_context():
affiliate = Affiliate.query.filter_by(name=self.affiliate).first()
if affiliate is None:
return False
history = History.query.filter_by(affiliate_id=affiliate.id,
created_at=date).first()
if history is None:
return False
else:
return True
return True
def run(self):
self.log("""
======================================================
====== Starting William Spider ======================
""")
if self.isExisting():
self.log("Scrapped for `{0}` already done. Skipping...".format(
self.affiliate))
else:
self.client = UBrowse()
if self.login():
if self.get_data():
if self.save():
self.log("Successfully stored to DB.")
else:
self.log("Something went wrong in DB manipulation.")
else:
self.log("Failed to get data")
else:
self.log("Login Failed in william spider", "error")
self.client.close()
class Affil(object):
"""docstring for Affil"""
def __init__(self):
self.client = UBrowse()
self.headers = {
'Host': 'program.uffiliates.com',
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0',
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate, br',
'X-Requested-With': 'XMLHttpRequest',
'Referer':
'https://program.uffiliates.com/en/reports/trafficstats',
}
def _create_params(self, from_date, to_date, media=False):
if media:
self.params = (
('operatorID', '1'),
('brandID', '-1'),
('mediaTypeId', '-1'),
('langId', '-1'),
('sizeId', '-1'),
('toDate', '%sT00:00:00Z' % to_date),
('fromDate', '%sT00:00:00Z' % from_date),
('_search', 'false'),
('rows', '50'),
('page', '1'),
('sidx', ''),
('sord', 'desc'),
)
else:
self.params = (
('operatorID', '1'),
('brandID', '-1'),
('trackingCode', '-1'),
('toDate', '%sT00:00:00Z' % to_date),
('fromDate', '%sT00:00:00Z' % from_date),
('isGrouping', 'false'),
('isFilterFirstTime', 'true'),
('isWithAnid', 'false'),
('isShowCountry', 'false'),
('isShowDevice', 'false'),
('_search', 'false'),
('rows', '50'),
('page', '1'),
('sidx', ''),
('sord', 'asc'),
)
def _get_cookies(self):
self.cookies = dict()
cookies = self.client.driver.get_cookies()
for i in cookies:
self.cookies[i['name']] = i['value']
def get_data(self,
week=None,
this_month=None,
pre_month=None,
year=None,
media=False):
date_now = datetime.datetime.now()
from_date = date_now.strftime('%Y-%m-%d')
to_date = date_now.strftime('%Y-%m-%d')
if week:
time_delta = datetime.timedelta(days=7)
from_date = (date_now - time_delta).strftime('%Y-%m-%d')
if this_month:
from_date = date_now.strftime('%Y-%m-01')
if pre_month:
time_delta = datetime.timedelta(days=date_now.day)
last_date_pre_month = date_now - time_delta
from_date = last_date_pre_month.strftime('%Y-%m-01')
to_date = last_date_pre_month.strftime('%Y-%m-%d')
if year:
from_date = datetime.datetime(date_now.year, 1,
1).strftime('%Y-%m-%d')
if media:
self._create_params(from_date, to_date, media=True)
url = 'https://program.uffiliates.com/en/Reports/GetMediaStatsReport'
else:
self._create_params(from_date, to_date)
url = 'https://program.uffiliates.com/en/Reports/GetTrafficStatsReport'
response = requests.get(url,
headers=self.headers,
params=self.params,
cookies=self.cookies)
return response
def get_media_data(self,
week=None,
this_month=None,
pre_month=None,
year=None):
date_now = datetime.datetime.now()
from_date = date_now.strftime('%Y-%m-%d')
to_date = date_now.strftime('%Y-%m-%d')
if week:
time_delta = datetime.timedelta(days=7)
from_date = (date_now - time_delta).strftime('%Y-%m-%d')
if this_month:
from_date = date_now.strftime('%Y-%m-01')
if pre_month:
time_delta = datetime.timedelta(days=date_now.day)
last_date_pre_month = date_now - time_delta
from_date = last_date_pre_month.strftime('%Y-%m-01')
to_date = last_date_pre_month.strftime('%Y-%m-%d')
if year:
from_date = datetime.datetime(date_now.year, 1,
1).strftime('%Y-%m-%d')
self._create_params(from_date, to_date)
url = 'https://program.uffiliates.com/en/Reports/GetMediaStatsReport'
response = requests.get(url,
headers=self.headers,
params=self.params,
cookies=self.cookies)
return response
def run(self):
self.client.open_url('https://program.uffiliates.com/en/Auth/Login')
self.client.set_loginform('//*[@id="userName"]')
self.client.set_passform('//*[@id="password"]')
self.client.set_loginbutton('//*[@id="btnLogin"]')
if self.client.login('betfyuk', 'LALB37hUhs') is True:
self._get_cookies()
return True