class Affil(object):
"""docstring for Affil"""
def __init__(self):
self.client = UBrowse()
self.headers = {
'Host': 'program.uffiliates.com',
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0',
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate, br',
'X-Requested-With': 'XMLHttpRequest',
'Referer':
'https://program.uffiliates.com/en/reports/trafficstats',
}
def _create_params(self, from_date, to_date, media=False):
if media:
self.params = (
('operatorID', '1'),
('brandID', '-1'),
('mediaTypeId', '-1'),
('langId', '-1'),
('sizeId', '-1'),
('toDate', '%sT00:00:00Z' % to_date),
('fromDate', '%sT00:00:00Z' % from_date),
('_search', 'false'),
('rows', '50'),
('page', '1'),
('sidx', ''),
('sord', 'desc'),
)
else:
self.params = (
('operatorID', '1'),
('brandID', '-1'),
('trackingCode', '-1'),
('toDate', '%sT00:00:00Z' % to_date),
('fromDate', '%sT00:00:00Z' % from_date),
('isGrouping', 'false'),
('isFilterFirstTime', 'true'),
('isWithAnid', 'false'),
('isShowCountry', 'false'),
('isShowDevice', 'false'),
('_search', 'false'),
('rows', '50'),
('page', '1'),
('sidx', ''),
('sord', 'asc'),
)
def _get_cookies(self):
self.cookies = dict()
cookies = self.client.driver.get_cookies()
for i in cookies:
self.cookies[i['name']] = i['value']
def get_data(self,
week=None,
this_month=None,
pre_month=None,
year=None,
media=False):
date_now = datetime.datetime.now()
from_date = date_now.strftime('%Y-%m-%d')
to_date = date_now.strftime('%Y-%m-%d')
if week:
time_delta = datetime.timedelta(days=7)
from_date = (date_now - time_delta).strftime('%Y-%m-%d')
if this_month:
from_date = date_now.strftime('%Y-%m-01')
if pre_month:
time_delta = datetime.timedelta(days=date_now.day)
last_date_pre_month = date_now - time_delta
from_date = last_date_pre_month.strftime('%Y-%m-01')
to_date = last_date_pre_month.strftime('%Y-%m-%d')
if year:
from_date = datetime.datetime(date_now.year, 1,
1).strftime('%Y-%m-%d')
if media:
self._create_params(from_date, to_date, media=True)
url = 'https://program.uffiliates.com/en/Reports/GetMediaStatsReport'
else:
self._create_params(from_date, to_date)
url = 'https://program.uffiliates.com/en/Reports/GetTrafficStatsReport'
response = requests.get(url,
headers=self.headers,
params=self.params,
cookies=self.cookies)
return response
def get_media_data(self,
week=None,
this_month=None,
pre_month=None,
year=None):
date_now = datetime.datetime.now()
from_date = date_now.strftime('%Y-%m-%d')
to_date = date_now.strftime('%Y-%m-%d')
if week:
time_delta = datetime.timedelta(days=7)
from_date = (date_now - time_delta).strftime('%Y-%m-%d')
if this_month:
from_date = date_now.strftime('%Y-%m-01')
if pre_month:
time_delta = datetime.timedelta(days=date_now.day)
last_date_pre_month = date_now - time_delta
from_date = last_date_pre_month.strftime('%Y-%m-01')
to_date = last_date_pre_month.strftime('%Y-%m-%d')
if year:
from_date = datetime.datetime(date_now.year, 1,
1).strftime('%Y-%m-%d')
self._create_params(from_date, to_date)
url = 'https://program.uffiliates.com/en/Reports/GetMediaStatsReport'
response = requests.get(url,
headers=self.headers,
params=self.params,
cookies=self.cookies)
return response
def run(self):
self.client.open_url('https://program.uffiliates.com/en/Auth/Login')
self.client.set_loginform('//*[@id="userName"]')
self.client.set_passform('//*[@id="password"]')
self.client.set_loginbutton('//*[@id="btnLogin"]')
if self.client.login('betfyuk', 'LALB37hUhs') is True:
self._get_cookies()
return True
class SkyBet(object):
"""docstring for SkyBet"""
def __init__(self):
self.report = SpiderReporter()
self.client = UBrowse()
self.login_url = 'https://affiliatehub.skybet.com/login.asp'
self.report_url = 'https://affiliatehub.skybet.com/reporting/quick_summary_report.asp'
self.username = '******'
self.password = '******'
self.items = []
self.quick_stats_timer = 0
self.YTD_stats_timer = 0
self.report_timer = 0
self.affiliate = "SkyBet"
self.headers = {
'Host': 'affiliatehub.skybet.com',
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0',
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate, br',
'X-Requested-With': 'XMLHttpRequest',
'Referer': 'https://affiliatehub.skybet.com/reporting/quick_summary_report.asp',
}
def _create_params(self, from_date, to_date, media=False):
if media:
self.params = (
('key', 'value'),
)
else:
self.params = (
('operatorID', '1'),
)
def _get_cookies(self):
self.cookies = dict()
cookies = self.client.driver.get_cookies()
for i in cookies:
self.cookies[i['name']] = i['value']
def get_delta_date(self, delta = DELTA_DAYS, format_string = "%Y/%m/%d"):
today = datetime.datetime.today()
diff = datetime.timedelta(days = delta)
return (today - diff).strftime(format_string)
def login(self):
self.client.open_url(self.login_url)
time.sleep(3)
self.client.set_loginform('//*[@id="username"]')
self.client.set_passform('//*[@id="password"]')
self.client.set_loginbutton('//button[@type="submit"]')
if self.client.login(self.username, self.password) is True:
self._get_cookies()
return True
else:
return False
def select_YTD_option(self):
try:
period_select = Select(self.client.driver.find_element_by_xpath('//*[@id="dashboard"]//select[@name="WRQSperiod"]'))
period_select.select_by_value('YTD')
return True
except:
return False
def log(self, message, type = "info"):
self.report.write_log("SkyBet", message, self.get_delta_date(), type)
def get_YTD_stats(self):
time.sleep(5)
try:
table = self.client.driver.find_element_by_xpath('//*[@id="dashboard_quick_stats"]//tr[@class="row_light_color"]')
for td in table.find_elements_by_tag_name('td'):
if td.text == u'':
raise ValueError("Value can't be empty.")
break
self.items.append(td.text)
return True
except:
self.log("Element not found. (in get_YTD_stats function)")
self.YTD_stats_timer += 1
if self.YTD_stats_timer < 10:
return self.get_YTD_stats()
else:
return False
def get_quick_stats(self):
time.sleep(5)
try:
table = self.client.driver.find_element_by_xpath('//*[@id="dashboard_quick_stats"]//tr[@class="row_light_color"]')
for td in table.find_elements_by_tag_name('td'):
if td.text == u'':
raise ValueError("Value can't be empty.")
break
self.items.append(td.text)
return True
except:
self.log("Element not found. (in get_quick_stats function)")
self.quick_stats_timer += 1
if self.quick_stats_timer < 6:
return self.get_quick_stats()
else:
return False
def parse_stats_report(self):
time.sleep(3)
param_date = self.get_delta_date()
try:
table = self.client.driver.find_element_by_xpath('//*[@id="internalreportdata"]/table')
todayVal = table.find_elements_by_tag_name("tr")
pattern = re.compile(r'[\-\d.\d]+')
imprto = pattern.search(todayVal[1].text).group(0)
self.items.append(imprto)
clito = pattern.search(todayVal[2].text).group(0)
self.items.append(clito)
regto = pattern.search(todayVal[5].text).group(0)
self.items.append(regto)
ndto = pattern.search(todayVal[8].text).group(0)
self.items.append(ndto)
commito = pattern.search(todayVal[-1].text).group(0)
self.items.append(commito)
self.items.append(param_date)
return True
except:
self.log("Element not found. (in parse_stats_report function)")
self.report_timer += 1
if self.report_timer < 4:
return self.parse_stats_report()
else:
return False
def get_stats_report(self):
self.client.open_url(self.report_url)
time.sleep(3)
merchant = Select(self.client.driver.find_element_by_xpath('//form[@id="FRMReportoptions"]//select[@name="merchantid"]'))
merchant.select_by_value('0')
param_date = self.get_delta_date()
self.client.driver.execute_script("document.getElementById('startdate').value = '{0}'".format(param_date))
self.client.driver.execute_script("document.getElementById('enddate').value = '{0}'".format(param_date))
self.client.driver.find_element_by_class_name("button").click()
self.parse_stats_report()
def save(self):
pattern = re.compile(r'[\-\d.\d]+')
monthly_click = int(self.items[2])
monthly_signup = int(self.items[3])
paid_signup = int(self.items[4])
commissionStr = str(self.items[5]).replace(',', '')
monthly_commission = float(pattern.search(commissionStr).group(0))
yearly_click = int(self.items[8])
yearly_signup = int(self.items[9])
commiytdStr = str(self.items[11]).replace(',', '')
yearly_commission = float(pattern.search(commiytdStr).group(0))
daily_click = int(self.items[13])
daily_signup = int(self.items[14])
daily_commission = float(self.items[16])
created_at = self.get_delta_date()
app = scheduler.app
with app.app_context():
affiliate = Affiliate.query.filter_by(name = self.affiliate).first()
if affiliate is None:
affiliate = Affiliate(name = self.affiliate)
db.session.add(affiliate)
db.session.commit()
history = History.query.filter_by(affiliate_id = affiliate.id, created_at = created_at).first()
if history is None:
history = History(
affiliate_id = affiliate.id,
daily_click = daily_click,
daily_signup = daily_signup,
daily_commission = daily_commission,
monthly_click = monthly_click,
monthly_signup = monthly_signup,
monthly_commission = monthly_commission,
yearly_click = yearly_click,
yearly_signup = yearly_signup,
yearly_commission = yearly_commission,
paid_signup = paid_signup,
created_at = created_at
)
db.session.add(history)
db.session.commit()
return True
def isExisting(self, date = None):
if date is None:
date = self.get_delta_date()
app = scheduler.app
with app.app_context():
affiliate = Affiliate.query.filter_by(name = self.affiliate).first()
if affiliate is None:
return False
history = History.query.filter_by(affiliate_id = affiliate.id, created_at = date).first()
if history is None:
return False
else:
return True
return True
def run(self):
self.log("""
======================================================
====== Starting SkyBet Spider ======================
""")
if self.isExisting:
self.log("Scrapped for `{0}` already done. Skipping...".format(self.affiliate))
return False
elif self.login() is True:
self.log("Successfully logged in. Parsing quick stats.")
self.get_quick_stats()
self.select_YTD_option()
self.get_YTD_stats()
self.log("Pulling quick stats reporting...")
self.get_stats_report()
if self.save() == True:
self.log("Pulled data successfully saved!")
return True
else:
self.log("Something went wrong in DB Query.", "error")
return False
else:
self.log("Failed to log in!!", "error")
return False
self.client.close()
class LadBrokes(object):
"""docstring for LadBrokes"""
def __init__(self):
self.report = SpiderReporter()
self.report_url = 'https://portal.ladbrokespartners.com/portal/#/statistics'
self.ajax_url = 'https://portal.ladbrokespartners.com/portal/rest/reports/run/atOnce'
self.data = {}
self.affiliate = "LadBrokes"
self.headers = {
'Accept':
'application/json, text/plain, */*',
'Accept-Encoding':
'gzip, deflate, br',
'Accept-Language':
'en-US,en;q=0.5',
'Content-Length':
'9167',
'Content-Type':
'application/json;charset=utf-8',
'Host':
'portal.ladbrokespartners.com',
'Referer':
'https://portal.ladbrokespartners.com/portal/',
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0',
}
def get_delta_date(self, delta=DELTA_DAYS, format_string="%Y/%m/%d"):
today = datetime.datetime.today()
diff = datetime.timedelta(days=delta)
return (today - diff).strftime(format_string)
def _create_params(self, mode='daily'):
delta = datetime.timedelta(days=1)
today = datetime.datetime.now()
yesterday = today - delta
end_date = yesterday.strftime('%Y-%m-%d')
start_date = yesterday.strftime('%Y-%m-%d')
if mode == 'monthly':
delta = dateutil.relativedelta.relativedelta(months=1)
start_date = (yesterday - delta).strftime('%Y-%m-%d')
elif mode == 'yearly':
delta = dateutil.relativedelta.relativedelta(years=1)
start_date = (yesterday - delta).strftime('%Y-%m-%d')
self.ajax_param = '{"report":{"campaignId":"245450","input":[{"name":"startDate","label":"Stat Date","type":"dateRange","required":true,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":"yyyy-MM-dd","jsDateTimeFormat":"yy-MM-dd","precision":3,"value":"custom","values":[],"sublist":[{"name":"startDate_list","label":null,"type":"list","required":true,"collapsed":false,"filter":true,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":null,"value":"today","values":[{"id":"custom","name":null},{"id":"today","name":null},{"id":"yesterday","name":null},{"id":"currweek","name":null},{"id":"prevweek","name":null},{"id":"currmonth","name":null},{"id":"prevmonth","name":null},{"id":"threemonths","name":null},{"id":"sixmonths","name":null},{"id":"curryear","name":null},{"id":"prevyear","name":null}],"sublist":[]},{"name":"startDate_from","label":null,"type":"date","required":true,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":"yyyy-MM-dd","jsDateTimeFormat":"yy-MM-dd","precision":null,"value":"%s","values":[],"sublist":[]},{"name":"startDate_to","label":null,"type":"date","required":true,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":"yyyy-MM-dd","jsDateTimeFormat":"yy-MM-dd","precision":null,"value":"%s","values":[],"sublist":[]}]},{"name":"profile","label":"Profile","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"reportBy1","label":"Report By","type":"list","required":true,"collapsed":false,"filter":true,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":"stat_date","values":[{"id":"stat_date","name":"Date"},{"id":"stat_month","name":"Month"},{"id":"profile","name":"Profile"},{"id":"product","name":"Product"},{"id":"banner","name":"Banner"},{"id":"platform","name":"Platform"},{"id":"country","name":"Country"},{"id":"var1","name":"Var 1"},{"id":"var2","name":"Var 2"},{"id":"var9","name":"Var 9"},{"id":"var10","name":"Var 10"}],"sublist":[]},{"name":"reportBy2","label":"Report By","type":"list","required":false,"collapsed":false,"filter":true,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[{"id":"stat_date","name":"Date"},{"id":"stat_month","name":"Month"},{"id":"profile","name":"Profile"},{"id":"product","name":"Product"},{"id":"banner","name":"Banner"},{"id":"platform","name":"Platform"},{"id":"country","name":"Country"},{"id":"var1","name":"Var 1"},{"id":"var2","name":"Var 2"},{"id":"var9","name":"Var 9"},{"id":"var10","name":"Var 10"}],"sublist":[]},{"name":"reportBy3","label":"Report By","type":"list","required":false,"collapsed":false,"filter":true,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[{"id":"stat_date","name":"Date"},{"id":"stat_month","name":"Month"},{"id":"profile","name":"Profile"},{"id":"product","name":"Product"},{"id":"banner","name":"Banner"},{"id":"platform","name":"Platform"},{"id":"country","name":"Country"},{"id":"var1","name":"Var 1"},{"id":"var2","name":"Var 2"},{"id":"var9","name":"Var 9"},{"id":"var10","name":"Var 10"}],"sublist":[]},{"name":"reportBy4","label":"Report By","type":"list","required":false,"collapsed":false,"filter":true,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[{"id":"stat_date","name":"Date"},{"id":"stat_month","name":"Month"},{"id":"profile","name":"Profile"},{"id":"product","name":"Product"},{"id":"banner","name":"Banner"},{"id":"platform","name":"Platform"},{"id":"country","name":"Country"},{"id":"var1","name":"Var 1"},{"id":"var2","name":"Var 2"},{"id":"var9","name":"Var 9"},{"id":"var10","name":"Var 10"}],"sublist":[]}],"output":[{"name":"stat_date","label":"Date","type":"date","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":"yyyy-MM-dd","jsDateTimeFormat":"yy-MM-dd","precision":3,"value":null,"values":[],"sublist":[]},{"name":"stat_month","label":"Month","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"platform","label":"Platform","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"profile","label":"Profile","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"product","label":"Product","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"banner","label":"Banner","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"country","label":"Country","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"var1","label":"Var 1","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"var2","label":"Var 2","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"var9","label":"Var 9","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"var10","label":"Var 10","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"REAL_IMPS","label":"Unique Impressions","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"RAW_IMPS","label":"Non-Unique Impressions","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"REAL_CLICKS","label":"Unique Clicks","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"RAW_CLICKS","label":"Non-Unique Clicks","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"sports_signups","label":"sports Signups Cnt","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"sports_acquired_count","label":"sports Acquired cnt","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"acquired_count","label":"Acquired cnt","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"WITHDRAWS_CNT","label":"Withdrawal cnt","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"sports_net_gaming_revenue","label":"Sports Net Gaming Revenue","type":"number","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"cpa_commission","label":"CPA Commission","type":"number","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"ng_commission","label":"Net Gaming Commission","type":"number","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"tlr_amount","label":"Top Level Revenue","type":"number","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]}],"language":"EN","name":"trafficStatsNew","id":"1712"},"sc":{"offset":0,"limit":1000,"order":[{"ascending":true,"column":"1"}],"namedQuery":""}}' % (
start_date, end_date)
def _get_auth_token(self):
try:
self.headers['X-Auth-Token'] = self.client.driver.execute_script(
"return localStorage.authToken")
except:
return False
return True
def _get_cookies(self):
self.cookies = dict()
cookies = self.client.driver.get_cookies()
for i in cookies:
self.cookies[i['name']] = i['value']
def log(self, message, type='info'):
self.report.write_log("LadBroke", message, self.get_delta_date(), type)
def get_yearly_data(self):
self._create_params('yearly')
response = requests.post(self.ajax_url,
headers=self.headers,
cookies=self.cookies,
data=self.ajax_param)
content = response.content
response = json.loads(content)
if (response['data']['total']['data']):
values = response['data']['total']['data']
self.data['yearly_click'] = int(values[3])
self.data['yearly_signup'] = int(values[5])
self.data['yearly_commission'] = float(values[10])
return True
else:
self.log("Something went wrong in Ajax for Yearly data", 'error')
return False
def get_monthly_data(self):
self._create_params('monthly')
response = requests.post(self.ajax_url,
headers=self.headers,
cookies=self.cookies,
data=self.ajax_param)
content = response.content
response = json.loads(content)
if (response['data']['total']['data']):
values = response['data']['total']['data']
self.data['monthly_click'] = int(values[3])
self.data['monthly_signup'] = int(values[5])
self.data['monthly_commission'] = float(values[10])
return True
else:
self.log("Something went wrong in Ajax for Monthly data", 'error')
return False
def get_daily_data(self):
self.client.open_url(self.report_url)
time.sleep(2)
response = requests.post(self.ajax_url,
headers=self.headers,
cookies=self.cookies,
data=self.ajax_param)
content = response.content
response = json.loads(content)
if (response['data']['total']['data']):
values = response['data']['total']['data']
self.data['created_at'] = self.get_delta_date()
self.data['daily_click'] = int(values[3])
self.data['daily_signup'] = int(values[5])
self.data['daily_commission'] = float(values[10])
self.data['paid_signup'] = 0
return True
else:
self.log("Something went wrong in Ajax for Daily data", 'error')
return False
def get_data(self):
time.sleep(5)
self._get_auth_token()
self.log("Getting daily data...")
self.get_daily_data()
self.log("Getting monthly data...")
self.get_monthly_data()
self.log("Getting yearly data...")
self.get_yearly_data()
return self.save()
def save(self):
try:
app = scheduler.app
with app.app_context():
affiliate = Affiliate.query.filter_by(
name=self.affiliate).first()
if affiliate is None:
affiliate = Affiliate(name=self.affiliate)
db.session.add(affiliate)
db.session.commit()
print(affiliate)
created_at = self.get_delta_date()
history = History.query.filter_by(
affiliate_id=affiliate.id, created_at=created_at).first()
if history is None:
history = History(
affiliate_id=affiliate.id,
daily_click=self.data['daily_click'],
daily_signup=self.data['daily_signup'],
daily_commission=self.data['daily_commission'],
monthly_click=self.data['monthly_click'],
monthly_signup=self.data['monthly_signup'],
monthly_commission=self.data['monthly_commission'],
yearly_click=self.data['yearly_click'],
yearly_signup=self.data['yearly_signup'],
yearly_commission=self.data['yearly_commission'],
paid_signup=self.data['paid_signup'],
created_at=created_at)
db.session.add(history)
db.session.commit()
return True
except Exception as e:
self.log(str(e), "error")
return False
def isExisting(self, date=None):
if date == None:
date = self.get_delta_date()
app = scheduler.app
with app.app_context():
affiliate = Affiliate.query.filter_by(name=self.affiliate).first()
if affiliate is None:
return False
history = History.query.filter_by(affiliate_id=affiliate.id,
created_at=date).first()
if history is None:
return False
else:
return True
return True
def run(self):
self.log("""
======================================================
====== Starting LadBrokes Spider ======================
""")
if self.isExisting() is False:
self.client = UBrowse()
self.client.open_url(
'https://portal.ladbrokespartners.com/portal/#/login')
self.client.set_loginform('//*[@id="userName"]')
self.client.set_passform('//*[@id="password"]')
self.client.set_loginbutton(
'/html/body/div[3]/section/div/form/div[5]/div/input')
if self.client.login('betfyuk', 'WjewEEUV') is True:
self._get_cookies()
self._create_params()
if self.get_data():
self.log("Data stored successfully")
else:
self.log("Failed to write to DB")
else:
self.log("Login Failed", 'error')
self.client.close()
else:
self.log("Already scrapped for `{0}`. Skipping...".format(
self.affiliate))