class Affil(object):
    """docstring for Affil"""
    def __init__(self):
        self.client = UBrowse()

        self.headers = {
            'Host': 'program.uffiliates.com',
            'User-Agent':
            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0',
            'Accept': 'application/json, text/javascript, */*; q=0.01',
            'Accept-Language': 'en-US,en;q=0.5',
            'Accept-Encoding': 'gzip, deflate, br',
            'X-Requested-With': 'XMLHttpRequest',
            'Referer':
            'https://program.uffiliates.com/en/reports/trafficstats',
        }

    def _create_params(self, from_date, to_date, media=False):

        if media:
            self.params = (
                ('operatorID', '1'),
                ('brandID', '-1'),
                ('mediaTypeId', '-1'),
                ('langId', '-1'),
                ('sizeId', '-1'),
                ('toDate', '%sT00:00:00Z' % to_date),
                ('fromDate', '%sT00:00:00Z' % from_date),
                ('_search', 'false'),
                ('rows', '50'),
                ('page', '1'),
                ('sidx', ''),
                ('sord', 'desc'),
            )
        else:
            self.params = (
                ('operatorID', '1'),
                ('brandID', '-1'),
                ('trackingCode', '-1'),
                ('toDate', '%sT00:00:00Z' % to_date),
                ('fromDate', '%sT00:00:00Z' % from_date),
                ('isGrouping', 'false'),
                ('isFilterFirstTime', 'true'),
                ('isWithAnid', 'false'),
                ('isShowCountry', 'false'),
                ('isShowDevice', 'false'),
                ('_search', 'false'),
                ('rows', '50'),
                ('page', '1'),
                ('sidx', ''),
                ('sord', 'asc'),
            )

    def _get_cookies(self):
        self.cookies = dict()
        cookies = self.client.driver.get_cookies()
        for i in cookies:
            self.cookies[i['name']] = i['value']

    def get_data(self,
                 week=None,
                 this_month=None,
                 pre_month=None,
                 year=None,
                 media=False):
        date_now = datetime.datetime.now()
        from_date = date_now.strftime('%Y-%m-%d')
        to_date = date_now.strftime('%Y-%m-%d')

        if week:
            time_delta = datetime.timedelta(days=7)
            from_date = (date_now - time_delta).strftime('%Y-%m-%d')
        if this_month:
            from_date = date_now.strftime('%Y-%m-01')
        if pre_month:
            time_delta = datetime.timedelta(days=date_now.day)
            last_date_pre_month = date_now - time_delta
            from_date = last_date_pre_month.strftime('%Y-%m-01')
            to_date = last_date_pre_month.strftime('%Y-%m-%d')
        if year:
            from_date = datetime.datetime(date_now.year, 1,
                                          1).strftime('%Y-%m-%d')

        if media:
            self._create_params(from_date, to_date, media=True)
            url = 'https://program.uffiliates.com/en/Reports/GetMediaStatsReport'
        else:
            self._create_params(from_date, to_date)
            url = 'https://program.uffiliates.com/en/Reports/GetTrafficStatsReport'

        response = requests.get(url,
                                headers=self.headers,
                                params=self.params,
                                cookies=self.cookies)
        return response

    def get_media_data(self,
                       week=None,
                       this_month=None,
                       pre_month=None,
                       year=None):
        date_now = datetime.datetime.now()
        from_date = date_now.strftime('%Y-%m-%d')
        to_date = date_now.strftime('%Y-%m-%d')

        if week:
            time_delta = datetime.timedelta(days=7)
            from_date = (date_now - time_delta).strftime('%Y-%m-%d')
        if this_month:
            from_date = date_now.strftime('%Y-%m-01')
        if pre_month:
            time_delta = datetime.timedelta(days=date_now.day)
            last_date_pre_month = date_now - time_delta
            from_date = last_date_pre_month.strftime('%Y-%m-01')
            to_date = last_date_pre_month.strftime('%Y-%m-%d')
        if year:
            from_date = datetime.datetime(date_now.year, 1,
                                          1).strftime('%Y-%m-%d')

        self._create_params(from_date, to_date)
        url = 'https://program.uffiliates.com/en/Reports/GetMediaStatsReport'
        response = requests.get(url,
                                headers=self.headers,
                                params=self.params,
                                cookies=self.cookies)
        return response

    def run(self):
        self.client.open_url('https://program.uffiliates.com/en/Auth/Login')
        self.client.set_loginform('//*[@id="userName"]')
        self.client.set_passform('//*[@id="password"]')
        self.client.set_loginbutton('//*[@id="btnLogin"]')

        if self.client.login('betfyuk', 'LALB37hUhs') is True:
            self._get_cookies()

        return True
Example #2
0
class SkyBet(object):
    """docstring for SkyBet"""
    def __init__(self):
        self.report = SpiderReporter()
        self.client = UBrowse()
        self.login_url = 'https://affiliatehub.skybet.com/login.asp'
        self.report_url = 'https://affiliatehub.skybet.com/reporting/quick_summary_report.asp'
        self.username = '******'
        self.password = '******'
        self.items = []
        self.quick_stats_timer = 0
        self.YTD_stats_timer = 0
        self.report_timer = 0
        self.affiliate = "SkyBet"

        self.headers = {
            'Host': 'affiliatehub.skybet.com',
            'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0',
            'Accept': 'application/json, text/javascript, */*; q=0.01',
            'Accept-Language': 'en-US,en;q=0.5',
            'Accept-Encoding': 'gzip, deflate, br',
            'X-Requested-With': 'XMLHttpRequest',
            'Referer': 'https://affiliatehub.skybet.com/reporting/quick_summary_report.asp',
        }

    def _create_params(self, from_date, to_date, media=False):
        if media:
            self.params = (
                ('key', 'value'),
                )
        else:
            self.params = (
                ('operatorID', '1'),
                )

    def _get_cookies(self):
        self.cookies = dict()
        cookies = self.client.driver.get_cookies()
        for i in cookies:
            self.cookies[i['name']] = i['value']

    def get_delta_date(self, delta = DELTA_DAYS, format_string = "%Y/%m/%d"):
        today = datetime.datetime.today()
        diff = datetime.timedelta(days = delta)
        return (today - diff).strftime(format_string)

    def login(self):
        self.client.open_url(self.login_url)
        time.sleep(3)
        self.client.set_loginform('//*[@id="username"]')
        self.client.set_passform('//*[@id="password"]')
        self.client.set_loginbutton('//button[@type="submit"]')

        if self.client.login(self.username, self.password) is True:
            self._get_cookies()
            return True
        else:
            return False

    def select_YTD_option(self):
        try:
            period_select = Select(self.client.driver.find_element_by_xpath('//*[@id="dashboard"]//select[@name="WRQSperiod"]'))
            period_select.select_by_value('YTD')
            return True
        except:
            return False

    def log(self, message, type = "info"):
        self.report.write_log("SkyBet", message, self.get_delta_date(), type)

    def get_YTD_stats(self):
        time.sleep(5)
        try:
            table = self.client.driver.find_element_by_xpath('//*[@id="dashboard_quick_stats"]//tr[@class="row_light_color"]')
            for td in table.find_elements_by_tag_name('td'):
                if td.text == u'':
                    raise ValueError("Value can't be empty.")
                    break
                self.items.append(td.text)
            return True

        except:
            self.log("Element not found. (in get_YTD_stats function)")
            self.YTD_stats_timer += 1
            if self.YTD_stats_timer < 10:
                return self.get_YTD_stats()
            else:
                return False

    def get_quick_stats(self):
        time.sleep(5)
        try:
            table = self.client.driver.find_element_by_xpath('//*[@id="dashboard_quick_stats"]//tr[@class="row_light_color"]')
            for td in table.find_elements_by_tag_name('td'):
                if td.text == u'':
                    raise ValueError("Value can't be empty.")
                    break
                self.items.append(td.text)
            return True
            
        except:
            self.log("Element not found. (in get_quick_stats function)")
            self.quick_stats_timer += 1
            if self.quick_stats_timer < 6:
                return self.get_quick_stats()
            else:
                return False

    def parse_stats_report(self):
        time.sleep(3)
        param_date = self.get_delta_date()
        try:
            table = self.client.driver.find_element_by_xpath('//*[@id="internalreportdata"]/table')
            todayVal = table.find_elements_by_tag_name("tr")

            pattern = re.compile(r'[\-\d.\d]+')
            imprto = pattern.search(todayVal[1].text).group(0)
            self.items.append(imprto)
            clito = pattern.search(todayVal[2].text).group(0)
            self.items.append(clito)
            regto = pattern.search(todayVal[5].text).group(0)
            self.items.append(regto)
            ndto = pattern.search(todayVal[8].text).group(0)
            self.items.append(ndto)
            commito = pattern.search(todayVal[-1].text).group(0)
            self.items.append(commito)
            self.items.append(param_date)
            return True

        except:
            self.log("Element not found. (in parse_stats_report function)")
            self.report_timer += 1
            if self.report_timer < 4:
                return self.parse_stats_report()
            else:
                return False

    def get_stats_report(self):
        self.client.open_url(self.report_url)
        time.sleep(3)
        merchant = Select(self.client.driver.find_element_by_xpath('//form[@id="FRMReportoptions"]//select[@name="merchantid"]'))
        merchant.select_by_value('0')
        param_date = self.get_delta_date()
        self.client.driver.execute_script("document.getElementById('startdate').value = '{0}'".format(param_date))
        self.client.driver.execute_script("document.getElementById('enddate').value = '{0}'".format(param_date))
        self.client.driver.find_element_by_class_name("button").click()
        self.parse_stats_report()

    def save(self):
        pattern = re.compile(r'[\-\d.\d]+')

        monthly_click = int(self.items[2])
        monthly_signup = int(self.items[3])
        paid_signup = int(self.items[4])
        commissionStr = str(self.items[5]).replace(',', '')
        monthly_commission = float(pattern.search(commissionStr).group(0))
        
        yearly_click = int(self.items[8])
        yearly_signup = int(self.items[9])
        commiytdStr = str(self.items[11]).replace(',', '')
        yearly_commission = float(pattern.search(commiytdStr).group(0))
        
        daily_click = int(self.items[13])
        daily_signup = int(self.items[14])
        daily_commission = float(self.items[16])
        
        created_at = self.get_delta_date()

        app = scheduler.app
        with app.app_context():
            affiliate = Affiliate.query.filter_by(name = self.affiliate).first()
            if affiliate is None:
                affiliate = Affiliate(name = self.affiliate)
                db.session.add(affiliate)
                db.session.commit()

            history = History.query.filter_by(affiliate_id = affiliate.id, created_at = created_at).first()
            if history is None:
                history = History(
                    affiliate_id = affiliate.id,
                    daily_click = daily_click,
                    daily_signup = daily_signup,
                    daily_commission = daily_commission,
                    monthly_click = monthly_click,
                    monthly_signup = monthly_signup,
                    monthly_commission = monthly_commission,
                    yearly_click = yearly_click,
                    yearly_signup = yearly_signup,
                    yearly_commission = yearly_commission,
                    paid_signup = paid_signup,
                    created_at = created_at
                )
                db.session.add(history)
                db.session.commit()
        return True

    def isExisting(self, date = None):
        if date is None:
            date = self.get_delta_date()
        app = scheduler.app
        with app.app_context():
            affiliate = Affiliate.query.filter_by(name = self.affiliate).first()

            if affiliate is None:
                return False

            history = History.query.filter_by(affiliate_id = affiliate.id, created_at = date).first()

            if history is None:
                return False
            else:
                return True
        
        return True

    def run(self):
        self.log("""
        ======================================================
        ======  Starting SkyBet Spider  ======================
        """)
        if self.isExisting:
            self.log("Scrapped for `{0}` already done. Skipping...".format(self.affiliate))
            return False
        elif self.login() is True:
            self.log("Successfully logged in. Parsing quick stats.")
            self.get_quick_stats()
            self.select_YTD_option()
            self.get_YTD_stats()

            self.log("Pulling quick stats reporting...")
            self.get_stats_report()

            if self.save() == True:
                self.log("Pulled data successfully saved!")
                return True
            else:
                self.log("Something went wrong in DB Query.", "error")
                return False
        else:
            self.log("Failed to log in!!", "error")
            return False

        self.client.close()
class LadBrokes(object):
    """docstring for LadBrokes"""
    def __init__(self):
        self.report = SpiderReporter()
        self.report_url = 'https://portal.ladbrokespartners.com/portal/#/statistics'
        self.ajax_url = 'https://portal.ladbrokespartners.com/portal/rest/reports/run/atOnce'
        self.data = {}
        self.affiliate = "LadBrokes"

        self.headers = {
            'Accept':
            'application/json, text/plain, */*',
            'Accept-Encoding':
            'gzip, deflate, br',
            'Accept-Language':
            'en-US,en;q=0.5',
            'Content-Length':
            '9167',
            'Content-Type':
            'application/json;charset=utf-8',
            'Host':
            'portal.ladbrokespartners.com',
            'Referer':
            'https://portal.ladbrokespartners.com/portal/',
            'User-Agent':
            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0',
        }

    def get_delta_date(self, delta=DELTA_DAYS, format_string="%Y/%m/%d"):
        today = datetime.datetime.today()
        diff = datetime.timedelta(days=delta)
        return (today - diff).strftime(format_string)

    def _create_params(self, mode='daily'):
        delta = datetime.timedelta(days=1)
        today = datetime.datetime.now()
        yesterday = today - delta
        end_date = yesterday.strftime('%Y-%m-%d')
        start_date = yesterday.strftime('%Y-%m-%d')

        if mode == 'monthly':
            delta = dateutil.relativedelta.relativedelta(months=1)
            start_date = (yesterday - delta).strftime('%Y-%m-%d')
        elif mode == 'yearly':
            delta = dateutil.relativedelta.relativedelta(years=1)
            start_date = (yesterday - delta).strftime('%Y-%m-%d')

        self.ajax_param = '{"report":{"campaignId":"245450","input":[{"name":"startDate","label":"Stat Date","type":"dateRange","required":true,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":"yyyy-MM-dd","jsDateTimeFormat":"yy-MM-dd","precision":3,"value":"custom","values":[],"sublist":[{"name":"startDate_list","label":null,"type":"list","required":true,"collapsed":false,"filter":true,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":null,"value":"today","values":[{"id":"custom","name":null},{"id":"today","name":null},{"id":"yesterday","name":null},{"id":"currweek","name":null},{"id":"prevweek","name":null},{"id":"currmonth","name":null},{"id":"prevmonth","name":null},{"id":"threemonths","name":null},{"id":"sixmonths","name":null},{"id":"curryear","name":null},{"id":"prevyear","name":null}],"sublist":[]},{"name":"startDate_from","label":null,"type":"date","required":true,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":"yyyy-MM-dd","jsDateTimeFormat":"yy-MM-dd","precision":null,"value":"%s","values":[],"sublist":[]},{"name":"startDate_to","label":null,"type":"date","required":true,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":"yyyy-MM-dd","jsDateTimeFormat":"yy-MM-dd","precision":null,"value":"%s","values":[],"sublist":[]}]},{"name":"profile","label":"Profile","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"reportBy1","label":"Report By","type":"list","required":true,"collapsed":false,"filter":true,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":"stat_date","values":[{"id":"stat_date","name":"Date"},{"id":"stat_month","name":"Month"},{"id":"profile","name":"Profile"},{"id":"product","name":"Product"},{"id":"banner","name":"Banner"},{"id":"platform","name":"Platform"},{"id":"country","name":"Country"},{"id":"var1","name":"Var 1"},{"id":"var2","name":"Var 2"},{"id":"var9","name":"Var 9"},{"id":"var10","name":"Var 10"}],"sublist":[]},{"name":"reportBy2","label":"Report By","type":"list","required":false,"collapsed":false,"filter":true,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[{"id":"stat_date","name":"Date"},{"id":"stat_month","name":"Month"},{"id":"profile","name":"Profile"},{"id":"product","name":"Product"},{"id":"banner","name":"Banner"},{"id":"platform","name":"Platform"},{"id":"country","name":"Country"},{"id":"var1","name":"Var 1"},{"id":"var2","name":"Var 2"},{"id":"var9","name":"Var 9"},{"id":"var10","name":"Var 10"}],"sublist":[]},{"name":"reportBy3","label":"Report By","type":"list","required":false,"collapsed":false,"filter":true,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[{"id":"stat_date","name":"Date"},{"id":"stat_month","name":"Month"},{"id":"profile","name":"Profile"},{"id":"product","name":"Product"},{"id":"banner","name":"Banner"},{"id":"platform","name":"Platform"},{"id":"country","name":"Country"},{"id":"var1","name":"Var 1"},{"id":"var2","name":"Var 2"},{"id":"var9","name":"Var 9"},{"id":"var10","name":"Var 10"}],"sublist":[]},{"name":"reportBy4","label":"Report By","type":"list","required":false,"collapsed":false,"filter":true,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[{"id":"stat_date","name":"Date"},{"id":"stat_month","name":"Month"},{"id":"profile","name":"Profile"},{"id":"product","name":"Product"},{"id":"banner","name":"Banner"},{"id":"platform","name":"Platform"},{"id":"country","name":"Country"},{"id":"var1","name":"Var 1"},{"id":"var2","name":"Var 2"},{"id":"var9","name":"Var 9"},{"id":"var10","name":"Var 10"}],"sublist":[]}],"output":[{"name":"stat_date","label":"Date","type":"date","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":"yyyy-MM-dd","jsDateTimeFormat":"yy-MM-dd","precision":3,"value":null,"values":[],"sublist":[]},{"name":"stat_month","label":"Month","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"platform","label":"Platform","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"profile","label":"Profile","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"product","label":"Product","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"banner","label":"Banner","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"country","label":"Country","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"var1","label":"Var 1","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"var2","label":"Var 2","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"var9","label":"Var 9","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"var10","label":"Var 10","type":"text","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"REAL_IMPS","label":"Unique Impressions","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"RAW_IMPS","label":"Non-Unique Impressions","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"REAL_CLICKS","label":"Unique Clicks","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"RAW_CLICKS","label":"Non-Unique Clicks","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"sports_signups","label":"sports Signups Cnt","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"sports_acquired_count","label":"sports Acquired cnt","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"acquired_count","label":"Acquired cnt","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"WITHDRAWS_CNT","label":"Withdrawal cnt","type":"long","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"sports_net_gaming_revenue","label":"Sports Net Gaming Revenue","type":"number","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"cpa_commission","label":"CPA Commission","type":"number","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"ng_commission","label":"Net Gaming Commission","type":"number","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]},{"name":"tlr_amount","label":"Top Level Revenue","type":"number","required":false,"collapsed":false,"filter":false,"internal":false,"dateTimeFormat":null,"jsDateTimeFormat":null,"precision":3,"value":null,"values":[],"sublist":[]}],"language":"EN","name":"trafficStatsNew","id":"1712"},"sc":{"offset":0,"limit":1000,"order":[{"ascending":true,"column":"1"}],"namedQuery":""}}' % (
            start_date, end_date)

    def _get_auth_token(self):
        try:
            self.headers['X-Auth-Token'] = self.client.driver.execute_script(
                "return localStorage.authToken")
        except:
            return False
        return True

    def _get_cookies(self):
        self.cookies = dict()
        cookies = self.client.driver.get_cookies()
        for i in cookies:
            self.cookies[i['name']] = i['value']

    def log(self, message, type='info'):
        self.report.write_log("LadBroke", message, self.get_delta_date(), type)

    def get_yearly_data(self):
        self._create_params('yearly')
        response = requests.post(self.ajax_url,
                                 headers=self.headers,
                                 cookies=self.cookies,
                                 data=self.ajax_param)
        content = response.content
        response = json.loads(content)

        if (response['data']['total']['data']):
            values = response['data']['total']['data']
            self.data['yearly_click'] = int(values[3])
            self.data['yearly_signup'] = int(values[5])
            self.data['yearly_commission'] = float(values[10])
            return True

        else:
            self.log("Something went wrong in Ajax for Yearly data", 'error')
            return False

    def get_monthly_data(self):
        self._create_params('monthly')
        response = requests.post(self.ajax_url,
                                 headers=self.headers,
                                 cookies=self.cookies,
                                 data=self.ajax_param)
        content = response.content
        response = json.loads(content)

        if (response['data']['total']['data']):
            values = response['data']['total']['data']
            self.data['monthly_click'] = int(values[3])
            self.data['monthly_signup'] = int(values[5])
            self.data['monthly_commission'] = float(values[10])
            return True

        else:
            self.log("Something went wrong in Ajax for Monthly data", 'error')
            return False

    def get_daily_data(self):
        self.client.open_url(self.report_url)
        time.sleep(2)
        response = requests.post(self.ajax_url,
                                 headers=self.headers,
                                 cookies=self.cookies,
                                 data=self.ajax_param)
        content = response.content
        response = json.loads(content)

        if (response['data']['total']['data']):
            values = response['data']['total']['data']
            self.data['created_at'] = self.get_delta_date()
            self.data['daily_click'] = int(values[3])
            self.data['daily_signup'] = int(values[5])
            self.data['daily_commission'] = float(values[10])
            self.data['paid_signup'] = 0
            return True

        else:
            self.log("Something went wrong in Ajax for Daily data", 'error')
            return False

    def get_data(self):
        time.sleep(5)
        self._get_auth_token()
        self.log("Getting daily data...")
        self.get_daily_data()
        self.log("Getting monthly data...")
        self.get_monthly_data()
        self.log("Getting yearly data...")
        self.get_yearly_data()
        return self.save()

    def save(self):
        try:
            app = scheduler.app
            with app.app_context():
                affiliate = Affiliate.query.filter_by(
                    name=self.affiliate).first()

                if affiliate is None:
                    affiliate = Affiliate(name=self.affiliate)
                    db.session.add(affiliate)
                    db.session.commit()

                print(affiliate)
                created_at = self.get_delta_date()

                history = History.query.filter_by(
                    affiliate_id=affiliate.id, created_at=created_at).first()

                if history is None:
                    history = History(
                        affiliate_id=affiliate.id,
                        daily_click=self.data['daily_click'],
                        daily_signup=self.data['daily_signup'],
                        daily_commission=self.data['daily_commission'],
                        monthly_click=self.data['monthly_click'],
                        monthly_signup=self.data['monthly_signup'],
                        monthly_commission=self.data['monthly_commission'],
                        yearly_click=self.data['yearly_click'],
                        yearly_signup=self.data['yearly_signup'],
                        yearly_commission=self.data['yearly_commission'],
                        paid_signup=self.data['paid_signup'],
                        created_at=created_at)
                    db.session.add(history)
                    db.session.commit()
            return True
        except Exception as e:
            self.log(str(e), "error")
            return False

    def isExisting(self, date=None):
        if date == None:
            date = self.get_delta_date()
        app = scheduler.app
        with app.app_context():
            affiliate = Affiliate.query.filter_by(name=self.affiliate).first()

            if affiliate is None:
                return False

            history = History.query.filter_by(affiliate_id=affiliate.id,
                                              created_at=date).first()

            if history is None:
                return False
            else:
                return True

        return True

    def run(self):
        self.log("""
        ======================================================
        ======  Starting LadBrokes Spider  ======================
        """)
        if self.isExisting() is False:
            self.client = UBrowse()
            self.client.open_url(
                'https://portal.ladbrokespartners.com/portal/#/login')
            self.client.set_loginform('//*[@id="userName"]')
            self.client.set_passform('//*[@id="password"]')
            self.client.set_loginbutton(
                '/html/body/div[3]/section/div/form/div[5]/div/input')

            if self.client.login('betfyuk', 'WjewEEUV') is True:
                self._get_cookies()
                self._create_params()

                if self.get_data():
                    self.log("Data stored successfully")
                else:
                    self.log("Failed to write to DB")
            else:
                self.log("Login Failed", 'error')
            self.client.close()

        else:
            self.log("Already scrapped for `{0}`. Skipping...".format(
                self.affiliate))