Пример #1
0
    def extract(self):
        '''对采集到的网页进行数据提取'''
        self.db = MySQL('job')
        jobs = self.findall('div.job-list>ul>li')
        for job in jobs:
            title = self.find('.job-title',job).text
            salary = self.find('.red',job).text

            job_link = self.find('.info-primary>h3.name>a',job).get_attribute('href')
            job_id = re.search(r'/job_detail/(.*).html',job_link).group(1)
            job_info = self.find('.info-primary>p',job).get_attribute('innerHTML')

            company_name = self.find(".info-company>div>h3>a",job).text
            company_link = self.find(".info-company>div>h3>a",job).get_attribute('href')
            company_id = re.search(r'/gongsi/(.*).html',company_link).group(1)

            html_str = self.find('.company-text>p',job).get_attribute('innerHTML')
            company_info = html_str.split('''<em class="vline"></em>''')
            company_industry = company_info[0]
            company_size = company_info.pop()
            publisher = self.find('.info-publis>h3',job).get_attribute('innerHTML').split('<em class="vline"></em>').pop()
            date_str = self.find('.info-publis p',job).text
            date_str = date_str.split('发布于')[1]
            try:
                pub_date = datetime.datetime.strptime(date_str,'%m月%d日')
                pub_date = pub_date.replace(year = 2019)
            except ValueError:
                pub_date = datetime.datetime.strptime('11月18日','%m月%d日')
                pub_date = pub_date.replace(year = 2019)
            data = dict(title=title,salary=salary,job_info=job_info.replace('\"','\''),job_id=job_id,company_name=company_name\
                ,company_id=company_id,company_industry=company_industry,company_size=company_size\
                ,publisher=publisher,pub_date=str(pub_date))
            self.write(data)
Пример #2
0
class ZhipinCrawler(IterateCrawler):
    '''继承浏览器迭代操作类,完成boss直聘数据采集'''
    def __init__(self, page_count):
        template = {
            'np_selector':'div.page>a.next',
        }
        super().__init__(template, page_count)

    def extract(self):
        '''对采集到的网页进行数据提取'''
        self.db = MySQL('job')
        jobs = self.findall('div.job-list>ul>li')
        for job in jobs:
            title = self.find('.job-title',job).text
            salary = self.find('.red',job).text

            job_link = self.find('.info-primary>h3.name>a',job).get_attribute('href')
            job_id = re.search(r'/job_detail/(.*).html',job_link).group(1)
            job_info = self.find('.info-primary>p',job).get_attribute('innerHTML')

            company_name = self.find(".info-company>div>h3>a",job).text
            company_link = self.find(".info-company>div>h3>a",job).get_attribute('href')
            company_id = re.search(r'/gongsi/(.*).html',company_link).group(1)

            html_str = self.find('.company-text>p',job).get_attribute('innerHTML')
            company_info = html_str.split('''<em class="vline"></em>''')
            company_industry = company_info[0]
            company_size = company_info.pop()
            publisher = self.find('.info-publis>h3',job).get_attribute('innerHTML').split('<em class="vline"></em>').pop()
            date_str = self.find('.info-publis p',job).text
            date_str = date_str.split('发布于')[1]
            try:
                pub_date = datetime.datetime.strptime(date_str,'%m月%d日')
                pub_date = pub_date.replace(year = 2019)
            except ValueError:
                pub_date = datetime.datetime.strptime('11月18日','%m月%d日')
                pub_date = pub_date.replace(year = 2019)
            data = dict(title=title,salary=salary,job_info=job_info.replace('\"','\''),job_id=job_id,company_name=company_name\
                ,company_id=company_id,company_industry=company_industry,company_size=company_size\
                ,publisher=publisher,pub_date=str(pub_date))
            self.write(data)

    def write(self, data):
        '''将提取到的数据写入到数据库'''
        self.db.run(sql.append('zhipin_guangzhou',data))

    def next_page(self):
        '''重写父类方法,实现翻页操作'''
        url = self.instance.current_url
        rst = re.search(r'page=([0-9]*)&',url)
        match = rst.group()
        num = int(rst.group(1))
        self.to(url.replace(match,'page=%d&'%(num+1)))

    def after(self):
        '''实现父类方法,采集完成后处理函数'''
        self.db.exit()
        t.say('job done')
Пример #3
0
 def __init__(self):
     self._headers = {
         'User-Agent':
         'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36',
         'Sec-Fetch-Mode': 'no-cors',
         'Host': 'arxiv.org'
     }
     self._sess = requests.Session()
     self._sleep_time = 5
     self._mysql = MySQL()
Пример #4
0
 def run(self):
     args = self.arguments()
     if (args.filenames):
         csvobject = CSVParser(args.filenames[0], args.filenames[1])
         csvobject.format_csv_files()
     elif (args.output):
         print('We are connecting to the database...')
         db = MySQL()  # Here is where you would pass in a MySQL connection
         db.write_to_file('SELECT * from foobar', args.output[0])
     else:
         assert False, "Unhandled"
Пример #5
0
 def __init__(self):
     self.user = sys.argv[1]
     self.password = sys.argv[2]
     self.timeframe = sys.argv[3]
     self.base_url = "%simportant" % (url)
     self.test_page = test_page
     self.test_page_cont = None
     self.art_data = {}
     self.instruments_dict = {}
     self.trades = {}
     self.all_trade_ideas = {}
     self.db = MySQL()
Пример #6
0
 def postDailySummary(self):
     '''Posts a daily summary to Twitter'''
     
     # Get current minutes from system time
     currentTime = datetime.now()
     # create time difference to be used to work out time period
     timeDiff = timedelta(days=1)
     # Get current minutes
     currentHour = currentTime.strftime("%H")
     # Check if the hours of the time is 00 which means midnight and the current day
     # has changed
     if(currentHour == "00" and (self.LAST_DAY_POST == "" or currentTime.strftime("%d") != self.LAST_DAY_POST)):
         Debug.writeOut("Daily condition met (hour of day:" + currentHour + " == 00 && day:" + currentTime.strftime("%d") + " == " + self.LAST_DAY_POST + "). Posting to Twitter")
         # Create SQL to get data for tweet
         sql = " SELECT COALESCE(ROUND(AVG(energy), 2), 0), COALESCE(MAX(energy), 0), COALESCE(ROUND(AVG(temperature), 1), 0) FROM historical_data WHERE date_time >= ADDDATE(NOW(), INTERVAL -1 DAY)"
         self.LOGGER.debug(sql)
         
         # Get statistics from DB
         stats = MySQL.executeOneUpdate(sql, None)
         # Create tweet
         message = (currentTime - timeDiff).strftime("%d-%b-%Y") + " Summary: " + str(stats[0]) +\
         "w was used. " +\
         "Energy usage peaked at " + str(stats[1]) + "w. " +\
         "The average temperature was " + str(stats[2]) + "c."
         
         # Save new day of tweet
         self.LAST_DAY_POST = currentTime.strftime("%d")
         
         # Check if tweet should be a Direct Message or just a tweet
         if self.CONFIG.getBooleanConfig("Twitter", "directMessagePost"):
             self.postDirectMessage(self.CONFIG.getBooleanConfig("Twitter", "directMessageUser"), message)
         else:
             # Post message to twitter
             self.tweet(message)
Пример #7
0
 def postHourlySummary(self):
     '''Posts an hourly summary to Twitter'''
     
     # Get current system time
     currentTime = datetime.now()
     # create time difference to be used to work out time period
     timeDiff = timedelta(hours=1)
     # Get current hour
     currentHour = currentTime.strftime("%H")
     # If current hour does not match last post hour then it's been a new hour since last post
     if(currentHour != self.LAST_HOURLY_POST):
         Debug.writeOut("Hourly condtion met (" + currentHour + " != " + self.LAST_HOURLY_POST + "). Posting to Twitter")
         # Create SQL to get data for tweet
         sql = "SELECT COALESCE(ROUND(AVG(energy), 2), 0), " +\
         "COALESCE(MAX(energy), 0), COALESCE(ROUND(AVG(temperature), 1), 0) " +\
         "FROM historical_data WHERE date_time >= ADDDATE(NOW(), INTERVAL -1 HOUR)"
         self.LOGGER.debug(sql)
         
         # Get statistics from DB
         stats = MySQL.executeOneUpdate(sql, None)
         # Create tweet
         message = (currentTime - timeDiff).strftime("%H:%M") + "-" + currentTime.strftime("%H:%M") +\
         " Summary: " + str(stats[0]) + "w was used." +\
         " Energy usage peaked at " + str(stats[1]) + "w" +\
         ". The average temperature was " + str(stats[2]) + "c."
         
         # Check if tweet should be a Direct Message or just a tweet
         if self.CONFIG.getBooleanConfig("Twitter", "directMessagePost"):
             self.postDirectMessage(self.CONFIG.getBooleanConfig("Twitter", "directMessageUser"), message)
         else:
             # Post message to twitter
             self.tweet(message)
             
         # Set last hourly post to current hour
         self.LAST_HOURLY_POST = currentHour
Пример #8
0
 def Authentication(self):
     path = MainCore()
     file = configparser.ConfigParser()
     file.read(path.resource('config.ini'))
     operation = MySQL(file.get('DATABASE', 'host'),
                       file.get('DATABASE', 'user'),
                       file.get('DATABASE', 'password'),
                       file.get('DATABASE', 'database'))
     result = operation.user_login(self.user_name.text(),
                                   self.password.text())
     if len(list(result[1])) == 0:
         self.label_error.setStyleSheet('color: red')
         self.label_error.setText('Access denied !')
     else:
         permission = result[1][0]
         RootWindow.show()
         self.close()
Пример #9
0
def ExampleGen(num_epochs=None):
    epoch = 0
    mysql=MySQL(sqldb="HWCom")
    mysql.login()
    cursor=mysql.get_cursor()
    while True:
        if num_epochs is not None and epoch >= num_epochs:
            break

        sent="select title,brief,content from news where brief !=''"
        cursor.execute(sent)
        for rows in cursor.fetchall():
            title, brief, content=rows
            content=extract_html(content)
            brief=extract_html(brief,False)
            yield (title,content,brief)

        epoch += 1
Пример #10
0
def genVocab(vocabfile):
    mysql=MySQL()
    mysql.login()
    cursor=mysql.get_cursor()

    vocab=defaultdict(int)
    def imdict(ab):
        for a in ab.split(" "):
            a=a.strip()
            # 去掉全是小写的英文单词
            if len(a)==0 or (rec.match(a) and a.islower()) or (rec0.match(a)):
                continue
            vocab[a]+=1

    urlset=set()
    dalist = []
    tables=["news","crawldata"]
    for table in tables:
        sent="select title,brief,content,url from %s where 1"%table
        cursor.execute(sent)

        for title, brief, content,url in cursor.fetchall():
            if url in urlset:
                continue
            else:
                urlset.add(url)
            title = Data.extract_html(title,False)
            imdict(title)

            if table=="news" and brief is not None:
                brief= re.sub("摘要:","",brief)
                brief = Data.extract_html(brief,False)
                imdict(brief)
                brieflen=len(brief)
            else:brieflen=0

            content=re.sub("资料图(图源:.*?)","",content)
            try:
               content=Data.extract_html(content)
            except:
                continue
            time.sleep(0.1)
            imdict(content)
            contentlen=len(content)
            dalist.append([brieflen,contentlen])

    data = pd.DataFrame(columns=["brief", "content"],data=dalist)
    data=data[data['brief']>0]
    data.to_csv("./data/len.csv",index=False)
    mysql.close()
    newvocab={Data.UNKNOWN_TOKEN:0,Data.PAD_TOKEN:-1,Data.SENTENCE_START:-1,Data.SENTENCE_END:-1}
    for key, value in vocab.items():
        if value >= 5:
            newvocab.update({key:value})
        else:
            newvocab[Data.UNKNOWN_TOKEN]+=value
    with open(vocabfile,'w') as f:
        for word,num in newvocab.items():
            f.write(word+" "+str(num)+"\n")
Пример #11
0
    def setUp(self):
        self.user = ''
        self.password = ''
        self._art_takestop = {}
	self.test_page = test_page
	self.test_page_cont = None
	self.art_data = {}
	self.trades = {}
	self.all_trade_ideas = {}
	self.db = MySQL()
Пример #12
0
    def __init__(self):
        self.user = sys.argv[1]
        self.password = sys.argv[2]
        self.timeframe = sys.argv[3]
        self.base_url = "%simportant" %(url)
	self.test_page = test_page
	self.test_page_cont = None
        self.art_data = {}
	self.instruments_dict = {}
	self.trades = {}
	self.all_trade_ideas = {}
	self.db = MySQL()
Пример #13
0
    def create_db(self):
        def field():
            for i in list(self.get_config):
                if i == '':
                    self.status_label.setStyleSheet("color: red;")
                    self.status_label.setText('Empty field !')
                    return False
                else:
                    return True

        if field():
            db = MySQL(self.get_config[0], self.get_config[1],
                       self.get_config[2], self.get_config[3])
            db.CreateDatabase()
            if db.CreateTables():
                self.status_label.setStyleSheet("color: green;")
                self.status_label.setText('Success Connect')
                self.save_btn.setEnabled(True)
            else:
                self.status_label.setStyleSheet("color: red;")
                self.status_label.setText("Access denied!")
Пример #14
0
class Spider(object):
    def __init__(self):
        self._headers = {
            'User-Agent':
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36',
            'Sec-Fetch-Mode': 'no-cors',
            'Host': 'arxiv.org'
        }
        self._sess = requests.Session()
        self._sleep_time = 5
        self._mysql = MySQL()

    def _get_detail(self, url):
        while 1:
            try:
                content = self._sess.get(url, headers=self._headers).content
            except Exception as e:
                print(e)
                self._sess.close()
                self._sess = requests.Session()
                time.sleep(self._sleep_time)
                continue
            html = etree.HTML(content)
            title = html.xpath(
                '//h1[@class="title mathjax"]/text()')[0].strip()
            #authors = ','.join(html.xpath('//div[@class="authors"]/a/text()')).strip()
            abstract = html.xpath(
                '//blockquote[@class="abstract mathjax"]/text()')[0].strip()
            subjects = html.xpath(
                'string(//td[@class="tablecell subjects"])').strip()
            arxiv = url.split('/')[-1]
            print(arxiv)
            return (arxiv, title, abstract, subjects)

    def crawl_arxiv_n(self, begin, stop):
        self._mysql.connect()
        # for month in ['07, 06, 05, 04, 03, 02, 01']:
        #     try:
        index_error_count = 0
        for i in range(begin, stop + 1):
            try:
                result = self._get_detail(
                    'https://arxiv.org/abs/1709.{:05d}'.format(i))
                index_error_count = 0
                self._mysql.execute(
                    'INSERT IGNORE INTO `rec_arxiv_paper` \
                    (`arxiv`, `title`, `abstract`, `subjects`) VALUES \
                    (%s, %s, %s, %s)', result)
                time.sleep(self._sleep_time // 5)
                if i % 150:
                    self._sess.close()
                    self._sess = requests.Session()
            except IndexError:
                index_error_count += 1
                if index_error_count > 5:
                    break
            # except IndexError:
            #     continue
        self._mysql.close()
Пример #15
0
    def post_woo_products(self):
        '''Post WooCommerce product'''
        try:
            # Auth
            auth = self.get_woo_request()
            # Upload image to media
            image = wp(self.book["image"])

            data = {
                "name":
                self.book["name"],
                "description":
                self.book["description"],
                "sku":
                self.book["isbn"],
                "categories": [],
                "tags": [],
                "attributes": [
                    {
                        "id": 1,
                        "name": "Tytuł",  # cspell: disable-line
                        "position": 1,
                        "visible": True,
                        "variation": True,
                        "options": [self.book["title"]]
                    },
                    {
                        "id": 2,
                        "name": "Autor",  # cspell: disable-line
                        "position": 2,
                        "visible": True,
                        "variation": True,
                        "options": [self.book["authors"]]
                    },
                    {
                        "id": 3,
                        "name": "Wydawnictwo",  # cspell: disable-line
                        "position": 3,
                        "visible": True,
                        "variation": True,
                        "options": [self.book["publisher"]]
                    },
                    {
                        "id": 4,
                        "name": "Rok wydania",  # cspell: disable-line
                        "position": 4,
                        "visible": True,
                        "variation": True,
                        "options": [self.book["publish_date"]]
                    },
                    {
                        "id": 5,
                        "name": "Okładka",  # cspell: disable-line
                        "position": 5,
                        "visible": True,
                        "variation": True,
                        "options": [self.book["binding"]]
                    },
                    {
                        "id": 6,
                        "name": "ISBN",
                        "position": 6,
                        "visible": True,
                        "variation": True,
                        "options": [self.book["isbn"]]
                    }
                ]
            }

            # Tags
            try:
                if self.book["tags"]:
                    tags = self.validate_tags()
                    for tag in tags:
                        data["tags"].append({'id': tag})
            except Exception as error:  # pylint: disable=broad-except
                logger.info(error)

            # Image
            try:
                if image:
                    data["images"] = [{"src": image}]
            except Exception as error:  # pylint: disable=broad-except
                logger.info(error)

            # Price
            try:
                if self.book["price"]:
                    data["regular_price"] = self.book["price"]
            except Exception as error:  # pylint: disable=broad-except
                logger.info(error)

            # Sale Price
            try:
                if self.book["sale_price"]:
                    data["sale_price"] = self.book["sale_price"]
            except Exception as error:  # pylint: disable=broad-except
                logger.info(error)

            # Amount
            try:
                if self.book["amount"]:
                    data["manage_stock"] = True
                    data["stock_quantity"] = self.book["amount"]
            except Exception as error:  # pylint: disable=broad-except
                logger.info(error)

            # Get category ID
            try:
                categories = self.validate_category()
                for category in categories:
                    data["categories"].append({'id': category})
            except Exception as error:  # pylint: disable=broad-except
                logger.info(error)

            # Send request
            response = auth.post("products", data).json()

            # Send none if status code found in error codes
            if "data" in response:
                if response.get("data", {}).get("status") in self.error_codes:
                    self.error_catch.append(inspect.getouterframes(inspect.currentframe())[0].function)  # pylint: disable=line-too-long
                    return None

            # Format output
            try:
                output = {
                    'id': response["id"],
                    'name': response["name"],
                    'link': response["permalink"],
                    'source': False
                }

            except Exception as error:  # pylint: disable=broad-except
                logger.info(error)

            if response["data"]["status"] == 400:

                try:
                    mysql_request = MySQL(isbn=self.book["isbn"])
                    request = mysql_request.db_mysql()

                except Exception as error:  # pylint: disable=broad-except
                    logger.info(error)

                if request:
                    product = self.get_woo_product(request)
                    if product["stock_quantity"]:
                        data["stock_quantity"] = int(data["stock_quantity"]) + product["stock_quantity"]  # pylint: disable=line-too-long

                    try:
                        response = self.update_woo_products(
                            product["id"], data)
                        output = {
                            'id': response["id"],
                            'name': response["name"],
                            'link': response["permalink"],
                            'source': True
                        }

                        return output

                    except Exception as error:  # pylint: disable=broad-except
                        logger.info(error)
                else:
                    return None

        except Exception as error:  # pylint: disable=broad-except
            logger.info(error)
        return output
Пример #16
0
def add_ladders_to_database(db: MySQL, ladders: list):
    for ladder in ladders:
        db.add_ladder(ladder)
Пример #17
0
def add_players_to_database(db: MySQL, players: list):
    for player in players:
        db.add_player(player)
Пример #18
0
def main():
    # verify that the necessary files exist
    battletag_from_cli = []
    if len(sys.argv) == 1:
        try:
            verify_files_exists(REGION_CODES)
        except FileNotFoundError:
            exit(1)
    elif len(sys.argv) == 2:
        if not os.path.exists(sys.argv[1]):
            Log.write_log_message("Specified file does not exist, exiting...",
                                  True)
        btags = open(sys.argv[1], "r")
        for btag in btags:
            battletag_from_cli.append(btag.strip())

    # get the API request parameters
    request_parameters = get_request_parameters()

    # get the current season ID
    season_id = -1
    try:
        season_id = API.get_current_season_id(request_parameters)
    except RequestError as e:
        print(e)
        exit(1)
    Log.write_log_message("Current Season ID: {}".format(season_id))

    db_handle = MySQL()

    for region in REGION_CODES:
        Log.write_log_message("Starting {} Region".format(region.upper()))

        # get ladders
        ladders = API.get_all_ladders(region, MAX_LEAGUE_ID, season_id,
                                      request_parameters)
        Log.write_log_message("Total Ladders Found: {}".format(len(ladders)))

        # add all of the ladders to the database
        try:
            add_ladders_to_database(db_handle, ladders)
        except MySQLdb.IntegrityError:
            Log.write_log_message(
                "Ladders are already in database for {}".format(
                    region.upper()))

        # read in btags to a list
        if len(battletag_from_cli) == 0:
            battletags = get_battletags(region)
        else:
            battletags = battletag_from_cli
        num_battletags = len(battletags)
        Log.write_log_message("Battletags Read In: {}".format(num_battletags))

        # go through every ladder looking for one of our players
        for ladder in ladders:
            # loop through every ladder between bronze and diamond

            # get all of the players in the ladder
            players = API.get_players_in_ladder(region, ladder,
                                                request_parameters)

            for player in players:
                # loop through every player in the ladder

                if [battletag.lower() for battletag in battletags
                    ].__contains__(player.battletag.lower()):
                    # a JSL contestant was found
                    db_handle.add_player(player)

                    for team in player.ladders:
                        db_handle.add_race(player, team)

                    for team in player.ladders:
                        Log.write_log_message(
                            "Found player: {} [{} {} {}]".format(
                                player.battletag, team.league, team.divison,
                                team.race))

    # get all players in database
    Log.write_log_message("Writing valid player data to disk")
    valid_players = db_handle.get_all_valid_players()
    write_valid_players(valid_players)

    # close database
    db_handle.close()
Пример #19
0
class get_idea_trade():

    def __init__(self):
        self.user = sys.argv[1]
        self.password = sys.argv[2]
        self.timeframe = sys.argv[3]
        self.base_url = "%simportant" %(url)
	self.test_page = test_page
	self.test_page_cont = None
        self.art_data = {}
	self.instruments_dict = {}
	self.trades = {}
	self.all_trade_ideas = {}
	self.db = MySQL()

    def import_instruments(self):
	'''
	Importing list of indtrument from instruments.yaml file
	'''
	try:
	    with open(instruments, 'r') as f:
		self.instruments_dict = yaml.load(f)
	    log.print_green('Instruments imported!')
	except:
	    log.print_error('Instruments import failed')

    def check_page(self):
	'''
	Main function which is called every time.
	It's some kind of start process function.
	'''
        with requests.Session() as c:
            self.main_page = c.get(self.base_url, headers={"User-Agent": useragent})
            self.do_soup(self.main_page, "article")

    def do_article(self, art):
	'''
	check articles from /important page and find keywords from trade_word
	'''
        for trade_list in trade_word.values():
	    for trade in trade_list:
                try:
                    assert trade in art.text.lower()
                    self.art_data["action"] = self.get_action(trade)
                    self.login(art.contents[5].find_all("a")[0].get("href"))
                except AssertionError:
                    pass

    def login(self, link):
	'''
	Login function. This function use detailes passed into tadebeat.py
	'''
        with requests.Session() as l:
            l.get(url+"user")
            self.phpid= l.cookies["PHPSESSID"]  #save token
            self.login_data = {"PHPSESSID": self.phpid, #create login dictionary with details
                               "T[_U]": self.user,
                               "T[_P]": self.password,
                               "T[_A]":0, "T[_A]":1,
                               "Action[login]":1, "T[_B]":''}
            l.post(url+"user", data=self.login_data, headers={"Referer": url+"user"}) # send post request to login page
            page = l.get(url+link[1:]) # send get request to article page
	    self.check_artid(page, {"class": "news urgent"}) # pass page content for checking process

    def update_database(self, what, onwhat, artID):
	self._result = self.db.update(what, onwhat, artID)
	if self._result:
	    log.print_green(what, onwhat, artID, ' Update OK!')
	else:
	    return

    def ask_database(self, what, artID):
	self._result = self.db.select(what, artID)
	if self._result:
	    for data in self._result:
		self._result_dict = yaml.load(data)
	    return self._result_dict
	else:
	    log.print_warning('Nothing found!')
	    return 0

    def check_artid(self, page, option):
	self.soup = BeautifulSoup(page.content, "lxml")
	self.art = self.soup.find('article', option)
	if not self.art: return
	if self.ask_database('time', self.art.get("id")):
	    #log.print_green('Found something with has same articleID in database already...')
	    self.check_timestamp(self.art.get("id"), self.art) #  checking if article is an update, or old one
	    return
	else:
	    log.print_warning('Found something new!')
	    self.do_soup(page, option)

    def check_timestamp(self, artID, content):
	'''
	compare timestamp of last entry in time key
	'''
	#log.print_green('I am going to compare timestamps')
	#self._old_time = self.all_trade_ideas.get(artID).get('time')[len(self.all_trade_ideas.get(artID).get('time'))]
	self._old_time = self.ask_database('time', artID)
	#print artID
	#print self._old_time
	#print len(self._old_time)
	#self._old_time = self._old_time.values()
	self._new_time = content.find("time").get("datetime")
	if self._new_time in self._old_time.values():
	    print('No updates')
	    return
	else:
	    log.print_green('Update of article has been detected!')
	    return self.do_update(self._new_time, artID)

    def do_soup(self, page, option):
        self.soup = BeautifulSoup(page.content, "lxml")
        if option == "article":
            for art in self.soup.find_all(option):
                self.do_article(art)
        else:
            self.art = self.soup.find('article', option)
	    self.do_collect_info(self.art, page.url.encode('ascii', 'ignore'))

    def do_collect_info(self, art, pageurl):
	#self.save_to_yaml('page', str(art))
        self._art_data = {} #internal variable to hold article data	
	log.print_green("Let's start with ", art.find("h1").text.lower())
	self._art_data = tradeClass()
	self._art_data.title = art.find("h1").text.lower().encode('ascii', 'ignore')
	self._art_data.add_time(art.find("time").get("datetime"))
	self._art_data.ID = art.get("id")
	self._art_data.page_adr = pageurl
	self._art_data.instrument = self.check_instrument(self._art_data.title)
	for p in art.find('div').find_all('p'): #.text.encode('ascii', 'ignore')
	    self._art_data.description += p.text.encode('ascii', 'ignore')
	log.print_warning('###### description ##### ', self._art_data.description, self._art_data.ID)
        self.find_tp_sl(self._art_data.description) # find take profit and sl values
	self._art_data.author = art.find('section', {'class' : 'autor'}).find('div', {'class' : 'about'}).find('h1').text.encode('ascii', 'ignore')
	#self._art_data.add_trade(self.art_data['action'], self._art_data.takestop)
	#log.print_warning(art.find('div').find('p').text.encode('ascii', 'ignore'))
	self._art_data.do_all_data()
	print 'take stop', self._art_data.takestop
	self.do_trade(self._art_data.takestop)
	if self._art_data.trade and (self._art_data.instrument != 'not found') : self.place_order(self._art_data.all_data, len(self._art_data.trade))
	#log.print_warning('trade: ', self._art_data.trade)
	#log.print_warning('len: ', len(self._art_data.trade))
	self._art_data.do_all_data()
	self.trades.update({self._art_data.ID : self._art_data.all_data})
	#self.save_to_yaml(self._art_data.ID, self._art_data.all_data)
	#self.save_to_yaml_all(self.trades)
	self.db.insert(self._art_data.all_data) #insert all data to database
	self.trades = {}
	return

    def do_trade(self, value):
	log.print_green('Do trade with value: ', value)
	try:
    	    for i in range(0,max(len(value.get('SL')), len(value.get('TP')))):
    	        try:
		    self._temp = {}
            	    self._temp.update({'SL' : value.get('SL', '0')[0], 'TP': value.get('TP', '0')[i]})
            	    log.print_green('temp1: ', self._temp)
		    self._art_data.add_trade(self.art_data['action'], self._temp)
    	    	except IndexError:
		    self._temp = {}
		    log.print_green('temp2: ', self._temp)
        	    self._temp.update({'SL' : value.get('SL', '0')[i], 'TP': value.get('TP', '0')[0]})
		    self._art_data.add_trade(self.art_data['action'], self._temp)
	except TypeError:
	    self._art_data.trade = ''
	    

    def check_instrument(self, title):
	log.print_green('checking instruments...')
	for i in self.instruments_dict.keys():
	    if i.lower() in title.lower():
		log.print_green('Found: ', i.lower())
		return i.lower()
	return 'not found'


    def find_tp_sl(self, info):
	log.print_green('finding TP and SL...')
	for action in trade_action.keys():
	    for t in trade_action.get(action):
		try:
		    if t in info.lower():
			self._tmp = self.find_digits(t, info.lower())
			self._art_data.add_takestop(action, self._tmp)
		except ValueError:
		    pass
	log.print_green('takestop: ', self._art_data.takestop)

    def do_update(self, new_time, artID):
	'''
	do update for article
	'''
	log.print_green("Let's check what has been updated")
	self._old_time = self.ask_database('time', artID)
	self._tmp = self._old_time
	print 'przed: ', self._tmp
	self._tmp2 = '{}'.format(len(self._old_time)+1)
	print 'tmp2: ', self._tmp2
	self._tmp.update({self._tmp2 : new_time})
	print 'po: ', self._tmp
	self.update_database('time', self._tmp, artID)
	return

    def get_action(self, action):
	'''
	get action like, buy or sell
	'''
	for t in trade_word.keys():
	    if action in trade_word.get(t):
		return t

    def find_digits(self, what, where):
	self._result = 0
	self._compile = "{}.[0-9\s.,]+".format(what)
	try:
	    self._result = re.findall(self._compile, where)[0]
	    self._result = self._result.split(':')[1]
	    self._result = self._result.split(', ')
	    for i in range(0, len(self._result)):
		self._result[i] = self._result[i].replace(' ', '')
		self._result[i] = self._result[i].replace(',', '.')
	    return self._result
	except IndexError:
	    return ['0']


    def save_to_yaml_all(self, data):
        self._time = time.ctime().split(' ')
	self._time.remove('')
        self._dir_name = "./%s-%s/" % (self._time[2], self._time[1])
        self._file_name = 'all_trade_ideas.yaml'
        try:
            with open(self._file_name, "ab") as f:
                yaml.dump(data, f)
		log.print_green('All data has been saved to: ', self._file_name)
        except IOError:
	    log.print_error('Error during saving: ', self._file_name)

    def save_to_yaml(self, fname,data):
        self._time = time.ctime().split(' ')
	self._time.remove('')
        self._dir_name = "./%s-%s/" % (self._time[2], self._time[1])
        self._file_name = self._dir_name + fname + '.yaml'
        try:
            with open(self._file_name, "ab") as f:
                yaml.dump(data, f)
        except IOError:
            os.makedirs(self._dir_name)
            with open(self._file_name, "ab") as f:
                yaml.dump(data, f)

    def place_order(self, data, i):
	'''
	Funtion to place an order with ST and TP parameters
	'''
	log.print_green('Placing an order...')
	print data
	try:
	    self._instr = self.instruments_dict.get(data.get('instrument').upper())[0].get('instrument')
	    self._unit = self.instruments_dict.get(data.get('instrument').upper())[3].get('tradeUnit')
	    for x in range(1,(i+1)):
	        self._action = data.get('trade')[x].keys()[0]
		if not data.get('trade')[x][self._action]['TP']:
		    self._tp = 0
		else:
    	    	    self._tp = data.get('trade')[x][self._action]['TP'] 
    	    	self._sl = data.get('trade')[x][self._action]['SL']
	    	log.print_warning(self._instr, self._unit, self._action, self._tp, self._sl)
	    	self.ordr = order.MyOanda(self._instr, self._unit, self._action, self._sl, self._tp)
	    	try:
	    	    self._art_data.add_oanda(self.ordr.create_order())
	    	except OandaError:
	    	    log.print_error('Placing a order failed')
	    return
	except AttributeError, TypeError:
	    log.print_error('AttributeError, data NoneType')
	    return
Пример #20
0
def get_product(book, gui):
    '''Get product form Woo'''

    dictionary = {}

    try:
        # Get product ID from DB.
        mysql_request = MySQL(isbn=book)
        mysql_response = mysql_request.db_mysql()

        if mysql_response:
            product = WooCommerce(book=book)
            request = product.get_woo_product(mysql_response)

            if request:
                try:
                    dictionary["id"] = request["id"]
                except Exception as error:  # pylint: disable=broad-except
                    logger.info(error)

                try:
                    for attribute in request["attributes"]:
                        dictionary[product.get_translation(attribute["name"],
                                                           "en")] = product.list_expander(attribute["options"]).replace("amp;", "")  # pylint: disable=line-too-long
                except Exception as error:  # pylint: disable=broad-except
                    logger.info(error)

                try:
                    dictionary["name"] = request["name"]
                except Exception as error:  # pylint: disable=broad-except
                    logger.info(error)

                try:
                    dictionary["description"] = request["description"]
                except Exception as error:  # pylint: disable=broad-except
                    logger.info(error)

                try:
                    categories_list = []
                    categories = request["categories"]
                    for category in categories:
                        categories_list.append(category["name"].replace(
                            "amp;", ""))

                    dictionary["categories"] = categories_list
                except Exception as error:  # pylint: disable=broad-except
                    logger.info(error)

                try:
                    tags_list = []
                    tags = request["tags"]
                    for tag in tags:
                        tags_list.append(tag["name"].replace("amp;", ""))

                    dictionary["tags"] = tags_list
                except Exception as error:  # pylint: disable=broad-except
                    logger.info(error)

                try:
                    dictionary["image"] = request["images"][0]["src"]
                except Exception as error:  # pylint: disable=broad-except
                    logger.info(error)

                try:
                    dictionary["price"] = request["regular_price"]
                except Exception as error:  # pylint: disable=broad-except
                    logger.info(error)

                try:
                    dictionary["sale_price"] = request["sale_price"]
                except Exception as error:  # pylint: disable=broad-except
                    logger.info(error)

                try:
                    dictionary["amount"] = request["stock_quantity"]
                except Exception as error:  # pylint: disable=broad-except
                    logger.info(error)

                for key in gui:
                    if "_box" in key and gui[key]:
                        if key.split('_box')[0] not in dictionary:
                            dictionary[key.split('_box')[0]] = None

                        if dictionary[key.split('_box')[0]] == "":
                            dictionary[key.split('_box')[0]] = None

    except Exception as error:  # pylint: disable=broad-except
        logger.info(error)

    return dictionary
Пример #21
0
class get_idea_trade():
    def __init__(self):
        self.user = sys.argv[1]
        self.password = sys.argv[2]
        self.timeframe = sys.argv[3]
        self.base_url = "%simportant" % (url)
        self.test_page = test_page
        self.test_page_cont = None
        self.art_data = {}
        self.instruments_dict = {}
        self.trades = {}
        self.all_trade_ideas = {}
        self.db = MySQL()

    def import_instruments(self):
        '''
	Importing list of indtrument from instruments.yaml file
	'''
        try:
            with open(instruments, 'r') as f:
                self.instruments_dict = yaml.load(f)
            log.print_green('Instruments imported!')
        except:
            log.print_error('Instruments import failed')

    def check_page(self):
        '''
	Main function which is called every time.
	It's some kind of start process function.
	'''
        with requests.Session() as c:
            self.main_page = c.get(self.base_url,
                                   headers={"User-Agent": useragent})
            self.do_soup(self.main_page, "article")

    def do_article(self, art):
        '''
	check articles from /important page and find keywords from trade_word
	'''
        for trade_list in trade_word.values():
            for trade in trade_list:
                try:
                    assert trade in art.text.lower()
                    self.art_data["action"] = self.get_action(trade)
                    self.login(art.contents[5].find_all("a")[0].get("href"))
                except AssertionError:
                    pass

    def login(self, link):
        '''
	Login function. This function use detailes passed into tadebeat.py
	'''
        with requests.Session() as l:
            l.get(url + "user")
            self.phpid = l.cookies["PHPSESSID"]  #save token
            self.login_data = {
                "PHPSESSID": self.phpid,  #create login dictionary with details
                "T[_U]": self.user,
                "T[_P]": self.password,
                "T[_A]": 0,
                "T[_A]": 1,
                "Action[login]": 1,
                "T[_B]": ''
            }
            l.post(url + "user",
                   data=self.login_data,
                   headers={"Referer":
                            url + "user"})  # send post request to login page
            page = l.get(url + link[1:])  # send get request to article page
            self.check_artid(page,
                             {"class": "news urgent"
                              })  # pass page content for checking process

    def update_database(self, what, onwhat, artID):
        self._result = self.db.update(what, onwhat, artID)
        if self._result:
            log.print_green(what, onwhat, artID, ' Update OK!')
        else:
            return

    def ask_database(self, what, artID):
        self._result = self.db.select(what, artID)
        if self._result:
            for data in self._result:
                self._result_dict = yaml.load(data)
            return self._result_dict
        else:
            log.print_warning('Nothing found!')
            return 0

    def check_artid(self, page, option):
        self.soup = BeautifulSoup(page.content, "lxml")
        self.art = self.soup.find('article', option)
        if not self.art: return
        if self.ask_database('time', self.art.get("id")):
            #log.print_green('Found something with has same articleID in database already...')
            self.check_timestamp(
                self.art.get("id"),
                self.art)  #  checking if article is an update, or old one
            return
        else:
            log.print_warning('Found something new!')
            self.do_soup(page, option)

    def check_timestamp(self, artID, content):
        '''
	compare timestamp of last entry in time key
	'''
        #log.print_green('I am going to compare timestamps')
        #self._old_time = self.all_trade_ideas.get(artID).get('time')[len(self.all_trade_ideas.get(artID).get('time'))]
        self._old_time = self.ask_database('time', artID)
        #print artID
        #print self._old_time
        #print len(self._old_time)
        #self._old_time = self._old_time.values()
        self._new_time = content.find("time").get("datetime")
        if self._new_time in self._old_time.values():
            print('No updates')
            return
        else:
            log.print_green('Update of article has been detected!')
            return self.do_update(self._new_time, artID)

    def do_soup(self, page, option):
        self.soup = BeautifulSoup(page.content, "lxml")
        if option == "article":
            for art in self.soup.find_all(option):
                self.do_article(art)
        else:
            self.art = self.soup.find('article', option)
            self.do_collect_info(self.art, page.url.encode('ascii', 'ignore'))

    def do_collect_info(self, art, pageurl):
        #self.save_to_yaml('page', str(art))
        self._art_data = {}  #internal variable to hold article data
        log.print_green("Let's start with ", art.find("h1").text.lower())
        self._art_data = tradeClass()
        self._art_data.title = art.find("h1").text.lower().encode(
            'ascii', 'ignore')
        self._art_data.add_time(art.find("time").get("datetime"))
        self._art_data.ID = art.get("id")
        self._art_data.page_adr = pageurl
        self._art_data.instrument = self.check_instrument(self._art_data.title)
        for p in art.find('div').find_all(
                'p'):  #.text.encode('ascii', 'ignore')
            self._art_data.description += p.text.encode('ascii', 'ignore')
        log.print_warning('###### description ##### ',
                          self._art_data.description, self._art_data.ID)
        self.find_tp_sl(
            self._art_data.description)  # find take profit and sl values
        self._art_data.author = art.find('section', {
            'class': 'autor'
        }).find('div', {
            'class': 'about'
        }).find('h1').text.encode('ascii', 'ignore')
        #self._art_data.add_trade(self.art_data['action'], self._art_data.takestop)
        #log.print_warning(art.find('div').find('p').text.encode('ascii', 'ignore'))
        self._art_data.do_all_data()
        print 'take stop', self._art_data.takestop
        self.do_trade(self._art_data.takestop)
        if self._art_data.trade and (self._art_data.instrument != 'not found'):
            self.place_order(self._art_data.all_data,
                             len(self._art_data.trade))
        #log.print_warning('trade: ', self._art_data.trade)
        #log.print_warning('len: ', len(self._art_data.trade))
        self._art_data.do_all_data()
        self.trades.update({self._art_data.ID: self._art_data.all_data})
        #self.save_to_yaml(self._art_data.ID, self._art_data.all_data)
        #self.save_to_yaml_all(self.trades)
        self.db.insert(self._art_data.all_data)  #insert all data to database
        self.trades = {}
        return

    def do_trade(self, value):
        log.print_green('Do trade with value: ', value)
        try:
            for i in range(0, max(len(value.get('SL')), len(value.get('TP')))):
                try:
                    self._temp = {}
                    self._temp.update({
                        'SL': value.get('SL', '0')[0],
                        'TP': value.get('TP', '0')[i]
                    })
                    log.print_green('temp1: ', self._temp)
                    self._art_data.add_trade(self.art_data['action'],
                                             self._temp)
                except IndexError:
                    self._temp = {}
                    log.print_green('temp2: ', self._temp)
                    self._temp.update({
                        'SL': value.get('SL', '0')[i],
                        'TP': value.get('TP', '0')[0]
                    })
                    self._art_data.add_trade(self.art_data['action'],
                                             self._temp)
        except TypeError:
            self._art_data.trade = ''

    def check_instrument(self, title):
        log.print_green('checking instruments...')
        for i in self.instruments_dict.keys():
            if i.lower() in title.lower():
                log.print_green('Found: ', i.lower())
                return i.lower()
        return 'not found'

    def find_tp_sl(self, info):
        log.print_green('finding TP and SL...')
        for action in trade_action.keys():
            for t in trade_action.get(action):
                try:
                    if t in info.lower():
                        self._tmp = self.find_digits(t, info.lower())
                        self._art_data.add_takestop(action, self._tmp)
                except ValueError:
                    pass
        log.print_green('takestop: ', self._art_data.takestop)

    def do_update(self, new_time, artID):
        '''
	do update for article
	'''
        log.print_green("Let's check what has been updated")
        self._old_time = self.ask_database('time', artID)
        self._tmp = self._old_time
        print 'przed: ', self._tmp
        self._tmp2 = '{}'.format(len(self._old_time) + 1)
        print 'tmp2: ', self._tmp2
        self._tmp.update({self._tmp2: new_time})
        print 'po: ', self._tmp
        self.update_database('time', self._tmp, artID)
        return

    def get_action(self, action):
        '''
	get action like, buy or sell
	'''
        for t in trade_word.keys():
            if action in trade_word.get(t):
                return t

    def find_digits(self, what, where):
        self._result = 0
        self._compile = "{}.[0-9\s.,]+".format(what)
        try:
            self._result = re.findall(self._compile, where)[0]
            self._result = self._result.split(':')[1]
            self._result = self._result.split(', ')
            for i in range(0, len(self._result)):
                self._result[i] = self._result[i].replace(' ', '')
                self._result[i] = self._result[i].replace(',', '.')
            return self._result
        except IndexError:
            return ['0']

    def save_to_yaml_all(self, data):
        self._time = time.ctime().split(' ')
        self._time.remove('')
        self._dir_name = "./%s-%s/" % (self._time[2], self._time[1])
        self._file_name = 'all_trade_ideas.yaml'
        try:
            with open(self._file_name, "ab") as f:
                yaml.dump(data, f)
                log.print_green('All data has been saved to: ',
                                self._file_name)
        except IOError:
            log.print_error('Error during saving: ', self._file_name)

    def save_to_yaml(self, fname, data):
        self._time = time.ctime().split(' ')
        self._time.remove('')
        self._dir_name = "./%s-%s/" % (self._time[2], self._time[1])
        self._file_name = self._dir_name + fname + '.yaml'
        try:
            with open(self._file_name, "ab") as f:
                yaml.dump(data, f)
        except IOError:
            os.makedirs(self._dir_name)
            with open(self._file_name, "ab") as f:
                yaml.dump(data, f)

    def place_order(self, data, i):
        '''
	Funtion to place an order with ST and TP parameters
	'''
        log.print_green('Placing an order...')
        print data
        try:
            self._instr = self.instruments_dict.get(
                data.get('instrument').upper())[0].get('instrument')
            self._unit = self.instruments_dict.get(
                data.get('instrument').upper())[3].get('tradeUnit')
            for x in range(1, (i + 1)):
                self._action = data.get('trade')[x].keys()[0]
                if not data.get('trade')[x][self._action]['TP']:
                    self._tp = 0
                else:
                    self._tp = data.get('trade')[x][self._action]['TP']
                self._sl = data.get('trade')[x][self._action]['SL']
                log.print_warning(self._instr, self._unit, self._action,
                                  self._tp, self._sl)
                self.ordr = order.MyOanda(self._instr, self._unit,
                                          self._action, self._sl, self._tp)
                try:
                    self._art_data.add_oanda(self.ordr.create_order())
                except OandaError:
                    log.print_error('Placing a order failed')
            return
        except AttributeError, TypeError:
            log.print_error('AttributeError, data NoneType')
            return
Пример #22
0
class FindInTextTest(unittest.TestCase):

    def setUp(self):
        self.user = ''
        self.password = ''
        self._art_takestop = {}
	self.test_page = test_page
	self.test_page_cont = None
	self.art_data = {}
	self.trades = {}
	self.all_trade_ideas = {}
	self.db = MySQL()


    def check_page(self):
	'''
	Main function which is called every time.
	It's some kind of start process function.
	'''
        with requests.Session() as c:
	    self.do_soup(self.test_page_cont, "article")
	
    def check_instrument(self, title):
	log.print_green('checking instruments...')
	for i in self.instruments_dict.keys():
	    print title
	    print 'i.lower() ', i.lower()
	    if i.lower() in title:
		print '2: ', title
		print '2: i.lower() ', i.lower()
		log.print_green('Found: ', i.lower())
		return i.lower()

    def do_update(self,new_time):
	'''
	do update for article
	'''
	print("Let's check what has been updated")
	self._art_data.add_time(new_time)
	return

    def get_action(self, action):
	'''
	get action like, buy or sell
	'''
	for t in trade_word.keys():
	    if action in trade_word.get(t):
		return t


    def do_article(self, art):
	'''
	check articles from /important page and find keywords from trade_word
	'''
        for trade_list in trade_word.values():
	    for trade in trade_list:
                try:
                    assert trade in art.text.lower()
                    self.art_data["action"] = self.get_action(trade)
		    self.check_artid(self.test_page, {"class": "news urgent"})
                except AssertionError:
                    pass


    def ask_database(self, what, artID):
	self._result = self.db.select(what, artID)
	if self._result:
	    for data in self._result:
		self._result_dict = yaml.load(data)
	    return self._result_dict
	else:
	    log.print_warning('Nothing found!')
	    return 0

    def check_artid(self, page, option):
	self.soup = BeautifulSoup(self.test_page_cont, "lxml")
	self.art = self.soup.find('article', option)
	if self.ask_database('time', self.art.get("id")):
	    log.print_green('Found something with has same articleID in database already...')
	    self.check_timestamp(self.art.get("id"), self.art) #  checking if article is an update, or old one
	    return
	else:
	    log.print_warning('Found something new!')
	    self.do_soup(page, option)

    def check_timestamp(self, artID, content):
	'''
	compare timestamp of last entry in time key
	'''
	print('I am going to compare timestamps')
	#self._old_time = self.all_trade_ideas.get(artID).get('time')[len(self.all_trade_ideas.get(artID).get('time'))]
	self._old_time = self.ask_database('time', artID)
	print artID
	print self._old_time
	print len(self._old_time)
	self._old_time = self._old_time.values()[-1]
	self._new_time = content.find("time").get("datetime")
	if self._old_time == self._new_time:
	    print('No updates')
	    return
	else:
	    print('Update of article has been detected!')
	    return self.do_update(self._new_time)

    def do_soup(self, page, option):
	self.soup = BeautifulSoup(page, "lxml")
        if option == "article":
            for art in self.soup.find_all(option):
                #self.do_article(art)
		print art
        else:
            self.art = self.soup.find('article', option)
	    self.do_collect_info(self.art, page.url.encode('ascii', 'ignore'))

    def do_collect_info(self, art, pageurl):
	self.save_to_yaml('page', str(art))
        self._art_data = {} #internal variable to hold article data	
	log.print_green("Let's start with ", art.find("h1").text.lower())
	self._art_data = tradeClass()
	self._art_data.title = art.find("h1").text.lower().encode('ascii', 'ignore')
	self._art_data.add_time(art.find("time").get("datetime"))
	self._art_data.ID = art.get("id")
	self._art_data.page_adr = pageurl
	self._art_data.instrument = self.check_instrument(self._art_data.title)
	for p in art.find('div').find_all('p'): #.text.encode('ascii', 'ignore')
	    self._art_data.description += p.text.encode('ascii', 'ignore')
	log.print_warning('###### description ##### ', self._art_data.description)
        self.find_tp_sl(self._art_data.description) # find take profit and sl values
	self._art_data.author = art.find('section', {'class' : 'autor'}).find('div', {'class' : 'about'}).find('h1').text.encode('ascii', 'ignore')
	#self._art_data.add_trade(self.art_data['action'], self._art_data.takestop)
	#log.print_warning(art.find('div').find('p').text.encode('ascii', 'ignore'))
	self._art_data.do_all_data()
	self.do_trade(self._art_data.takestop)
	if self._art_data.trade: self.place_order(self._art_data.all_data, len(self._art_data.trade))
	#log.print_warning('trade: ', self._art_data.trade)
	#log.print_warning('len: ', len(self._art_data.trade))
	self._art_data.do_all_data()
	self.trades.update({self._art_data.ID : self._art_data.all_data})
	self.save_to_yaml(self._art_data.ID, self._art_data.all_data)
	self.save_to_yaml_all(self.trades)
	self.db.insert(self._art_data.all_data) #insert all data to database
	self.trades = {}
	return

    def do_trade(self, value):
	log.print_green('Do trade with value: ', value)
	try:
    	    for i in range(0,max(len(value.get('SL')), len(value.get('TP')))):
	        log.print_green(i)
    	        try:
		    self._temp = {}
            	    self._temp.update({'SL' : value.get('SL')[0], 'TP': value.get('TP')[i]})
            	    log.print_green('temp1: ', self._temp)
		    self._art_data.add_trade(self.art_data['action'], self._temp)
    	    	except IndexError:
		    self._temp = {}
		    log.print_green('temp2: ', self._temp)
        	    self._temp.update({'SL' : value.get('SL')[i], 'TP': value.get('TP')[0]})
		    self._art_data.add_trade(self.art_data['action'], self._temp)
	except TypeError:
	    self._art_data.trade = ''
	   


    def import_test_page(self):
	'''
	Importing test_page from mainpage.yaml file
	'''
	try:
	    with open(self.test_page, 'r') as f:
		self.test_page_cont = f.read()
	    log.print_green('Test page imported!')
	except:
	    log.print_error('Test page import failed')

    def import_instruments(self):
	'''
	Importing list of indtrument from instruments.yaml file
	'''
	try:
	    with open('./instruments.yaml', 'r') as f:
		self.instruments_dict = yaml.load(f)
	    log.print_green('Instruments imported!')
	except:
	    log.print_error('Instruments import failed')

    def find_tp_sl(self, info):
	print('finding TP and SL...')
	for action in trade_action.keys():
	    for t in trade_action.get(action):
		try:
		    if t in info.lower():
			self._tmp = self.find_digits(t, info.lower())
			self._art_takestop.update({action : self._tmp})
		except ValueError:
		    pass
	log.print_green(self._art_takestop)

    def find_digits(self, what, where):
	self._result = 0
	self._compile = "{}.[0-9\s.,]+".format(what)
	try:
	    self._result = re.findall(self._compile, where)[0]
	    self._result = self._result.split(':')[1]
	    self._result = self._result.split(', ')
	    for i in range(0, len(self._result)):
		self._result[i] = self._result[i].replace(' ', '')
		self._result[i] = self._result[i].replace(',', '.')
	    print 'new result: ', self._result
	    return self._result
	except IndexError:
	    return []


    @unittest.skip("skipping")
    def test_01_value_with_one_comma(self):
	self.description = description.format('(open price 0,67580)','Take profit: 0,6200', 'Stop Loss:0,7050')
	print self.description
	self.find_tp_sl(self.description)
	self.assertEqual(self._art_takestop, {'TP': ['0.6200'], 'SL': ['0.7050']}, self._art_takestop) 

    @unittest.skip("skipping")
    def test_02_value_with_two_comma_sl(self):
	self.description = description.format('(open price 0,67580)','Take profit: 0,6200', 'Stop Loss:0,7050, 0,7150')
	print self.description
	self.find_tp_sl(self.description)
	self.assertEqual(self._art_takestop, {'TP': ['0.6200'], 'SL': ['0.7050', '0.7150']}, self._art_takestop) 

    @unittest.skip("skipping")
    def test_03_value_with_two_comma(self):
	self.description = description.format('(open price 0,67580)','Take profit: 0,6200, 0,6500', 'Stop Loss:0,7050')
	print self.description
	self.find_tp_sl(self.description)
	self.assertEqual(self._art_takestop, {'TP': ['0.6200', '0.6500'], 'SL': ['0.7050']}, self._art_takestop)

    @unittest.skip("skipping")
    def test_04_value_with_three_comma(self):
	self.description = description.format('(open price 0,67580)','Take profit: 0,6200, 0,6500, 0,6700', 'Stop Loss:0,7050')
	print self.description
	self.find_tp_sl(self.description)
	self.assertEqual(self._art_takestop, {'TP': ['0.6200', '0.6500', '0.6700'], 'SL': ['0.7050']}, self._art_takestop)

    @unittest.skip("skipping")
    def test_05_value_with_three_comma_sl(self):
	self.description = description.format('(open price 0,67580)','Take profit: 0,6200', 'Stop Loss:0,7050, 0,7150, 0,7350')
	print self.description
	self.find_tp_sl(self.description)
	self.assertEqual(self._art_takestop, {'TP': ['0.6200'], 'SL': ['0.7050', '0.7150', '0.7350']}, self._art_takestop) 

    @unittest.skip("skipping")
    def test_06_value_with_one_dot(self):
	self.description = description.format('(open price 0.67580)','Take profit: 0.6200', 'Stop Loss:0.7050')
	print self.description
	self.find_tp_sl(self.description)
	self.assertEqual(self._art_takestop, {'TP': ['0.6200'], 'SL': ['0.7050']}, self._art_takestop)

    @unittest.skip("skipping")
    def test_07_value_with_two_dot(self):
	self.description = description.format('(open price 0.67580)','Take profit: 0.6200, 0.6500', 'Stop Loss:0.7050')
	print self.description
	self.find_tp_sl(self.description)
	self.assertEqual(self._art_takestop, {'TP': ['0.6200', '0.6500'], 'SL': ['0.7050']}, self._art_takestop)

    @unittest.skip("skipping")
    def test_08_value_with_three_dot(self):
	self.description = description.format('(open price 0.67580)','Take profit: 0.6200, 0.6500, 0.6700', 'Stop Loss:0.7050')
	print self.description
	self.find_tp_sl(self.description)
	self.assertEqual(self._art_takestop, {'TP': ['0.6200', '0.6500', '0.6700'], 'SL': ['0.7050']}, self._art_takestop)

    @unittest.skip("skipping")
    def test_09_no_tp_dot(self):
	self.description = description.format('(open price 0.67580)','Take profit,', 'Stop Loss:0.7050')
	print self.description
	self.find_tp_sl(self.description)
	self.assertEqual(self._art_takestop, {'TP': [], 'SL': ['0.7050']}, self._art_takestop)

    @unittest.skip("skipping")
    def test_10_no_sl_dot(self):
	self.description = description.format('(open price 0.67580)','Take profit: 0.6200', 'Stop Loss')
	print self.description
	self.find_tp_sl(self.description)
	self.assertEqual(self._art_takestop, {'TP': ['0.6200'], 'SL': []}, self._art_takestop)

    @unittest.skip("skipping")
    def test_11_no_sl_and_tp(self):
	self.description = description.format('(open price 0.67580)','Take profit:', 'Stop Loss:')
	print self.description
	self.find_tp_sl(self.description)
	self.assertEqual(self._art_takestop, {'TP': [], 'SL': []}, self._art_takestop)

    @unittest.skip("skipping")
    def test_12_value_with_one_comma_short(self):
	self.description = description.format('(open price 0,67580)','TP: 0,6200', 'SL:0,7050')
	print self.description
	self.find_tp_sl(self.description)
	self.assertEqual(self._art_takestop, {'TP': ['0.6200'], 'SL': ['0.7050']}, self._art_takestop) 

    @unittest.skip("skipping")
    def test_13_value_with_two_comma_short(self):
	self.description = description.format('(open price 0,67580)','TP: 0,6200', 'SL:0,7050, 0,7150')
	print self.description
	self.find_tp_sl(self.description)
	self.assertEqual(self._art_takestop, {'TP': ['0.6200'], 'SL': ['0.7050', '0.7150']}, self._art_takestop) 

    @unittest.skip("skipping")
    def test_14_value_with_one_comma_mixed(self):
	self.description = description.format('(open price 0,67580)','Take profit: 0,6200', 'sl: 0,7050')
	print self.description
	self.find_tp_sl(self.description)
	self.assertEqual(self._art_takestop, {'TP': ['0.6200'], 'SL': ['0.7050']}, self._art_takestop) 

    @unittest.skip("skipping")
    def test_15_value_with_two_comma_mixed(self):
	self.description = description.format('(open price 0,67580)','Take profit: 0,6200', 'SL:0,7050, 0,7150')
	print self.description
	self.find_tp_sl(self.description)
	self.assertEqual(self._art_takestop, {'TP': ['0.6200'], 'SL': ['0.7050', '0.7150']}, self._art_takestop) 

    @unittest.skip("skipping")
    def test_16_title(self):
	self.instruments_dict = {}
	self.import_instruments()
	self._title = 'caduiyjpy - trade idea 02/03'
	self._instr = self.check_instrument(self._title)
	self.assertEqual(self._instr, 'cadjpy' , self._instr)

    def test_17_check_page(self):
    	self.import_instruments()
        self.import_test_page()
	self.do_soup(self.test_page_cont, 'article')