def extract(self): '''对采集到的网页进行数据提取''' self.db = MySQL('job') jobs = self.findall('div.job-list>ul>li') for job in jobs: title = self.find('.job-title',job).text salary = self.find('.red',job).text job_link = self.find('.info-primary>h3.name>a',job).get_attribute('href') job_id = re.search(r'/job_detail/(.*).html',job_link).group(1) job_info = self.find('.info-primary>p',job).get_attribute('innerHTML') company_name = self.find(".info-company>div>h3>a",job).text company_link = self.find(".info-company>div>h3>a",job).get_attribute('href') company_id = re.search(r'/gongsi/(.*).html',company_link).group(1) html_str = self.find('.company-text>p',job).get_attribute('innerHTML') company_info = html_str.split('''<em class="vline"></em>''') company_industry = company_info[0] company_size = company_info.pop() publisher = self.find('.info-publis>h3',job).get_attribute('innerHTML').split('<em class="vline"></em>').pop() date_str = self.find('.info-publis p',job).text date_str = date_str.split('发布于')[1] try: pub_date = datetime.datetime.strptime(date_str,'%m月%d日') pub_date = pub_date.replace(year = 2019) except ValueError: pub_date = datetime.datetime.strptime('11月18日','%m月%d日') pub_date = pub_date.replace(year = 2019) data = dict(title=title,salary=salary,job_info=job_info.replace('\"','\''),job_id=job_id,company_name=company_name\ ,company_id=company_id,company_industry=company_industry,company_size=company_size\ ,publisher=publisher,pub_date=str(pub_date)) self.write(data)
class ZhipinCrawler(IterateCrawler): '''继承浏览器迭代操作类,完成boss直聘数据采集''' def __init__(self, page_count): template = { 'np_selector':'div.page>a.next', } super().__init__(template, page_count) def extract(self): '''对采集到的网页进行数据提取''' self.db = MySQL('job') jobs = self.findall('div.job-list>ul>li') for job in jobs: title = self.find('.job-title',job).text salary = self.find('.red',job).text job_link = self.find('.info-primary>h3.name>a',job).get_attribute('href') job_id = re.search(r'/job_detail/(.*).html',job_link).group(1) job_info = self.find('.info-primary>p',job).get_attribute('innerHTML') company_name = self.find(".info-company>div>h3>a",job).text company_link = self.find(".info-company>div>h3>a",job).get_attribute('href') company_id = re.search(r'/gongsi/(.*).html',company_link).group(1) html_str = self.find('.company-text>p',job).get_attribute('innerHTML') company_info = html_str.split('''<em class="vline"></em>''') company_industry = company_info[0] company_size = company_info.pop() publisher = self.find('.info-publis>h3',job).get_attribute('innerHTML').split('<em class="vline"></em>').pop() date_str = self.find('.info-publis p',job).text date_str = date_str.split('发布于')[1] try: pub_date = datetime.datetime.strptime(date_str,'%m月%d日') pub_date = pub_date.replace(year = 2019) except ValueError: pub_date = datetime.datetime.strptime('11月18日','%m月%d日') pub_date = pub_date.replace(year = 2019) data = dict(title=title,salary=salary,job_info=job_info.replace('\"','\''),job_id=job_id,company_name=company_name\ ,company_id=company_id,company_industry=company_industry,company_size=company_size\ ,publisher=publisher,pub_date=str(pub_date)) self.write(data) def write(self, data): '''将提取到的数据写入到数据库''' self.db.run(sql.append('zhipin_guangzhou',data)) def next_page(self): '''重写父类方法,实现翻页操作''' url = self.instance.current_url rst = re.search(r'page=([0-9]*)&',url) match = rst.group() num = int(rst.group(1)) self.to(url.replace(match,'page=%d&'%(num+1))) def after(self): '''实现父类方法,采集完成后处理函数''' self.db.exit() t.say('job done')
def __init__(self): self._headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', 'Sec-Fetch-Mode': 'no-cors', 'Host': 'arxiv.org' } self._sess = requests.Session() self._sleep_time = 5 self._mysql = MySQL()
def run(self): args = self.arguments() if (args.filenames): csvobject = CSVParser(args.filenames[0], args.filenames[1]) csvobject.format_csv_files() elif (args.output): print('We are connecting to the database...') db = MySQL() # Here is where you would pass in a MySQL connection db.write_to_file('SELECT * from foobar', args.output[0]) else: assert False, "Unhandled"
def __init__(self): self.user = sys.argv[1] self.password = sys.argv[2] self.timeframe = sys.argv[3] self.base_url = "%simportant" % (url) self.test_page = test_page self.test_page_cont = None self.art_data = {} self.instruments_dict = {} self.trades = {} self.all_trade_ideas = {} self.db = MySQL()
def postDailySummary(self): '''Posts a daily summary to Twitter''' # Get current minutes from system time currentTime = datetime.now() # create time difference to be used to work out time period timeDiff = timedelta(days=1) # Get current minutes currentHour = currentTime.strftime("%H") # Check if the hours of the time is 00 which means midnight and the current day # has changed if(currentHour == "00" and (self.LAST_DAY_POST == "" or currentTime.strftime("%d") != self.LAST_DAY_POST)): Debug.writeOut("Daily condition met (hour of day:" + currentHour + " == 00 && day:" + currentTime.strftime("%d") + " == " + self.LAST_DAY_POST + "). Posting to Twitter") # Create SQL to get data for tweet sql = " SELECT COALESCE(ROUND(AVG(energy), 2), 0), COALESCE(MAX(energy), 0), COALESCE(ROUND(AVG(temperature), 1), 0) FROM historical_data WHERE date_time >= ADDDATE(NOW(), INTERVAL -1 DAY)" self.LOGGER.debug(sql) # Get statistics from DB stats = MySQL.executeOneUpdate(sql, None) # Create tweet message = (currentTime - timeDiff).strftime("%d-%b-%Y") + " Summary: " + str(stats[0]) +\ "w was used. " +\ "Energy usage peaked at " + str(stats[1]) + "w. " +\ "The average temperature was " + str(stats[2]) + "c." # Save new day of tweet self.LAST_DAY_POST = currentTime.strftime("%d") # Check if tweet should be a Direct Message or just a tweet if self.CONFIG.getBooleanConfig("Twitter", "directMessagePost"): self.postDirectMessage(self.CONFIG.getBooleanConfig("Twitter", "directMessageUser"), message) else: # Post message to twitter self.tweet(message)
def postHourlySummary(self): '''Posts an hourly summary to Twitter''' # Get current system time currentTime = datetime.now() # create time difference to be used to work out time period timeDiff = timedelta(hours=1) # Get current hour currentHour = currentTime.strftime("%H") # If current hour does not match last post hour then it's been a new hour since last post if(currentHour != self.LAST_HOURLY_POST): Debug.writeOut("Hourly condtion met (" + currentHour + " != " + self.LAST_HOURLY_POST + "). Posting to Twitter") # Create SQL to get data for tweet sql = "SELECT COALESCE(ROUND(AVG(energy), 2), 0), " +\ "COALESCE(MAX(energy), 0), COALESCE(ROUND(AVG(temperature), 1), 0) " +\ "FROM historical_data WHERE date_time >= ADDDATE(NOW(), INTERVAL -1 HOUR)" self.LOGGER.debug(sql) # Get statistics from DB stats = MySQL.executeOneUpdate(sql, None) # Create tweet message = (currentTime - timeDiff).strftime("%H:%M") + "-" + currentTime.strftime("%H:%M") +\ " Summary: " + str(stats[0]) + "w was used." +\ " Energy usage peaked at " + str(stats[1]) + "w" +\ ". The average temperature was " + str(stats[2]) + "c." # Check if tweet should be a Direct Message or just a tweet if self.CONFIG.getBooleanConfig("Twitter", "directMessagePost"): self.postDirectMessage(self.CONFIG.getBooleanConfig("Twitter", "directMessageUser"), message) else: # Post message to twitter self.tweet(message) # Set last hourly post to current hour self.LAST_HOURLY_POST = currentHour
def Authentication(self): path = MainCore() file = configparser.ConfigParser() file.read(path.resource('config.ini')) operation = MySQL(file.get('DATABASE', 'host'), file.get('DATABASE', 'user'), file.get('DATABASE', 'password'), file.get('DATABASE', 'database')) result = operation.user_login(self.user_name.text(), self.password.text()) if len(list(result[1])) == 0: self.label_error.setStyleSheet('color: red') self.label_error.setText('Access denied !') else: permission = result[1][0] RootWindow.show() self.close()
def ExampleGen(num_epochs=None): epoch = 0 mysql=MySQL(sqldb="HWCom") mysql.login() cursor=mysql.get_cursor() while True: if num_epochs is not None and epoch >= num_epochs: break sent="select title,brief,content from news where brief !=''" cursor.execute(sent) for rows in cursor.fetchall(): title, brief, content=rows content=extract_html(content) brief=extract_html(brief,False) yield (title,content,brief) epoch += 1
def genVocab(vocabfile): mysql=MySQL() mysql.login() cursor=mysql.get_cursor() vocab=defaultdict(int) def imdict(ab): for a in ab.split(" "): a=a.strip() # 去掉全是小写的英文单词 if len(a)==0 or (rec.match(a) and a.islower()) or (rec0.match(a)): continue vocab[a]+=1 urlset=set() dalist = [] tables=["news","crawldata"] for table in tables: sent="select title,brief,content,url from %s where 1"%table cursor.execute(sent) for title, brief, content,url in cursor.fetchall(): if url in urlset: continue else: urlset.add(url) title = Data.extract_html(title,False) imdict(title) if table=="news" and brief is not None: brief= re.sub("摘要:","",brief) brief = Data.extract_html(brief,False) imdict(brief) brieflen=len(brief) else:brieflen=0 content=re.sub("资料图(图源:.*?)","",content) try: content=Data.extract_html(content) except: continue time.sleep(0.1) imdict(content) contentlen=len(content) dalist.append([brieflen,contentlen]) data = pd.DataFrame(columns=["brief", "content"],data=dalist) data=data[data['brief']>0] data.to_csv("./data/len.csv",index=False) mysql.close() newvocab={Data.UNKNOWN_TOKEN:0,Data.PAD_TOKEN:-1,Data.SENTENCE_START:-1,Data.SENTENCE_END:-1} for key, value in vocab.items(): if value >= 5: newvocab.update({key:value}) else: newvocab[Data.UNKNOWN_TOKEN]+=value with open(vocabfile,'w') as f: for word,num in newvocab.items(): f.write(word+" "+str(num)+"\n")
def setUp(self): self.user = '' self.password = '' self._art_takestop = {} self.test_page = test_page self.test_page_cont = None self.art_data = {} self.trades = {} self.all_trade_ideas = {} self.db = MySQL()
def __init__(self): self.user = sys.argv[1] self.password = sys.argv[2] self.timeframe = sys.argv[3] self.base_url = "%simportant" %(url) self.test_page = test_page self.test_page_cont = None self.art_data = {} self.instruments_dict = {} self.trades = {} self.all_trade_ideas = {} self.db = MySQL()
def create_db(self): def field(): for i in list(self.get_config): if i == '': self.status_label.setStyleSheet("color: red;") self.status_label.setText('Empty field !') return False else: return True if field(): db = MySQL(self.get_config[0], self.get_config[1], self.get_config[2], self.get_config[3]) db.CreateDatabase() if db.CreateTables(): self.status_label.setStyleSheet("color: green;") self.status_label.setText('Success Connect') self.save_btn.setEnabled(True) else: self.status_label.setStyleSheet("color: red;") self.status_label.setText("Access denied!")
class Spider(object): def __init__(self): self._headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', 'Sec-Fetch-Mode': 'no-cors', 'Host': 'arxiv.org' } self._sess = requests.Session() self._sleep_time = 5 self._mysql = MySQL() def _get_detail(self, url): while 1: try: content = self._sess.get(url, headers=self._headers).content except Exception as e: print(e) self._sess.close() self._sess = requests.Session() time.sleep(self._sleep_time) continue html = etree.HTML(content) title = html.xpath( '//h1[@class="title mathjax"]/text()')[0].strip() #authors = ','.join(html.xpath('//div[@class="authors"]/a/text()')).strip() abstract = html.xpath( '//blockquote[@class="abstract mathjax"]/text()')[0].strip() subjects = html.xpath( 'string(//td[@class="tablecell subjects"])').strip() arxiv = url.split('/')[-1] print(arxiv) return (arxiv, title, abstract, subjects) def crawl_arxiv_n(self, begin, stop): self._mysql.connect() # for month in ['07, 06, 05, 04, 03, 02, 01']: # try: index_error_count = 0 for i in range(begin, stop + 1): try: result = self._get_detail( 'https://arxiv.org/abs/1709.{:05d}'.format(i)) index_error_count = 0 self._mysql.execute( 'INSERT IGNORE INTO `rec_arxiv_paper` \ (`arxiv`, `title`, `abstract`, `subjects`) VALUES \ (%s, %s, %s, %s)', result) time.sleep(self._sleep_time // 5) if i % 150: self._sess.close() self._sess = requests.Session() except IndexError: index_error_count += 1 if index_error_count > 5: break # except IndexError: # continue self._mysql.close()
def post_woo_products(self): '''Post WooCommerce product''' try: # Auth auth = self.get_woo_request() # Upload image to media image = wp(self.book["image"]) data = { "name": self.book["name"], "description": self.book["description"], "sku": self.book["isbn"], "categories": [], "tags": [], "attributes": [ { "id": 1, "name": "Tytuł", # cspell: disable-line "position": 1, "visible": True, "variation": True, "options": [self.book["title"]] }, { "id": 2, "name": "Autor", # cspell: disable-line "position": 2, "visible": True, "variation": True, "options": [self.book["authors"]] }, { "id": 3, "name": "Wydawnictwo", # cspell: disable-line "position": 3, "visible": True, "variation": True, "options": [self.book["publisher"]] }, { "id": 4, "name": "Rok wydania", # cspell: disable-line "position": 4, "visible": True, "variation": True, "options": [self.book["publish_date"]] }, { "id": 5, "name": "Okładka", # cspell: disable-line "position": 5, "visible": True, "variation": True, "options": [self.book["binding"]] }, { "id": 6, "name": "ISBN", "position": 6, "visible": True, "variation": True, "options": [self.book["isbn"]] } ] } # Tags try: if self.book["tags"]: tags = self.validate_tags() for tag in tags: data["tags"].append({'id': tag}) except Exception as error: # pylint: disable=broad-except logger.info(error) # Image try: if image: data["images"] = [{"src": image}] except Exception as error: # pylint: disable=broad-except logger.info(error) # Price try: if self.book["price"]: data["regular_price"] = self.book["price"] except Exception as error: # pylint: disable=broad-except logger.info(error) # Sale Price try: if self.book["sale_price"]: data["sale_price"] = self.book["sale_price"] except Exception as error: # pylint: disable=broad-except logger.info(error) # Amount try: if self.book["amount"]: data["manage_stock"] = True data["stock_quantity"] = self.book["amount"] except Exception as error: # pylint: disable=broad-except logger.info(error) # Get category ID try: categories = self.validate_category() for category in categories: data["categories"].append({'id': category}) except Exception as error: # pylint: disable=broad-except logger.info(error) # Send request response = auth.post("products", data).json() # Send none if status code found in error codes if "data" in response: if response.get("data", {}).get("status") in self.error_codes: self.error_catch.append(inspect.getouterframes(inspect.currentframe())[0].function) # pylint: disable=line-too-long return None # Format output try: output = { 'id': response["id"], 'name': response["name"], 'link': response["permalink"], 'source': False } except Exception as error: # pylint: disable=broad-except logger.info(error) if response["data"]["status"] == 400: try: mysql_request = MySQL(isbn=self.book["isbn"]) request = mysql_request.db_mysql() except Exception as error: # pylint: disable=broad-except logger.info(error) if request: product = self.get_woo_product(request) if product["stock_quantity"]: data["stock_quantity"] = int(data["stock_quantity"]) + product["stock_quantity"] # pylint: disable=line-too-long try: response = self.update_woo_products( product["id"], data) output = { 'id': response["id"], 'name': response["name"], 'link': response["permalink"], 'source': True } return output except Exception as error: # pylint: disable=broad-except logger.info(error) else: return None except Exception as error: # pylint: disable=broad-except logger.info(error) return output
def add_ladders_to_database(db: MySQL, ladders: list): for ladder in ladders: db.add_ladder(ladder)
def add_players_to_database(db: MySQL, players: list): for player in players: db.add_player(player)
def main(): # verify that the necessary files exist battletag_from_cli = [] if len(sys.argv) == 1: try: verify_files_exists(REGION_CODES) except FileNotFoundError: exit(1) elif len(sys.argv) == 2: if not os.path.exists(sys.argv[1]): Log.write_log_message("Specified file does not exist, exiting...", True) btags = open(sys.argv[1], "r") for btag in btags: battletag_from_cli.append(btag.strip()) # get the API request parameters request_parameters = get_request_parameters() # get the current season ID season_id = -1 try: season_id = API.get_current_season_id(request_parameters) except RequestError as e: print(e) exit(1) Log.write_log_message("Current Season ID: {}".format(season_id)) db_handle = MySQL() for region in REGION_CODES: Log.write_log_message("Starting {} Region".format(region.upper())) # get ladders ladders = API.get_all_ladders(region, MAX_LEAGUE_ID, season_id, request_parameters) Log.write_log_message("Total Ladders Found: {}".format(len(ladders))) # add all of the ladders to the database try: add_ladders_to_database(db_handle, ladders) except MySQLdb.IntegrityError: Log.write_log_message( "Ladders are already in database for {}".format( region.upper())) # read in btags to a list if len(battletag_from_cli) == 0: battletags = get_battletags(region) else: battletags = battletag_from_cli num_battletags = len(battletags) Log.write_log_message("Battletags Read In: {}".format(num_battletags)) # go through every ladder looking for one of our players for ladder in ladders: # loop through every ladder between bronze and diamond # get all of the players in the ladder players = API.get_players_in_ladder(region, ladder, request_parameters) for player in players: # loop through every player in the ladder if [battletag.lower() for battletag in battletags ].__contains__(player.battletag.lower()): # a JSL contestant was found db_handle.add_player(player) for team in player.ladders: db_handle.add_race(player, team) for team in player.ladders: Log.write_log_message( "Found player: {} [{} {} {}]".format( player.battletag, team.league, team.divison, team.race)) # get all players in database Log.write_log_message("Writing valid player data to disk") valid_players = db_handle.get_all_valid_players() write_valid_players(valid_players) # close database db_handle.close()
class get_idea_trade(): def __init__(self): self.user = sys.argv[1] self.password = sys.argv[2] self.timeframe = sys.argv[3] self.base_url = "%simportant" %(url) self.test_page = test_page self.test_page_cont = None self.art_data = {} self.instruments_dict = {} self.trades = {} self.all_trade_ideas = {} self.db = MySQL() def import_instruments(self): ''' Importing list of indtrument from instruments.yaml file ''' try: with open(instruments, 'r') as f: self.instruments_dict = yaml.load(f) log.print_green('Instruments imported!') except: log.print_error('Instruments import failed') def check_page(self): ''' Main function which is called every time. It's some kind of start process function. ''' with requests.Session() as c: self.main_page = c.get(self.base_url, headers={"User-Agent": useragent}) self.do_soup(self.main_page, "article") def do_article(self, art): ''' check articles from /important page and find keywords from trade_word ''' for trade_list in trade_word.values(): for trade in trade_list: try: assert trade in art.text.lower() self.art_data["action"] = self.get_action(trade) self.login(art.contents[5].find_all("a")[0].get("href")) except AssertionError: pass def login(self, link): ''' Login function. This function use detailes passed into tadebeat.py ''' with requests.Session() as l: l.get(url+"user") self.phpid= l.cookies["PHPSESSID"] #save token self.login_data = {"PHPSESSID": self.phpid, #create login dictionary with details "T[_U]": self.user, "T[_P]": self.password, "T[_A]":0, "T[_A]":1, "Action[login]":1, "T[_B]":''} l.post(url+"user", data=self.login_data, headers={"Referer": url+"user"}) # send post request to login page page = l.get(url+link[1:]) # send get request to article page self.check_artid(page, {"class": "news urgent"}) # pass page content for checking process def update_database(self, what, onwhat, artID): self._result = self.db.update(what, onwhat, artID) if self._result: log.print_green(what, onwhat, artID, ' Update OK!') else: return def ask_database(self, what, artID): self._result = self.db.select(what, artID) if self._result: for data in self._result: self._result_dict = yaml.load(data) return self._result_dict else: log.print_warning('Nothing found!') return 0 def check_artid(self, page, option): self.soup = BeautifulSoup(page.content, "lxml") self.art = self.soup.find('article', option) if not self.art: return if self.ask_database('time', self.art.get("id")): #log.print_green('Found something with has same articleID in database already...') self.check_timestamp(self.art.get("id"), self.art) # checking if article is an update, or old one return else: log.print_warning('Found something new!') self.do_soup(page, option) def check_timestamp(self, artID, content): ''' compare timestamp of last entry in time key ''' #log.print_green('I am going to compare timestamps') #self._old_time = self.all_trade_ideas.get(artID).get('time')[len(self.all_trade_ideas.get(artID).get('time'))] self._old_time = self.ask_database('time', artID) #print artID #print self._old_time #print len(self._old_time) #self._old_time = self._old_time.values() self._new_time = content.find("time").get("datetime") if self._new_time in self._old_time.values(): print('No updates') return else: log.print_green('Update of article has been detected!') return self.do_update(self._new_time, artID) def do_soup(self, page, option): self.soup = BeautifulSoup(page.content, "lxml") if option == "article": for art in self.soup.find_all(option): self.do_article(art) else: self.art = self.soup.find('article', option) self.do_collect_info(self.art, page.url.encode('ascii', 'ignore')) def do_collect_info(self, art, pageurl): #self.save_to_yaml('page', str(art)) self._art_data = {} #internal variable to hold article data log.print_green("Let's start with ", art.find("h1").text.lower()) self._art_data = tradeClass() self._art_data.title = art.find("h1").text.lower().encode('ascii', 'ignore') self._art_data.add_time(art.find("time").get("datetime")) self._art_data.ID = art.get("id") self._art_data.page_adr = pageurl self._art_data.instrument = self.check_instrument(self._art_data.title) for p in art.find('div').find_all('p'): #.text.encode('ascii', 'ignore') self._art_data.description += p.text.encode('ascii', 'ignore') log.print_warning('###### description ##### ', self._art_data.description, self._art_data.ID) self.find_tp_sl(self._art_data.description) # find take profit and sl values self._art_data.author = art.find('section', {'class' : 'autor'}).find('div', {'class' : 'about'}).find('h1').text.encode('ascii', 'ignore') #self._art_data.add_trade(self.art_data['action'], self._art_data.takestop) #log.print_warning(art.find('div').find('p').text.encode('ascii', 'ignore')) self._art_data.do_all_data() print 'take stop', self._art_data.takestop self.do_trade(self._art_data.takestop) if self._art_data.trade and (self._art_data.instrument != 'not found') : self.place_order(self._art_data.all_data, len(self._art_data.trade)) #log.print_warning('trade: ', self._art_data.trade) #log.print_warning('len: ', len(self._art_data.trade)) self._art_data.do_all_data() self.trades.update({self._art_data.ID : self._art_data.all_data}) #self.save_to_yaml(self._art_data.ID, self._art_data.all_data) #self.save_to_yaml_all(self.trades) self.db.insert(self._art_data.all_data) #insert all data to database self.trades = {} return def do_trade(self, value): log.print_green('Do trade with value: ', value) try: for i in range(0,max(len(value.get('SL')), len(value.get('TP')))): try: self._temp = {} self._temp.update({'SL' : value.get('SL', '0')[0], 'TP': value.get('TP', '0')[i]}) log.print_green('temp1: ', self._temp) self._art_data.add_trade(self.art_data['action'], self._temp) except IndexError: self._temp = {} log.print_green('temp2: ', self._temp) self._temp.update({'SL' : value.get('SL', '0')[i], 'TP': value.get('TP', '0')[0]}) self._art_data.add_trade(self.art_data['action'], self._temp) except TypeError: self._art_data.trade = '' def check_instrument(self, title): log.print_green('checking instruments...') for i in self.instruments_dict.keys(): if i.lower() in title.lower(): log.print_green('Found: ', i.lower()) return i.lower() return 'not found' def find_tp_sl(self, info): log.print_green('finding TP and SL...') for action in trade_action.keys(): for t in trade_action.get(action): try: if t in info.lower(): self._tmp = self.find_digits(t, info.lower()) self._art_data.add_takestop(action, self._tmp) except ValueError: pass log.print_green('takestop: ', self._art_data.takestop) def do_update(self, new_time, artID): ''' do update for article ''' log.print_green("Let's check what has been updated") self._old_time = self.ask_database('time', artID) self._tmp = self._old_time print 'przed: ', self._tmp self._tmp2 = '{}'.format(len(self._old_time)+1) print 'tmp2: ', self._tmp2 self._tmp.update({self._tmp2 : new_time}) print 'po: ', self._tmp self.update_database('time', self._tmp, artID) return def get_action(self, action): ''' get action like, buy or sell ''' for t in trade_word.keys(): if action in trade_word.get(t): return t def find_digits(self, what, where): self._result = 0 self._compile = "{}.[0-9\s.,]+".format(what) try: self._result = re.findall(self._compile, where)[0] self._result = self._result.split(':')[1] self._result = self._result.split(', ') for i in range(0, len(self._result)): self._result[i] = self._result[i].replace(' ', '') self._result[i] = self._result[i].replace(',', '.') return self._result except IndexError: return ['0'] def save_to_yaml_all(self, data): self._time = time.ctime().split(' ') self._time.remove('') self._dir_name = "./%s-%s/" % (self._time[2], self._time[1]) self._file_name = 'all_trade_ideas.yaml' try: with open(self._file_name, "ab") as f: yaml.dump(data, f) log.print_green('All data has been saved to: ', self._file_name) except IOError: log.print_error('Error during saving: ', self._file_name) def save_to_yaml(self, fname,data): self._time = time.ctime().split(' ') self._time.remove('') self._dir_name = "./%s-%s/" % (self._time[2], self._time[1]) self._file_name = self._dir_name + fname + '.yaml' try: with open(self._file_name, "ab") as f: yaml.dump(data, f) except IOError: os.makedirs(self._dir_name) with open(self._file_name, "ab") as f: yaml.dump(data, f) def place_order(self, data, i): ''' Funtion to place an order with ST and TP parameters ''' log.print_green('Placing an order...') print data try: self._instr = self.instruments_dict.get(data.get('instrument').upper())[0].get('instrument') self._unit = self.instruments_dict.get(data.get('instrument').upper())[3].get('tradeUnit') for x in range(1,(i+1)): self._action = data.get('trade')[x].keys()[0] if not data.get('trade')[x][self._action]['TP']: self._tp = 0 else: self._tp = data.get('trade')[x][self._action]['TP'] self._sl = data.get('trade')[x][self._action]['SL'] log.print_warning(self._instr, self._unit, self._action, self._tp, self._sl) self.ordr = order.MyOanda(self._instr, self._unit, self._action, self._sl, self._tp) try: self._art_data.add_oanda(self.ordr.create_order()) except OandaError: log.print_error('Placing a order failed') return except AttributeError, TypeError: log.print_error('AttributeError, data NoneType') return
def get_product(book, gui): '''Get product form Woo''' dictionary = {} try: # Get product ID from DB. mysql_request = MySQL(isbn=book) mysql_response = mysql_request.db_mysql() if mysql_response: product = WooCommerce(book=book) request = product.get_woo_product(mysql_response) if request: try: dictionary["id"] = request["id"] except Exception as error: # pylint: disable=broad-except logger.info(error) try: for attribute in request["attributes"]: dictionary[product.get_translation(attribute["name"], "en")] = product.list_expander(attribute["options"]).replace("amp;", "") # pylint: disable=line-too-long except Exception as error: # pylint: disable=broad-except logger.info(error) try: dictionary["name"] = request["name"] except Exception as error: # pylint: disable=broad-except logger.info(error) try: dictionary["description"] = request["description"] except Exception as error: # pylint: disable=broad-except logger.info(error) try: categories_list = [] categories = request["categories"] for category in categories: categories_list.append(category["name"].replace( "amp;", "")) dictionary["categories"] = categories_list except Exception as error: # pylint: disable=broad-except logger.info(error) try: tags_list = [] tags = request["tags"] for tag in tags: tags_list.append(tag["name"].replace("amp;", "")) dictionary["tags"] = tags_list except Exception as error: # pylint: disable=broad-except logger.info(error) try: dictionary["image"] = request["images"][0]["src"] except Exception as error: # pylint: disable=broad-except logger.info(error) try: dictionary["price"] = request["regular_price"] except Exception as error: # pylint: disable=broad-except logger.info(error) try: dictionary["sale_price"] = request["sale_price"] except Exception as error: # pylint: disable=broad-except logger.info(error) try: dictionary["amount"] = request["stock_quantity"] except Exception as error: # pylint: disable=broad-except logger.info(error) for key in gui: if "_box" in key and gui[key]: if key.split('_box')[0] not in dictionary: dictionary[key.split('_box')[0]] = None if dictionary[key.split('_box')[0]] == "": dictionary[key.split('_box')[0]] = None except Exception as error: # pylint: disable=broad-except logger.info(error) return dictionary
class get_idea_trade(): def __init__(self): self.user = sys.argv[1] self.password = sys.argv[2] self.timeframe = sys.argv[3] self.base_url = "%simportant" % (url) self.test_page = test_page self.test_page_cont = None self.art_data = {} self.instruments_dict = {} self.trades = {} self.all_trade_ideas = {} self.db = MySQL() def import_instruments(self): ''' Importing list of indtrument from instruments.yaml file ''' try: with open(instruments, 'r') as f: self.instruments_dict = yaml.load(f) log.print_green('Instruments imported!') except: log.print_error('Instruments import failed') def check_page(self): ''' Main function which is called every time. It's some kind of start process function. ''' with requests.Session() as c: self.main_page = c.get(self.base_url, headers={"User-Agent": useragent}) self.do_soup(self.main_page, "article") def do_article(self, art): ''' check articles from /important page and find keywords from trade_word ''' for trade_list in trade_word.values(): for trade in trade_list: try: assert trade in art.text.lower() self.art_data["action"] = self.get_action(trade) self.login(art.contents[5].find_all("a")[0].get("href")) except AssertionError: pass def login(self, link): ''' Login function. This function use detailes passed into tadebeat.py ''' with requests.Session() as l: l.get(url + "user") self.phpid = l.cookies["PHPSESSID"] #save token self.login_data = { "PHPSESSID": self.phpid, #create login dictionary with details "T[_U]": self.user, "T[_P]": self.password, "T[_A]": 0, "T[_A]": 1, "Action[login]": 1, "T[_B]": '' } l.post(url + "user", data=self.login_data, headers={"Referer": url + "user"}) # send post request to login page page = l.get(url + link[1:]) # send get request to article page self.check_artid(page, {"class": "news urgent" }) # pass page content for checking process def update_database(self, what, onwhat, artID): self._result = self.db.update(what, onwhat, artID) if self._result: log.print_green(what, onwhat, artID, ' Update OK!') else: return def ask_database(self, what, artID): self._result = self.db.select(what, artID) if self._result: for data in self._result: self._result_dict = yaml.load(data) return self._result_dict else: log.print_warning('Nothing found!') return 0 def check_artid(self, page, option): self.soup = BeautifulSoup(page.content, "lxml") self.art = self.soup.find('article', option) if not self.art: return if self.ask_database('time', self.art.get("id")): #log.print_green('Found something with has same articleID in database already...') self.check_timestamp( self.art.get("id"), self.art) # checking if article is an update, or old one return else: log.print_warning('Found something new!') self.do_soup(page, option) def check_timestamp(self, artID, content): ''' compare timestamp of last entry in time key ''' #log.print_green('I am going to compare timestamps') #self._old_time = self.all_trade_ideas.get(artID).get('time')[len(self.all_trade_ideas.get(artID).get('time'))] self._old_time = self.ask_database('time', artID) #print artID #print self._old_time #print len(self._old_time) #self._old_time = self._old_time.values() self._new_time = content.find("time").get("datetime") if self._new_time in self._old_time.values(): print('No updates') return else: log.print_green('Update of article has been detected!') return self.do_update(self._new_time, artID) def do_soup(self, page, option): self.soup = BeautifulSoup(page.content, "lxml") if option == "article": for art in self.soup.find_all(option): self.do_article(art) else: self.art = self.soup.find('article', option) self.do_collect_info(self.art, page.url.encode('ascii', 'ignore')) def do_collect_info(self, art, pageurl): #self.save_to_yaml('page', str(art)) self._art_data = {} #internal variable to hold article data log.print_green("Let's start with ", art.find("h1").text.lower()) self._art_data = tradeClass() self._art_data.title = art.find("h1").text.lower().encode( 'ascii', 'ignore') self._art_data.add_time(art.find("time").get("datetime")) self._art_data.ID = art.get("id") self._art_data.page_adr = pageurl self._art_data.instrument = self.check_instrument(self._art_data.title) for p in art.find('div').find_all( 'p'): #.text.encode('ascii', 'ignore') self._art_data.description += p.text.encode('ascii', 'ignore') log.print_warning('###### description ##### ', self._art_data.description, self._art_data.ID) self.find_tp_sl( self._art_data.description) # find take profit and sl values self._art_data.author = art.find('section', { 'class': 'autor' }).find('div', { 'class': 'about' }).find('h1').text.encode('ascii', 'ignore') #self._art_data.add_trade(self.art_data['action'], self._art_data.takestop) #log.print_warning(art.find('div').find('p').text.encode('ascii', 'ignore')) self._art_data.do_all_data() print 'take stop', self._art_data.takestop self.do_trade(self._art_data.takestop) if self._art_data.trade and (self._art_data.instrument != 'not found'): self.place_order(self._art_data.all_data, len(self._art_data.trade)) #log.print_warning('trade: ', self._art_data.trade) #log.print_warning('len: ', len(self._art_data.trade)) self._art_data.do_all_data() self.trades.update({self._art_data.ID: self._art_data.all_data}) #self.save_to_yaml(self._art_data.ID, self._art_data.all_data) #self.save_to_yaml_all(self.trades) self.db.insert(self._art_data.all_data) #insert all data to database self.trades = {} return def do_trade(self, value): log.print_green('Do trade with value: ', value) try: for i in range(0, max(len(value.get('SL')), len(value.get('TP')))): try: self._temp = {} self._temp.update({ 'SL': value.get('SL', '0')[0], 'TP': value.get('TP', '0')[i] }) log.print_green('temp1: ', self._temp) self._art_data.add_trade(self.art_data['action'], self._temp) except IndexError: self._temp = {} log.print_green('temp2: ', self._temp) self._temp.update({ 'SL': value.get('SL', '0')[i], 'TP': value.get('TP', '0')[0] }) self._art_data.add_trade(self.art_data['action'], self._temp) except TypeError: self._art_data.trade = '' def check_instrument(self, title): log.print_green('checking instruments...') for i in self.instruments_dict.keys(): if i.lower() in title.lower(): log.print_green('Found: ', i.lower()) return i.lower() return 'not found' def find_tp_sl(self, info): log.print_green('finding TP and SL...') for action in trade_action.keys(): for t in trade_action.get(action): try: if t in info.lower(): self._tmp = self.find_digits(t, info.lower()) self._art_data.add_takestop(action, self._tmp) except ValueError: pass log.print_green('takestop: ', self._art_data.takestop) def do_update(self, new_time, artID): ''' do update for article ''' log.print_green("Let's check what has been updated") self._old_time = self.ask_database('time', artID) self._tmp = self._old_time print 'przed: ', self._tmp self._tmp2 = '{}'.format(len(self._old_time) + 1) print 'tmp2: ', self._tmp2 self._tmp.update({self._tmp2: new_time}) print 'po: ', self._tmp self.update_database('time', self._tmp, artID) return def get_action(self, action): ''' get action like, buy or sell ''' for t in trade_word.keys(): if action in trade_word.get(t): return t def find_digits(self, what, where): self._result = 0 self._compile = "{}.[0-9\s.,]+".format(what) try: self._result = re.findall(self._compile, where)[0] self._result = self._result.split(':')[1] self._result = self._result.split(', ') for i in range(0, len(self._result)): self._result[i] = self._result[i].replace(' ', '') self._result[i] = self._result[i].replace(',', '.') return self._result except IndexError: return ['0'] def save_to_yaml_all(self, data): self._time = time.ctime().split(' ') self._time.remove('') self._dir_name = "./%s-%s/" % (self._time[2], self._time[1]) self._file_name = 'all_trade_ideas.yaml' try: with open(self._file_name, "ab") as f: yaml.dump(data, f) log.print_green('All data has been saved to: ', self._file_name) except IOError: log.print_error('Error during saving: ', self._file_name) def save_to_yaml(self, fname, data): self._time = time.ctime().split(' ') self._time.remove('') self._dir_name = "./%s-%s/" % (self._time[2], self._time[1]) self._file_name = self._dir_name + fname + '.yaml' try: with open(self._file_name, "ab") as f: yaml.dump(data, f) except IOError: os.makedirs(self._dir_name) with open(self._file_name, "ab") as f: yaml.dump(data, f) def place_order(self, data, i): ''' Funtion to place an order with ST and TP parameters ''' log.print_green('Placing an order...') print data try: self._instr = self.instruments_dict.get( data.get('instrument').upper())[0].get('instrument') self._unit = self.instruments_dict.get( data.get('instrument').upper())[3].get('tradeUnit') for x in range(1, (i + 1)): self._action = data.get('trade')[x].keys()[0] if not data.get('trade')[x][self._action]['TP']: self._tp = 0 else: self._tp = data.get('trade')[x][self._action]['TP'] self._sl = data.get('trade')[x][self._action]['SL'] log.print_warning(self._instr, self._unit, self._action, self._tp, self._sl) self.ordr = order.MyOanda(self._instr, self._unit, self._action, self._sl, self._tp) try: self._art_data.add_oanda(self.ordr.create_order()) except OandaError: log.print_error('Placing a order failed') return except AttributeError, TypeError: log.print_error('AttributeError, data NoneType') return
class FindInTextTest(unittest.TestCase): def setUp(self): self.user = '' self.password = '' self._art_takestop = {} self.test_page = test_page self.test_page_cont = None self.art_data = {} self.trades = {} self.all_trade_ideas = {} self.db = MySQL() def check_page(self): ''' Main function which is called every time. It's some kind of start process function. ''' with requests.Session() as c: self.do_soup(self.test_page_cont, "article") def check_instrument(self, title): log.print_green('checking instruments...') for i in self.instruments_dict.keys(): print title print 'i.lower() ', i.lower() if i.lower() in title: print '2: ', title print '2: i.lower() ', i.lower() log.print_green('Found: ', i.lower()) return i.lower() def do_update(self,new_time): ''' do update for article ''' print("Let's check what has been updated") self._art_data.add_time(new_time) return def get_action(self, action): ''' get action like, buy or sell ''' for t in trade_word.keys(): if action in trade_word.get(t): return t def do_article(self, art): ''' check articles from /important page and find keywords from trade_word ''' for trade_list in trade_word.values(): for trade in trade_list: try: assert trade in art.text.lower() self.art_data["action"] = self.get_action(trade) self.check_artid(self.test_page, {"class": "news urgent"}) except AssertionError: pass def ask_database(self, what, artID): self._result = self.db.select(what, artID) if self._result: for data in self._result: self._result_dict = yaml.load(data) return self._result_dict else: log.print_warning('Nothing found!') return 0 def check_artid(self, page, option): self.soup = BeautifulSoup(self.test_page_cont, "lxml") self.art = self.soup.find('article', option) if self.ask_database('time', self.art.get("id")): log.print_green('Found something with has same articleID in database already...') self.check_timestamp(self.art.get("id"), self.art) # checking if article is an update, or old one return else: log.print_warning('Found something new!') self.do_soup(page, option) def check_timestamp(self, artID, content): ''' compare timestamp of last entry in time key ''' print('I am going to compare timestamps') #self._old_time = self.all_trade_ideas.get(artID).get('time')[len(self.all_trade_ideas.get(artID).get('time'))] self._old_time = self.ask_database('time', artID) print artID print self._old_time print len(self._old_time) self._old_time = self._old_time.values()[-1] self._new_time = content.find("time").get("datetime") if self._old_time == self._new_time: print('No updates') return else: print('Update of article has been detected!') return self.do_update(self._new_time) def do_soup(self, page, option): self.soup = BeautifulSoup(page, "lxml") if option == "article": for art in self.soup.find_all(option): #self.do_article(art) print art else: self.art = self.soup.find('article', option) self.do_collect_info(self.art, page.url.encode('ascii', 'ignore')) def do_collect_info(self, art, pageurl): self.save_to_yaml('page', str(art)) self._art_data = {} #internal variable to hold article data log.print_green("Let's start with ", art.find("h1").text.lower()) self._art_data = tradeClass() self._art_data.title = art.find("h1").text.lower().encode('ascii', 'ignore') self._art_data.add_time(art.find("time").get("datetime")) self._art_data.ID = art.get("id") self._art_data.page_adr = pageurl self._art_data.instrument = self.check_instrument(self._art_data.title) for p in art.find('div').find_all('p'): #.text.encode('ascii', 'ignore') self._art_data.description += p.text.encode('ascii', 'ignore') log.print_warning('###### description ##### ', self._art_data.description) self.find_tp_sl(self._art_data.description) # find take profit and sl values self._art_data.author = art.find('section', {'class' : 'autor'}).find('div', {'class' : 'about'}).find('h1').text.encode('ascii', 'ignore') #self._art_data.add_trade(self.art_data['action'], self._art_data.takestop) #log.print_warning(art.find('div').find('p').text.encode('ascii', 'ignore')) self._art_data.do_all_data() self.do_trade(self._art_data.takestop) if self._art_data.trade: self.place_order(self._art_data.all_data, len(self._art_data.trade)) #log.print_warning('trade: ', self._art_data.trade) #log.print_warning('len: ', len(self._art_data.trade)) self._art_data.do_all_data() self.trades.update({self._art_data.ID : self._art_data.all_data}) self.save_to_yaml(self._art_data.ID, self._art_data.all_data) self.save_to_yaml_all(self.trades) self.db.insert(self._art_data.all_data) #insert all data to database self.trades = {} return def do_trade(self, value): log.print_green('Do trade with value: ', value) try: for i in range(0,max(len(value.get('SL')), len(value.get('TP')))): log.print_green(i) try: self._temp = {} self._temp.update({'SL' : value.get('SL')[0], 'TP': value.get('TP')[i]}) log.print_green('temp1: ', self._temp) self._art_data.add_trade(self.art_data['action'], self._temp) except IndexError: self._temp = {} log.print_green('temp2: ', self._temp) self._temp.update({'SL' : value.get('SL')[i], 'TP': value.get('TP')[0]}) self._art_data.add_trade(self.art_data['action'], self._temp) except TypeError: self._art_data.trade = '' def import_test_page(self): ''' Importing test_page from mainpage.yaml file ''' try: with open(self.test_page, 'r') as f: self.test_page_cont = f.read() log.print_green('Test page imported!') except: log.print_error('Test page import failed') def import_instruments(self): ''' Importing list of indtrument from instruments.yaml file ''' try: with open('./instruments.yaml', 'r') as f: self.instruments_dict = yaml.load(f) log.print_green('Instruments imported!') except: log.print_error('Instruments import failed') def find_tp_sl(self, info): print('finding TP and SL...') for action in trade_action.keys(): for t in trade_action.get(action): try: if t in info.lower(): self._tmp = self.find_digits(t, info.lower()) self._art_takestop.update({action : self._tmp}) except ValueError: pass log.print_green(self._art_takestop) def find_digits(self, what, where): self._result = 0 self._compile = "{}.[0-9\s.,]+".format(what) try: self._result = re.findall(self._compile, where)[0] self._result = self._result.split(':')[1] self._result = self._result.split(', ') for i in range(0, len(self._result)): self._result[i] = self._result[i].replace(' ', '') self._result[i] = self._result[i].replace(',', '.') print 'new result: ', self._result return self._result except IndexError: return [] @unittest.skip("skipping") def test_01_value_with_one_comma(self): self.description = description.format('(open price 0,67580)','Take profit: 0,6200', 'Stop Loss:0,7050') print self.description self.find_tp_sl(self.description) self.assertEqual(self._art_takestop, {'TP': ['0.6200'], 'SL': ['0.7050']}, self._art_takestop) @unittest.skip("skipping") def test_02_value_with_two_comma_sl(self): self.description = description.format('(open price 0,67580)','Take profit: 0,6200', 'Stop Loss:0,7050, 0,7150') print self.description self.find_tp_sl(self.description) self.assertEqual(self._art_takestop, {'TP': ['0.6200'], 'SL': ['0.7050', '0.7150']}, self._art_takestop) @unittest.skip("skipping") def test_03_value_with_two_comma(self): self.description = description.format('(open price 0,67580)','Take profit: 0,6200, 0,6500', 'Stop Loss:0,7050') print self.description self.find_tp_sl(self.description) self.assertEqual(self._art_takestop, {'TP': ['0.6200', '0.6500'], 'SL': ['0.7050']}, self._art_takestop) @unittest.skip("skipping") def test_04_value_with_three_comma(self): self.description = description.format('(open price 0,67580)','Take profit: 0,6200, 0,6500, 0,6700', 'Stop Loss:0,7050') print self.description self.find_tp_sl(self.description) self.assertEqual(self._art_takestop, {'TP': ['0.6200', '0.6500', '0.6700'], 'SL': ['0.7050']}, self._art_takestop) @unittest.skip("skipping") def test_05_value_with_three_comma_sl(self): self.description = description.format('(open price 0,67580)','Take profit: 0,6200', 'Stop Loss:0,7050, 0,7150, 0,7350') print self.description self.find_tp_sl(self.description) self.assertEqual(self._art_takestop, {'TP': ['0.6200'], 'SL': ['0.7050', '0.7150', '0.7350']}, self._art_takestop) @unittest.skip("skipping") def test_06_value_with_one_dot(self): self.description = description.format('(open price 0.67580)','Take profit: 0.6200', 'Stop Loss:0.7050') print self.description self.find_tp_sl(self.description) self.assertEqual(self._art_takestop, {'TP': ['0.6200'], 'SL': ['0.7050']}, self._art_takestop) @unittest.skip("skipping") def test_07_value_with_two_dot(self): self.description = description.format('(open price 0.67580)','Take profit: 0.6200, 0.6500', 'Stop Loss:0.7050') print self.description self.find_tp_sl(self.description) self.assertEqual(self._art_takestop, {'TP': ['0.6200', '0.6500'], 'SL': ['0.7050']}, self._art_takestop) @unittest.skip("skipping") def test_08_value_with_three_dot(self): self.description = description.format('(open price 0.67580)','Take profit: 0.6200, 0.6500, 0.6700', 'Stop Loss:0.7050') print self.description self.find_tp_sl(self.description) self.assertEqual(self._art_takestop, {'TP': ['0.6200', '0.6500', '0.6700'], 'SL': ['0.7050']}, self._art_takestop) @unittest.skip("skipping") def test_09_no_tp_dot(self): self.description = description.format('(open price 0.67580)','Take profit,', 'Stop Loss:0.7050') print self.description self.find_tp_sl(self.description) self.assertEqual(self._art_takestop, {'TP': [], 'SL': ['0.7050']}, self._art_takestop) @unittest.skip("skipping") def test_10_no_sl_dot(self): self.description = description.format('(open price 0.67580)','Take profit: 0.6200', 'Stop Loss') print self.description self.find_tp_sl(self.description) self.assertEqual(self._art_takestop, {'TP': ['0.6200'], 'SL': []}, self._art_takestop) @unittest.skip("skipping") def test_11_no_sl_and_tp(self): self.description = description.format('(open price 0.67580)','Take profit:', 'Stop Loss:') print self.description self.find_tp_sl(self.description) self.assertEqual(self._art_takestop, {'TP': [], 'SL': []}, self._art_takestop) @unittest.skip("skipping") def test_12_value_with_one_comma_short(self): self.description = description.format('(open price 0,67580)','TP: 0,6200', 'SL:0,7050') print self.description self.find_tp_sl(self.description) self.assertEqual(self._art_takestop, {'TP': ['0.6200'], 'SL': ['0.7050']}, self._art_takestop) @unittest.skip("skipping") def test_13_value_with_two_comma_short(self): self.description = description.format('(open price 0,67580)','TP: 0,6200', 'SL:0,7050, 0,7150') print self.description self.find_tp_sl(self.description) self.assertEqual(self._art_takestop, {'TP': ['0.6200'], 'SL': ['0.7050', '0.7150']}, self._art_takestop) @unittest.skip("skipping") def test_14_value_with_one_comma_mixed(self): self.description = description.format('(open price 0,67580)','Take profit: 0,6200', 'sl: 0,7050') print self.description self.find_tp_sl(self.description) self.assertEqual(self._art_takestop, {'TP': ['0.6200'], 'SL': ['0.7050']}, self._art_takestop) @unittest.skip("skipping") def test_15_value_with_two_comma_mixed(self): self.description = description.format('(open price 0,67580)','Take profit: 0,6200', 'SL:0,7050, 0,7150') print self.description self.find_tp_sl(self.description) self.assertEqual(self._art_takestop, {'TP': ['0.6200'], 'SL': ['0.7050', '0.7150']}, self._art_takestop) @unittest.skip("skipping") def test_16_title(self): self.instruments_dict = {} self.import_instruments() self._title = 'caduiyjpy - trade idea 02/03' self._instr = self.check_instrument(self._title) self.assertEqual(self._instr, 'cadjpy' , self._instr) def test_17_check_page(self): self.import_instruments() self.import_test_page() self.do_soup(self.test_page_cont, 'article')