def getCommander(driver, commander, threshold): print(commander) commanderHtml = getCommanderHtml(driver, commander) soup = BeautifulSoup(commanderHtml, 'html.parser') cardsDivs = soup.findAll("div", attrs={"class": "cardlist"}) cardContainers = [] for cardsDiv in cardsDivs: divs = cardsDiv.findAll("div", attrs={"class": "card__container"}) for cardContainer in divs: cardContainers.append(cardContainer) cards = [] for cardContainer in cardContainers: cardNameDiv = cardContainer.find("div", attrs={"class": "card__name"}) cardName = cardNameDiv.text cardLabelDiv = cardContainer.find("div", attrs={"class": "card__label"}) cardLabel = cardLabelDiv.text (percent, synergy) = re.findall(r"([-]?\d+)%", cardLabel) if (int(percent) > threshold): cardName = CardUtil.standardizeCardName(cardName) card = Card(cardName, int(percent), int(synergy)) cards.append(card) return cards
def getCardData(filePath): file = open(filePath, "r") cards = json.load(file) file.close() data = cards["data"] allCardNames = set(data.keys()) for cardName in allCardNames: standardCardName = CardUtil.standardizeCardName(cardName) if (cardName != standardCardName): data[standardCardName] = data[cardName] del data[cardName] return data
def getColorIdentity(cardName): cardName = CardUtil.standardizeCardName(cardName) cards = cardData[cardName] atomicCard = cards[0] colorIdentity = atomicCard["colorIdentity"] types = atomicCard["types"] if "Land" in types: return "Land" elif (len(colorIdentity) == 0): return "Colorless" elif (len(colorIdentity) == 1): return colorIdentity[0] else: return "Multi"
def main(): db = mysql.connector.connect(user=DatabaseConfig.database_user, password=DatabaseConfig.database_password, host=DatabaseConfig.database_host, database=DatabaseConfig.database_name) CardUtil.updateStart('cards',db) CardUtil.scrapeMarket('trading+card', db, processItem) CardUtil.updateEnd('cards',db) db.close()
def main(): db = mysql.connector.connect(user=DatabaseConfig.database_user, password=DatabaseConfig.database_password, host=DatabaseConfig.database_host, database=DatabaseConfig.database_name) CardUtil.updateStart('cardvolume',db) cursor = db.cursor() cursor.execute("SELECT app_id, name, link FROM cards ORDER BY last_updated ASC") cardlist = cursor.fetchall() count = 0 numrows = len(cardlist) for (app_id, name, link) in cardlist: try: cleanlink = re.search('\/753\/(.*)', link).group(1) volumeitem = CardUtil.getMarketVolume( cleanlink ) if (not volumeitem.success): print(cleanlink) elif (volumeitem.lowest_price != 0): cursor.execute("UPDATE cards SET lowest_price = %s, volume = %s, median_price = %s, last_updated = NOW() WHERE app_id = %s AND name = %s", (volumeitem.lowest_price, volumeitem.volume, volumeitem.median_price, app_id, name)) else: cursor.execute("UPDATE cards SET quantity = 0, volume = %s, median_price = %s, last_updated = NOW() WHERE app_id = %s AND name = %s", (0, 0, app_id, name)) except KeyboardInterrupt: raise SystemExit except: print sys.exc_info()[0] if (count%100 == 0): db.commit() print (str(count)+" / "+str(numrows)) count += 1 cursor.close() db.commit() CardUtil.updateEnd('cardvolume',db) db.close()