def __init__(self):
self.initWindow()
self.initComponents()
self.ubicateComponents()
self.dataFetcher = DataFetcher()
self.dataCache = DataCache()
self.retrieveUserName()
self.rootWindow.mainloop()
def perform(keyword, pageRange):
print "Received keyword : " + keyword
keywordDB = {}
page = 1
while page <= pageRange:
if page == 1:
searchURL = "http://www.amazon.in/s/?url=search-alias%3Daps&field-keywords=" + keyword
else:
searchURL = "http://www.amazon.in/s/?url=search-alias%3Daps&field-keywords=" + keyword + "&page=" + str(page)
response = DataFetcher.getPageContent(searchURL)
urls = APIs.AmazonAPI.getListURLSofListPage(response)
def something(pageURL):
pageContent = DataFetcher.getPageContent(pageURL)
productDetails = APIs.AmazonAPI.getProductDetails(pageContent)
if productDetails and productDetails.has_key("name"):
keywordDB[productDetails["name"]] = productDetails["price"]
print productDetails
finalDB[keyword] = keywordDB
else:
failedURLS.append(pageURL)
for each in urls:
t = threading.Thread(target=something, args=(each, ))
threads.append(t)
t.start()
page += 1
print keywordDB
def something(pageURL):
pageContent = DataFetcher.getPageContent(pageURL)
productDetails = APIs.AmazonAPI.getProductDetails(pageContent)
if productDetails and productDetails.has_key("name"):
keywordDB[productDetails["name"]] = productDetails["price"]
print productDetails
finalDB[keyword] = keywordDB
else:
failedURLS.append(pageURL)
def get_player_kda(self, root_url = str(), region = str(), player_name = str()):
'''
This method returns player's kda data searching by name
'''
try:
full_url = root_url + region + '/' + player_name
raw_html = self.simple_get(full_url)
soup = DataFetcher.BeautifulSoup(raw_html, 'html.parser')
player_kda = soup.find("div",{"class":"kda tooltip"})
player_kda_result = player_kda.text.replace('\n','').replace('\t','').split('/')
return player_kda_result
except Exception:
print('Player {player_name} not found on url'.format(player_name = player_name))
return [-1 for i in range(3)]
def store_leaderboard_data(self, root_url = str(), region = str()):
'''
This method fetches player data from leaderboard and populate
the local players data dictionary
'''
full_url = root_url + region + '/' # after / there will be the number of the page appended
page_number = 0
exit_loop = False
while True:
raw_html = self.simple_get(full_url + str(page_number))
soup = DataFetcher.BeautifulSoup(raw_html, 'html.parser')
current_leaderboard = self.get_current_page_leaderboard(soup)
#check if we get to the end of leaderboard
for current_list in current_leaderboard:
# the list contains only ['No summoners recorded.'] in case the leaderboard is empty
if len(current_list) <= 1:
exit_loop = True
if exit_loop is True:
break
for current_list in current_leaderboard:
#get rid of extra ',' that may corupt our csv
current_list = [item.replace(',', '') for item in current_list]
#remove points from Division field
current_list[LoLFetcher.FETCHER_INDEX_DIVISION] = current_list[LoLFetcher.FETCHER_INDEX_DIVISION].replace(str(current_list[LoLFetcher.FETCHER_INDEX_POINTS]), '')
#remove text from Points field
current_list[LoLFetcher.FETCHER_INDEX_POINTS] = current_list[LoLFetcher.FETCHER_INDEX_POINTS].split(' ')[0]
#remove # from player rank
current_list[LoLFetcher.FETCHER_INDEX_PLAYER_RANK] = current_list[LoLFetcher.FETCHER_INDEX_PLAYER_RANK].replace('#', '')
player_data_list = ['0' for iter in range(LoLFetcher.FETCHER_INDEX_MAX_SIZE)]
player_data_list[LoLFetcher.DATABASE_INDEX_NAME] = current_list[LoLFetcher.FETCHER_INDEX_NAME]
player_data_list[LoLFetcher.DATABASE_INDEX_WINS] = current_list[LoLFetcher.FETCHER_INDEX_WINS]
player_data_list[LoLFetcher.DATABASE_INDEX_LOSSES] = current_list[LoLFetcher.FETCHER_INDEX_LOSSES]
player_data_list[LoLFetcher.DATABASE_INDEX_DIVISION] = current_list[LoLFetcher.FETCHER_INDEX_DIVISION]
player_data_list[LoLFetcher.DATABASE_INDEX_POINTS] = current_list[LoLFetcher.FETCHER_INDEX_POINTS]
player_muc = self.get_player_muc(LoLFetcher.PLAYER_ROOT_URL, LoLFetcher.NA, player_data_list[LoLFetcher.DATABASE_INDEX_NAME])
player_data_list[LoLFetcher.DATABASE_INDEX_MOST_USED_CHAMPS] = player_muc
player_kda = self.get_player_kda(LoLFetcher.PLAYER_ROOT_URL, LoLFetcher.NA, player_data_list[LoLFetcher.DATABASE_INDEX_NAME])
player_data_list[LoLFetcher.DATABASE_INDEX_KILLS] = player_kda[0]
player_data_list[LoLFetcher.DATABASE_INDEX_DEATHS] = player_kda[1]
player_data_list[LoLFetcher.DATABASE_INDEX_ASSISTS] = player_kda[2]
player_data_list[LoLFetcher.DATABASE_INDEX_PLAYER_RANK] = current_list[LoLFetcher.FETCHER_INDEX_PLAYER_RANK]
#print(player_data_list)
self.__exporter.upsert_lol_player_data(LoLFetcher.DB_NAME, player_data_list)
print(page_number)
page_number += 1
def get_player_muc(self, root_url = str(), region = str(), player_name = str()):
'''
This method returns player's most used champs data searching by name
'''
try:
full_url = root_url + region + '/' + player_name
raw_html = self.simple_get(full_url)
soup = DataFetcher.BeautifulSoup(raw_html, 'html.parser')
player_kda = soup.find("div",{"class":"s-block2"})
muchamps_html = player_kda.find_all('span', {'class':'champid'})
muchamps_list = [iter.text for iter in muchamps_html]
#return top 5 most used champs
return ','.join(muchamps_list[:5])
except Exception:
return 'No record.'
def save_leaderboard_to_csv(self, root_url = str(), region = str()):
full_url = root_url + region + '/' # after / there will be the number of the page appended
page_number = 0
exit_loop = False
header_added = False
while True:
raw_html = self.simple_get(full_url + str(page_number))
soup = DataFetcher.BeautifulSoup(raw_html, 'html.parser')
current_leaderboard = self.get_current_page_leaderboard(soup)
if header_added is False:
csv_header = self.get_leaderboard_table_head(soup)
with open('output.csv', 'a+') as output_file:
for item in csv_header:
not_last_index = csv_header.index(item) < len(csv_header) - 1
item = item.replace(',', '')
output_file.write(str(item))
if not_last_index is True:
output_file.write(',')
output_file.write('\n')
header_added = True
#check if we get to the end of leaderboard
for current_list in current_leaderboard:
# the list contains only ['No summoners recorded.'] in case the leaderboard is empty
if len(current_list) <= 1:
exit_loop = True
if exit_loop is True:
break
with open('output.csv', 'a+') as output_file:
for current_list in current_leaderboard:
#get rid of extra ',' that may corupt our csv
current_list = [item.replace(',', '') for item in current_list]
#remove points from Tier
current_list[4] = current_list[4].replace(str(current_list[5]), '')
for item in current_list:
not_last_index = current_list.index(item) < len(current_list) - 1
output_file.write(str(item))
if not_last_index is True:
output_file.write(',')
output_file.write('\n')
print(page_number)
page_number += 1
def create_task(taskname):
print "Received task - " + taskname
values = request.form.keys()[0]
values = json.loads(values)
url = values["url"]
url = convertAsciiTOString(url)
print "Received url : " + url
data = DataFetcher.getPageContent(url)
result = {}
result["ProductName"] = APIs.AmazonAPI.getProductName(data)
result["ProductPrice"] = APIs.AmazonAPI.getPrice(data)
result["ProductImage"] = APIs.AmazonAPI.getProductImage(data)
result = json.dumps(result)
print result
resp = Response(result)
resp.headers['Access-Control-Allow-Origin'] = '*'
return resp
def fetchData(self):
movieData = {}
if self.movieNameTextField.text() != "":
tmp = DataFetcher.fetchMovieContent(self.movieNameTextField.text())
movieData = tmp[0]
imageFileName = tmp[1]
else:
print(
'DIGITE ALGUMA COISA!') # TODO: trocar para janela de warning
if movieData != {}:
movieTitle = movieData['Title']
movieYear = movieData['Year']
movieDirector = movieData['Director']
movieGenre = movieData['Genre']
#Este conjunto cria uma janela e associa a uma interface
#Cria uma janela (QDialog), e armazena na variavel
movieDetailsWindow = QtWidgets.QDialog()
#construtor da classe de interface do designer
ui = movieInfoWindow.Ui_Dialog()
#carrega a interface pra dentro da janela criada
ui.setupUi(movieDetailsWindow)
#Fim do conjunto
ui.setMovieTitle(movieTitle)
ui.setMovieYear(movieYear)
ui.setMovieDirector(movieDirector)
ui.setMovieGenre(movieGenre)
ui.setMoviePoster(imageFileName)
movieDetailsWindow.exec_()
else:
print('O SERVIDOR NÃO RETORNOU DADOS'
) # TODO: trocar para janela de warning
def __init__(self):
self.players = {}
self.dataFetcher = DataFetcher.DataFetcher()
[tf.placeholder(tf.int32, shape=(None, 2)) for _ in range(num_blocks)],
'lape_idx':
[tf.placeholder(tf.int32, shape=(None, 10))
for _ in range(num_blocks)], #for laplace term
'pool_idx':
[tf.placeholder(tf.int32, shape=(None, 2))
for _ in range(num_blocks - 1)], #for unpooling
'dropout':
tf.placeholder_with_default(0., shape=()),
'num_features_nonzero':
tf.placeholder(tf.int32) # helper variable for sparse dropout
}
model = GCN(placeholders, logging=True)
# Load data, initialize session
data = DataFetcher(FLAGS.data_list)
data.setDaemon(True) ####
data.start()
config = tf.ConfigProto()
#config.gpu_options.allow_growth=True
config.allow_soft_placement = True
sess = tf.Session(config=config)
sess.run(tf.global_variables_initializer())
#model.load(sess)
# Train graph model
train_loss = open('tmp/train_loss_record.txt', 'a')
train_loss.write('Start training, lr = %f\n' % (FLAGS.learning_rate))
pkl = pickle.load(open('data/utils/eccv_final_version.dat', 'rb'))
train_number = data.number
__author__ = 'kh1911'
import DataFetcher
import APIs.AmazonAPI
data = DataFetcher.getPageContent(
"http://www.flipkart.com/search?q=shirts&as=off&as-show=off&otracker=start"
)
print data
class Window:
def __init__(self):
self.initWindow()
self.initComponents()
self.ubicateComponents()
self.dataFetcher = DataFetcher()
self.dataCache = DataCache()
self.retrieveUserName()
self.rootWindow.mainloop()
def retrieveUserName(self):
name = self.dataCache.getSavedUserName()
self.txtAnilistUserName.insert(0, name)
def initWindow(self):
self.rootWindow = Tk()
self.rootWindow.title("Anilist Challenge Helper")
self.rootWindow.geometry(WINDOWSIZE)
self.rootWindow.resizable(False, False)
def initComponents(self):
self.lblAnilistUserName = CustomLabel(self.rootWindow,
text="Anilist Username: "******"Anime Link: ")
self.txtLink = Entry(self.rootWindow, width=ENTRYWIDTH)
self.lblChallenge = CustomLabel(self.rootWindow,
text="Challenge Text: ")
self.txtChallenge = Entry(self.rootWindow, width=ENTRYWIDTH)
self.actionPanel = Frame(self.rootWindow, width=RESULTWIDTH)
self.btnBuild = Button(self.actionPanel,
text="Generate Line",
command=self.onGenerateClick,
anchor=CENTER)
self.txtResult = Text(self.actionPanel, width=50, height=10)
self.txtResult.configure(state=DISABLED)
self.rootWindow.iconbitmap(r'icon.ico')
def ubicateComponents(self):
self.lblAnilistUserName.grid(row=0, column=0)
self.txtAnilistUserName.grid(row=0, column=1)
self.lblLink.grid(row=1, column=0)
self.txtLink.grid(row=1, column=1)
self.lblChallenge.grid(row=2, column=0)
self.txtChallenge.grid(row=2, column=1)
self.actionPanel.grid(row=3, column=0, columnspan=2)
self.btnBuild.grid(row=0, column=0, columnspan=2)
self.txtResult.grid(row=1, column=0, columnspan=2)
def onGenerateClick(self):
resultado = ''
try:
userName = self.txtAnilistUserName.get()
self.dataCache.saveUserName(userName)
mediaURL = self.txtLink.get()
mediaId = self.dataFetcher.getIdFromURL(mediaURL)
challenge = self.txtChallenge.get()
animeData = self.dataFetcher.runQuery(userName, mediaId)
title = animeData[0]
startDate = animeData[1]
endDate = animeData[2]
resultado = self.dataFetcher.getResponse(startDate, endDate,
challenge, title,
mediaURL)
except Exception as e:
resultado = e
self.setResultText(resultado)
def setResultText(self, text):
self.txtResult.configure(state=NORMAL)
self.txtResult.delete(1.0, END)
self.txtResult.insert(END, text)
self.txtResult.configure(state=DISABLED)
