def format_results(start_date, end_date):
for i in range(start_date,end_date+1,1):
for j in range(1,32,1):
current_path = Path('C:\\Users\\Nicholas\\Documents\\GitHub\\Fantasy\\data\\%s.%s\\results%s.%s.csv' % (str(i),str(j),str(i),str(j)))
if current_path.is_file():
list = []
with open('C:\\Users\\Nicholas\\Documents\\GitHub\\Fantasy\\data\\%s.%s\\results%s.%s.csv' % (str(i),str(j),str(i),str(j)), 'r', encoding='utf-8', errors="ignore") as csvf:
csvreader = csv.reader(csvf, delimiter=',', lineterminator='\n')
for row in csvreader:
if row[7]:
list.append(row)
del list[0]
for item in list:
del item[:7]
del item[1]
with open('C:\\Users\\Nicholas\\Documents\\GitHub\\Fantasy\\results\\results%s.%s.csv' % (str(i),str(j)), 'w') as csvf:
csvwriter = csv.writer(csvf, delimiter=',', lineterminator='\n')
for row in list:
if row:
row[0] = clean_string.cleanstr(row[0])
if row[0] in dict:
row[0] = dict.get(row[0])
csvwriter.writerow(row)
def format_results_single(start_date):
list = []
with open('C:\\Users\\Nicholas\\Documents\\GitHub\\Fantasy\\data\\2017\\1.%s\\results1.%s.csv' % (str(start_date),str(start_date)), 'r', encoding='utf-8', errors="ignore") as csvf:
csvreader = csv.reader(csvf, delimiter=',', lineterminator='\n')
for row in csvreader:
if row[7]:
list.append(row)
del list[0]
for item in list:
del item[:7]
del item[1]
with open('C:\\Users\\Nicholas\\Documents\\GitHub\\Fantasy\\results\\results1.%s.csv' % str(start_date), 'w') as csvf:
csvwriter = csv.writer(csvf, delimiter=',', lineterminator='\n')
for row in list:
if row:
row[0] = clean_string.cleanstr(row[0])
if row[0] in dict:
row[0] = dict.get(row[0])
csvwriter.writerow(row)
def format_template(num): list = [] filename = 'C:\\Users\\Nicholas\\Documents\\GitHub\\Fantasy\\template.csv' filename_results = 'C:\\Users\\Nicholas\\Documents\\GitHub\\Fantasy\\template1.%s.csv' % str(num) with open(filename, 'r') as csvf: csvreader = csv.reader(csvf,delimiter=',',lineterminator='\n') for row in csvreader: list.append(row) del list[0] del list[1][:40] for i in range(len(list)): del list[i][:11] del list[i][2:] del list[:7] with open(filename_results, 'w') as csvf: csvwriter = csv.writer(csvf,delimiter=',',lineterminator='\n') for row in list: row[0] = clean_string.cleanstr(row[0]) csvwriter.writerow(row)
def format_sal_single(date):
list = []
with open('C:\\Users\\Nicholas\\Documents\\GitHub\\Fantasy\\data\\2017\\1.%s\\salaries1.%s.csv' % (str(date),str(date)), 'r') as csvf:
csvreader = csv.reader(csvf, delimiter=',', lineterminator='\n')
for row in csvreader:
list.append(row)
del list[0]
copy_list = [_ for _ in list]
new_list = []
for i in range(len(copy_list)):
semaphore1 = 0
semaphore2 = 0
if copy_list[i][0].find('/') > 0:
temp_list = []
copyrow = copy_list[i]
temp_list.append(copyrow[0][copyrow[0].find('/')+1:])
for row in copyrow[1:]:
temp_list.append(row)
new_list.append(temp_list)
copyrow.insert(0,copyrow[0][:copyrow[0].find('/')])
del copyrow[1]
if temp_list[0] == 'PG' or temp_list[0] == 'SG':
semaphore1 = 1
temp_list_2 = []
temp_list_2.append('G')
for row in temp_list[1:]:
temp_list_2.append(row)
new_list.append(temp_list_2)
if temp_list[0] == 'SF' or temp_list[0] == 'PF':
semaphore2 = 1
temp_list_2 = []
temp_list_2.append('F')
for row in temp_list[1:]:
temp_list_2.append(row)
new_list.append(temp_list_2)
if (copy_list[i][0] == 'PG' or copy_list[i][0] == 'SG') and semaphore1 == 0:
temp_list = []
temp_list.append('G')
for row in copy_list[i][1:]:
temp_list.append(row)
new_list.append(temp_list)
if (copy_list[i][0] == 'SF' or copy_list[i][0] == 'PF') and semaphore2 == 0:
temp_list = []
temp_list.append('F')
for row in copy_list[i][1:]:
temp_list.append(row)
new_list.append(temp_list)
temp_list = []
temp_list.append('UTIL')
for row in copy_list[i][1:]:
temp_list.append(row)
new_list.append(temp_list)
with open('C:\\Users\\Nicholas\\Documents\\GitHub\\Fantasy\\salaries\\salaries1.%s.format.csv' % str(date), 'w') as csvf:
csvwriter = csv.writer(csvf, delimiter=',', lineterminator='\n')
for row in copy_list:
row[1] = clean_string.cleanstr(row[1])
if row[1] in dict:
row[1] = dict.get(row[1])
if row[5] in dict_team:
row[5] = dict_team.get(row[5])
csvwriter.writerow(row)
for row in new_list:
row[1] = clean_string.cleanstr(row[1])
if row[1] in dict:
row[1] = dict.get(row[1])
if row[5] in dict_team:
row[5] = dict_team.get(row[5])
csvwriter.writerow(row)
def format_sal(start_date, end_date):
for j in range(start_date,end_date+1,1):
for k in range(1,32,1):
list = []
current_path = Path('C:\\Users\\Nicholas\\Documents\\GitHub\\Fantasy\\data\\%s.%s\\salaries%s.%s.csv' % (str(j),str(k),str(j),str(k)))
if current_path.is_file():
with open('C:\\Users\\Nicholas\\Documents\\GitHub\\Fantasy\\data\\%s.%s\\salaries%s.%s.csv' % (str(j),str(k),str(j),str(k)), 'r') as csvf:
csvreader = csv.reader(csvf, delimiter=',', lineterminator='\n')
for row in csvreader:
list.append(row)
del list[0]
copy_list = [_ for _ in list]
new_list = []
for i in range(len(copy_list)):
semaphore = 0
if copy_list[i][0].find('/') > 0:
temp_list = []
copyrow = copy_list[i]
temp_list.append(copyrow[0][copyrow[0].find('/')+1:])
for row in copyrow[1:]:
temp_list.append(row)
new_list.append(temp_list)
copyrow.insert(0,copyrow[0][:copyrow[0].find('/')])
del copyrow[1]
if temp_list[0] == 'PG' or temp_list[0] == 'SG':
semaphore = 1
temp_list_2 = []
temp_list_2.append('G')
for row in temp_list[1:]:
temp_list_2.append(row)
new_list.append(temp_list_2)
if temp_list[0] == 'SF' or temp_list[0] == 'PF':
semaphore = 1
temp_list_2 = []
temp_list_2.append('F')
for row in temp_list[1:]:
temp_list_2.append(row)
new_list.append(temp_list_2)
if (copy_list[i][0] == 'PG' or copy_list[i][0] == 'SG') and semaphore == 0:
temp_list = []
temp_list.append('G')
for row in copy_list[i][1:]:
temp_list.append(row)
new_list.append(temp_list)
if (copy_list[i][0] == 'SF' or copy_list[i][0] == 'PF') and semaphore == 0:
temp_list = []
temp_list.append('F')
for row in copy_list[i][1:]:
temp_list.append(row)
new_list.append(temp_list)
temp_list = []
temp_list.append('UTIL')
for row in copy_list[i][1:]:
temp_list.append(row)
new_list.append(temp_list)
with open('C:\\Users\\Nicholas\\Documents\\GitHub\\Fantasy\\salaries\\salaries%s.%s.format.csv' % (str(j),str(k)), 'w') as csvf:
csvwriter = csv.writer(csvf, delimiter=',', lineterminator='\n')
for row in copy_list:
row[1] = clean_string.cleanstr(row[1])
if row[1] in dict:
row[1] = dict.get(row[1])
if row[5] in dict_team:
row[5] = dict_team.get(row[5])
csvwriter.writerow(row)
for row in new_list:
row[1] = clean_string.cleanstr(row[1])
if row[1] in dict:
row[1] = dict.get(row[1])
if row[5] in dict_team:
row[5] = dict_team.get(row[5])
csvwriter.writerow(row)
def parse_single(current_date):
next_page = str(baseurl) + str(0) + str(dateurl) + str(current_date) + str(
endurl) + str(0)
list = []
j = 0
nextindex = 0
while True:
r = requests.get(next_page)
soup = BeautifulSoup(r.content, 'html.parser')
button = soup.find('div', class_='paginationNav')
table = soup.find('table', class_='playerTableTable tableBody')
tr_count = 0
semaphore = 0
for row in table.findAll('tr'):
#discard first two tr
tr_count += 1
cells = [c.get_text() for c in row.findAll('td')]
del cells[1:4]
playername = clean_string.cleanstr(str(cells[0]))
cells[0] = playername
stringcells = clean_string.cleanstr(str(cells))
if '--' in stringcells:
semaphore = 1
break
if tr_count > 2:
if 'PLAYER' not in stringcells and '--' not in stringcells and len(
stringcells) != 4:
cells.append(positions[j])
loc = stringcells.find(', ') + 2
teamname = stringcells[loc:stringcells.find(' ', loc + 2)]
if teamname == 'Nor':
teamname = 'NO'
cells.append(teamname)
if 'DTD' in stringcells:
stringcells = stringcells.replace(' DTD', '')
cells.append('DTD')
elif '*' in stringcells:
stringcells = stringcells.replace('*', '')
stringcells = stringcells.replace(' O', '')
cells.append('Out')
else:
cells.append('G')
del (cells[1])
list.append(cells)
#elif 'PLAYER' in stringcells and j == 0 and nextindex == 0:
#list.append(cells)
if 'NEXT' in button.get_text() and semaphore == 0:
nextindex += 50
ending = endurl + str(nextindex)
next_page = str(baseurl) + str(
pagenums[j]) + str(dateurl) + str(current_date) + str(ending)
print(next_page)
else:
if j + 1 >= len(positions):
print(next_page)
break #exit loop
else:
j += 1
nextindex = 0
ending = endurl + str(0)
next_page = str(baseurl) + str(pagenums[j]) + str(
dateurl) + str(current_date) + str(ending)
print(next_page)
with open(
r'C:\Users\Nicholas\Documents\GitHub\Fantasy\espn\data%s.csv' %
str(current_date), 'w') as csvf:
csvwriter = csv.writer(csvf, delimiter=',', lineterminator='\n')
for i in range(len(list)):
list[i][0] = clean_string.cleanstr(
list[i][0][:list[i][0].find(',')])
csvwriter.writerow(list[i])