def scrap_items():
for itemlist in ITEMLIST:
soup = BS(urllib2.urlopen(''.join([LOLWIKI, itemlist])).read())
item_table = soup.find('table', class_='stdt sortable')
for tr in item_table.find_all('tr'):
tds = tr.find_all('td')
if len(tds) < 1:
continue
if tr.find('p') == None:
continue
item_name = tr.find('p').text.strip()
item_url = tr.find('img')['src']
if item_url.split(':')[0] == 'data':
item_url = tr.find('img')['data-src']
if not HOOKED:
continue
#store item in database
d_item = Item()
d_item.name = item_name
t_img = NamedTemporaryFile(delete=True)
t_img.write(urllib2.urlopen(item_url).read())
t_img.flush()
t_img.name = '.'.join([item_name, 'jpg'])
d_item.picture = File(t_img)
d_item.save()
def save_post(post):
"""
Сохраняет пост
"""
from_id = abs(post['from_id'])
photos = get_all_photos_from_post(post)
if not photos:
return
text = replace_vk_links_on_markdown(post['text'])
text = from_tags_to_links(text)
news = models.News.objects.create(
author=f'group{from_id}',
text=text,
date_to=timezone.localtime() + timedelta(7),
id_vk=post['id'],
)
for photo_photo in photos:
# Загружаю картинку по ссылке из вк
img_temp = NamedTemporaryFile(
delete=True,
suffix='.jpg',
)
img_temp.write(urlopen(photo_photo).read())
img_temp.flush()
img_temp.name = img_temp.name.split('/')[-1]
# Создаю модель изображения
models.Image.objects.create(
image=File(img_temp),
news=news,
)
def create_submission_for(submission_exercise, aplus_submission, user):
"""
1. check if there's an user with the aplus submitter's email
2. if not, crate one and set temporary = True
3. get the submissions file from aplus API
4. create a new original submission with the file and submitter
"""
file_url = aplus_submission["files"][0]["url"]
filename = aplus_submission["files"][0]["filename"]
file_blob = requests.get(file_url, headers={ 'Authorization': f'Token {submission_exercise.course.aplus_apikey}' })
temp_file = NamedTemporaryFile(delete=True)
temp_file.name = filename
temp_file.write(file_blob.content)
temp_file.flush()
new_orig_sub = OriginalSubmission(
course=submission_exercise.course,
submitter_user=user,
exercise=submission_exercise,
file=File(temp_file)
)
new_orig_sub.save()
logger.info(new_orig_sub)
def generate_csv_html_file_response(name, list_data, keys):
# use of named temporary file here is to handle deletion of file
# after we return the file, after the new file object is evicted
# it gets deleted
# http://stackoverflow.com/questions/3582414/removing-tmp-file-
# after-return-httpresponse-in-django
newfile = NamedTemporaryFile(suffix='.csv', delete=False)
newfile.name = name
dict_writer = csv.DictWriter(newfile, keys)
dict_writer.writeheader()
dict_writer.writerows(list_data)
newfile.seek(0)
wrapper = FileWrapper(newfile)
httpresponse = HttpResponse(wrapper, content_type="text/csv")
httpresponse['Content-Disposition'] = 'attachment; filename=%s' \
% newfile.name
return httpresponse
def scrap_heroes():
soup = BS(urllib2.urlopen(''.join([LOLWIKI, HEROLIST])).read())
#get hero names and urls
hero_table = soup.find_all('table')[1]
heroes = {}
for tr in hero_table.find_all('tr'):
tds = tr.find_all('td')
if len(tds) < 1:
continue;
hero_data = tds[0]
hero_useful = hero_data.find('a')
hero_name = hero_useful['title'].strip()
hero_url = hero_useful['href'].split('/')[-1]
heroes[hero_name] = {}
heroes[hero_name]['url'] = hero_url
#get hero skills
heroes_skills = {}
for name, data in heroes.items():
soup = BS(urllib2.urlopen(''.join([LOLWIKI, data['url']])).read())
hero_info_table = soup.find('table', id='champion_info-upper')
data['pic'] = hero_info_table.find('img')['src']
#get skill data
skills_data = {}
skills_data['q'] = soup.find('div', class_='skill skill_q')
skills_data['w'] = soup.find('div', class_='skill skill_w')
skills_data['e'] = soup.find('div', class_='skill skill_e')
skills = {}
for key, sdata in skills_data.items():
skills[key] = {
'name': sdata.find('span').text.strip(),
'pic': sdata.find('img')['data-src']
}
data['skills'] = skills
if not HOOKED:
continue
#upload hero image
t_img = NamedTemporaryFile(delete=True)
t_img.write(urllib2.urlopen(data['pic']).read())
t_img.flush()
t_img.name = '.'.join(['temp', 'jpg'])
#create hero data
d_champ = Champion()
d_champ.name = name
d_champ.picture = File(t_img)
d_champ.save()
#create skill data and upload skill images
for key, skill in data['skills'].items():
t_img = NamedTemporaryFile(delete=True)
t_img.write(urllib2.urlopen(skill['pic']).read())
t_img.name = '.'.join([skill['name'], 'jpg'])
d_skill = Skill()
d_skill.name = skill['name']
d_skill.key = key.upper()
d_skill.picture = File(t_img)
d_skill.champion = d_champ
d_skill.save()
