def hepsiburada_linkleri(key):
session = HTMLSession()
liste = []
pattern = re.compile(r'p-[a-zA-Z]{1,}[0-9]{1,}[a-zA-Z0-9]{1,}'
) # regex şablonumuz # p-HBV00000Y8E0O
###############################3
req = session.get(URL[key])
soup = BeautifulSoup(req.html.html, 'html.parser')
sayfalama = soup.find('div', {'id': 'pagination'})('li')[-1].text.strip()
####################################
for a in range(1, int(sayfalama)):
try:
url = URL[key] + "?sayfa={}".format(a)
req = session.get(url)
time.sleep(1.5)
container = req.html.find('.product-list')[0]
for i in container.absolute_links: # tüm linkler , all links
r = re.search(pattern, i)
if r is not None:
urun_kimligi = r.group(
) # çıktı, output, example: p-HBV00000Y8E0O
liste.append(urun_kimligi)
except:
break
json_save(liste, key)
def app_view_scrapy(name1, name2):
items = {}
for link in scrapy(name1, name2):
url = requests.get(link)
bs = soup(url.content, 'html.parser')
###
download = bs.find('div', {'id': 'indir'}).find("a", {'itemprop': 'downloadURL'})
baslik = bs.find('div', {'id': 'baslik'})
app_information = bs.find('div', {'id': 'detayads'}).find_all('div', {'class': 'dl'})
###
img_link = baslik.find('img', {'itemprop': 'image'})
baslik_name = baslik.find('span', {'itemprop': 'name'}).text.replace(' ', '_').lower().replace('ı','i')
rating_info = baslik.find('div', {'class': 'rating'}).find('strong', {'class', 'ratingval'})
rating_star = rating_info.span.text
rating_oy = rating_info.find('span', {'itemprop': 'ratingCount'})
data = {
#'virus_scan':app_information[7]('span').text,
'system':app_information[6]('span')[-1].text,
'update':app_information[5]('span')[0].text,
'interface': app_information[4]('span')[0].text.strip().replace('\n','').replace('İ','i').replace('ç','c').replace('I','i').replace('ü','u'),
'person':app_information[3]('span')[0].text.strip().replace('\n','').replace('ç','c'),
'size':app_information[2].text[6:].strip().replace('\n',''),
'license' : app_information[0]('span')[0].text.strip().lower().replace('\n','').replace('Ü','u').replace('ç','c').replace('ü','u'),
'publisher' : app_information[1]('span')[0].text.lower(),
'rating_star': rating_star,
'rating': [rating_oy.text if rating_oy is not None else '0'][0],
'img_link': [img_link['src'] if img_link is not None else None][0],
'download_link': [download.get('onclick')[11:-13] if download is not None else None][0],
}
items[baslik_name] = data
json_save(items,"unix-linux")
return items
def tum_kategori_linkleri():
session = HTMLSession()
r = session.get('https://www.hepsiburada.com/tum-kategoriler')
soup = BeautifulSoup(r.html.html, 'html.parser')
a = 0
items = {}
while True:
try:
container = soup.find_all('div', class_='categories')
group = container[a].find_all('div', class_='group')
for i in group:
anabaslik = i.find('a')
if anabaslik is not None:
it = {}
for j in i('a'):
baslik = j.text.lower().replace('&', '').replace(
' ',
'').replace('/',
'').replace('ö',
'o').replace('ü', 'u')
link = "https://www.hepsiburada.com" + str(
j.get('href'))
it[baslik] = link
items[anabaslik.text.lower().replace('.',
'').replace('/',
'')] = it
a += 1
except:
break
json_save(items, 'tüm_kategoriler')
def android_and_ios_scrapy(kategori,key, path):
android = "https://www.gezginler.net/android/"
ios = "https://www.gezginler.net/ios/"
if kategori == 'android':
kategori = android
elif kategori == 'ios':
kategori = ios
else:
print('Sadece android ve ios kategorisi vardır...')
return 0
url = kategori + key
driver = webdriver.Chrome()
driver.get(url)
height = driver.execute_script("return document.body.scrollHeight")
while True:
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(1)
new_height = driver.execute_script("return document.body.scrollHeight")
if new_height == height:
break
height = new_height
time.sleep(1)
source = driver.page_source
driver.close()
bs = soup(source,'html.parser')
items = {}
for app in bs.find_all('div',{'class':'masonry-brick'}):
href = app.find('a').get('href')
title = app.find('a').strong.text
title = title.lower().replace(' ','_').replace('-','').replace('!','').replace('ü','u').replace('ç','c').replace('İ','i').replace('ğ','g').replace('ö','o')
#info = app.find('a')[-1].text
items[title.replace('ş','s').replace('ı','i')] = href
json_save(items, path)
def lncanon_characters_scrapy():
req = requests.get(URL['characters']['lncanon'], headers=HEADERS)
soup = bs(req.content, 'html.parser')
items = {}
for x in range(2):
for tr in soup('table', class_='wikitable')[x]('tr')[1:]:
name = tr('td')[1].text
data = {
'name_url':
'https://onepiece.fandom.com' + tr('td')[1].a.get('href'),
'chapter_no':
tr('td')[2].text.replace('\n', ''),
'chapter_no_url': [
'https://onepiece.fandom.com' + tr('td')[2].a.get('href')
if tr('td')[2].a is not None else None
][0],
'epsido_no':
tr('td')[3].text.replace('\n', ''),
'epsido_no_url': [
'https://onepiece.fandom.com' + tr('td')[3].a.get('href')
if tr('td')[3].a is not None else None
][0],
'year':
tr('td')[4].text.replace('\n', ''),
'note':
tr('td')[5].text.replace('\n', '')
}
items[name.lower().replace(' ', '_').replace('\n', '')] = data
json_save(items, 'lncanon')
def create(self, text_lines):
all_word = []
for sent in text_lines:
for word in sent:
all_word.append(word)
self.__vocab_list.extend(list(sorted(set(all_word))))
word_to_index = {word: i for i, word in enumerate(self.__vocab_list)}
u.json_save(c.vocab_path, word_to_index)
def dizi_listesi_scrapy():
# dizilistesi/yerlidizi.json
# dizilistesi/trdublaj.json
# dizilistesi/anime.json
# dizilistesi/asya.json
# dizilistesi/yerlidizi.json
with open('dizilistesi/yerlidizi.json', 'r') as f:
data = json.load(f)
json_save(view_scrapy(data),'data/yerlidizi_')
def dizi_listesi(file_name):
req = requests.get(URL[file_name])
soup = bs(req.content, 'html.parser')
items = {}
for i in soup.find('div', {'class': 'dizi_turleri'})('a'):
name = i.font.text.lower().replace('ü', 'u').replace('ğ', 'g').replace('ı', 'i').replace('ç', 'c').replace('•', '').replace('!', '').replace('(anime)', '')
link = i.get('href')
items[name.strip().replace(':', '.').replace('.', '').replace('(tã¼rkã§e dublaj)', '')] = link
json_save(items,"dizilistesi/"+ file_name)
def organizations_marines_scrapy():
req = requests.get(URL['organizations']['marines'], headers=HEADERS)
soup = bs(req.content, 'html.parser')
items = {}
for i in soup('div', class_='Gallery-pic'):
name = i.a.get('title')
# data = {
# 'name_url': ,
# #'image_url':i.img.get('data-src')
# }
items[name.lower()] = 'https://onepiece.fandom.com/' + i.a.get('href')
json_save(items, 'marines')
def pirate_crews_scrapy():
req = requests.get(URL['pirate_crews']['whitebeard'], headers=HEADERS)
soup = bs(req.content, 'html.parser')
items = {}
liste = [
'status', 'age', 'birthday', 'height', 'bounty', 'occupations',
'epithet', 'japanese_name', 'devil_fruit', 'fruit'
]
#print(soup('table',class_='cs')[0]('th')[0].text)
for j in soup('div', class_='Gallery-pic')[:-2]:
name = j.a.get('title')
name_url = 'https://onepiece.fandom.com' + j.a.get('href')
rq = requests.get(name_url, headers=HEADERS)
view_soup = bs(rq.content, 'html.parser')
aside = view_soup('aside')[0]
# data verisinin içinde önce listedeki elemanları none olarak atıyoruz sebebi ise bazı karakterlerin bazı özellikleri yok
data = {}
for li in liste:
data[li] = None
try:
section0 = aside('section', class_='pi-item')[0]
section1 = aside('section', class_='pi-item')[1]
except:
section1 = ''
for i in section0('div')[:-2]:
if i('h3') != []:
h3 = i('h3')[0].text.lower().replace(':', '').replace(' ', '')
for h3_name in liste:
if h3 == h3_name:
print(h3_name, i('div')[0].text)
data[h3_name] = sup_cite_parser2(
i('div')[0].text.replace('\"', '')).strip()
if section1 == '':
data['devil_fruit'] = None
else:
data['devil_fruit'] = 'yes'
data['fruit'] = sup_cite_parser2(
section1('div')[0].find('div').text)
items[name] = data
json_save(items, 'whitebeard')
def straw_hat_prites_galery_scrapy(name, path):
session = HTMLSession()
req = session.get(f'https://onepiece.fandom.com/wiki/{name}/Gallery')
titles = bs(req.html.find('#toc')[0].html, 'html.parser')
items = {}
a = 0
for x in range(2, len(titles('li')) * 2 - 1, 2):
image_table = req.html.xpath(
f'//*[@id="mw-content-text"]/div/table[2]/tbody/tr/td/div[{x}]')
soup = bs(image_table[0].html, 'html.parser')
liste = []
for i in soup('span'):
data = {
'explanation':
i('div')[0].text.lower().replace('\"', '').replace('\n', ''),
'img_url':
i('td')[0].a.get('href')
}
liste.append(data)
items[req.html.find('.mw-headline')[a].text.lower().replace(
'&', '-').replace(' (', '_').replace(')', '')] = liste
a += 1
json_save(items, path)
def straw_hat_prites_scrapy(
path): #https://onepiece.fandom.com/wiki/Straw_Hat_Pirates --> table
url = URL['strawhatpirates']
req = requests.get(url, headers=HEADERS)
soup = bs(req.content, 'html.parser')
table = soup('table', class_='sortable')
items = {}
for i in range(1, len(table[0]('tr')), 2):
name = table[0]('tr')[i].td.text.strip()
unvan = table[0]('tr')[i]('td')[5]
item = {
'wiki_url':
'https://onepiece.fandom.com' + table[0]('tr')[i].td.a.get('href'),
'profession':
table[0]('tr')[i]('td')[1].text.lower().strip(),
'capabilities':
table[0]('tr')[i]('td')[2].text.strip().lower().replace(
"", '').split('\n'),
'epithet':
[unvan.b.text.replace('"', '') if unvan.b else unvan.text
][0].lower()
}
items[name] = item
json_save(items, path)
def non_canon_characters_scrapy():
req = requests.get(URL['characters']['noncanon'], headers=HEADERS)
soup = bs(req.content, 'html.parser')
items = {}
for tr in soup('table', class_='wikitable')[0]('tr')[1:]:
name = tr('td')[1].text
data = {
'name_url':
'https://onepiece.fandom.com' + tr('td')[1].a.get('href'),
'type':
tr('td')[2].text,
'number':
tr('td')[3].text,
'year':
tr('td')[4].text,
'appears_in':
tr('td')[5].text.replace('\n', ''),
'appears_in_url': [
'https://onepiece.fandom.com' + tr('td')[5].a.get('href')
if tr('td')[5].a is not None else None
][0]
}
items[name.lower()] = data
json_save(items, 'noncanon1')
def rewrite_site_projects(files_and_data):
parents_seen = set()
files_seen = set()
for file in files_and_data.keys():
parent_dir = pathlib.Path(*file.parts[:3])
if parent_dir not in parents_seen:
for file in (WORKING_DIR / parent_dir).glob('*.json'):
files_seen.add(file.relative_to(WORKING_DIR))
parents_seen.add(parent_dir)
for file, data in files_and_data.items():
json_save(data, WORKING_DIR / file)
files_to_delete = files_seen.difference(files_and_data)
print(f'{len(files_to_delete)} files to be deleted')
print('Committing updated projects')
try:
git_fs.update_localization(list(files_and_data), list(files_to_delete))
except Exception as e:
logging.exception(e)
logging.error('An error occured but this may not be a problem')
print('Creating PR')
git_fs.publish_localization(list(files_and_data), list(files_to_delete))
def update_segment(segment, user):
"""
segment looks like:
{
"segmentId": "dn1:1.1",
"field": "translation-en-sujato",
"value": "..", "oldValue": "..."
}
"""
segment_id = segment["segmentId"]
if not is_id_legal(segment_id):
logging.error(f"Invalid Segment ID: {segment_id}")
return {"error": "Invalid Segment ID"}
uid, _ = segment_id.split(":")
parent_uid = get_parent_uid(uid)
long_id = f'{parent_uid}_{segment["field"]}'
try:
filepath = get_file_path(long_id)
except KeyError as e:
logging.exception(e)
logging.error('f"{long_id}" not found, {segment}')
return {"error": "file not found"}
file = get_file(filepath)
permission = get_permissions(filepath, user['login'])
if permission != Permission.EDIT:
logging.error("User not allowed to edit")
return {"error": "Inadequate Permission"}
with git_fs._lock:
try:
file_data = json_load(file)
except FileNotFoundError:
file.parent.mkdir(parents=True, exist_ok=True)
file_data = {}
current_value = file_data.get(segment_id)
result = {}
if current_value and current_value != segment.get("oldValue"):
result["clobbered"] = current_value
if current_value != segment["value"]:
result["changed"] = True
file_data[segment_id] = segment["value"]
sorted_data = dict(sorted(file_data.items(), key=bilarasortkey))
try:
json_save(sorted_data, file)
result["success"] = True
except Exception:
logging.exception(f"could not write segment: {segment}")
return {"error": "could not write file"}
executor.submit(background_update, filepath, user, segment)
return result