def getNsuIds(titleIds, type='title', region = 'US', language = 'en', shop_id = 4):
j = ids(titleIds, type, region, language, shop_id)
lst = {}
try:
for i in j['id_pairs']:
titleId = i['title_id'].upper()
nsuId = int(i['id'])
lst[titleId] = nsuId
title = Titles.getNsuid(nsuId, region, language)
title.setId(titleId)
try:
pass
if title.isDLC:
title.parseShogunJson(getDlcByNsuid(nsuId, region, language), region, language, True)
elif not title.isUpdate:
title.parseShogunJson(getTitleByNsuid(nsuId, region, language), region, language, True)
except:
Print.error(str(e))
pass
except BaseException as e:
Print.error(str(e))
return lst
def scrapeLangTitles(region='US', language='en', shop_id=4):
Print.info('Scraping %s %s' % (region, language))
pageSize = 50
offset = 0
total = 1
c = 0
while offset < total:
url = 'https://bugyo.hac.%s.eshop.nintendo.net/shogun/v1/titles?shop_id=%d&lang=%s&country=%s&sort=new&limit=%d&offset=%d' % (
Config.cdn.environment, shop_id, language, region, pageSize,
offset)
#print(url)
j = makeJsonRequest(
'GET', url, {}, '%d/%s/%s/titles/index/%d-%d.json' %
(shop_id, language, region, pageSize, offset))
if not j:
break
total = int(j['total'])
try:
for i in j['contents']:
title = Titles.getNsuid(i['id'], region, language)
n = getTitleByNsuid(i['id'], region, language)
title.parseShogunJson(n)
try:
if n and "applications" in n and len(
n["applications"]) > 0:
titleId = n["applications"][0]["id"].upper()
if titleId:
title.setId(titleId)
for x in cdn.Superfly.getAddOns(titleId):
getNsuIds(x, 'aoc', region, language)
scrapeDlc(i['id'], region, language)
else:
print('Could not get title json!')
else:
#print('no title id found in json!')
pass
except Exception as e:
print(str(e))
raise
pass
except Exception as e:
print(str(e))
raise
break
offset = offset + len(j['contents'])
Titles.saveRegion(region, language)
def scrapeDlc(baseNsuid, region='US', language='en', shop_id=3):
pageSize = 50
offset = 0
total = 1
while offset < total:
url = 'https://bugyo.hac.%s.eshop.nintendo.net/shogun/v1/titles/%d/aocs?shop_id=%d&lang=%s&country=%s' % (
Config.cdn.environment, baseNsuid, shop_id, language, region)
#print(url)
#exit(0)
j = makeJsonRequest(
'GET', url, {}, '%d/%s/%s/titles/aocs/%d.json' %
(shop_id, language, region, baseNsuid))
if not j:
break
total = int(j['total'])
if total == 0:
break
try:
for i in j['contents']:
title = Titles.getNsuid(i['id'], region, language)
n = getDlcByNsuid(i['id'], region, language)
if n and "applications" in n and len(n["applications"]) > 0:
title.setId(n["applications"][0]["id"].upper())
title.parseShogunJson(n, region)
except Exception as e:
print(str(e))
raise
break
offset = offset + len(j['contents'])
def parseShogunJson(self,
_json,
region=None,
language=None,
canGrabFromShogun=False):
if 'hero_banner_url' in _json:
self.bannerUrl = _json['hero_banner_url']
if "release_date_on_eshop" in _json:
try:
self.releaseDate = int(_json["release_date_on_eshop"].replace(
'-', ''))
except:
pass
if "id" in _json:
try:
self.setNsuId("%s" % _json["id"])
except:
pass
if "formal_name" in _json:
self.name = _json["formal_name"].strip()
if 'screenshots' in _json:
self.screenshots = []
for i, k in enumerate(_json["screenshots"]):
self.screenshots.append(k["images"][0]["url"])
if "languages" in _json:
self.languages = []
for language in _json["languages"]:
self.languages.append(language['iso_code'])
if "genre" in _json:
self.category = _json["genre"].split(' / ')
if "total_rom_size" in _json:
self.size = _json["total_rom_size"]
if "rating_info" in _json:
if "rating" in _json["rating_info"]:
if "age" in _json["rating_info"]['rating']:
self.rating = _json["rating_info"]['rating']['age']
if "content_descriptors" in _json["rating_info"]:
content = []
for descriptor in _json["rating_info"]["content_descriptors"]:
content.append(descriptor['name'])
self.ratingContent = content
if "player_number" in _json:
if 'local_max' in _json["player_number"]:
self.numberOfPlayers = _json["player_number"]["local_max"]
if 'offline_max' in _json["player_number"]:
self.numberOfPlayers = _json["player_number"]["offline_max"]
if "publisher" in _json:
if 'name' in _json["publisher"]:
self.publisher = _json["publisher"]["name"]
if 'title' in _json["publisher"]:
self.publisher = _json["publisher"]["title"]
if "applications" in _json and isinstance(_json["applications"], list):
for a in _json["applications"]:
if "id" in a:
self.setId(a['id'])
if "image_url" in a:
self.iconUrl = a['image_url']
break
if "applications" in _json and "image_url" in _json["applications"]:
self.iconUrl = _json["applications"]['image_url']
if "catch_copy" in _json:
intro = re.sub('(?<!\n)\n(?!\n)', ' ', _json["catch_copy"])
intro = re.sub(' ', ' ', intro)
self.intro = intro
if "description" in _json:
desc = re.sub('(?<!\n)\n(?!\n)', ' ', _json["description"])
desc = re.sub(' ', ' ', desc)
self.description = desc
try:
if "target_titles" in _json:
for nsu in _json["target_titles"]:
if not Titles.hasNsuid(nsu["id"], region, language):
if canGrabFromShogun:
baseTitle = Titles.getNsuid(
nsu["id"], region, language)
baseTitle.parseShogunJson(
cdn.Shogun.getTitleByNsuid(
nsu["id"], region, language), region,
language, True)
#Titles.saveRegion(region, language)
else:
baseTitle = Titles.getNsuid(nsu["id"], region,
language)
if baseTitle.id:
pass
#self.setId(baseTitle.id)
#Print.info("setting appid " + str(baseTitle.id))
#else:
# Print.error("Could not find title for " + str(nsu["id"]))
except:
Print.error('target titles error')
raise
def scrapeDlc(baseNsuid, region='US', shop_id=3):
pageSize = 50
offset = 0
total = 1
while offset < total:
url = 'https://bugyo.hac.%s.eshop.nintendo.net/shogun/v1/titles/%d/aocs?shop_id=%d&lang=%s&country=%s' % (
Config.cdn.environment, baseNsuid, shop_id,
countryLanguage(region), region)
#print(url)
#exit(0)
j = makeJsonRequest(
'GET', url, {}, '%d/%s/%s/titles/aocs/%d.json' %
(shop_id, countryLanguage(region), region, baseNsuid))
if not j:
break
total = int(j['total'])
if total == 0:
break
try:
for i in j['contents']:
title = Titles.getNsuid(i['id'])
n = getDlcByNsuid(i['id'])
if title:
title.parseShogunJson(n, region)
rt = Title.Title()
rt.setId(title.id)
rt.setRegion(region)
rt.parseShogunJson(n)
Titles.set(title.id, rt, region)
else:
try:
if n and len(n["applications"]) > 0:
titleId = n["applications"][0]["id"].upper()
if titleId:
if Titles.contains(titleId):
title = Titles.get(titleId)
title.setId(titleId)
title.parseShogunJson(n, region)
#print('existing title found!')
else:
title = Title.Title()
title.setId(titleId)
title.parseShogunJson(n, region)
Titles.set(titleId, title)
print('added new DLC %s %s' %
(title.id, title.name))
rt = Title.Title()
rt.setId(titleId)
rt.setRegion(region)
rt.parseShogunJson(n, region)
Titles.set(titleId, rt, region)
else:
print('Could not get title json!')
else:
#print('no title id found in json!')
pass
except Exception as e:
#print(str(e))
pass
except Exception as e:
print(str(e))
raise
break
offset = offset + len(j['contents'])
def scrapeTitles(region='US', shop_id=4):
Print.info('Scraping %s' % region)
pageSize = 50
offset = 0
total = 1
c = 0
while offset < total:
url = 'https://bugyo.hac.%s.eshop.nintendo.net/shogun/v1/titles?shop_id=%d&lang=%s&country=%s&sort=new&limit=%d&offset=%d' % (
Config.cdn.environment, shop_id, countryLanguage(region), region,
pageSize, offset)
#print(url)
j = makeJsonRequest(
'GET', url, {}, '%d/%s/%s/titles/index/%d-%d.json' %
(shop_id, countryLanguage(region), region, pageSize, offset))
if not j:
break
total = int(j['total'])
try:
for i in j['contents']:
title = Titles.getNsuid(i['id'])
n = getTitleByNsuid(i['id'], region)
if title:
for x in cdn.Superfly.getAddOns(title.id):
getNsuIds(x, 'aoc', region)
title.parseShogunJson(n)
rt = Title.Title()
rt.setId(title.id)
rt.setRegion(region)
rt.parseShogunJson(n)
Titles.set(title.id, rt, region)
scrapeDlc(i['id'], region)
else:
try:
if n and len(n["applications"]) > 0:
titleId = n["applications"][0]["id"].upper()
for x in cdn.Superfly.getAddOns(titleId):
getNsuIds(x, 'aoc', region)
if titleId:
if Titles.contains(titleId):
title = Titles.get(titleId)
title.setId(titleId)
title.parseShogunJson(n)
#print('existing title found!')
else:
title = Title.Title()
title.setId(titleId)
title.parseShogunJson(n)
Titles.set(titleId, title)
print('added new title %s %s' %
(title.id, title.name))
rt = Title.Title()
rt.setId(titleId)
rt.setRegion(region)
rt.parseShogunJson(n)
Titles.set(titleId, rt, region)
scrapeDlc(i['id'], region)
else:
print('Could not get title json!')
else:
#print('no title id found in json!')
pass
except Exception as e:
#print(str(e))
pass
except Exception as e:
print(str(e))
raise
break
offset = offset + len(j['contents'])
#c = c + 1
#if c % 100 == 0:
# Print.info('.')
# Titles.save()
Titles.save()