def collect_background_data(self):
collected = {}
background_pool = None
background_urls = []
background_data_expected = {}
for background_id in tqdm(range(0, 99)):
if self.stop_requested is True:
break
background_url = self._create_background_url(background_id)
if self.event_exists(background_id):
continue
data = {
'event_id': background_id,
'url': background_url,
'type': 'backgrounds'
}
background_urls.append(background_url)
background_data_expected[background_url] = data
background_pool = pool.Pool.from_urls(background_urls, num_processes=20)
print("Awaiting background Requests to Finish")
background_pool.join_all()
for response in background_pool.responses():
if response.status_code != requests.codes.ok:
continue
url = response.request_kwargs['url']
data = background_data_expected[url]
if self.event_exists(data['event_id']):
continue
image_file_location = os.path.join(self.files_folder,
extract_file_name_from_url(response.request_kwargs['url']))
with open(image_file_location, 'wb') as f:
f.write(response.content)
# image = Image.open(BytesIO(response.content))
# image.save(image_file_location)
self.results['backgrounds'][data['event_id']] = {
'url': url,
'time': "{:%B %d, %Y}".format(datetime.datetime.now()),
'image': image_file_location
}
collected['{0}'.format(data['event_id'])] = image_file_location
return collected
def collect_legends_data(self):
collected = {}
legends_urls = []
legends_data_expected = {}
print("Generating Legends Requests")
for lid in tqdm(range(0, 999)):
if self.stop_requested is True:
break
if lid < 10:
legend_id = "00{0}".format(lid)
elif 10 <= lid < 100:
legend_id = "0{0}".format(lid)
elif lid >= 100:
legend_id = "{0}".format(lid)
legends_url = self._create_legends_url(legend_id)
if self.event_exists(legend_id):
continue
data = {
'event_id': legend_id,
'url': legends_url,
'type': 'legends'
}
legends_urls.append(legends_url)
legends_data_expected[legends_url] = data
legends_pool = pool.Pool.from_urls(legends_urls, num_processes=20)
print("Awaiting Legends Requests to Finish")
legends_pool.join_all()
for response in legends_pool.responses():
if response.status_code != requests.codes.ok:
continue
url = response.request_kwargs['url']
data = legends_data_expected[url]
if self.event_exists(data['event_id']):
continue
image_file_location = os.path.join(self.files_folder,
extract_file_name_from_url(response.request_kwargs['url']))
with open(image_file_location, 'wb') as f:
f.write(response.content)
# image = Image.open(BytesIO(response.content))
# image.save(image_file_location)
self.results['legends'][data['event_id']] = {
'url': url,
'time': "{:%B %d, %Y}".format(datetime.datetime.now()),
'image': image_file_location
}
collected['1021{0}'.format(data['event_id'])] = image_file_location
return collected
def collect_challenge_data(self, year=2018):
collected = {}
challenge_pool = None
challenge_urls = []
challenge_data_expected = {}
config = _FIFA_CONFIG[str(year)]
zfill = config['zfill']
_max = config['max']
print("Generating Challenge Reqeusts")
for cid in tqdm(range(0, _max)):
if self.stop_requested is True:
break
challenge_id = str(cid).zfill(zfill)
challenge_url = build_url('challenge', year=2018, item_id=challenge_id)
if self.event_exists(challenge_id):
continue
data = {
'event_id': challenge_id,
'url': challenge_url,
'type': 'challenges'
}
challenge_urls.append(challenge_url)
challenge_data_expected[challenge_url] = data
challenge_pool = pool.Pool.from_urls(challenge_urls, num_processes=20)
print("Awaiting Challenge Requests to Finish")
challenge_pool.join_all()
for response in challenge_pool.responses():
if response.status_code != requests.codes.ok:
continue
url = response.request_kwargs['url']
data = challenge_data_expected[url]
if self.event_exists(data['event_id']):
continue
image_file_location = os.path.join(self.files_folder,
extract_file_name_from_url(response.request_kwargs['url']))
with open(image_file_location, 'wb') as f:
f.write(response.content)
# image = Image.open(BytesIO(response.content))
# image.save(image_file_location)
self.results['challenges'][data['event_id']] = {
'url': url,
'time': "{:%B %d, %Y}".format(datetime.datetime.now()),
'image': image_file_location
}
collected[data['event_id']] = image_file_location
return collected
def collect_player_of_the_month_data(self):
collected = {}
potm_urls = []
potm_data_expected = {}
print("Generating Player of the Month Requests")
for pid in tqdm(range(0, 999)):
if self.stop_requested is True:
break
if pid < 10:
potm_id = "00{0}".format(pid)
elif 10 <= pid < 100:
potm_id = "0{0}".format(pid)
elif pid >= 100:
potm_id = "{0}".format(pid)
potm_url = self._create_player_of_the_month_url(potm_id)
if self.event_exists(potm_id):
continue
data = {
'event_id': potm_id,
'url': potm_url,
'type': 'potm'
}
potm_urls.append(potm_url)
potm_data_expected[potm_url] = data
potm_pool = pool.Pool.from_urls(potm_urls, num_processes=20)
print("Awaiting Player of the Month Requests to Finish")
potm_pool.join_all()
for response in potm_pool.responses():
if response.status_code != requests.codes.ok:
continue
url = response.request_kwargs['url']
data = potm_data_expected[url]
if self.event_exists(data['event_id']):
continue
image_file_location = os.path.join(self.files_folder,
extract_file_name_from_url(response.request_kwargs['url']))
with open(image_file_location, 'wb') as f:
f.write(response.content)
# image = Image.open(BytesIO(response.content))
# image.save(image_file_location)
self.results['potm'][data['event_id']] = {
'url': url,
'time': "{:%B %d, %Y}".format(datetime.datetime.now()),
'image': image_file_location
}
collected['1027{0}'.format(data['event_id'])] = image_file_location
return collected
def collect_set_data(self, year=2018):
# Iterate Sets
print("Generating Set Requests")
set_urls = []
set_data_expected = {}
collected = {
}
_max = _FIFA_CONFIG[str(year)]['max']
_zfill = _FIFA_CONFIG[str(year)]['zfill']
for set_id in tqdm(range(0, _max)):
if self.stop_requested is True:
break
item_id = str(set_id).zfill(_FIFA_18_ZFILL)
set_url = build_url(type='set', year=2018, item_id=item_id)
if self.event_exists(item_id):
continue
data = {
'event_id': item_id,
'url': set_url,
'type': 'sets'
}
set_urls.append(set_url)
set_data_expected[set_url] = data
set_pool = pool.Pool.from_urls(set_urls, num_processes=20)
print("Awaiting Set Requests to Finish")
set_pool.join_all()
for response in set_pool.responses():
if response.status_code != requests.codes.ok:
continue
url = response.request_kwargs['url']
data = set_data_expected[url]
if self.event_exists(data['event_id']):
continue
image_file_location = os.path.join(self.files_folder,
extract_file_name_from_url(response.request_kwargs['url']))
with open(image_file_location, 'wb') as f:
f.write(response.content)
# image = Image.open(BytesIO(response.content))
# image.save(image_file_location)
self.results['sets'][data['event_id']] = {
'url': url,
'time': "{:%B %d, %Y}".format(datetime.datetime.now()),
'image': image_file_location
}
collected[data['event_id']] = image_file_location
return collected