def _upsert_results(self, results: List[FetchedPlatform]) -> None:
errors = []
count = 0
for platform in results:
self.stdout.write("{}:".format(platform.source_platform_id),
ending="")
platform.name = clean_string_field(platform.name)
platform.shortname = clean_string_field(platform.shortname)
try:
existing_platform = FetchedPlatform.objects.get(
source_platform_id=platform.source_platform_id,
source_id=platform.source_id)
existing_platform.name = platform.name
existing_platform.source_platform_id = platform.source_platform_id
existing_platform.source_id = platform.source_id
existing_platform.source_url = platform.source_url
if platform.publish_date != self.default_publish_date:
existing_platform.publish_date = platform.publish_date
last_modified_date = existing_platform.last_modified_date
existing_platform.save()
if existing_platform.last_modified_date != last_modified_date:
self.stdout.write(self.style.SUCCESS("☑ "), ending="")
else:
self.stdout.write("☐ ", ending="")
except FetchedPlatform.DoesNotExist:
platform.save()
self.stdout.write(self.style.SUCCESS("✓ "), ending="")
except Exception as error:
errors.append(str(error))
self.stdout.write(self.style.ERROR("✗ "), ending="")
count += 1
if count % 10 == 0:
self.stdout.write("")
if errors:
self.stdout.write(self.style.ERROR("\nErrors:"))
for error_item in errors:
self.stdout.write(self.style.ERROR(error_item))
def import_fetched_platform(
name: str,
shortname: str,
publish_date_string: str,
fetched_platform_id: int,
platform_id: int = None,
update_fields_filter: Optional[List[str]] = None,
) -> None:
if platform_id:
platform = Platform.objects.filter(id=platform_id).get()
else:
platform = Platform()
include_all_fields = not platform_id or update_fields_filter is None
# cast is like a NOP outside type checking
if include_all_fields or "name" in cast(List[str],
update_fields_filter):
platform.name = clean_string_field(name)
if include_all_fields or "shortname" in cast(List[str],
update_fields_filter):
platform.shortname = clean_string_field(shortname)
if include_all_fields or "publish_date" in cast(
List[str], update_fields_filter):
platform.publish_date = publish_date_string
try:
platform.save()
except Exception as error:
raise PlatformImportSaveError(str(error))
# Update always linked platform
fetched_platform = FetchedPlatform.objects.filter(
id=fetched_platform_id).get()
fetched_platform.fg_platform_id = platform.id
fetched_platform.save(update_fields=["fg_platform_id"])
def sanitize(self, games: List[Union[FetchedGame, Game]], model_name: str,
block_size_for_feedback: int) -> None:
count = 0
self.stdout.write(
self.style.WARNING("> Going to sanitize {}".format(model_name)))
for game in games:
game.name = clean_string_field(game.name)
try:
game.save()
except IntegrityError as error:
self.stdout.write(
self.style.ERROR("{} - {}".format(game.name, error)))
count += 1
if count % block_size_for_feedback == 0:
self.stdout.write(str(count))
def _upsert_results(self, results: List[Tuple[FetchedGame, List[FetchedPlatform]]]) -> None:
errors = []
count = 0
for (game, platforms) in results:
self.stdout.write("{}:".format(game.source_game_id), ending="")
game.name = clean_string_field(game.name)
try:
existing_game = FetchedGame.objects.get(source_game_id=game.source_game_id, source_id=game.source_id)
existing_game.name = game.name
if not existing_game.cover and game.cover is not None:
existing_game.cover = game.cover
existing_game.source_game_id = game.source_game_id
existing_game.source_id = game.source_id
existing_game.source_url = game.source_url
# wasn't released and now it is
if game.publish_date > self.default_publish_date:
existing_game.publish_date = game.publish_date
existing_game.platforms.set(platforms)
last_modified_date = existing_game.last_modified_date
existing_game.save()
if existing_game.last_modified_date != last_modified_date:
self.stdout.write(self.style.SUCCESS("☑ "), ending="")
else:
self.stdout.write("☐ ", ending="")
except FetchedGame.DoesNotExist:
game.save()
# Need to have an id before can change a many-to-many field
game.platforms.set(platforms)
game.save()
self.stdout.write(self.style.SUCCESS("✓ "), ending="")
except Exception as error:
errors.append(str(error))
self.stdout.write(self.style.ERROR("✗ "), ending="")
count += 1
if count % 10 == 0:
self.stdout.write("")
if errors:
self.stdout.write(self.style.ERROR("\nErrors:"))
for error_item in errors:
self.stdout.write(self.style.ERROR(error_item))
def import_fetched_game(
platforms: List[int],
fetched_game_id: int,
name: Optional[str] = None,
publish_date_string: Optional[str] = None,
dlc_or_expansion: Optional[bool] = None,
cover: Optional[str] = None,
game_id: Optional[int] = None,
parent_game_id: Optional[int] = None,
source_display_name: Optional[str] = None,
source_url: Optional[str] = None,
update_fields_filter: Optional[List[str]] = None,
) -> None:
if game_id:
game = Game.objects.filter(id=game_id).get()
else:
game = Game()
include_all_fields = not game_id or update_fields_filter is None
# cast is like a NOP outside type checking
if include_all_fields or "name" in cast(List[str],
update_fields_filter):
if not name:
raise GameImportSaveError("Name field missing")
game.name = clean_string_field(name)
if include_all_fields or "publish_date" in cast(
List[str], update_fields_filter):
if not publish_date_string:
raise GameImportSaveError("Publish Date field missing")
game.publish_date = publish_date_string
if include_all_fields or "dlc_or_expansion" in cast(
List[str], update_fields_filter):
if dlc_or_expansion is None:
raise GameImportSaveError("DLC field missing")
game.dlc_or_expansion = dlc_or_expansion
if include_all_fields or "parent_game" in cast(List[str],
update_fields_filter):
if parent_game_id:
game.parent_game_id = parent_game_id
else:
game.parent_game = None
if include_all_fields or "cover" in cast(List[str],
update_fields_filter):
# do not error if 'cover' empty (source import might not have it)
# Remark: only sets cover if not had one, never updates it
if game.cover is None and cover:
try:
source_path = os.path.join(settings.COVERS_IMPORT_PATH,
cover + ".png")
destination_path = os.path.join(settings.COVERS_PATH,
cover + ".png")
copyfile(source_path, destination_path)
except Exception as error:
raise GameImportSaveError(str(error))
game.cover = cover
# update always the url for this source
if source_display_name and source_url:
game.upsert_url(display_name=source_display_name, url=source_url)
try:
game.save()
except Exception as error:
raise GameImportSaveError(str(error))
# many to many need an id to be set, so platforms added after initial save
if include_all_fields or "platforms" in cast(List[str],
update_fields_filter):
try:
platforms = Platform.objects.filter(id__in=platforms)
# Add new platforms if proceed, not removing existing ones (and if already added, nothing happens).
# Also, because one catalog source might only include a subset of the platforms for a game.
game.platforms.add(*platforms)
game.save()
except Exception as error:
raise GameImportSaveError(str(error))
# Update always linked game
fetched_game = FetchedGame.objects.filter(id=fetched_game_id).get()
fetched_game.fg_game_id = game.id
fetched_game.mark_as_synchronized()
fetched_game.save(update_fields=["fg_game_id", "last_sync_date"])
def test_field_cleaning_helper_with_typical_string_field(self):
unclean_name = " _A name needing cleaning: ñ-:!"
cleaned_name = "A name needing cleaning: ñ"
self.assertEqual(clean_string_field(unclean_name), cleaned_name)
def test_field_cleaning_helper_with_null_field(self):
# should not error
self.assertEqual(clean_string_field(None), None)