def fetch_component(self, component: ComponentMeta) -> None:
download_url = f"{self.HOST}/download/crosscode/{component.id}/{PROJECT_LOCALE_WEBLATE}"
with internal_utils.http_request(download_url,
timeout=NETWORK_TIMEOUT) as (_,
reader):
with open(DOWNLOADS_DIR / f"{component.id}.po",
"wb") as output_file:
shutil.copyfileobj(reader, output_file)
def fetch_list(self) -> List[ComponentMeta]:
components: List[ComponentMeta] = []
api_url = f"{self.HOST}/__json__/~weblate/download/crosscode/{PROJECT_LOCALE}/components/"
with internal_utils.http_request(api_url,
timeout=NETWORK_TIMEOUT) as (_,
reader):
for file_meta in json.load(reader):
if file_meta["type"] == "file" and file_meta["name"].endswith(
".po"):
mtime = parse_rfc2822_date(file_meta["mtime"])
components.append(
ComponentMeta(id=file_meta["name"][:-3],
modification_timestamp=mtime))
return components
def download_dependency(url: str, filename: Optional[str] = None) -> Path:
if filename is None:
filename = urllib.parse.urlparse(url).path
filename = filename[filename.rfind("/") + 1:]
download_path = DOWNLOADS_DIR / filename
try:
if download_path.exists():
return download_path
with internal_utils.http_request(url, timeout=NETWORK_TIMEOUT) as (_, reader):
with open(download_path, "wb") as file:
shutil.copyfileobj(reader, file)
except BaseException:
download_path.unlink()
raise
return download_path
def fetch_list(self) -> List[ComponentMeta]:
components: List[ComponentMeta] = []
next_api_url = f"{self.HOST}/api/projects/crosscode/statistics/{PROJECT_LOCALE_WEBLATE}/"
while next_api_url is not None:
with internal_utils.http_request(
next_api_url, timeout=NETWORK_TIMEOUT) as (_, reader):
api_response = json.load(reader)
next_api_url = api_response["next"]
for stats_obj in api_response["results"]:
c_id = stats_obj["component"]
if c_id == "glossary":
continue
mtime: Optional[datetime] = None
mtime_str: Optional[str] = stats_obj["last_change"]
if mtime_str is not None:
mtime = datetime.fromisoformat(
mtime_str[:-1] +
"+00:00" if mtime_str.endswith("Z") else mtime_str)
components.append(
ComponentMeta(id=c_id, modification_timestamp=mtime))
return components
def main() -> None:
WORK_DIR.mkdir(exist_ok=True)
DOWNLOADS_DIR.mkdir(exist_ok=True)
print("==> reading the downloads state")
downloads_state: Dict[str, ComponentMeta] = {}
try:
with open(DOWNLOADS_STATE_FILE, "r") as file:
downloads_state_json = json.load(file)
if downloads_state_json["version"] == 1:
for c_id, c_data in downloads_state_json["data"].items():
c_mtime: Optional[float] = c_data["mtime"]
downloads_state[c_id] = ComponentMeta(
id=c_id,
modification_timestamp=datetime.fromtimestamp(
c_mtime, timezone.utc)
if c_mtime is not None else None,
)
except FileNotFoundError:
pass
downloader: ComponentDownloader = WeblateApiComponentDownloader()
print("==> downloading the list of components")
remote_components_list = downloader.fetch_list()
components_to_fetch_list: List[ComponentMeta] = []
for remote_meta in remote_components_list:
local_meta = downloads_state.get(remote_meta.id)
should_fetch: bool
if local_meta is None:
# We don't have this component download, it has probably been created
should_fetch = True
else:
# Notice that missing timestamps mean that the component has never been
# modified so far
remote_mtime = remote_meta.modification_timestamp
local_mtime = local_meta.modification_timestamp
if remote_mtime is None and local_mtime is None:
# We have downloaded an empty component and there are still no changes
# affecting it
should_fetch = False
elif remote_mtime is not None and local_mtime is None:
# Got the first ever changes!
should_fetch = True
elif remote_mtime is None and local_mtime is not None:
# We have already downloaded some changed version of the component, but
# it has probably been reset to an empty state since.
should_fetch = True
elif remote_mtime is not None and local_mtime is not None:
# The sane and normal code path.
should_fetch = remote_mtime > local_mtime
else:
raise Exception("unreachable")
if should_fetch:
components_to_fetch_list.append(remote_meta)
with ThreadPool(NETWORK_THREADS) as pool:
print(
f"==> downloading {len(components_to_fetch_list)} component(s) from Weblate"
)
def callback(component: ComponentMeta) -> ComponentMeta:
downloader.fetch_component(component)
return component
for component in pool.imap_unordered(callback,
components_to_fetch_list):
print(f"downloaded {component.id}")
downloads_state[component.id] = component
if not CROSSLOCALE_SCAN_FILE.exists():
print(
f"==> downloading the scan database for v{PROJECT_TARGET_GAME_VERSION}"
)
download_urls = [
"https://raw.githubusercontent.com/dmitmel/crosscode-localization-data/main/scan.json",
f"https://raw.githubusercontent.com/dmitmel/crosslocale-scans/main/scan-{PROJECT_TARGET_GAME_VERSION}.json"
]
for i, download_url in enumerate(download_urls):
try:
with internal_utils.http_request(
download_url, timeout=NETWORK_TIMEOUT) as (_, reader):
with open(CROSSLOCALE_SCAN_FILE, "wb") as output_file:
shutil.copyfileobj(reader, output_file)
break
except urllib.error.HTTPError:
if i < len(download_url) - 1:
traceback.print_exc()
else:
raise
print("==> starting the translation pack compiler")
crosslocale_bin = PROJECT_DIR / CROSSLOCALE_BIN_NAME
if os.name == "nt":
crosslocale_bin = crosslocale_bin.with_suffix(".exe")
if not crosslocale_bin.exists():
# fall back to finding the binary in PATH
crosslocale_bin = CROSSLOCALE_BIN_NAME
subprocess.run(
[
crosslocale_bin,
"convert",
f"--scan={CROSSLOCALE_SCAN_FILE}",
"--format=po",
f"--original-locale={PROJECT_ORIGINAL_LOCALE}",
"--remove-untranslated",
"--compact",
"--output-format=lm-tr-pack",
f"--output={LOCALIZE_ME_PACKS_DIR}",
"--splitter=lm-file-tree",
"--mapping-lm-paths",
f"--mapping-output={LOCALIZE_ME_MAPPING_FILE}",
"--",
DOWNLOADS_DIR,
],
check=True,
)
print("==> writing the downloads state")
with open(DOWNLOADS_STATE_FILE, "w") as file:
downloads_state_json: Any = {
"version": 1,
"data": {
component_meta.id: {
"mtime":
component_meta.modification_timestamp.timestamp()
if component_meta.modification_timestamp is not None else
None,
}
for component_meta in downloads_state.values()
},
}
json.dump(downloads_state_json, file, ensure_ascii=False, indent=2)
file.write("\n")
print("==> done!")