def process(
path: Path,
locale: str,
re_download_link: re.Pattern,
re_old_versions: re.Pattern,
re_change_log: re.Pattern,
change_log: str,
):
print(f"Processing {path}")
with open(path, "r") as fi:
text = fi.read()
mt = re_download_link.search(text)
if mt is None:
print(f"Download link not found in: {path}")
return
plugin_name = mt.groups()[0]
major_version = mt.groups()[1]
minor_version = mt.groups()[2]
patch_version = mt.groups()[3]
download_url = mt.groups()[4]
# Add old download link to Old Versions section.
old_version = f"{major_version}.{minor_version}.{patch_version}"
old_version_link = f"- [{plugin_name} {old_version} - VST 3 (github.com)]({download_url})"
text = re_old_versions.sub(
lambda exp: f"{exp.group()}\n{old_version_link}", text, count=1)
# Update download link.
new_version = f"{major_version}.{minor_version}.{int(patch_version) + 1}"
new_downlaod_url = f"https://github.com/ryukau/VSTPlugins/releases/download/{release_name}/{plugin_name}{new_version}.zip"
new_link = compose_download_link(locale, plugin_name, new_version,
new_downlaod_url)
if new_link is None:
return
text = re_download_link.sub(new_link, text, count=1)
# Add change log.
text = re_change_log.sub(
lambda exp: f"{exp.group()}\n- {new_version}{change_log}",
text,
count=1)
out_dir = Path("out") / Path(path.parts[-2])
out_dir.mkdir(parents=True, exist_ok=True)
with open(out_dir / Path(path.name), "w") as fi:
fi.write(text)
def delete_length_to_flags(plugin: up.utauplugin.UtauPlugin,
pattern: re.Pattern):
"""フラグ内のLengthを消す
'【480】g-2H40' -> 'g-2H40'
'g-2【480】H40' -> 'g-2H40'
"""
for note in plugin.notes:
note.flags = pattern.sub('', note.flags)
def update_version(pattern: re.Pattern, v: str, file_path: str):
print(f"Replacing {pattern} to {version} in {file_path}")
with open(file_path, "r+") as f:
file_content = f.read()
if not pattern.search(file_content):
raise Exception(
f"Pattern {pattern!r} doesn't found in {file_path!r} file")
new_content = pattern.sub(fr'\g<1>{v}\g<2>', file_content)
if file_content == new_content:
return
f.seek(0)
f.truncate()
f.write(new_content)
def get_drug_names_by_suffix(drug_name: str, suffixes: List[str],
split_chars: re.Pattern,
remove_chars: re.Pattern):
drug_name_token_list = []
drug_name = drug_name.lower()
drug_token = split_chars.split(drug_name)
drug_token = [remove_chars.sub("", token) for token in drug_token]
for token in drug_token:
for suffix in suffixes:
if token.endswith(suffix):
drug_name_token_list.append(token)
break
return drug_name_token_list
def presentable_filename(filename: str, root_filter: re.Pattern) -> str:
"""mangle a filename so that it is suitable for a report"""
normalized = root_filter.sub("", filename)
if filename.endswith(normalized):
# remove any slashes between the removed prefix and the normalized name
if filename != normalized:
while normalized.startswith(os.path.sep):
normalized = normalized[len(os.path.sep) :]
else:
# Do no truncation if the filter does not start matching
# at the beginning of the string
normalized = filename
return normalized.replace("\\", "/")
def search_and_replace(
site: mwclient.Site,
search: str,
regex: re.Pattern,
replace: str,
namespace: int = 0,
) -> None:
for result in site.search(search, namespace=namespace, what='text'):
page = site.pages[result['title']]
text = page.text()
newtext = regex.sub(replace, text)
if newtext != text:
page.edit(newtext,
summary=f'replace {regex.pattern} with {replace} (auto)')
logger.info('page %s updated.', page.page_title)
else:
logger.warning('page %s not updated.', page.page_title)
def replace(self, search_value: re.Pattern, replace_value: str, replace_replaced_words: bool = False):
if not replace_replaced_words and self.search_value_contains_replaced_words(search_value, replace_value):
return self
replacing_word = self.word
if search_value.search(self.word) is not None:
replacing_word = search_value.sub(replace_value, self.word)
collection = search_value.findall(self.word)
replaced_words: List[str]
if len(collection) > 1:
replaced_words = list(map(lambda s: s.replace(s, replace_value), collection))
else:
replaced_words = []
if replacing_word != self.word:
for word in replaced_words:
self.replaced_words.add(word)
self.word = replacing_word
return self
def add_symbols(
text: str,
symbol: str,
searcher: re.Pattern, # type: ignore
ignore_tags: Iterable = frozenset([
'html', '[document]', 'title', 'style', 'head', 'script', 'noscript',
'pre', 'code', 'meta', 'img'
]),
ignore_tag_types: Iterable = (element.Doctype, element.Comment,
element.Declaration),
) -> str:
soup = BeautifulSoup(text, "html.parser")
for tag in soup.findAll(text=True):
if tag.parent.name in ignore_tags or type(tag) in ignore_tag_types:
continue
edited_text = searcher.sub(lambda matched: matched.group(0) + symbol,
tag.string)
tag.replace_with(edited_text)
return soup.prettify()
def build_referral_map_helper(
mtgjson_object: Union[MtgjsonCardObject, MtgjsonSealedProductObject],
string_regex: re.Pattern,
) -> List[Tuple[str, str]]:
"""
Helps construct the referral map contents
:param mtgjson_object: MTGJSON Set or Card object
:param string_regex: compiled scryfall regex data
:return: tuple to append
"""
return_list = []
for service, url in mtgjson_object.purchase_urls.to_json().items():
if service not in mtgjson_object.raw_purchase_urls:
LOGGER.info(f"Service {service} not found for {mtgjson_object.name}")
continue
return_list.append(
(
url.split("/")[-1],
string_regex.sub("mtgjson", mtgjson_object.raw_purchase_urls[service]),
)
)
return return_list
def replace_from_dict_and_pattern(string: str, replacements: Dict,
pattern: re.Pattern) -> str:
""" Given a pattern which matches keys in replacements, replace keys found in string with their values"""
return pattern.sub(lambda match: replacements[match.group(0)], string)
def process_line(line: Line, re_pat: re.Pattern,
re_map: Dict[str, str]) -> Iterable[str]:
"""Replace links according to map and pattern."""
yield re_pat.sub(lambda m: re_map[re.escape(m.group(0))], line.text)
async def worker(repo_owner: str, submodule_regex: re.Pattern, q: asyncio.Queue):
"""
Clones repository locally, if necessary updates submodules urls to new
remote, creates new remote repository and pushes to it, when done deletes
local repository.
"""
(repo_name, repo_url) = await q.get()
print(f"Started processing {repo_name}")
try:
# Check if repository with same name already exists on GitHub
proc: asyncio.subprocess.Process = await asyncio.create_subprocess_shell(
f"hub api repos/{repo_owner}/{repo_name}",
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
await proc.wait()
if proc.returncode == 0:
raise Exception(
f"Stopped processing {repo_name}: repo with same name already exists for this owner on GitHub"
)
# If we don't need to update submodules we mirror the repository so it's faster
mirror: str = "--mirror" if not submodule_regex else ""
# Clones repo
proc: asyncio.subprocess.Process = await asyncio.create_subprocess_shell(
f"git clone {mirror} {repo_url}",
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
)
await proc.wait()
repo_folder: str = f"{repo_name}.git" if mirror else repo_name
# No need to update submodules so skip the whole step
if not mirror:
proc: asyncio.subprocess.Process = await asyncio.create_subprocess_shell(
"git branch -r",
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.DEVNULL,
cwd=repo_folder,
)
(stdout, _) = await proc.communicate()
# Get all remote branches of current repo
branches = set()
for line in stdout.decode().splitlines():
if "origin/HEAD" in line:
line: str = line.split("->")[1]
branch = line.replace("origin/", "").strip()
branches.add(branch)
for branch in branches:
# Checkout branch
proc: asyncio.subprocess.Process = await asyncio.create_subprocess_shell(
f"git checkout {branch}",
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
cwd=repo_folder,
)
await proc.wait()
# No submodules in this branch
if not os.path.exists(f"{repo_folder}/.gitmodules"):
continue
# Replaces submodules URLs
with open(f"{repo_folder}/.gitmodules", "r+") as f:
text = ""
for line in f.readlines():
text += submodule_regex.sub(
fr"github.com:{repo_owner}/\2.git", line
)
f.seek(0)
f.write(text)
f.truncate()
# Commit change
proc: asyncio.subprocess.Process = await asyncio.create_subprocess_shell(
"git commit -a -m '[Migra] Updated submodules'",
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
cwd=repo_folder,
)
await proc.wait()
# Removes existing remote
proc: asyncio.subprocess.Process = await asyncio.create_subprocess_shell(
"git remote remove origin",
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
cwd=repo_folder,
)
await proc.wait()
# Creates repository on GitHub
proc: asyncio.subprocess.Process = await asyncio.create_subprocess_shell(
f"hub create -p {repo_owner}/{repo_name}",
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
cwd=repo_folder,
)
await proc.wait()
proc: asyncio.subprocess.Process = await asyncio.create_subprocess_shell(
"git push --mirror",
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
cwd=repo_folder,
)
await proc.wait()
# Removes local repository
proc: asyncio.subprocess.Process = await asyncio.create_subprocess_shell(
f"rm -rf {repo_folder}",
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
)
await proc.wait()
print(f"{repo_name} migrated")
except Exception as e:
print(e)
finally:
q.task_done()
def delete_tag_from_label(plugin: up.utauplugin.UtauPlugin, pattern: re.Pattern):
"""
ラベル内のノート番号を消す
"""
for note in plugin.notes:
note.label = pattern.sub('', note.label)
def _to_snake_case(camel_case_key: str, pattern: Pattern) -> str:
return pattern.sub('_', camel_case_key).lower()
def sub_or_none(pattern: Pattern,
repl: Union[AnyStr, Callable[[Match[AnyStr]], AnyStr]],
string: AnyStr,
count: int = 0) -> AnyStr:
return pattern.sub(repl, string, count) if string is not None else None
def process(
path: Path,
locale: str,
re_download_link: re.Pattern,
re_old_versions: re.Pattern,
re_no_old_versions: re.Pattern,
re_change_log: re.Pattern,
change_log: str,
):
# print(f"Processing {path}") # debug
with open(path, "r", encoding="utf-8") as fi:
text = fi.read()
text = re_no_old_versions.sub("", text)
matches = list(re_download_link.finditer(text))
if len(matches) == 0:
print(f"Download link not found in: {path}")
return
hasSinglePlugin = len(matches) == 1
for mt in matches:
plugin_name = mt.groups()[0]
major_version = mt.groups()[1]
minor_version = mt.groups()[2]
patch_version = mt.groups()[3]
download_url = mt.groups()[4]
source_version = get_source_version(plugin_name)
if (major_version != source_version[0]
or minor_version != source_version[1]
or int(patch_version) + 1 != int(source_version[2])):
src_ver = ".".join(source_version)
man_ver = ".".join([major_version, minor_version, patch_version])
print(
f"Warning: {plugin_name} version mismatch. source {src_ver} manual {man_ver}"
)
# Update download link.
new_version = f"{major_version}.{minor_version}.{int(patch_version) + 1}"
new_downlaod_url = f"https://github.com/ryukau/VSTPlugins/releases/download/{release_name}/{plugin_name}_{new_version}.zip"
new_link = compose_download_link(locale, plugin_name, new_version,
new_downlaod_url)
if new_link is None:
continue
pos = mt.start()
text = text[:pos] + re_download_link.sub(new_link, text[pos:], count=1)
# Add change log.
if hasSinglePlugin:
text = re_change_log.sub(
lambda exp: f"{exp.group()}\n- {new_version}{change_log}",
text,
count=1)
else:
pos = re_change_log.search(text).end()
text = text[:pos] + re.sub(
f"### {plugin_name}",
f"### {plugin_name}\n- {new_version}{change_log}",
text[pos:],
count=1)
# Add old download link to Old Versions section.
old_version = f"{major_version}.{minor_version}.{patch_version}"
old_version_link = f"- [{plugin_name} {old_version} - VST 3 (github.com)]({download_url})"
if hasSinglePlugin:
text = re_old_versions.sub(
lambda exp: f"{exp.group()}\n{old_version_link}",
text,
count=1)
else:
pos = re_old_versions.search(text).end()
text = text[:pos] + re.sub(
f"### {plugin_name}",
f"### {plugin_name}\n{old_version_link}",
text[pos:],
count=1)
out_dir = Path("out") / Path(path.parts[-2])
out_dir.mkdir(parents=True, exist_ok=True)
with open(out_dir / Path(path.name), "w", encoding="utf-8") as fi:
fi.write(text)
