def env2dict(env_data: t.AnyStr) -> dict:
"""
This function transforms the data loaded from a file to this
generalized format:
{
"source_filename_A": {"KEY1": "value1", "KEY2": "value2"},
"...": {...}
}
While this is most certainly not necessary, it serves as a
safeguard against badly formatted input files.
>>> env2dict('KEY1=VAL1\\nKEY2=VAL2')
{'KEY1': 'VAL1', 'KEY2': 'VAL2'}
>>> env2dict('KEY1= #VAL1\\n#KEY2=VAL2')
{'KEY1': ''}
"""
logger.debug("transforming the env to dict-class")
dict_data = {}
line_data = env_data.split("\n")
for line in line_data:
line = re.sub("\s*#.*", "", line)
if line:
key, value = line.split("=")
dict_data[key] = value
return dict_data
def should_report(self, path: ty.AnyStr, *, is_dir: bool) -> bool:
# A final slash means “only match directories”
if self._dir_only and not is_dir:
return False
labels = path.split(self._sep) # type: ty.List[ty.AnyStr]
return self._match(labels, idx_pat=0, idx_path=0, is_dir=is_dir)
def download_exp_data_if_not_exist(
exp_data_url: typing.AnyStr,
exp_data_destination: typing.AnyStr,
) -> typing.AnyStr:
"""
Downloads & extract config/weights for a model if the provided destination does not exist.
The provided URL will be assumed to be a Google Drive download URL. The download will be
skipped entirely if the destination folder already exists. This function will return the
path to the existing folder, or to the newly created folder.
Args:
exp_data_url: the zip URL (under the `https://drive.google.com/file/d/ID` format).
exp_data_destination: where to extract the model data.
Returns:
The path to the model data.
"""
assert exp_data_url.startswith("https://drive.google.com/file/d/")
gdrive_file_id = exp_data_url.split("/")[-1]
output_data_path = os.path.join(exp_data_destination, gdrive_file_id)
downloaded_zip_path = os.path.join(exp_data_destination,
f"{gdrive_file_id}.zip")
if os.path.isfile(downloaded_zip_path) and os.path.isdir(output_data_path):
return output_data_path
os.makedirs(output_data_path, exist_ok=True)
zip_path = download_file_from_google_drive(gdrive_file_id,
downloaded_zip_path)
with zipfile.ZipFile(zip_path, "r") as zip_ref:
zip_ref.extractall(output_data_path)
return output_data_path
def append_text(o: t.IO[str], text: t.AnyStr, *, type_="stdout") -> int:
if text == "\n":
return 0
data = {"type": type_, "text": text.split("\n")}
n = o.write(json.dumps(data))
m = o.write("\n")
return n + m
def url_decode(
s: t.AnyStr,
charset: str = "utf-8",
decode_keys: None = None,
include_empty: bool = True,
errors: str = "replace",
separator: str = "&",
cls: t.Optional[t.Type["ds.MultiDict"]] = None,
) -> "ds.MultiDict[str, str]":
"""Parse a query string and return it as a :class:`MultiDict`.
:param s: The query string to parse.
:param charset: Decode bytes to string with this charset. If not
given, bytes are returned as-is.
:param include_empty: Include keys with empty values in the dict.
:param errors: Error handling behavior when decoding bytes.
:param separator: Separator character between pairs.
:param cls: Container to hold result instead of :class:`MultiDict`.
.. versionchanged:: 2.0
The ``decode_keys`` parameter is deprecated and will be removed
in Werkzeug 2.1.
.. versionchanged:: 0.5
In previous versions ";" and "&" could be used for url decoding.
Now only "&" is supported. If you want to use ";", a different
``separator`` can be provided.
.. versionchanged:: 0.5
The ``cls`` parameter was added.
"""
if decode_keys is not None:
warnings.warn(
"'decode_keys' is deprecated and will be removed in Werkzeug 2.1.",
DeprecationWarning,
stacklevel=2,
)
if cls is None:
from .datastructures import MultiDict # noqa: F811
cls = MultiDict
if isinstance(s, str) and not isinstance(separator, str):
separator = separator.decode(charset or "ascii")
elif isinstance(s, bytes) and not isinstance(separator, bytes):
separator = separator.encode(charset or "ascii") # type: ignore
return cls(
_url_decode_impl(
s.split(separator),
charset,
include_empty,
errors # type: ignore
))
def deserialize(self, s: t.AnyStr) -> Deck:
maindeck = Multiset()
sideboard = Multiset()
pattern = re.compile('({}\s+)?(\d+) \[([A-Z0-9]*)\] (.*?)\s*$'.format(
self._sideboard_indicator.rstrip()))
for ln in s.split('\n'):
m = pattern.match(ln)
if m:
is_sideboard, qty, expansion, name = m.groups()
(sideboard if is_sideboard else maindeck).add(
self._get_printing(name.replace('/', '//'), expansion),
int(qty),
)
return Deck(
maindeck,
sideboard,
)
def should_descend(self, path: ty.AnyStr) -> bool:
for idx, label in enumerate(path.split(self._sep)):
# Always descend into any directory below a recursive pattern as we
# cannot predict what we will later do a tail match on
if self._pat[idx] is None:
return True
# Do not descend further if we reached the last label of the pattern
# (unless the final pattern label is a recursive match, see above)
#
# This is independent of whether this *directory* will be included
# or not.
if idx == (len(self._pat) - 1):
return False
# Match the current pattern to decide whether to keep looking or not
if not self._pat[idx].match(label):
return False
# The given path matched part of this pattern, so we should include this
# directory to go further
return True
def __init__(self, pat: ty.AnyStr, *, period_special: bool = True):
"""
Arguments
---------
pat
The glob pattern to use for matching
period_special
Whether a leading period in file/directory names should be matchable by
``*``, ``?`` and ``[…]`` – traditionally they are not, but many modern
shells allow one to disable this behaviour
"""
self.period_special = period_special # type: bool
self._sep = utils.maybe_fsencode(os.path.sep, pat) # type: ty.AnyStr
dblstar = utils.maybe_fsencode("**", pat) # type: ty.AnyStr
dot = utils.maybe_fsencode(".", pat) # type: ty.AnyStr
pat_ndot = utils.maybe_fsencode(r"(?![.])", pat) # type: ty.AnyStr
# Normalize path separator
if os.path.altsep:
pat = pat.replace(utils.maybe_fsencode(os.path.altsep, pat),
self._sep)
# Sanity checks for stuff that will definitely NOT EVER match
# (there is another one in the loop below)
assert not os.path.isabs(
pat), "Absolute matching patterns will never match"
# Note the extra final slash for its effect of only matching directories
#
# (TBH, I find it hard to see how that is useful, but everybody does it
# and it keeps things consistent overall – something to only match files
# would be nice however.)
self._dir_only = pat.endswith(self._sep) # type: bool
self._pat = [] # type: ty.List[ty.Optional[re_pattern_t]]
for label in pat.split(self._sep):
# Skip over useless path components
if len(label) < 1 or label == dot:
continue
assert label != dot + dot, 'Matching patterns containing ".." will never match'
if label == dblstar:
self._pat.append(None)
elif dblstar in label:
raise NotImplementedError(
"Using double-star (**) and other characters in the same glob "
"path label ({0}) is not currently supported – please do file "
"an issue if you need this!".format(os.fsdecode(label)))
else:
#re_expr: ty.AnyStr
if not isinstance(label, bytes):
re_expr = fnmatch.translate(label)
else:
re_expr = fnmatch.translate(
label.decode("latin-1")).encode("latin-1")
if period_special and not label.startswith(dot):
re_expr = pat_ndot + re_expr
self._pat.append(re.compile(re_expr))
def args_line2dict(argv: typing.AnyStr, output_dict: typing.Dict):
r = argv.split('=', maxsplit=1)
if len(r) != 2:
return
output_dict[r[0]] = r[1]
