def test_allow_double_underscore():
gamla.pipe(
'd.__getitem__("bla")',
ast.parse,
dead_code.detect,
gamla.check(gamla.complement(gamla.count), AssertionError),
)
def test_allow_unused_public():
gamla.pipe(
'I_AM_A_CONSTANT = "asd"',
ast.parse,
dead_code.detect,
gamla.check(gamla.complement(gamla.count), AssertionError),
)
def test_disallow_unused_async_private_function():
gamla.pipe(
"async def _hi():\n return 1",
ast.parse,
dead_code.detect,
gamla.check(gamla.count, AssertionError),
)
def test_allow_unused_public_function():
gamla.pipe(
"def hi():\n return 1",
ast.parse,
dead_code.detect,
gamla.check(gamla.complement(gamla.count), AssertionError),
)
def test_private_class():
gamla.pipe(
"class _Something: pass; A = _Something()",
ast.parse,
dead_code.detect,
gamla.check(gamla.complement(gamla.count), AssertionError),
)
def test_disallow_unused_private():
gamla.pipe(
'_I_AM_A_CONSTANT = "asd"',
ast.parse,
dead_code.detect,
gamla.check(gamla.count, AssertionError),
)
def test_class_methods_disallowed():
gamla.pipe(
"""@dataclasses.dataclass(frozen=True)
class SomeClass:
# Some comment.
text: Text
_private_thing: Text = "bla"
""",
ast.parse,
dead_code.detect,
gamla.check(gamla.count, AssertionError),
)
def deploy_schedule(tag: Text, dry_run: bool, job_configs: Iterable[Dict]):
job_configs = tuple(job_configs) # Defend from generators.
for config in job_configs:
run = config["run"]
gamla.pipe(
kubernetes_connector.make_job_spec(run, tag, None),
kubernetes_connector.create_cron_job(
run["pod_name"],
config["schedule"],
dry_run,
),
)
def test_bad():
for bad_comment in [
"# todo(uri): I am a bad comment.",
"#todo: do something",
"# TODO ROM: uncomment when vaccine faq fixed",
"# TODO (rachel): Remove if not relevant after rescraping novant's vaccine faq.",
"#no leading space",
]:
gamla.pipe(
bad_comment,
comment.detect,
gamla.check(gamla.count, AssertionError),
)
def test_class_methods_allowed():
gamla.pipe(
"""@dataclasses.dataclass(frozen=True)
class SomeClass:
# Some comment.
text: Text
_private_thing: Text = "bla"
def is_something(self) -> bool:
return self._private_thing in []
""",
ast.parse,
dead_code.detect,
gamla.check(gamla.complement(gamla.count), AssertionError),
)
def _load_item(bucket_name: Text, hash_to_load: Text):
return gamla.pipe(
hash_to_load,
utils.hash_to_filename,
storage.download_blob_as_string(bucket_name),
json.loads,
)
def load_by_hash(environment: Text, bucket_name: Text, object_hash: Text) -> Dict:
try:
return gamla.pipe(
object_hash,
local_path_for_hash,
lambda x: x.open("r"),
json.load,
gamla.log_text(f"Loaded {object_hash} from local cache."),
)
except FileNotFoundError:
return gamla.pipe(
object_hash,
gamla.log_text(f"Loading {object_hash} from bucket..."),
_load_item(bucket_name),
gamla.side_effect(save_local(environment, object_hash)),
)
def _write_to_cache_file(
cache_file_name: str,
identifier: str,
extra_fields: Dict,
hash_to_load: str,
):
cache_file = file_store.open_file("r+")(cache_file_name)
new_versions_dict = gamla.pipe(
cache_file,
json.load,
gamla.add_key_value(
identifier,
{
_RESULT_HASH_KEY: hash_to_load,
_LAST_RUN_TIMESTAMP: datetime.datetime.now().isoformat(),
**extra_fields,
},
),
dict.items,
sorted,
dict,
)
cache_file.seek(0)
json.dump(new_versions_dict, cache_file, indent=2)
cache_file.write("\n")
cache_file.truncate()
def test_good():
for good_comment in [
"# TODO(uri): I am a good comment.",
"# I am good too.",
"a = 3 # I am good.",
"from nlu.config import logging_config # noqa: F401",
" for element in filter( # type: ignore",
' url=render.add_utm_param("https://example.example.org/#screening"),',
" # https://example.com/example/index.html#/example/",
"#: Bla bla.",
]:
gamla.pipe(
good_comment,
comment.detect,
gamla.check(gamla.complement(gamla.count), AssertionError),
)
def local_path_for_hash(object_hash: Text) -> pathlib.Path:
local_path = gamla.pipe(
object_hash,
utils.hash_to_filename,
_LOCAL_CACHE_PATH.joinpath,
)
local_path.parent.mkdir(parents=True, exist_ok=True)
return local_path
def deploy_jobs(
tag: Text,
dry_run: bool,
job_configs: Iterable[Dict],
extra_arg: Text,
wait_minutes_for_completion: int,
):
job_configs = tuple(job_configs) # Defend from generators.
for config in job_configs:
run = config["run"]
gamla.pipe(
kubernetes_connector.make_job_spec(run, tag, extra_arg),
kubernetes_connector.create_job(
run["pod_name"],
dry_run,
wait_minutes_for_completion,
),
)
def test_detect_redundant_lambda():
gamla.pipe(
"""something = functools.lru_cache(maxsize=1024)(
gamla.compose_left(
gamla.pipe(
_CONFIDENCE_ELEMENTS, gamla.map(_collect_type), gamla.star(gamla.juxtcat)
),
gamla.filter(lambda element: element.confidence is not None),
gamla.map(lambda element: element.confidence),
tuple,
gamla.check(lambda x: gamla.identity(x), ValueError),
gamla.average,
)
)""",
ast.parse,
redundant_lambda.detect,
tuple,
gamla.check(gamla.identity, AssertionError),
)
def database_collection(
mongodb_uri: Text,
collection_name: Text,
database: Text,
) -> motor_asyncio.AsyncIOMotorCollection:
return gamla.pipe(
mongodb_uri,
motor_asyncio.AsyncIOMotorClient,
gamla.get_in([database, collection_name]),
)
def _get_call_arg_names(call: ast.Call) -> Iterable[str]:
return gamla.pipe(
gamla.concat([call.args, call.keywords]),
gamla.map(
gamla.ternary(
# Generator expressions can also be given as arguments.
gamla.is_instance(ast.Name),
gamla.attrgetter("id"),
gamla.just(None),
), ),
)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
parser.add_argument("filenames", nargs="*")
return gamla.pipe(
argv,
parser.parse_args,
gamla.attrgetter("filenames"),
gamla.map(_format_file),
tuple,
gamla.ternary(gamla.allmap(gamla.identity), gamla.just(0),
gamla.just(1)),
)
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser()
parser.add_argument("filenames", nargs="*")
return gamla.pipe(
argv,
parser.parse_args,
gamla.attrgetter("filenames"),
gamla.map(gamla.pair_with(_file_contents_to_messages)),
gamla.filter(gamla.head),
gamla.map(gamla.side_effect(gamla.star(_pretty_print_findings))),
tuple,
gamla.ternary(gamla.identity, gamla.just(1), gamla.just(0)),
)
def _make_pod_manifest(
env_variables: List[Dict[Text, Text]],
secrets: List[Dict[Text, Text]],
command: List[Text],
args: List[Text],
extra_arg: Text,
base_pod_spec: Dict[Text, Any],
) -> client.V1PodSpec:
if secrets:
for secret in secrets:
_create_secret(secret)
if extra_arg:
command[-1] += f" {extra_arg}"
return gamla.pipe(
base_pod_spec,
gamla.assoc_in(
keys=["containers", 0, "env"],
value=gamla.pipe(
env_variables,
gamla.map(
lambda env_var: gamla.update_in(
env_var,
["value"],
lambda value: os.getenv(value, value),
), ),
tuple,
),
),
gamla.compose_left(
gamla.juxt(*map(_add_volume_from_secret, secrets)),
gamla.merge,
) if secrets else gamla.identity,
gamla.assoc_in(keys=["containers", 0, "command"], value=command)
if command else gamla.identity,
gamla.assoc_in(keys=["containers", 0, "args"], value=args)
if args else gamla.identity,
)
def upload_blob(bucket_name: Text, blob_name: Text, obj: Any):
return gamla.pipe(
obj,
gamla.to_json,
_to_bytes,
gzip.compress,
io.BytesIO,
lambda stream: _blob_service().create_blob_from_stream(
bucket_name,
blob_name,
stream,
timeout=1800,
content_settings=blob.ContentSettings(content_encoding="gzip"),
),
)
def make_job_spec(
run: Dict[Text, Text],
tag: Text,
extra_arg: Optional[Text],
) -> client.V1JobSpec:
return gamla.pipe(
_make_base_pod_spec(
run["pod_name"],
run["image"],
tag,
run.get("node_selector"),
),
_make_pod_manifest(
run.get("env_variables"),
run.get("secrets"),
run.get("command"),
run.get("args"),
extra_arg,
),
_make_job_spec(run.get("labels")),
)
def _format_file(filename):
with open(filename, mode="r") as file_processed:
content_before = file_processed.readlines()
with open(filename, mode="r") as file_processed:
rows = tuple(csv.reader(file_processed))
with open(filename, mode="w") as file_processed:
csv.writer(file_processed).writerows(
gamla.pipe(
rows,
gamla.filter(gamla.identity),
gamla.sort_by(gamla.head),
tuple,
), )
with open(filename, mode="r") as file_processed:
content_after = file_processed.readlines()
identical = content_before == content_after
if not identical:
print(f"File {filename} has been modified.") # noqa:T001
return identical
def load_file_from_bucket(file_name: Text, bucket_name: Text):
return gamla.pipe(
file_name,
gamla.log_text("Loading {} from bucket..."),
download_blob_as_string(bucket_name),
)
async def releases():
return gamla.pipe(
await _run_in_shell(["helm", "list", "-q"], "./"),
lambda output: output.split("\n"),
frozenset,
)
def _gen_lambda_arg_names(l: ast.Lambda) -> Iterable[str]:
return gamla.pipe(l, lambda l: l.args.args,
gamla.map(gamla.attrgetter("arg")))
def _is_lambda_internal_invocation(l: ast.Lambda) -> bool:
return gamla.pipe(l.body, gamla.is_instance(ast.Call))
