def test_allow_double_underscore(): gamla.pipe( 'd.__getitem__("bla")', ast.parse, dead_code.detect, gamla.check(gamla.complement(gamla.count), AssertionError), )
def test_allow_unused_public(): gamla.pipe( 'I_AM_A_CONSTANT = "asd"', ast.parse, dead_code.detect, gamla.check(gamla.complement(gamla.count), AssertionError), )
def test_disallow_unused_async_private_function(): gamla.pipe( "async def _hi():\n return 1", ast.parse, dead_code.detect, gamla.check(gamla.count, AssertionError), )
def test_allow_unused_public_function(): gamla.pipe( "def hi():\n return 1", ast.parse, dead_code.detect, gamla.check(gamla.complement(gamla.count), AssertionError), )
def test_private_class(): gamla.pipe( "class _Something: pass; A = _Something()", ast.parse, dead_code.detect, gamla.check(gamla.complement(gamla.count), AssertionError), )
def test_disallow_unused_private(): gamla.pipe( '_I_AM_A_CONSTANT = "asd"', ast.parse, dead_code.detect, gamla.check(gamla.count, AssertionError), )
def test_class_methods_disallowed(): gamla.pipe( """@dataclasses.dataclass(frozen=True) class SomeClass: # Some comment. text: Text _private_thing: Text = "bla" """, ast.parse, dead_code.detect, gamla.check(gamla.count, AssertionError), )
def deploy_schedule(tag: Text, dry_run: bool, job_configs: Iterable[Dict]): job_configs = tuple(job_configs) # Defend from generators. for config in job_configs: run = config["run"] gamla.pipe( kubernetes_connector.make_job_spec(run, tag, None), kubernetes_connector.create_cron_job( run["pod_name"], config["schedule"], dry_run, ), )
def test_bad(): for bad_comment in [ "# todo(uri): I am a bad comment.", "#todo: do something", "# TODO ROM: uncomment when vaccine faq fixed", "# TODO (rachel): Remove if not relevant after rescraping novant's vaccine faq.", "#no leading space", ]: gamla.pipe( bad_comment, comment.detect, gamla.check(gamla.count, AssertionError), )
def test_class_methods_allowed(): gamla.pipe( """@dataclasses.dataclass(frozen=True) class SomeClass: # Some comment. text: Text _private_thing: Text = "bla" def is_something(self) -> bool: return self._private_thing in [] """, ast.parse, dead_code.detect, gamla.check(gamla.complement(gamla.count), AssertionError), )
def _load_item(bucket_name: Text, hash_to_load: Text): return gamla.pipe( hash_to_load, utils.hash_to_filename, storage.download_blob_as_string(bucket_name), json.loads, )
def load_by_hash(environment: Text, bucket_name: Text, object_hash: Text) -> Dict: try: return gamla.pipe( object_hash, local_path_for_hash, lambda x: x.open("r"), json.load, gamla.log_text(f"Loaded {object_hash} from local cache."), ) except FileNotFoundError: return gamla.pipe( object_hash, gamla.log_text(f"Loading {object_hash} from bucket..."), _load_item(bucket_name), gamla.side_effect(save_local(environment, object_hash)), )
def _write_to_cache_file( cache_file_name: str, identifier: str, extra_fields: Dict, hash_to_load: str, ): cache_file = file_store.open_file("r+")(cache_file_name) new_versions_dict = gamla.pipe( cache_file, json.load, gamla.add_key_value( identifier, { _RESULT_HASH_KEY: hash_to_load, _LAST_RUN_TIMESTAMP: datetime.datetime.now().isoformat(), **extra_fields, }, ), dict.items, sorted, dict, ) cache_file.seek(0) json.dump(new_versions_dict, cache_file, indent=2) cache_file.write("\n") cache_file.truncate()
def test_good(): for good_comment in [ "# TODO(uri): I am a good comment.", "# I am good too.", "a = 3 # I am good.", "from nlu.config import logging_config # noqa: F401", " for element in filter( # type: ignore", ' url=render.add_utm_param("https://example.example.org/#screening"),', " # https://example.com/example/index.html#/example/", "#: Bla bla.", ]: gamla.pipe( good_comment, comment.detect, gamla.check(gamla.complement(gamla.count), AssertionError), )
def local_path_for_hash(object_hash: Text) -> pathlib.Path: local_path = gamla.pipe( object_hash, utils.hash_to_filename, _LOCAL_CACHE_PATH.joinpath, ) local_path.parent.mkdir(parents=True, exist_ok=True) return local_path
def deploy_jobs( tag: Text, dry_run: bool, job_configs: Iterable[Dict], extra_arg: Text, wait_minutes_for_completion: int, ): job_configs = tuple(job_configs) # Defend from generators. for config in job_configs: run = config["run"] gamla.pipe( kubernetes_connector.make_job_spec(run, tag, extra_arg), kubernetes_connector.create_job( run["pod_name"], dry_run, wait_minutes_for_completion, ), )
def test_detect_redundant_lambda(): gamla.pipe( """something = functools.lru_cache(maxsize=1024)( gamla.compose_left( gamla.pipe( _CONFIDENCE_ELEMENTS, gamla.map(_collect_type), gamla.star(gamla.juxtcat) ), gamla.filter(lambda element: element.confidence is not None), gamla.map(lambda element: element.confidence), tuple, gamla.check(lambda x: gamla.identity(x), ValueError), gamla.average, ) )""", ast.parse, redundant_lambda.detect, tuple, gamla.check(gamla.identity, AssertionError), )
def database_collection( mongodb_uri: Text, collection_name: Text, database: Text, ) -> motor_asyncio.AsyncIOMotorCollection: return gamla.pipe( mongodb_uri, motor_asyncio.AsyncIOMotorClient, gamla.get_in([database, collection_name]), )
def _get_call_arg_names(call: ast.Call) -> Iterable[str]: return gamla.pipe( gamla.concat([call.args, call.keywords]), gamla.map( gamla.ternary( # Generator expressions can also be given as arguments. gamla.is_instance(ast.Name), gamla.attrgetter("id"), gamla.just(None), ), ), )
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument("filenames", nargs="*") return gamla.pipe( argv, parser.parse_args, gamla.attrgetter("filenames"), gamla.map(_format_file), tuple, gamla.ternary(gamla.allmap(gamla.identity), gamla.just(0), gamla.just(1)), )
def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument("filenames", nargs="*") return gamla.pipe( argv, parser.parse_args, gamla.attrgetter("filenames"), gamla.map(gamla.pair_with(_file_contents_to_messages)), gamla.filter(gamla.head), gamla.map(gamla.side_effect(gamla.star(_pretty_print_findings))), tuple, gamla.ternary(gamla.identity, gamla.just(1), gamla.just(0)), )
def _make_pod_manifest( env_variables: List[Dict[Text, Text]], secrets: List[Dict[Text, Text]], command: List[Text], args: List[Text], extra_arg: Text, base_pod_spec: Dict[Text, Any], ) -> client.V1PodSpec: if secrets: for secret in secrets: _create_secret(secret) if extra_arg: command[-1] += f" {extra_arg}" return gamla.pipe( base_pod_spec, gamla.assoc_in( keys=["containers", 0, "env"], value=gamla.pipe( env_variables, gamla.map( lambda env_var: gamla.update_in( env_var, ["value"], lambda value: os.getenv(value, value), ), ), tuple, ), ), gamla.compose_left( gamla.juxt(*map(_add_volume_from_secret, secrets)), gamla.merge, ) if secrets else gamla.identity, gamla.assoc_in(keys=["containers", 0, "command"], value=command) if command else gamla.identity, gamla.assoc_in(keys=["containers", 0, "args"], value=args) if args else gamla.identity, )
def upload_blob(bucket_name: Text, blob_name: Text, obj: Any): return gamla.pipe( obj, gamla.to_json, _to_bytes, gzip.compress, io.BytesIO, lambda stream: _blob_service().create_blob_from_stream( bucket_name, blob_name, stream, timeout=1800, content_settings=blob.ContentSettings(content_encoding="gzip"), ), )
def make_job_spec( run: Dict[Text, Text], tag: Text, extra_arg: Optional[Text], ) -> client.V1JobSpec: return gamla.pipe( _make_base_pod_spec( run["pod_name"], run["image"], tag, run.get("node_selector"), ), _make_pod_manifest( run.get("env_variables"), run.get("secrets"), run.get("command"), run.get("args"), extra_arg, ), _make_job_spec(run.get("labels")), )
def _format_file(filename): with open(filename, mode="r") as file_processed: content_before = file_processed.readlines() with open(filename, mode="r") as file_processed: rows = tuple(csv.reader(file_processed)) with open(filename, mode="w") as file_processed: csv.writer(file_processed).writerows( gamla.pipe( rows, gamla.filter(gamla.identity), gamla.sort_by(gamla.head), tuple, ), ) with open(filename, mode="r") as file_processed: content_after = file_processed.readlines() identical = content_before == content_after if not identical: print(f"File {filename} has been modified.") # noqa:T001 return identical
def load_file_from_bucket(file_name: Text, bucket_name: Text): return gamla.pipe( file_name, gamla.log_text("Loading {} from bucket..."), download_blob_as_string(bucket_name), )
async def releases(): return gamla.pipe( await _run_in_shell(["helm", "list", "-q"], "./"), lambda output: output.split("\n"), frozenset, )
def _gen_lambda_arg_names(l: ast.Lambda) -> Iterable[str]: return gamla.pipe(l, lambda l: l.args.args, gamla.map(gamla.attrgetter("arg")))
def _is_lambda_internal_invocation(l: ast.Lambda) -> bool: return gamla.pipe(l.body, gamla.is_instance(ast.Call))