def test_parse_args_glueetl():
argv = [
"script_2020-04-15-11-21-57.py",
"--JOB_NAME",
"glue-spark-job",
"--APP_SETTINGS_ENVIRONMENT",
"dev",
"--JOB_ID",
"j_3456789",
"--JOB_RUN_ID",
"jr_3456789",
"--job-bookmark-option",
"job-bookmark-disable",
"--TempDir",
"s3://bucker/Key/dir",
]
sys.modules["dynamicframe"] = ModuleType("DynamicFrame")
sys.modules["dynamicframe"].DynamicFrame = None
sys.modules["awsglue.utils"] = ModuleType("awsglue.utils")
sys.modules[
"awsglue.utils"].getResolvedOptions = lambda arguments, options=None: {
"APP_SETTINGS_ENVIRONMENT": "dev",
"JOB_ID": "j_3456789",
}
actual = parse_args(argv, ["APP_SETTINGS_ENVIRONMENT", "JOB_ID"])
expected = {"APP_SETTINGS_ENVIRONMENT": "dev", "JOB_ID": "j_3456789"}
assert actual == expected
def test_parse_args_pyshell():
argv = [
"/tmp/glue-python-scripts-jm72zh6c/jan_pyshell_job.py",
"--APP_SETTINGS_ENVIRONMENT",
"dev",
"--job-bookmark-option",
"job-bookmark-disable",
"--job-language",
"python",
]
actual = parse_args(argv, ["APP_SETTINGS_ENVIRONMENT"])
expected = {"APP_SETTINGS_ENVIRONMENT": "dev"}
assert actual == expected
"""
Config file for iot refined to curated Glue job.
"""
import logging.config
import sys
from glue_shared import parse_args
from glue_shared.defaults import default_logging_config
region = "us-east-1"
arguments = parse_args(sys.argv, ["APP_SETTINGS_ENVIRONMENT", "LOG_LEVEL"])
LOGGING_CONFIG = default_logging_config(arguments["LOG_LEVEL"])
logging.config.dictConfig(LOGGING_CONFIG)
JOB_CONFIG = dict(arguments)
# must be hard-coded because glue does not provide this in PyShell jobs
JOB_CONFIG["JOB_NAME"] = JOB_CONFIG.get("JOB_NAME") or "refined-to-curated"
JOB_CONFIG["JOB_ID"] = JOB_CONFIG.get("JOB_ID")
JOB_CONFIG["JOB_RUN_ID"] = JOB_CONFIG.get("JOB_RUN_ID")
JOB_CONFIG["s3_bucket"] = "your-awsglue-bucket"
JOB_CONFIG["s3_prefix"] = "ds1/refined"