script_path = os.path.abspath(os.path.join(os.getcwd(), 'check.py'))
check_stock = "python -u %s" % script_path
root_logger = logging.getLogger('cron_executor')
root_logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stderr)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(message)s')
ch.setFormatter(formatter)
root_logger.addHandler(ch)
cloud_handler = CloudLoggingHandler(on_gce=True, logname="task_runner")
root_logger.addHandler(cloud_handler)
# create the executor that watches the topic, and will run the job task
test_executor = Executor(topic=TOPIC,
project=PROJECT,
task_cmd=check_stock,
subname='sample_task_task')
# add a cloud logging handler and stderr logging handler
job_cloud_handler = CloudLoggingHandler(on_gce=True,
logname=test_executor.subname)
test_executor.job_log.addHandler(job_cloud_handler)
test_executor.job_log.addHandler(ch)
test_executor.job_log.setLevel(logging.DEBUG)
# watches indefinitely
test_executor.watch_topic()
script_path = os.path.abspath(os.path.join(os.getcwd(), 'logger_sample_task.py'))
sample_task = "python -u %s" % script_path
root_logger = logging.getLogger('cron_executor')
root_logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stderr)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
root_logger.addHandler(ch)
cloud_handler = CloudLoggingHandler(on_gce=True, logname="task_runner")
root_logger.addHandler(cloud_handler)
# create the executor that watches the topic, and will run the job task
test_executor = Executor(topic=TOPIC, project=PROJECT, task_cmd=sample_task, subname='sample_task_task')
# add a cloud logging handler and stderr logging handler
job_cloud_handler = CloudLoggingHandler(on_gce=True, logname=test_executor.subname)
test_executor.job_log.addHandler(job_cloud_handler)
test_executor.job_log.addHandler(ch)
test_executor.job_log.setLevel(logging.DEBUG)
# watches indefinitely
test_executor.watch_topic()
filename = os.path.join(dirname, 'runner_pipelines/senate_members_df.py')
dataflow_task = "git pull origin " \
"&& python ../../runner_pipelines/senate_members_df.py " \
" --setup=../../runner_pipelines/setup.py --experiments=allow_non_updatable_job parameter " \
"&& python ../../runner_pipelines/house_members_df.py " \
" --setup=../../runner_pipelines/setup.py --experiments=allow_non_updatable_job parameter "
root_logger = logging.getLogger('cron_executor')
root_logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stderr)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
root_logger.addHandler(ch)
cloud_handler = CloudLoggingHandler(on_gce=True, logname="task_runner")
root_logger.addHandler(cloud_handler)
# create the executor that watches the topic, and will run the job task
df_executor = Executor(topic=TOPIC, project=PROJECT, task_cmd=dataflow_task, subname='dataflow_task_task')
# add a cloud logging handler and stderr logging handler
job_cloud_handler = CloudLoggingHandler(on_gce=True, logname=df_executor.subname)
df_executor.job_log.addHandler(job_cloud_handler)
df_executor.job_log.addHandler(ch)
df_executor.job_log.setLevel(logging.DEBUG)
# watches indefinitely
df_executor.watch_topic()
TOPIC = cr.TOPIC
script_path = cr.script_path
sample_task = "python3 %s" % script_path
root_logger = logging.getLogger('cron_executor')
root_logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stderr)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
root_logger.addHandler(ch)
cloud_handler = CloudLoggingHandler(on_gce=True, logname="task_runner")
root_logger.addHandler(cloud_handler)
# create the executor that watches the topic, and will run the job task
test_executor = Executor(topic=TOPIC, project=PROJECT, task_cmd=sample_task)
# add a cloud logging handler and stderr logging handler
job_cloud_handler = CloudLoggingHandler(on_gce=True,
logname=test_executor.subname)
test_executor.job_log.addHandler(job_cloud_handler)
test_executor.job_log.addHandler(ch)
test_executor.job_log.setLevel(logging.DEBUG)
# watches indefinitely
test_executor.watch_topic()