def test_monitor_no_message(self, mock_post):
with monitor(slack_url='mock://slack',
events=['DEPENDENCY_MISSING']) as m:
luigi.run(main_task_cls=TestSuccessTask,
local_scheduler=True,
cmdline_args=['--num', '1'])
self.assertDictEqual(m.recorded_events, {})
self.assertEqual(mock_post.call_count, 0, 'Slack webhook called')
def test_monitor_missing_message(self, mock_post):
with monitor(slack_url='mock://slack',
events=['DEPENDENCY_MISSING']) as m:
luigi.run(main_task_cls=TestMissingTask,
local_scheduler=True,
cmdline_args=[])
self.assertDictEqual(m.recorded_events,
{'DEPENDENCY_MISSING': ['TestMissingTask()']})
call_data = mock_post.call_args[1]['data']
call_url = mock_post.call_args[0][0]
expected = '{"text": "Status report for ' + os.path.basename(inspect.stack()[-1][1]) + '\\n' \
'*Tasks with missing dependencies:*' \
'\\n' \
'TestMissingTask()"}'
self.assertEqual(call_data, expected)
def test_monitor_success_message(self, mock_post):
expected = defaultdict(list)
expected['SUCCESS'] = ['TestSuccessTask(num=2)']
expected['FAILURE']
expected['DEPENDENCY_MISSING']
with monitor(slack_url='mock://slack') as m:
luigi.run(main_task_cls=TestSuccessTask,
local_scheduler=True,
cmdline_args=['--num', '2'])
self.assertDictContainsSubset(expected, m.recorded_events)
call_data = mock_post.call_args[1]['data']
call_url = mock_post.call_args[0][0]
expected = '{"text": "Status report for ' + os.path.basename(
inspect.stack()[-1][1]) + '\\nJob ran successfully!"}'
self.assertEqual(call_data, expected)
print("Out_files cleared")
def requires(self):
return FilterText(self.db, self.collection)
# The _requires method lets StoreLabel depend on StoreText's completion without consuming StoreText's output
def _requires(self):
return flatten(
[
StoreExtraction(self.db, self.collection),
StoreTranslation(self.db, self.collection),
super(StoreLabel, self)._requires(),
]
)
if __name__ == "__main__":
with monitor(
slack_url=os.environ.get("SLACK_WEBHOOK"),
max_print=10,
username="******",
):
luigi.build(
[
StoreLabel(
db=os.environ.get("SHARECHAT_DB_NAME"),
collection=os.environ.get("SHARECHAT_DB_COLLECTION"),
)
]
)
client_name = luigi.Parameter()
def requires(self):
return MonthFTPDOwnload(start_date=self.start_date,
save_path=self.save_path)
@property
def path_extract(self):
path_extract = os.path.join(self.save_path, 'csv_extract',
start_date.strptime('%Y_%M'))
return path_extract
def output(self):
return luigi.LocalTarget(self.path_extract)
def run(self):
return docker_execute(client_name=self.client_name,
path_dir=self.path_extract,
message_code=self.message_extract)
if __name__ == '__main__':
config = luigi.configuration.get_config()
slack_url = config.get('luigi-monitor', 'slack_url', None)
max_print = config.get('luigi-monitor', 'max_print', 5)
username = config.get('luigi-monitor', 'username', None)
with monitor(slack_url=slack_url, username=username, max_print=max_print):
luigi.run(main_task_cls=OrderDownload)
class GetList(luigi.Task):
def __init__(self):
super(GetList, self).__init__()
if not os.path.exists(URL_LIST_DIR):
os.makedirs(URL_LIST_DIR)
def output(self):
return luigi.LocalTarget(os.path.join(URL_LIST_DIR, "url.json"))
def run(self):
url_list = list()
for item in range(1, RANGE):
try:
response = RequestProxy().request_get(BASIC_URL + str(item))
except Exception, e:
raise e
if response.status_code == 200:
url_list.append(BASIC_URL + str(item))
else:
print "Error occur for url: %s, ", BASIC_URL + str(item)
time.sleep(2)
with self.output().open("w") as f:
json.dump(url_list, f, indent=4)
if __name__ == "__main__":
# create slack app and bound app link to the specified channel or people
# https://api.slack.com/tutorials/slack-apps-hello-world
with monitor(slack_url="https://hooks.slack.com/services/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", max_print=10, username="******"):
luigi.run(main_task_cls=Save)