def examples(args): c = fetch_config(args) app = BuildApp(c) example_tasks(c, app) app.run()
def mine(args): conf = fetch_config(args) app = BuildApp(conf) app.pool = 'thread' gh = get_connection(conf) pprint(mine_github_pulls(gh, app, conf))
def push_translations(args): conf = fetch_config(args) app = BuildApp(conf) push_tasks(conf, app) update(conf) app.run()
def build_translation_model(args): conf = fetch_config(args) if args.t_translate_config is None: tconf = conf.system.files.data.translate elif os.path.isfile(args.t_translate_config): tconf = TranslateConfig(ingest_yaml_doc(args.t_translate_config), conf) else: logger.error(args.t_translate_config + " doesn't exist") return if os.path.exists(tconf.paths.project) is False: os.makedirs(tconf.paths.project) elif os.path.isfile(tconf.paths.project): logger.error(tconf.paths.project + " is a file") sys.exit(1) elif os.listdir(tconf.paths.project) != []: logger.error(tconf.paths.project + " must be empty") sys.exit(1) with open(os.path.join(tconf.paths.project, "translate.yaml"), 'w') as f: yaml.dump(tconf.dict(), f, default_flow_style=False) tconf.conf.runstate.pool_size = tconf.settings.pool_size run_args = get_run_args(tconf) app = BuildApp(conf) os.environ['IRSTLM'] = tconf.paths.irstlm setup_train(tconf) setup_tune(tconf) setup_test(tconf) for idx, parameter_set in enumerate(run_args): parameter_set = list(parameter_set) parameter_set.append(idx) parameter_set.append(tconf) t = app.add() t.job = build_model t.args = parameter_set t.description = "model_" + str(parameter_set[9]) app.run() aggregate_model_data(tconf.paths.project) from_addr = "*****@*****.**" to_addr = [tconf.settings.email] with open(tconf.paths.project+"/data.csv") as data: msg = MIMEText(data.read()) msg['Subject'] = "Model Complete" msg['From'] = from_addr msg['To'] = ", ".join(to_addr) server = smtplib.SMTP("localhost") server.sendmail(from_addr, to_addr, msg.as_string()) server.quit()
def options(args): c = fetch_config(args) if c.runstate.clean_generated is True: option_clean(c) else: app = BuildApp(c) option_tasks(c, app) app.run()
def toc(args): c = fetch_config(args) if c.runstate.clean_generated is True: toc_clean(c) else: app = BuildApp(c) toc_tasks(c, app) app.run()
def primer(args): c = fetch_config(args) if c.runstate.clean_generated is True: primer_clean(c) else: app = BuildApp(c) primer_migration_tasks(c, app) app.run()
def intersphinx(args): c = fetch_config(args) app = BuildApp(c) if c.runstate.clean_generated is True: intersphinx_clean(c, app) else: intersphinx_tasks(c, app) app.run()
def assets(args): c = fetch_config(args) app = BuildApp(c) if c.runstate.clean_generated is True: assets_clean(c, app) else: assets_tasks(c, app) app.run()
def redirects(args): c = fetch_config(args) if args.dry_run is True: print(''.join(make_redirect(c))) else: app = BuildApp(c) redirect_tasks(c, app) app.run()
def images(args): c = fetch_config(args) app = BuildApp(c) if c.runstate.clean_generated is True: image_clean(c, app) else: image_tasks(c, app) app.run()
def test_single_runner_app(self): self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) app = BuildApp(self.c) t = app.add('task') t.job = sum t.args = [[ 1 , 2 ], 0] self.app._run_single(app) self.assertEqual(self.app.results[0], 3)
def triage(args): conf = fetch_config(args) app = BuildApp(conf) app.pool = 'thread' j = JeerahClient(conf) j.connect() query_data = giza.jeerah.triage.query(j, app, conf) pprint(giza.jeerah.triage.report(query_data, conf))
def actions(args): conf = fetch_config(args) app = BuildApp(conf) app.pool = 'thread' gh = get_connection(conf) results = [] for pull in mine_github_pulls(gh, app, conf): if pull['merge_safe'] is True: results.append(pull) pprint(results)
def test_single_runner_app_with_many_subtasks(self): self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) app = BuildApp(self.c) for _ in range(10): t = app.add('task') t.job = sum t.args = [[ 1 , 2 ], 0] self.app._run_single(app) self.assertEqual(len(self.app.results), 10) self.assertEqual(self.app.results[0], 3) self.assertEqual(sum(self.app.results), 30)
def sphinx_publication(c, args, app): build_prep_tasks(c, app) # this loop will produce an app for each language/edition/builder combination build_source_copies = set() sphinx_app = BuildApp(c) sphinx_app.pool = app.pool jobs = itertools.product(args.editions_to_build, args.languages_to_build, args.builder) for edition, language, builder in jobs: args.language = language args.edition = edition args.builder = builder build_config = fetch_config(args) prep_app = app.add('app') prep_app.conf = build_config primer_app = prep_app.add('app') primer_migration_tasks(build_config, primer_app) sconf = render_sconf(edition, builder, language, build_config) if build_config.paths.branch_source not in build_source_copies: build_source_copies.add(build_config.paths.branch_source) source_tasks(build_config, sconf, prep_app) source_app = prep_app.add('app') build_content_generation_tasks(build_config, source_app) refresh_dependency_tasks(build_config, prep_app) sphinx_tasks(sconf, build_config, sphinx_app) logger.info("adding builder job for {0} ({1}, {2})".format(builder, language, edition)) app.add(sphinx_app) logger.info("sphinx build setup, running now.") app.run() logger.info("sphinx build complete.") ret_code, sphinx_output = post_build_operations(sphinx_app.results, app) logger.info('builds finalized. sphinx output and errors to follow') output_sphinx_stream(sphinx_output, c) return ret_code
def stats(args): conf = fetch_config(args) app = BuildApp(conf) app.pool = 'thread' gh = get_connection(conf) users = set() result = {'merge_safe': 0, 'total': 0 } for pull in mine_github_pulls(gh, app, conf): result['total'] += 1 if pull['merge_safe'] is True: result['merge_safe'] += 1 users.add(pull['user']) result['user_count'] = len(users) result['users'] = list(users) pprint(result)
def clean(args): c = Configuration() c.ingest(args.conf_path) c.runstate = args app = BuildApp(c) to_remove = [] if c.runstate.builder_to_delete is not None: builder = c.runstate.builder_to_delete to_remove.append( os.path.join(c.paths.branch_output, 'doctrees-' + builder)) to_remove.append( os.path.join(c.paths.branch_output, builder)) m = 'remove artifacts associated with the {0} builder in {1}' logger.debug(m.format(builder, c.git.branches.current)) if c.runstate.days_to_save is not None: published_branches = [ 'docs-tools', 'archive', 'public', 'primer', c.git.branches.current ] published_branches.extend(c.git.branches.published) for build in os.listdir(os.path.join(c.paths.projectroot, c.paths.output)): build = os.path.join(c.paths.projectroot, c.paths.output, build) branch = os.path.split(build)[1] if branch in published_branches: continue elif not os.path.isdir(build): continue elif os.stat(build).st_mtime > c.runstate.days_to_save: to_remove.append(build) to_remove.append(os.path.join(c.paths.projectroot, c.paths.output, 'public', branch)) logger.debug('removed stale artifacts: "{0}" and "build/public/{0}"'.format(branch)) for fn in to_remove: t = app.add() t.job = rm_rf t.args = fn m = 'removing artifact: {0}'.format(fn) t.description = m logger.critical(m) app.run()
class TestBuildApp(TestCase): @classmethod def setUp(self): self.c = Configuration() self.c.runstate = RuntimeStateConfig() self.app = BuildApp(self.c) def test_add_make_test_default(self): self.assertEqual(self.app.queue, []) self.app.add() self.assertTrue(len(self.app.queue) == 1) self.assertTrue(isinstance(self.app.queue[0], Task)) def test_add_make_test_task(self): self.assertEqual(self.app.queue, []) self.app.add('task') self.assertTrue(len(self.app.queue) == 1) self.assertTrue(isinstance(self.app.queue[0], Task)) def test_add_make_test_app(self): self.assertEqual(self.app.queue, []) self.app.add('app') self.assertTrue(len(self.app.queue) == 1) self.assertIsInstance(self.app.queue[0], BuildApp) def test_add_existing_task_object(self): self.assertEqual(self.app.queue, []) t = Task() self.app.add(t) self.assertIs(t, self.app.queue[0]) self.assertIsNot(t, Task()) self.assertIsNot(Task(), self.app.queue[0]) def test_add_existing_app_object(self): self.assertEqual(self.app.queue, []) app = BuildApp(self.c) self.app.add(app) self.assertIs(app, self.app.queue[0]) self.assertIsNot(app, BuildApp(self.c)) self.assertIsNot(BuildApp(self.c), self.app.queue[0]) def test_conf_objet_consistent_in_task(self): self.assertEqual(self.app.queue, []) t = self.app.add('task') self.assertIs(self.c, t.conf) self.assertIs(self.c, self.app.queue[0].conf) def test_conf_objet_consistent_in_app(self): self.assertEqual(self.app.queue, []) app = self.app.add('app') self.assertIs(self.c, app.conf) self.assertIs(self.c, self.app.queue[0].conf) def test_conf_objet_consistent_in_task(self): self.assertEqual(self.app.queue, []) t = Task() self.assertIsNone(t.conf) self.app.add(t) self.assertIsNotNone(t.conf) self.assertIs(self.c, self.app.queue[0].conf) self.assertIs(self.c, t.conf) def test_pool_setter_default(self): self.assertIsNone(self.app.worker_pool) self.app.pool = None self.assertIsNotNone(self.app.worker_pool) self.assertIsInstance(self.app.pool, ProcessPool) def test_pool_setter_process(self): self.assertIsNone(self.app.worker_pool) a = self.app.pool = 'process' self.assertIsNotNone(self.app.worker_pool) self.assertIsInstance(self.app.pool, ProcessPool) def test_pool_setter_thread(self): self.assertIsNone(self.app.worker_pool) a = self.app.pool = 'thread' self.assertIsNotNone(self.app.worker_pool) self.assertIsInstance(self.app.pool, ThreadPool) def test_pool_setter_serial(self): self.assertIsNone(self.app.worker_pool) a = self.app.pool = 'serial' self.assertIsNotNone(self.app.worker_pool) self.assertIsInstance(self.app.pool, SerialPool) def test_pool_setter_process_by_ref(self): self.assertIsNone(self.app.worker_pool) a = self.app.pool = ProcessPool self.assertIsNotNone(self.app.worker_pool) self.assertIsInstance(self.app.pool, ProcessPool) def test_pool_setter_thread_by_ref(self): self.assertIsNone(self.app.worker_pool) self.app.pool = ThreadPool self.assertIsNotNone(self.app.worker_pool) self.assertIsInstance(self.app.pool, ThreadPool) def test_pool_setter_serial_by_ref(self): self.assertIsNone(self.app.worker_pool) self.app.pool = SerialPool self.assertIsNotNone(self.app.worker_pool) self.assertIsInstance(self.app.pool, SerialPool) def test_pool_setter_invalid_input(self): self.assertIsNone(self.app.worker_pool) a = self.app.pool = 1 self.assertIsInstance(self.app.pool, ProcessPool) def test_pool_setter_existing_pool_thread(self): self.assertIsNone(self.app.worker_pool) p = ThreadPool(self.c) self.app.pool = p self.assertIs(self.app.pool, p) def test_pool_setter_existing_pool_process(self): self.assertIsNone(self.app.worker_pool) p = ProcessPool(self.c) self.app.pool = p self.assertIs(self.app.pool, p) def test_pool_setter_existing_pool_serial(self): self.assertIsNone(self.app.worker_pool) p = SerialPool(self.c) self.app.pool = p self.assertIs(self.app.pool, p) def test_pool_closer(self): self.assertIsNone(self.app.worker_pool) self.app.pool = 'thread' self.assertIsInstance(self.app.pool, ThreadPool) self.app.close_pool() self.assertIsNone(self.app.worker_pool) def test_pool_type_checker_thread(self): self.assertTrue(self.app.is_pool_type('thread')) def test_pool_type_checker_process(self): self.assertTrue(self.app.is_pool_type('process')) def test_pool_type_checker_serial(self): self.assertTrue(self.app.is_pool_type('serial')) def test_pool_type_checker_serial_invalid(self): self.assertFalse(self.app.is_pool_type('serialized')) def test_pool_type_checker_process_invalid(self): self.assertFalse(self.app.is_pool_type('proc')) def test_pool_type_checker_thread_invalid(self): self.assertFalse(self.app.is_pool_type('threaded')) def test_is_pool_predicate_thead(self): self.assertTrue(self.app.is_pool(ThreadPool())) def test_is_pool_predicate_process(self): self.assertTrue(self.app.is_pool(ProcessPool())) def test_is_pool_predicate_serial(self): self.assertTrue(self.app.is_pool(SerialPool())) def test_is_pool_predicate_invalid(self): self.assertFalse(self.app.is_pool(self.c)) self.assertFalse(self.app.is_pool(self.app)) def test_add_invalid_object(self): with self.assertRaises(TypeError): self.app.add(1) def test_run_invalid_task(self): self.app.queue.append(1) with self.assertRaises(TypeError): self.app.run() def test_single_runner_task(self): self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) t = Task() t.job = sum t.args = [[ 1 , 2 ], 0] self.app._run_single(t) self.assertEqual(self.app.results[0], 3) def test_single_runner_task_integrated(self): self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) t = self.app.add('task') t.job = sum t.args = [[ 1 , 2 ], 0] self.app.run() self.assertEqual(self.app.results[0], 3) def test_single_runner_app(self): self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) app = BuildApp(self.c) t = app.add('task') t.job = sum t.args = [[ 1 , 2 ], 0] self.app._run_single(app) self.assertEqual(self.app.results[0], 3) def test_single_runner_app_integrated(self): self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) app = self.app.add('app') t = app.add('task') t.job = sum t.args = [[ 1 , 2 ], 0] self.app.run() self.assertEqual(self.app.results[0], 3) def test_single_runner_app_with_many_subtasks(self): self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) app = BuildApp(self.c) for _ in range(10): t = app.add('task') t.job = sum t.args = [[ 1 , 2 ], 0] self.app._run_single(app) self.assertEqual(len(self.app.results), 10) self.assertEqual(self.app.results[0], 3) self.assertEqual(sum(self.app.results), 30) def test_results_ordering(self): expected_results = [12, 13, 14, 15, 7, 17, 18, 10, 20, 12] self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) for inc in range(10): t = self.app.add('task') t.job = sum if inc in (4, 7, 9): t.args = [[ 1 , 2, inc ], 0] else: t.args = [[ 20 , 2, inc - 10 ], 0] self.app.run() self.assertEqual(self.app.results, expected_results) def test_single_runner_app_integrated_with_many_subtasks(self): self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) app = self.app.add('app') for _ in range(10): t = app.add('task') t.job = sum t.args = [[ 1 , 2 ], 0] self.app.run() self.assertEqual(len(self.app.results), 10) self.assertEqual(self.app.results[0], 3) self.assertEqual(sum(self.app.results), 30) def test_has_apps_predicate_single(self): self.assertEqual(self.app.queue, []) self.app.queue.append(None) self.assertFalse(self.app.queue_has_apps) def test_has_apps_predicate_empty(self): self.assertEqual(self.app.queue, []) self.assertFalse(self.app.queue_has_apps) def test_has_apps_predicate_all_tasks(self): self.assertEqual(self.app.queue, []) for _ in range(10): self.app.add('task') self.assertEqual(len(self.app.queue), 10) self.assertFalse(self.app.queue_has_apps) def test_has_apps_predicate_all_apps(self): self.assertEqual(self.app.queue, []) for _ in range(10): self.app.add('app') self.assertEqual(len(self.app.queue), 10) self.assertTrue(self.app.queue_has_apps) def test_has_apps_predicate_mixed(self): self.assertEqual(self.app.queue, []) for _ in range(10): self.app.add('task') for _ in range(10): self.app.add('app') self.assertEqual(len(self.app.queue), 20) self.assertTrue(self.app.queue_has_apps) def test_running_mixed_queue_all_apps_integrated(self): self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) self.app.pool = 'serial' for _ in range(10): app = self.app.add('app') for _ in range(10): t = app.add('task') t.job = sum t.args = [[1,2], 0] self.app.run() self.assertEqual(len(self.app.queue), 0) self.assertEqual(sum(self.app.results), 300) def test_running_mixed_queue_mixed_queue_integrated(self): self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) self.app.pool = 'serial' for _ in range(10): t = self.app.add('task') t.job = sum t.args = [[1,2], 0] for _ in range(10): app = self.app.add('app') for _ in range(10): t = app.add('task') t.job = sum t.args = [[1,2], 0] self.app.run() self.assertEqual(len(self.app.queue), 0) self.assertEqual(sum(self.app.results), 330) def test_running_mixed_queue_all_apps_direct(self): self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) self.app.pool = 'serial' for _ in range(10): app = self.app.add('app') for _ in range(10): t = app.add('task') t.job = sum t.args = [[1,2], 0] self.app._run_mixed_queue() self.assertEqual(sum(self.app.results), 300) self.assertEqual(len(self.app.queue), 10) def test_running_mixed_queue_mixed_queue_direct(self): self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) self.app.pool = 'serial' for _ in range(10): t = self.app.add('task') t.job = sum t.args = [[1,2], 0] for _ in range(10): app = self.app.add('app') for _ in range(10): t = app.add('task') t.job = sum t.args = [[1,2], 0] self.app._run_mixed_queue() self.assertEqual(len(self.app.queue), 20) self.assertEqual(sum(self.app.results), 330) def test_running_tasks_ordering_serial(self): self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) self.app.pool = 'serial' for _ in range(5): t = self.app.add('task') t.job = sum t.args = [[1,2], 0] for _ in range(5): t = self.app.add('task') t.job = sum t.args = [[2,2], 0] self.app.run() self.assertEqual(len(self.app.queue), 0) self.assertEqual(self.app.results, [ 3, 3, 3, 3, 3, 4, 4, 4, 4, 4 ]) def test_task_results_ordering_with_apps(self): self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) for _ in range(3): app = self.app.add('app') for _ in range(5): t = app.add('task') t.job = sum t.args = [[1,2], 0] for _ in range(5): t = app.add('task') t.job = sum t.args = [[2,2], 0] self.app.run() self.assertEqual(self.app.results, [ 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4 ]) def test_task_results_ordering_varried_with_apps(self): self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) app = self.app.add('app') t = app.add('task') t.job = sum t.args = [[1, 8], 0] for _ in range(3): app = self.app.add('app') for _ in range(5): t = app.add('task') t.job = sum t.args = [[1,2], 0] for _ in range(5): t = app.add('task') t.job = sum t.args = [[2,2], 0] app = self.app.add('app') t = app.add('task') t.job = sum t.args = [[2, 8], 0] for _ in range(5): t = app.add('task') t.job = sum t.args = [[2,2], 0] self.app.run() self.assertEqual(self.app.results, [ 9, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 10, 4, 4, 4, 4, 4 ]) def test_task_results_lack_of_order(self): self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) for _ in range(5): t = self.app.add('task') t.job = sum t.args = [[1,2], 0] for _ in range(5): t = self.app.add('task') t.job = sum t.args = [[2,2], 0] self.app.run() # there's a small chance that this could randomly fail without # indicating a correctness bug. self.assertNotEqual(self.app.results, [ 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4 ]) def test_task_results_task_and_apps0(self): self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) for _ in range(6): t = self.app.add('task') t.job = sum t.args = [[1,1], 0] for _ in range(3): app0 = self.app.add('app') for _ in range(5): t = app0.add('task') t.job = sum t.args = [[1,2], 0] t = self.app.add('task') t.job = sum t.args = [[1,1], 0] app1 = self.app.add('app') for _ in range(5): t = app1.add('task') t.job = sum t.args = [[2,2], 0] for _ in range(10): t = self.app.add('task') t.job = sum t.args = [[1,1], 0] self.app.run() print(self.app.results) self.assertEqual(self.app.results, [ 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 2, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 2, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 2, 4, 4, 4, 4, 4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 ]) def test_task_results_task_and_apps1(self): self.assertEqual(self.app.queue, []) self.assertEqual(self.app.results, []) for _ in range(6): t = self.app.add('task') t.job = sum t.args = [[1,1], 0] for _ in range(3): app = self.app.add('app') for _ in range(5): t = app.add('task') t.job = sum t.args = [[1,2], 0] t = self.app.add('task') t.job = sum t.args = [[1,1], 0] for _ in range(5): t = app.add('task') t.job = sum t.args = [[2,2], 0] for _ in range(10): t = self.app.add('task') t.job = sum t.args = [[1,1], 0] self.app.run() print(self.app.results) self.assertEqual(self.app.results, [2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 ])
def setUp(self): self.c = Configuration() self.c.runstate = RuntimeStateConfig() self.app = BuildApp(self.c)