def scorer(self): def from_attr(): if self._scorer is not None: return self._scorer else: raise ValueError def from_sklearn_metrics(): return sklearn.metrics.get_scorer(self.name) def from_import(): module_name, scorer_name = self.name.rsplit('.', maxsplit=1) mod = import_module_from_modname(module_name) return getattr(mod, scorer_name) self._scorer = fallback( from_attr, from_sklearn_metrics, from_import, raiser( BalletError, 'Could not get a scorer with configuration {}'.format( self.name)), ) return self._scorer
def follow(bot: Bot, nodes, args): max = float(args['max']) if 'max' in args else float('inf') count = 0 def increment(): nonlocal count count += 1 stop = raiser(StopIteration) process = rcompose( # lambda x: tap(x, lambda: bot.logger.warn('{}._data: \n{}'.format(x, unmask(x._data)))), lambda x: stop() if x and count >= max else x, # lambda node: node \ # if bot.suitable(node) \ # else tap(None,lambda: bot.logger.warn('{} not suitable'.format(node))), lambda node: follow_user(node, bot=bot) \ if node else None, lambda x: tap(x, increment) if x else None, ) followed = map(process, nodes) followed = filter(lambda x: x, followed) return followed, {}
def evaluate(expression, variables): names = { **simpleeval.DEFAULT_NAMES, 'data': variables, } def update(x, y): op.setitem(variables, x, y) return '' functions = dict( **simpleeval.DEFAULT_FUNCTIONS, update=update, print=print, choice=random.choice, stop=funcy.raiser(Stop), ) evaluator_class = simpleeval.EvalWithCompoundTypes(names=names, functions=functions) try: res = evaluator_class.eval(expression) # print(res) return res except Exception as e: raise e from None
def like(bot, nodes, args): max = float(args['max']) if 'max' in args else float('inf') count = 0 def increment(): nonlocal count count += 1 return True stop = raiser(StopIteration) process = rcompose( lambda x: stop() if x and count >= max else x, # lambda node: node \ # if bot.suitable(node) \ # else tap(None,lambda: bot.logger.warn('{} not suitable'.format(node))), lambda node: like_media(node, bot=bot) \ if node else None, lambda x: x and increment() and x, ) liked = map(process, nodes) liked = filter(lambda x: x, liked) return liked, {}
def test_remote_missing_deps_are_correctly_reported(tmp_dir, caplog, mocker, pkg, msg): error = RemoteMissingDepsError(FileSystem(), "proto", "proto://", ["deps"]) mocker.patch("dvc.utils.pkg.PKG", pkg) mocker.patch( "dvc.cli.parse_args", return_value=Namespace( func=raiser(error), quiet=False, verbose=True, ), ) assert main() == 255 expected = ( "URL 'proto://' is supported but requires these missing dependencies: " "['deps']. ") if pkg: expected += ("To install dvc with those dependencies, run:\n\n" f"\t{msg}\n\n" "See <https://dvc.org/doc/install> for more info.") else: expected += ("\nPlease report this bug to " "<https://github.com/iterative/dvc/issues>. " "Thank you!") assert expected in caplog.text
def message(bot, nodes, args): try: max = float(args['max']) if 'max' in args else float('inf') messages = args['messages'] media = args.get('media_share') profile = args.get('profile') hashtag = args.get('hashtag') except Exception: bot.logger.error( 'please add all necessary args, {} isn\'t enought'.format(args)) return [], {} count = 0 events = [] def increment(): nonlocal count count += 1 return True def add_event(pair): node, text = pair events.append({ 'type': 'message', 'metadata': bot.metadata, 'args': { 'message': text, }, 'node': { 'type': 'user', 'username': node.username, }, 'timestamp': int(datetime.utcnow().timestamp()) }) return node stop = raiser(StopIteration) listmap = rcompose(map, list) process = rcompose( lambda x: stop() if x and count >= max else x, lambda x: (x, listmap(choice, messages)), lambda pair: listmap(lambda m: send_message(bot, m, pair[0]), pair[1]), # lambda x: print(x) or x, lambda x: listmap(add_event, x), lambda x: x and x[0], lambda x: x and increment() and x, ) result = map(process, nodes) result = filter(lambda x: x, result) return result, {'events': events}
def test_collect_optimization_on_stage_name(tmp_dir, dvc, mocker, run_copy): tmp_dir.dvc_gen("foo", "foo") stage = run_copy("foo", "bar", name="copy-foo-bar") # Forget cached stages and graph and error out on collection dvc._reset() mocker.patch( "dvc.repo.Repo.stages", property(raiser(Exception("Should not collect"))), ) # Should read stage directly instead of collecting the whole graph assert dvc.stage.collect("copy-foo-bar") == [stage] assert dvc.stage.collect_granular("copy-foo-bar") == [(stage, None)]
def test_collect_optimization(tmp_dir, dvc, mocker): (stage, ) = tmp_dir.dvc_gen("foo", "foo text") # Forget cached stages and graph and error out on collection dvc._reset() mocker.patch( "dvc.repo.Repo.stages", property(raiser(Exception("Should not collect"))), ) # Should read stage directly instead of collecting the whole graph dvc.stage.collect(stage.path) dvc.stage.collect_granular(stage.path)
def test_ignore_in_collected_dir_error_is_logged(tmp_dir, caplog, mocker): error = IgnoreInCollectedDirError(".dvcignore", "dir") mocker.patch( "dvc.cli.parse_args", return_value=Namespace( func=raiser(error), quiet=False, verbose=True, ), ) assert main() == 255 expected = ".dvcignore file should not be in collected dir path: 'dir'" assert expected in caplog.text
def test_state_pickle_errors_are_correctly_raised(tmp_dir, caplog, mocker): path = tmp_dir / "dir" / "test" mocker.patch( "dvc.cli.parse_args", return_value=Namespace( func=raiser(DiskError(path, "md5s")), quiet=False, verbose=True, ), ) assert main() == 255 assert (f"Could not open pickled 'md5s' cache.\n" f"Remove the '{path.relative_to(tmp_dir)}' directory " "and then retry this command.\n" "See <https://error.dvc.org/pickle> for more information." ) in caplog.text
def comment(bot: Bot, nodes, args): try: max = float(args['max']) if 'max' in args else float('inf') comments = args['comments'] except: bot.logger.error( 'please add all necessary args, {} isn\'t enought'.format(args)) return [], {} count = 0 def increment(): nonlocal count count += 1 stop = raiser(StopIteration) return_if_suitable = lambda node: node \ if bot.suitable(node, table='commented', specifier=str(comments)) \ else tap(None,lambda: bot.logger.warn('{} not suitable'.format(node))) discard_if_reached_limit = lambda node: node \ if not bot.reached_limit('comments') \ else tap(None, bot.logger.error('reached commenting daily limit')) do_comment_from_groups = lambda node: map( lambda cmnts: do_comment(bot, choice(cmnts), node), comments) and node \ if node else None process = rcompose( lambda x: stop() if x and count >= max else x, # return_if_suitable, # discard_if_reached_limit, do_comment_from_groups, lambda x: tap(x, increment) if x else None, ) result = map(process, nodes) result = filter(lambda x: x, result) return result, {}
def text(bot, nodes, args): try: max = float(args['max']) if 'max' in args else float('inf') messages = args['messages'] except Exception: bot.logger.error( 'please add all necessary args, {} isn\'t enought'.format(args)) return [], {} count = 0 def increment(): bot.total['texts'] += 1 nonlocal count count += 1 stop = raiser(StopIteration) return_if_suitable = lambda node: node \ if bot.suitable(node, table='texted', specifier=str(messages)) \ else tap(None,lambda: bot.logger.warning('{} not suitable'.format(node))) discard_if_reached_limit = lambda node: node \ if not bot.reached_limit('texts') \ else tap(None, bot.logger.error('reached texting daily limit')) send_msg_from_groups = lambda node: map( lambda msgs: send_message(bot, choice(msgs), node), messages) \ if node else [] process = rcompose( lambda x: stop() if x and count >= max else x, # return_if_suitable, discard_if_reached_limit, send_msg_from_groups, lambda arr: list(arr)[0] if arr else None, lambda x: tap(x, increment) if x else None, ) result = map(process, nodes) result = filter(lambda x: x, result) return result, {}
def like(bot, nodes, args): max = float(args['max']) if 'max' in args else float('inf') count = 0 events = [] def increment(): nonlocal count count += 1 return True def add_event(node: Media): events.append({ 'type': 'like', 'metadata': bot.metadata, 'args': {}, 'node': { 'type': 'media', 'url': node.url, }, 'timestamp': int(datetime.utcnow().timestamp()) }) return node stop = raiser(StopIteration) process = rcompose( lambda x: stop() if x and count >= max else x, # lambda node: node \ # if bot.suitable(node) \ # else tap(None,lambda: bot.logger.warning('{} not suitable'.format(node))), lambda node: like_media(node, bot=bot) \ if node else None, lambda x: x and increment() and x, lambda x: x and add_event(x), ) liked = map(process, nodes) liked = filter(lambda x: x, liked) return liked, { 'events': events }
def unfollow(bot: Bot, nodes, args): max = float(args['max']) if 'max' in args else float('inf') count = 0 events = [] def add_event(node: User): events.append({ 'type': 'unfollow', 'metadata': bot.metadata, 'args': {}, 'node': { 'type': 'user', 'username': node.username, }, 'timestamp': int(datetime.utcnow().timestamp()) }) return node def increment(): nonlocal count count += 1 return True stop = raiser(StopIteration) process = rcompose( lambda x: stop() if x and count >= max else x, lambda node: unfollow_user(node, bot=bot) \ if node else None, lambda x: x and increment() and x, lambda x: x and add_event(x) and x ) unfollowed = map(process, nodes) unfollowed = filter(lambda x: x, unfollowed) return unfollowed, { 'events': events }
def test_cancel_on_error_context_manager(mocker): executor = ThreadPoolExecutor(max_workers=2, cancel_on_error=True) spy = mocker.spy(executor, "shutdown") with pytest.raises(RuntimeError), executor: future1 = executor.submit(operator.mul, 2, 21) future2 = executor.submit(time.sleep, 0.1) future3 = executor.submit(raiser(RuntimeError), "This is an error") fs = [executor.submit(time.sleep, 0.1) for _ in range(50)] assert future1.result() == 42 assert future2.result() is None _ = future3.result() spy.assert_called_once_with(wait=True, cancel_futures=True) cancelled = [fut for fut in fs if fut.cancelled()] others = [fut for fut in fs if not fut.cancelled()] for fut in others: assert fut.done() assert fut.exception() is None assert len(cancelled) > 20 assert len(others) > 0
def message(bot, nodes, args): try: max = float(args['max']) if 'max' in args else float('inf') messages = args['messages'] media = args.get('media_share') profile = args.get('profile') hashtag = args.get('hashtag') except: bot.logger.error('please add all necessary args, {} isn\'t enought'.format(args)) return [], {} count = 0 def increment(): nonlocal count count += 1 return True stop = raiser(StopIteration) send_msg_from_groups = lambda node: list(map( lambda msgs: send_message(bot, choice(msgs), node), messages )) if node else [] process = rcompose( lambda x: stop() if x and count >= max else x, send_msg_from_groups, lambda x: len(x) and x[0] and increment(), ) result = map(process, nodes) result = filter(lambda x: x, result) return result, {}