def scorer(self):
def from_attr():
if self._scorer is not None:
return self._scorer
else:
raise ValueError
def from_sklearn_metrics():
return sklearn.metrics.get_scorer(self.name)
def from_import():
module_name, scorer_name = self.name.rsplit('.', maxsplit=1)
mod = import_module_from_modname(module_name)
return getattr(mod, scorer_name)
self._scorer = fallback(
from_attr,
from_sklearn_metrics,
from_import,
raiser(
BalletError,
'Could not get a scorer with configuration {}'.format(
self.name)),
)
return self._scorer
def follow(bot: Bot, nodes, args):
max = float(args['max']) if 'max' in args else float('inf')
count = 0
def increment():
nonlocal count
count += 1
stop = raiser(StopIteration)
process = rcompose(
# lambda x: tap(x, lambda: bot.logger.warn('{}._data: \n{}'.format(x, unmask(x._data)))),
lambda x: stop() if x and count >= max else x,
# lambda node: node \
# if bot.suitable(node) \
# else tap(None,lambda: bot.logger.warn('{} not suitable'.format(node))),
lambda node: follow_user(node, bot=bot) \
if node else None,
lambda x: tap(x, increment) if x else None,
)
followed = map(process, nodes)
followed = filter(lambda x: x, followed)
return followed, {}
def evaluate(expression, variables):
names = {
**simpleeval.DEFAULT_NAMES,
'data': variables,
}
def update(x, y):
op.setitem(variables, x, y)
return ''
functions = dict(
**simpleeval.DEFAULT_FUNCTIONS,
update=update,
print=print,
choice=random.choice,
stop=funcy.raiser(Stop),
)
evaluator_class = simpleeval.EvalWithCompoundTypes(names=names,
functions=functions)
try:
res = evaluator_class.eval(expression)
# print(res)
return res
except Exception as e:
raise e from None
def like(bot, nodes, args):
max = float(args['max']) if 'max' in args else float('inf')
count = 0
def increment():
nonlocal count
count += 1
return True
stop = raiser(StopIteration)
process = rcompose(
lambda x: stop() if x and count >= max else x,
# lambda node: node \
# if bot.suitable(node) \
# else tap(None,lambda: bot.logger.warn('{} not suitable'.format(node))),
lambda node: like_media(node, bot=bot) \
if node else None,
lambda x: x and increment() and x,
)
liked = map(process, nodes)
liked = filter(lambda x: x, liked)
return liked, {}
def test_remote_missing_deps_are_correctly_reported(tmp_dir, caplog, mocker,
pkg, msg):
error = RemoteMissingDepsError(FileSystem(), "proto", "proto://", ["deps"])
mocker.patch("dvc.utils.pkg.PKG", pkg)
mocker.patch(
"dvc.cli.parse_args",
return_value=Namespace(
func=raiser(error),
quiet=False,
verbose=True,
),
)
assert main() == 255
expected = (
"URL 'proto://' is supported but requires these missing dependencies: "
"['deps']. ")
if pkg:
expected += ("To install dvc with those dependencies, run:\n\n"
f"\t{msg}\n\n"
"See <https://dvc.org/doc/install> for more info.")
else:
expected += ("\nPlease report this bug to "
"<https://github.com/iterative/dvc/issues>. "
"Thank you!")
assert expected in caplog.text
def message(bot, nodes, args):
try:
max = float(args['max']) if 'max' in args else float('inf')
messages = args['messages']
media = args.get('media_share')
profile = args.get('profile')
hashtag = args.get('hashtag')
except Exception:
bot.logger.error(
'please add all necessary args, {} isn\'t enought'.format(args))
return [], {}
count = 0
events = []
def increment():
nonlocal count
count += 1
return True
def add_event(pair):
node, text = pair
events.append({
'type': 'message',
'metadata': bot.metadata,
'args': {
'message': text,
},
'node': {
'type': 'user',
'username': node.username,
},
'timestamp': int(datetime.utcnow().timestamp())
})
return node
stop = raiser(StopIteration)
listmap = rcompose(map, list)
process = rcompose(
lambda x: stop() if x and count >= max else x,
lambda x: (x, listmap(choice, messages)),
lambda pair: listmap(lambda m: send_message(bot, m, pair[0]), pair[1]),
# lambda x: print(x) or x,
lambda x: listmap(add_event, x),
lambda x: x and x[0],
lambda x: x and increment() and x,
)
result = map(process, nodes)
result = filter(lambda x: x, result)
return result, {'events': events}
def test_collect_optimization_on_stage_name(tmp_dir, dvc, mocker, run_copy):
tmp_dir.dvc_gen("foo", "foo")
stage = run_copy("foo", "bar", name="copy-foo-bar")
# Forget cached stages and graph and error out on collection
dvc._reset()
mocker.patch(
"dvc.repo.Repo.stages",
property(raiser(Exception("Should not collect"))),
)
# Should read stage directly instead of collecting the whole graph
assert dvc.stage.collect("copy-foo-bar") == [stage]
assert dvc.stage.collect_granular("copy-foo-bar") == [(stage, None)]
def test_collect_optimization(tmp_dir, dvc, mocker):
(stage, ) = tmp_dir.dvc_gen("foo", "foo text")
# Forget cached stages and graph and error out on collection
dvc._reset()
mocker.patch(
"dvc.repo.Repo.stages",
property(raiser(Exception("Should not collect"))),
)
# Should read stage directly instead of collecting the whole graph
dvc.stage.collect(stage.path)
dvc.stage.collect_granular(stage.path)
def test_ignore_in_collected_dir_error_is_logged(tmp_dir, caplog, mocker):
error = IgnoreInCollectedDirError(".dvcignore", "dir")
mocker.patch(
"dvc.cli.parse_args",
return_value=Namespace(
func=raiser(error),
quiet=False,
verbose=True,
),
)
assert main() == 255
expected = ".dvcignore file should not be in collected dir path: 'dir'"
assert expected in caplog.text
def test_state_pickle_errors_are_correctly_raised(tmp_dir, caplog, mocker):
path = tmp_dir / "dir" / "test"
mocker.patch(
"dvc.cli.parse_args",
return_value=Namespace(
func=raiser(DiskError(path, "md5s")),
quiet=False,
verbose=True,
),
)
assert main() == 255
assert (f"Could not open pickled 'md5s' cache.\n"
f"Remove the '{path.relative_to(tmp_dir)}' directory "
"and then retry this command.\n"
"See <https://error.dvc.org/pickle> for more information."
) in caplog.text
def comment(bot: Bot, nodes, args):
try:
max = float(args['max']) if 'max' in args else float('inf')
comments = args['comments']
except:
bot.logger.error(
'please add all necessary args, {} isn\'t enought'.format(args))
return [], {}
count = 0
def increment():
nonlocal count
count += 1
stop = raiser(StopIteration)
return_if_suitable = lambda node: node \
if bot.suitable(node, table='commented', specifier=str(comments)) \
else tap(None,lambda: bot.logger.warn('{} not suitable'.format(node)))
discard_if_reached_limit = lambda node: node \
if not bot.reached_limit('comments') \
else tap(None, bot.logger.error('reached commenting daily limit'))
do_comment_from_groups = lambda node: map(
lambda cmnts: do_comment(bot, choice(cmnts), node),
comments) and node \
if node else None
process = rcompose(
lambda x: stop() if x and count >= max else x,
# return_if_suitable,
# discard_if_reached_limit,
do_comment_from_groups,
lambda x: tap(x, increment) if x else None,
)
result = map(process, nodes)
result = filter(lambda x: x, result)
return result, {}
def text(bot, nodes, args):
try:
max = float(args['max']) if 'max' in args else float('inf')
messages = args['messages']
except Exception:
bot.logger.error(
'please add all necessary args, {} isn\'t enought'.format(args))
return [], {}
count = 0
def increment():
bot.total['texts'] += 1
nonlocal count
count += 1
stop = raiser(StopIteration)
return_if_suitable = lambda node: node \
if bot.suitable(node, table='texted', specifier=str(messages)) \
else tap(None,lambda: bot.logger.warning('{} not suitable'.format(node)))
discard_if_reached_limit = lambda node: node \
if not bot.reached_limit('texts') \
else tap(None, bot.logger.error('reached texting daily limit'))
send_msg_from_groups = lambda node: map(
lambda msgs: send_message(bot, choice(msgs), node),
messages) \
if node else []
process = rcompose(
lambda x: stop() if x and count >= max else x,
# return_if_suitable,
discard_if_reached_limit,
send_msg_from_groups,
lambda arr: list(arr)[0] if arr else None,
lambda x: tap(x, increment) if x else None,
)
result = map(process, nodes)
result = filter(lambda x: x, result)
return result, {}
def like(bot, nodes, args):
max = float(args['max']) if 'max' in args else float('inf')
count = 0
events = []
def increment():
nonlocal count
count += 1
return True
def add_event(node: Media):
events.append({
'type': 'like',
'metadata': bot.metadata,
'args': {},
'node': {
'type': 'media',
'url': node.url,
},
'timestamp': int(datetime.utcnow().timestamp())
})
return node
stop = raiser(StopIteration)
process = rcompose(
lambda x: stop() if x and count >= max else x,
# lambda node: node \
# if bot.suitable(node) \
# else tap(None,lambda: bot.logger.warning('{} not suitable'.format(node))),
lambda node: like_media(node, bot=bot) \
if node else None,
lambda x: x and increment() and x,
lambda x: x and add_event(x),
)
liked = map(process, nodes)
liked = filter(lambda x: x, liked)
return liked, { 'events': events }
def unfollow(bot: Bot, nodes, args):
max = float(args['max']) if 'max' in args else float('inf')
count = 0
events = []
def add_event(node: User):
events.append({
'type': 'unfollow',
'metadata': bot.metadata,
'args': {},
'node': {
'type': 'user',
'username': node.username,
},
'timestamp': int(datetime.utcnow().timestamp())
})
return node
def increment():
nonlocal count
count += 1
return True
stop = raiser(StopIteration)
process = rcompose(
lambda x: stop() if x and count >= max else x,
lambda node: unfollow_user(node, bot=bot) \
if node else None,
lambda x: x and increment() and x,
lambda x: x and add_event(x) and x
)
unfollowed = map(process, nodes)
unfollowed = filter(lambda x: x, unfollowed)
return unfollowed, { 'events': events }
def test_cancel_on_error_context_manager(mocker):
executor = ThreadPoolExecutor(max_workers=2, cancel_on_error=True)
spy = mocker.spy(executor, "shutdown")
with pytest.raises(RuntimeError), executor:
future1 = executor.submit(operator.mul, 2, 21)
future2 = executor.submit(time.sleep, 0.1)
future3 = executor.submit(raiser(RuntimeError), "This is an error")
fs = [executor.submit(time.sleep, 0.1) for _ in range(50)]
assert future1.result() == 42
assert future2.result() is None
_ = future3.result()
spy.assert_called_once_with(wait=True, cancel_futures=True)
cancelled = [fut for fut in fs if fut.cancelled()]
others = [fut for fut in fs if not fut.cancelled()]
for fut in others:
assert fut.done()
assert fut.exception() is None
assert len(cancelled) > 20
assert len(others) > 0
def message(bot, nodes, args):
try:
max = float(args['max']) if 'max' in args else float('inf')
messages = args['messages']
media = args.get('media_share')
profile = args.get('profile')
hashtag = args.get('hashtag')
except:
bot.logger.error('please add all necessary args, {} isn\'t enought'.format(args))
return [], {}
count = 0
def increment():
nonlocal count
count += 1
return True
stop = raiser(StopIteration)
send_msg_from_groups = lambda node: list(map(
lambda msgs: send_message(bot, choice(msgs), node),
messages
)) if node else []
process = rcompose(
lambda x: stop() if x and count >= max else x,
send_msg_from_groups,
lambda x: len(x) and x[0] and increment(),
)
result = map(process, nodes)
result = filter(lambda x: x, result)
return result, {}
