def styled(self) -> Styled:
if self is Coordinator.MonitorOption.status:
style = Format().green()
elif self is Coordinator.MonitorOption.quit:
style = Format().red()
else:
style = Format().blue()
return CustomStyled(text=self.option_text, style=style)
def display_content(self, redis: Union[Redis, Pipeline]) -> str:
description = Styleds(parts=[
CustomStyled(self.display_name, Format().cyan()),
CustomStyled(f' {self.display_summary}', Format().blue()),
CustomStyled(f'\n{self.display_metadata(redis=redis)}', Format().cyan()),
])
df = self.get_data_frame(redis=redis)
with pd.option_context('display.max_rows', None, 'display.max_columns', df.shape[1]):
return f'{description.styled}\n{df}'
def click_command(targets):
elements = [
e for t in ElementTarget if not targets or t.value in targets
for e in t.get_elements(redis=self.context.redis)
]
return [
Styleds(parts=[
CustomStyled(e.display_name,
Format().cyan()),
CustomStyled(f' {e.display_summary}',
Format().blue()),
]).styled for e in elements
]
def start(self):
assert not self.running
if self.should_define:
for definition in self.definitions:
self.user.present_message(
Format().blue()(f'Defining {definition.identifier}'))
self.define_structure(element=definition)
self.running = True
should_print_menu = True
if self.user.interactive:
self.user.present_message(
Format().green()('Starting in interactive mode.'))
next_update = time.time() + 1
while True:
if next_update < time.time():
should_print_menu = self.update_listeners(
) or should_print_menu
should_print_menu = self.update_subprocesses(
) or should_print_menu
next_update = time.time() + 1
command_result = self.try_command()
sys.stdout.flush()
sys.stderr.flush()
if command_result is True:
should_print_menu = True
elif command_result is False:
break
if self.user.interactive:
try:
if should_print_menu:
self.user.present_message('\n'.join(
c.display_styled.styled for c in self.commands
if c.can_run))
print('µ—>', end=' ')
sys.stdout.flush()
wait_read(sys.stdin.fileno(),
timeout=0.01,
timeout_exc=MicraInputTimeout)
user_command = sys.stdin.readline().strip()
if user_command:
self.commands_to_run.append(user_command)
except MicraInputTimeout:
should_print_menu = False
self.running = False
def try_command(self) -> Optional[bool]:
command = None
if self.commands_to_run:
command = self.commands_to_run.pop(0)
from_queue = False
elif self.listeners:
try:
command = self.queue.get(block=not self.user.interactive)
from_queue = True
except Empty:
pass
elif not self.subprocesses and not self.user.interactive:
self.user.present_message(Format().yellow()('Nothing to do.'))
return False
if command is None:
return None
try:
self.run_command(command=command)
except (KeyboardInterrupt, SystemExit):
raise
except MicraQuit:
return False
except Exception as e:
self.user.present_message(
message=f'Error running command: {command}', error=e)
if self.pdb_enabled:
pdb.post_mortem()
if from_queue:
self.queue.task_done()
return True
def stop_listener(self, thread_id: int):
items = list(
filter(lambda i: i[1].ident == thread_id, self.listeners.items()))
assert len(items) <= 1
for listener, thread in items:
if not listener.stop():
self.user.present_message(Format().red()(
f'Cannot stop listener {listener.name} ({thread.ident}).'))
def start_listener(self, listener: Listener, force: bool = False):
if not self.should_listen and not force:
return
thread = threading.Thread(target=listener.runner)
thread.setDaemon(True)
thread.start()
self.user.present_message(Format().cyan()(
f'Starting listener {listener.name} ({thread.ident}).'))
self.listeners[listener] = thread
def update_subprocesses(self) -> bool:
updated = False
for pid in sorted(self.subprocesses.keys()):
try:
os.getpgid(pid)
except ProcessLookupError:
self.user.present_message(
Format().cyan()(f'Subprocess {pid} ended.'))
del self.subprocesses[pid]
updated = True
return updated
def update_listeners(self) -> bool:
updated = False
for listener, thread in list(self.listeners.items()):
if not thread.is_alive():
self.user.present_message(
Format().cyan()(f'Listener {thread.ident} ended.'))
try:
listener.clean()
except (KeyboardInterrupt, SystemExit):
raise
except Exception as e:
self.user.present_message(
f'An error occurred while cleaning listener {listener.name} ({thread.ident})',
error=e)
del self.listeners[listener]
updated = True
return updated
def bot_remove(scrape: Scrape, should_remove_configuration: bool,
should_remove_script_link: bool, project: str):
user = UserInteractor()
scrape.configure_user_interactivity(user=user)
project_path = Path(__file__).parent / 'bots' / project
if not user.present_confirmation(
f'Remove the {project} bot and all of its files at {project_path}',
default_response=True):
return
paths = []
if project_path.exists():
paths.append(project_path)
if should_remove_configuration:
configuration_path = Path(
__file__).parent / 'configurations' / f'{project}_configuration'
if configuration_path.exists():
paths.append(configuration_path)
if should_remove_script_link:
scripts_path = Path(__file__).parent / 'output' / 'python' / 'scripts'
for root, _, files in os.walk(str(scripts_path), topdown=False):
for file in files:
path = Path(root) / file
if path.is_symlink() and project_path.absolute(
) in path.resolve().parents:
paths.append(path)
if paths:
base_path = Path(__file__).parent
user.present_message(
Styleds([
CustomStyled(
f'Discovered {len(paths)} directories and files related to bot ',
Format().yellow()),
CustomStyled(project,
Format().yellow().bold()),
CustomStyled(
f'\n{" ".join([str(p.relative_to(base_path)) for p in paths])}',
Format().red()),
CustomStyled(f'\nPlease remove them manually',
Format().yellow()),
]).styled)
else:
user.present_message(
Styleds([
CustomStyled(f'No files discovered related to bot ',
Format().yellow()),
CustomStyled(project,
Format().yellow().bold()),
]).styled)
def add_subprocess(self, pid: int, command: str):
self.subprocesses[pid] = command
self.user.present_message(
Format().cyan()(f'Adding subprocess {command} ({pid}).'))
def api_test_rule(data_dragon: DataDragon, test_url: Optional[str], credential_url: Optional[str], rule_url: Optional[str], user_id: Optional[str], rule_id: Optional[str], channel: Optional[str], from_date: Optional[str], to_date: Optional[str], granularity: Optional[str]):
assert channel is not None or test_url is not None or rule_url is not None or rule_id is not None, 'One of --channel --test-url, --rule-url, or --rule-id is required'
data_dragon.configure_encryption()
if test_url is None and channel is not None:
test_path = Path(__file__).parent.parent / 'input' / 'test' / 'rule' / f'test_{channel}.json'
test_url = str(test_path) if test_path.exists() else None
test_configuration = locator_factory(url=test_url).get().decode() if test_url is not None else {}
test = io_pruned_structure({
**json.loads(test_configuration),
**({'credential_url': credential_url} if credential_url is not None else {}),
**({'rule_id': rule_id} if rule_id is not None else {}),
**({'user_id': user_id} if user_id is not None else {}),
**({'rule_url': rule_url} if rule_url is not None else {}),
**({'channel': channel} if channel is not None else {}),
**({'from_date': from_date} if from_date is not None else {}),
**({'to_date': to_date} if to_date is not None else {}),
**({'granularity': granularity} if granularity is not None else {}),
})
test_format = Format().bold().cyan()
data_dragon.user.present_message(test_format(f'††† Running test configuration\n{json.dumps(test, indent=2)}'))
if 'channel' in test:
if 'credential_url' not in test:
test['credential_url'] = f'alias://credentials/test/test_{test["channel"]}.{"zip" if test["channel"] == "apple_search_ads" else "json"}'
if 'rule_url' not in test and 'rule_id' not in test:
test['rule_url'] = str(Path(__file__).parent.parent / 'input' / 'test' / 'rule' / f'test_{test["channel"]}_rule.json')
run_context = APIRunContext(data_dragon=data_dragon)
password = data_dragon.generate_password()
if 'user_id' not in test:
user = run_context.run_api_command(
command=['user', 'create'],
command_args=[
'-q',
'-t',
'-w', password,
'{"local":{"email":"*****@*****.**"},"name":"TestUser"}',
],
load_output=True
)
data_dragon.user.present_message(test_format(f'††† Created test user {user["_id"]}'))
else:
user = {'_id': test['user_id']}
if 'credential_url' in test:
if channel == 'apple_search_ads':
credential_json = '{"name":"AppleTestCredential","target":"apple_search_ads"}'
certificate_locator = locator_factory(url=test['credential_url'])
certificate_locator.safe = False
certificate_contents = certificate_locator.get()
certificate_fd, certificate_file_path = tempfile.mkstemp(prefix=str(Path(__file__).parent.parent / 'output' / 'temp' / 'test_'))
try:
os.write(certificate_fd, certificate_contents)
os.close(certificate_fd)
credential = run_context.run_api_command(
command=['credential', 'create'],
command_args=[
'-q',
'-t',
'-u', user['_id'],
'-c', certificate_file_path,
credential_json,
],
load_output=True
)
finally:
Path(certificate_file_path).unlink()
else:
credential_json = locator_factory(url=test['credential_url']).get().decode()
credential = run_context.run_api_command(
command=['credential', 'create'],
command_args=[
'-q',
'-t',
'-u', user['_id'],
credential_json,
],
load_output=True
)
data_dragon.user.present_message(test_format(f'††† Created test credential {credential["_id"]}'))
else:
credential = None
if 'rule_url' in test and 'rule_id' not in test:
rule_locator = locator_factory(url=test['rule_url'])
rule_json = rule_locator.get().decode()
rule = run_context.run_api_command(
command=['rule', 'create'],
command_args=[
'-q',
'-t',
*([
'-u', user['_id'],
'-c', credential['path'],
] if credential is not None else []),
rule_json,
],
load_output=True
)
data_dragon.user.present_message(test_format(f'††† Created {channel} test rule {rule["_id"]}'))
elif 'rule_id' in test:
rule = {'_id': test['rule_id']}
else:
rule = {}
data_dragon.user.present_message(test_format(f'††† Performing live run of {channel} test rule {rule["_id"]}'))
run_overrides = [
*(['-g', test['granularity']] if 'granularity' in test else []),
*(['-f', test['from_date']] if 'from_date' in test else []),
*(['-t', test['to_date']] if 'to_date' in test else []),
]
run_context.run_api_command(
command=['rule', 'run'],
command_args=[
*run_overrides,
'--allow-non-dry-run',
rule['_id'],
],
)
data_dragon.user.present_message(test_format(f'††† Retrieving live actions from {channel} test rule {rule["_id"]}'))
history = run_context.run_api_command(
command=['rule', 'show-history'],
command_args=[
'-q',
rule['_id'],
],
load_output=True
)
actions = list(filter(lambda h: h['historyType'] == 'action', history))
assert actions, 'No actions in test rule history'
def check_apple_search_ads_actions(actions: List[Dict[str, any]]):
for action in actions:
match = re.search(r'from ([^ ]+) to ([^ ]+)', action['actionDescription'])
action['adjustmentFrom'] = float(match.group(1))
action['adjustmentTo'] = float(match.group(2))
assert action['adjustmentFrom'] != action['adjustmentTo'], f'No adjustment made for action {action}'
if channel == 'apple_search_ads':
check_apple_search_ads_actions(actions)
data_dragon.user.present_message(test_format(f'††† Clearing live actions for {channel} test rule {rule["_id"]}'))
run_context.run_api_command(
command=['rule', 'clear-history'],
command_args=[
rule['_id'],
],
)
data_dragon.user.present_message(test_format(f'††† Performing dry run of {channel} test rule {rule["_id"]}'))
run_context.run_api_command(
command=['rule', 'run'],
command_args=[
'-g', 'DAILY',
'-f', '2020-05-01',
'-t', '2020-05-07',
rule['_id'],
],
)
data_dragon.user.present_message(test_format(f'††† Retrieving dry run actions from {channel} test rule {rule["_id"]}'))
dry_run_history = run_context.run_api_command(
command=['rule', 'show-history'],
command_args=[
'-q',
rule['_id'],
],
load_output=True
)
dry_run_actions = list(filter(lambda h: h['historyType'] == 'action', dry_run_history))
if channel == 'apple_search_ads':
check_apple_search_ads_actions(dry_run_actions)
def check_live_and_dry_actions(actions: List[Dict[str, any]], dry_run_actions: List[Dict[str, any]]):
def check_adjustment_difference(actual: any, expected: any):
if type(actual) is float and type(expected) is float:
return actual == expected or (abs(expected - actual) < 0.001 and abs((expected - actual) / expected)) < 0.001
return actual == expected
assert len(actions) == len(dry_run_actions), f'{len(actions)} action count does not match {len(dry_run_actions)} dry run action count'
for action in actions:
dry_actions = list(filter(lambda a: a['targetType'] == action['targetType'] and a['targetID'] == action['targetID'], dry_run_actions))
assert dry_actions, f'No matching dry run action found for target {action["targetType"]} {action["targetID"]}'
assert check_adjustment_difference(dry_actions[0]['adjustmentFrom'], action['adjustmentTo']), f'Dry run found {action["targetType"]} {action["targetID"]} in state {dry_actions[0]["adjustmentFrom"]}, which does not match live adjusment to state {action["adjustmentTo"]}'
check_live_and_dry_actions(actions, dry_run_actions)
adjustment_output = '\n'.join(f'{a["targetType"]} {a["targetID"]} {a["adjustmentType"]} from {a["adjustmentFrom"]} to {a["adjustmentTo"]}' for a in actions)
data_dragon.user.present_message(test_format(f'††† Finished test with {channel} test rule {rule["_id"]}\nConfiguration:\n{json.dumps(test, indent=2)}\nAdjustments:\n{adjustment_output}'))
def bot_install(scrape: Scrape,
project: str,
should_run_install: bool = True,
should_copy_configuration: bool = True,
should_link_script: bool = True,
nickname: Optional[str] = None,
url: Optional[str] = None,
_dry_run: bool = False):
assert url is None, 'Installing from a URL is not yet supported'
if not nickname:
nickname = project
project_path = Path(__file__).parent / 'bots' / project
user = UserInteractor()
scrape.configure_user_interactivity(user=user)
if not _dry_run and not project_path.exists():
user.present_message(
Format().red()(f'Project path {project_path} does not exist'))
raise click.Abort()
configuration_path = Path(
__file__).parent / 'configurations' / f'{project}_configuration'
if should_copy_configuration and configuration_path.exists():
user.present_message(Format().red()(
f'Configuration path {configuration_path} already exists'))
raise click.Abort()
script_path = Path(
__file__).parent / 'output' / 'python' / 'scripts' / f'{nickname}.py'
if should_link_script and script_path.exists():
user.present_message(
Format().red()(f'Script link path {script_path} already exists'))
raise click.Abort()
if _dry_run:
return
if should_run_install:
install_path = project_path / 'install.sh'
if install_path.exists():
subprocess.call(args=['./install.sh'],
cwd=str(install_path.parent))
user.present_message(
CustomStyled(f'\nRan the install script at \n{install_path}',
Format().yellow()).styled)
if should_copy_configuration:
configuration_source_path = project_path / f'{project}_configuration'
if configuration_source_path.exists():
shutil.copytree(str(configuration_source_path),
str(configuration_path))
configuration_path.chmod(0o700)
user.present_message(
CustomStyled(
f'\nAdded a configuration directory at\n> {configuration_path}\nSet local configuration options here',
Format().blue()).styled)
if should_link_script:
script_source_path = project_path / f'{project}_maneuver.py'
if script_source_path.exists():
relative_path = os.path.relpath(str(script_source_path.absolute()),
str(script_path.parent.absolute()))
script_path.symlink_to(relative_path)
user.present_message(
CustomStyled(
f'\nAdded a script symlink at\n{script_path} to\n> {script_source_path}\nImplement the bot\'s maneuevers here',
Format().cyan()).styled)
user.present_message(
Styleds([
CustomStyled(f'\nRun the ',
Format().green()),
CustomStyled(project,
Format().green().bold()),
CustomStyled(
f' bot with the command\n> {Path(__file__).parent.resolve() / "run.sh"} -i bot start {project} default',
Format().green()),
]).styled)
def init(ctx: any, scrape: Scrape, project: str, nickname: Optional[str],
prefix: Optional[str], type_prefix: Optional[str],
template_directory: Optional[str], template_prefix: Optional[str],
template_type_prefix: Optional[str], should_install: bool):
if not nickname:
nickname = project
if not prefix:
prefix = project
if not type_prefix:
type_prefix = ''.join(s.title() for s in project.split('_'))
if not template_directory:
template_directory = str(
Path(__file__).parent / 'bots' / 'raspador_template')
if not template_prefix:
template_prefix = 'raspador_template'
if not template_type_prefix:
template_type_prefix = 'RaspadorTemplate'
# Make sure that installation will succeed
if should_install:
ctx.invoke(bot_install,
project=project,
nickname=nickname,
_dry_run=True)
project_path = Path(__file__).parent / 'bots' / project
user = UserInteractor()
scrape.configure_user_interactivity(user=user)
if project_path.exists():
user.present_message(
Format().red()(f'Project path {project_path} already exists'))
raise click.Abort()
template_path = Path(template_directory)
if not template_path.exists():
user.present_message(
Format().red()(f'Template path {template_path} does not exist'))
raise click.Abort()
template_name = template_path.name
shutil.copytree(str(template_path), str(project_path))
for root, directories, files in os.walk(str(project_path), topdown=False):
if Path(root).name == '__pycache__':
continue
for file in files:
template_file_path = Path(root) / file
if template_file_path.suffix == '.pyc':
template_file_path.unlink()
continue
if not template_file_path.name.startswith(
template_prefix
) and template_file_path.name != '__init__.py':
continue
template_text = template_file_path.read_text()
file_path = template_file_path.parent / template_file_path.name.replace(
template_name, project)
template_file_path.rename(file_path)
text = template_text.replace(template_prefix, prefix).replace(
template_type_prefix, type_prefix)
file_path.write_text(text)
for directory in directories:
template_directory_path = Path(root) / directory
if not template_directory_path.name.startswith(template_prefix):
continue
directory_path = template_directory_path.parent / template_directory_path.name.replace(
template_name, project)
template_directory_path.rename(directory_path)
user.present_message(
Styleds([
CustomStyled(f'Created the ',
Format().green()),
CustomStyled(project,
Format().green().bold()),
CustomStyled(f' project at\n{project_path}',
Format().green()),
]).styled)
if should_install:
ctx.invoke(
bot_install,
project=project,
nickname=nickname,
)
else:
user.present_message(
Styleds([
CustomStyled(f'Install the ',
Format().green()),
CustomStyled(project,
Format().green().bold()),
CustomStyled(
f' bot with the command\n> {Path(__file__).parent.resolve() / "run.sh"} bot install {project}',
Format().green()),
]).styled)