async def add(self, parsed, message):
"""The add command adds a proposition, and it's response is the status message"""
# Pull the params out
try:
parsedAdd = self.addParser.parse_args(parsed.parameters)
except:
raise errors.UserError(
parsed.parameters,
"There was an error with the add command's parameters. Check `_DDD help -c add` for help."
)
#await self.logger.error("There was an error with the add command's parameters. Check `_DDD help -c add` for help.", message.channel)
#return
# Add the prop to the DB
# I wish there was a way that wasn't if/else chains...
propType = parsedAdd.type
propAction = None
if propType == "kick":
# Check that the target exists
uid = utils.mentionToId(parsedAdd.target[0])
if not message.server.get_member(uid):
raise errors.UserError(
parsedAdd.target[0],
"Could not find member `%s`." % parsedAdd.target[0])
#await self.logger.error("Could not find member `%s`." % parsedAdd.target[0],message.channel)
# Instantiate action
propAction = action.DDDAction.KickAction(message, message.author,
parsedAdd.target[0])
elif propType == "ban":
# Check that the target exists
uid = utils.mentionToId(parsedAdd.target[0])
if not message.server.get_member(uid):
raise errors.UserError(
parsedAdd.target[0],
"Could not find member `%s`." % parsedAdd.target[0])
#await self.logger.error("Could not find member `%s`. Is the user offline?" % parsedAdd.target[0])
propAction = action.DDDAction.BanAction(
message, message.author, parsedAdd.target[0],
utils.toSeconds(parsedAdd.duration[0]))
else:
raise errors.UserError(
propType,
"There was an error with the prop type %s. Check `_DDD help -c add` for a list of prop types"
% propType)
#await self.logger.error("There was an error with the prop type %s. Check `_DDD help -c add` for a list of prop types" % propType, message.channel)
#return
# Print status message for prop
# NOTE: As the status message id is added to the prop, this must be done before storing the prop
# If you do not, it default's to the command message
propAction.messageId = await self.logger.status(
propAction, message.channel)
# Store prop
await self.db.store_action(propAction)
async def help(self, parsed, message):
"""The help command prints out help info on commands"""
helpMessage = None
if not parsed.helpCommand:
# General help
helpMessage = "%s\n%s" % (self.topParser.format_help(),
helpMessages.genGeneralHelp(
message.server, utils))
elif parsed.helpCommand == "add":
helpMessage = "%s\n%s" % (self.addParser.format_help(),
helpMessages.genAddHelp(message.server))
#NOTE: the remove command has been removed (hahaha)
#NOTE: the status command has been removed
#NOTE: Both of these commands required the internalID field, which I decided not to implement
elif parsed.helpCommand == "admin":
helpMessage = "%s\n%s" % (self.topSubparser_admin.format_help(),
helpMessages.adminHelp)
elif parsed.helpCommand == "about":
helpMessage = "%s\n%s" % (self.topSubparser_about.format_help(),
helpMessages.aboutHelp)
else:
raise errors.UserError(
parsed.helpCommand,
"Help for command `%s` not found" % parsed.helpCommand)
#await self.logger.log("Help for command `%s` not found" % parsed.helpCommand, message.channel)
await self.logger.log(helpMessage, message.channel)
def extract_name_and_suite(test_name: str) -> str:
# Test names always have a suite name and test name, separated by '.'s. They
# may also have extra slash-separated parts on the beginning and the end,
# for parameterised tests.
for part in test_name.split('/'):
if '.' in part:
return part
raise errors.UserError(f"Couldn't parse test name: {test_name}")
def get_package_dependencies(package_name):
dist = pkg_resources.working_set.by_key.get(package_name)
if not dist:
raise errors.UserError("Package %s is not installed" % package_name)
deps = (pkg_resources.working_set.by_key.get(req.key)
for req in dist.requires())
deps = ((dist.key, PackageInfo(dist.key, dist.version, dist.location))
for dist in deps if dist)
deps = dict(deps)
return deps
def disable_test(test_id: str, cond_strs: List[str]):
conds = conditions.parse(cond_strs)
# If the given ID starts with "ninja:", then it's a full test ID. If not,
# assume it's a test name, and transform it into a query that will match the
# full ID.
if not test_id.startswith('ninja:'):
test_id = f'ninja://.*/{extract_name_and_suite(test_id)}(/.*)?'
test_name, filename = resultdb.get_test_metadata(test_id)
test_name = extract_name_and_suite(test_name)
# Paths returned from ResultDB look like //foo/bar, where // refers to the
# root of the chromium/src repo.
full_path = os.path.join(SRC_ROOT, filename.lstrip('/'))
_, extension = os.path.splitext(full_path)
extension = extension.lstrip('.')
if extension == 'html':
full_path = expectations.search_for_expectations(full_path, test_name)
try:
with open(full_path, 'r') as f:
source_file = f.read()
except FileNotFoundError as e:
raise errors.UserError(
f"Couldn't open file {filename}. Either this test has moved file very"
+ "recently, or your checkout isn't up-to-date.") from e
if extension == 'cc':
disabler = gtest.disabler
elif extension == 'html':
disabler = expectations.disabler
else:
raise errors.UserError(
f"Don't know how to disable tests for this file format ({extension})"
)
new_content = disabler(test_name, source_file, conds)
with open(full_path, 'w') as f:
f.write(new_content)
async def admin(self, parsed, message):
"""An admin command to set server-wide values"""
# Check to see if the user is an admin
if not message.author.server_permissions.administrator:
raise errors.UserError(
None,
"You have insufficient permissions to execute this command. You must have the `Administrator` permission to execute this command."
)
#await self.logger.error("You have insufficient permissions to execute this command. You must have the `Administrator` permission to execute this command.",message.channel)
quorum = parsed.quorum
delay = parsed.delay
# Pre processing
if delay:
delay = utils.toSeconds(delay)
if quorum:
quorum = int(quorum) / 100
if (not quorum) and (not delay):
raise errors.UserError(
None,
"Please specify a quorum, a delay, or both. See `_DDD help -c admin` for help"
)
#await self.logger.error("Please specify a quorum, a delay, or both. See `_DDD help -c admin` for help",message.channel)
# Check if the quorum is in range
if quorum and (quorum > 1 or quorum < 0.01):
raise errors.UserError(
quorum,
"There was an error with the admin command's quorum parameter. See `_DDD help -c admin` for help."
)
#await self.logger.error("There was an error with the admin command's quorum parameter. See `_DDD help -c admin` for help.", message.channel)
# Execute the update and wait for status
status = await self.serverWrapper.updateServerData(
message.server, quorum, delay)
if status:
await self.logger.success("Successfully changed server values",
message.channel)
async def fetch_activity(self, request):
if request.id == None:
raise errors.Unauthorized("A valid token is required")
activity_id = int(request.rel_url.query["itemid"])
if activity_id >= len(self.activities_df):
raise errors.UserError("this id is too big")
with open(get_path("../../interactions.csv"), "a") as output_csv:
writer = csv.writer(output_csv)
writer.writerow([request.id, activity_id, 1, int(time.time())])
output = self.activities_df.iloc[activity_id].to_json()
return web.Response(text=output)
async def middleware(request):
request.id = None
jwt_token = request.headers.get("Authorization", None)
if jwt_token:
try:
if jwt_token in self.queries.EXPIRED_TOKENS:
raise errors.UserError("blacklisted token")
payload = jwt.decode(
jwt_token,
self.queries.JWT_SECRET,
algorithms=[self.queries.JWT_ALGORITHM],
)
request.id = payload["id"]
except (jwt.DecodeError, jwt.ExpiredSignatureError):
pass
return await handler(request)
def FromText(cls, text: str, atoms=OPENCL_ATOMS) -> 'GreedyAtomizer':
"""Instantiate and an atomizer from a corpus text.
Args:
text: Text corpus
atoms: A list of multi-character tokens.
Returns:
An atomizer instance.
"""
if not atoms:
raise errors.UserError('No atoms specified')
# Instantiate a greedy atomizer using the full vocabulary.
full_vocab = dict(zip(atoms, range(len(atoms))))
c = GreedyAtomizer(full_vocab, determine_chars=True)
# Derive the subset of the vocabulary required to encode the given text.
tokens = sorted(list(set(c.TokenizeString(text))))
vocab_subset = dict(zip(tokens, range(len(tokens))))
# Return a new atomizer using the subset vocabulary.
return GreedyAtomizer(vocab_subset)
def parse(condition_strs: List[str]) -> Condition:
"""Parse a list of condition strings, as passed on the command line.
Each element of condition_strs is a set of Terminal names joined with '&'s.
The list is implicitly 'or'ed together.
"""
# When no conditions are given, this is taken to mean "always".
if not condition_strs:
return ALWAYS
try:
return op_of('or', [
op_of('and', [get_term(x.strip()) for x in cond.split('&')])
for cond in condition_strs
])
except ValueError as e:
# Catching the exception raised by get_term.
valid_conds = '\n'.join(sorted(f'\t{term.name}' for term in TERMINALS))
raise errors.UserError(f"{e}\nValid conditions are:\n{valid_conds}")
async def handleMessage(self, message):
"""Routes and executes command methods"""
if message.content[:4] != '_DDD':
return
try:
parsedCommand = self.topParser.parse_args(
message.content[4:].split())
except: # User error
#NOTE: Better error messages (e.g the help messages from the argparse library) aren't possible
# due to the design of the argparse library not returning those messages in the exception :(
#await self.logger.error("There was an error in your command. See `_DDD help` for help.", message.channel)
raise errors.UserError(
message.content[4:].split(),
"There was an error in your command. See `_DDD help` for help."
)
route = {
"add": self.add,
"help": self.help,
"admin": self.admin,
"about": self.about
}
await route[parsedCommand.command](parsedCommand, message)
def get_test_metadata(test_id: str) -> Tuple[str, str]:
"""Fetch test metadata from ResultDB.
Args:
test_id: The full ID of the test to fetch. For Chromium tests this will
begin with "ninja://"
Returns:
A tuple of (test name, filename). The test name will for example have the
form SuitName.TestName for GTest tests. The filename is the location in the
source tree where this test is defined.
"""
history = get_test_result_history(test_id, 1)
if 'entries' not in history:
raise errors.UserError(
f"ResultDB query couldn't find test with ID: {test_id}")
# TODO: Are there other statuses we need to exclude?
if history['entries'][0]['result'].get('status', '') == 'SKIP':
# Test metadata isn't populated for skipped tests. Ideally this wouldn't be
# the case, but for now we just work around it.
# On the off-chance this test is conditionally disabled, we request a bit
# more history in the hopes of finding a test run where it isn't skipped.
history = get_test_result_history(test_id, 10)
for entry in history['entries'][1:]:
if entry['result'].get('status', '') != 'SKIP':
break
else:
raise errors.UserError(
f"Unable to fetch metadata for test {test_id}, as the last 10 runs "
+
"in ResultDB have status 'SKIP'. Is the test already disabled?"
)
else:
entry = history['entries'][0]
try:
inv_name = entry['result']['name']
except KeyError as e:
raise errors.InternalError(
f"Malformed GetTestResultHistory response: no key {e}") from e
# Ideally GetTestResultHistory would return metadata so we could avoid making
# this second RPC.
result = get_test_result(inv_name)
try:
name = result['testMetadata']['name']
loc = result['testMetadata']['location']
repo, filename = loc['repo'], loc['fileName']
except KeyError as e:
raise errors.InternalError(
f"Malformed GetTestResult response: no key {e}") from e
if repo != 'https://chromium.googlesource.com/chromium/src':
raise errors.UserError(
f"Test is in repo '{repo}', this tool can only disable tests in " +
"chromium/chromium/src")
return name, filename
def deploy_pkg(egg_or_req, python=sys.executable, package_base=None,
console_script_base=None, pypi_index_url=None):
""" Deploys a package to a versioned, isolated virtual environment in a package
base directory. Maintains a 'current' symlink to the last version that
was deployed.
Parameters
----------
egg_or_req : `str`
Either a string requirement (eg, 'foo==1.2') or the path to an egg file.
python : `str` or `pyenv.PythonInstallation`
Python interpreter from which the python interpreter is derived
package_base: `str`
Path to package base directory under which the versioned virtual environment
is created.
console_script_base: `str`
Path to directory into which all the console scripts for the deployed package
are symlinked.
"""
package_base = package_base or CONFIG.deploy_path
console_script_base = console_script_base or CONFIG.deploy_bin
try:
req = pkg_resources.Requirement.parse(egg_or_req)
except ValueError:
req = pkg_resources.Distribution.from_filename(egg_or_req).as_requirement()
version = next(ver for op, ver in req.specs if op == '==')
package_dir = os.path.join(package_base, req.project_name)
if not isinstance(python, pyenv.PythonInstallation):
python = pyenv.PythonInstallation(python)
pyenv_dir = os.path.join(package_dir, version,
python.short_version(3, prefix="PYTHON_"))
if os.path.isdir(pyenv_dir):
raise errors.UserError("Package [%s] version [%s] is already installed at: %s"
% (req.project_name, version, pyenv_dir))
# Set umask for file creation: 0022 which is 'rwx r.x r.x'
with cmdline.umask(0o022):
virtualenv = pyenv.VirtualEnv(force_dir=os.path.abspath(pyenv_dir),
delete=False, python=python, verbose=False)
virtualenv.install_pkgutils(verbose=False,
pypi_index_url=pypi_index_url)
virtualenv.install_package(egg_or_req, installer='pyinstall',
debug=True, verbose=False,
pypi_index_url=pypi_index_url, dev=False)
current_link = os.path.join(package_dir, 'current')
if os.path.islink(current_link):
get_log().info("removing current link %s" % current_link)
os.unlink(current_link)
base = os.path.abspath(os.path.join(pyenv_dir, os.pardir, os.pardir))
relative_link = pyenv_dir[len(base):].lstrip(os.path.sep)
get_log().info("creating current link %s -> %s" % (current_link, relative_link))
os.symlink(relative_link, current_link)
if not os.path.isdir(console_script_base):
get_log().info("creating console script base: %s" % console_script_base)
os.makedirs(console_script_base)
scripts = virtualenv.run_python_cmd(['-c', r"""
from pkg_resources import working_set
dist = working_set.by_key[{0!r}]
print('\n'.join(dist.get_entry_map().get('console_scripts', {{}}).keys()))
""".format(req.key)]).split()
get_log().warn(scripts)
for item in scripts:
src = os.path.join(console_script_base, item)
dest = os.path.join(current_link, 'bin', item)
if os.path.islink(src):
get_log().info("Removing console script link: %s" % src)
os.unlink(src)
get_log().info("linking console script %s -> %s" % (src, dest))
os.symlink(dest, src)