from matrix_api_async.api_asyncio import AsyncHTTPAPI
from matrix_client.errors import MatrixRequestError
from voluptuous import Required
from opsdroid.connector import Connector, register_event
from opsdroid import events
from .html_cleaner import clean
from .create_events import MatrixEventCreator
from . import events as matrixevents
_LOGGER = logging.getLogger(__name__)
CONFIG_SCHEMA = {
Required("mxid"): str,
Required("password"): str,
Required("rooms"): dict,
"homeserver": str,
"nick": str,
"room_specific_nicks": bool,
}
__all__ = ["ConnectorMatrix"]
def ensure_room_id_and_send(func):
"""
Ensure that the target for the event is a matrix room id.
Also retry the function call if the server disconnects.
},
'pine': {
'target_tasks_method': 'pine_tasks',
},
'kaios': {
'target_tasks_method': 'kaios_tasks',
},
# the default parameters are used for projects that do not match above.
'default': {
'target_tasks_method': 'default',
}
}
try_task_config_schema = Schema({
Required('tasks'): [text_type],
Optional('browsertime'):
bool,
Optional('chemspill-prio'):
bool,
Optional('disable-pgo'):
bool,
Optional('env'): {
text_type: text_type
},
Optional('gecko-profile'):
bool,
Optional("optimize-strategies",
description="Alternative optimization strategies to use instead of the default. "
"A module path pointing to a dict to be use as the `strategy_override` "
"argument in `taskgraph.optimize.optimize_task_graph`."):
from taskgraph.transforms.job.common import (
docker_worker_add_workspace_cache,
setup_secrets,
docker_worker_add_artifacts,
add_tooltool,
generic_worker_add_artifacts,
generic_worker_hg_commands,
support_vcs_checkout,
)
from taskgraph.transforms.task import (
get_branch_repo,
get_branch_rev,
)
mozharness_run_schema = Schema({
Required('using'): 'mozharness',
# the mozharness script used to run this task, relative to the testing/
# directory and using forward slashes even on Windows
Required('script'): basestring,
# Additional paths to look for mozharness configs in. These should be
# relative to the base of the source checkout
Optional('config-paths'): [basestring],
# the config files required for the task, relative to
# testing/mozharness/configs or one of the paths specified in
# `config-paths` and using forward slashes even on Windows
Required('config'): [basestring],
# any additional actions to pass to the mozharness command
'stylo-disabled',
'stylo-sequential',
'qr',
'ccov',
]
def get_variant(test_platform):
for v in VARIANTS:
if '-{}/'.format(v) in test_platform:
return v
return ''
mozharness_test_run_schema = Schema({
Required('using'):
'mozharness-test',
Required('test'):
test_description_schema,
# Base work directory used to set up the task.
Required('workdir'):
basestring,
})
def test_packages_url(taskdesc):
"""Account for different platforms that name their test packages differently"""
artifact_url = get_artifact_url(
'<build>', get_artifact_path(taskdesc, 'target.test_packages.json'))
# for android nightly we need to add 'en-US' to the artifact url
test = taskdesc['run']['test']
import os
import logging
import attr
from six import text_type
from mozpack import path
from .util.schema import validate_schema, Schema, optionally_keyed_by
from voluptuous import Required, Optional, Any
from .util.yaml import load_yaml
logger = logging.getLogger(__name__)
graph_config_schema = Schema({
# The trust-domain for this graph.
# (See https://firefox-source-docs.mozilla.org/taskcluster/taskcluster/taskgraph.html#taskgraph-trust-domain) # noqa
Required('trust-domain'):
basestring,
# This specifes the prefix for repo parameters that refer to the project being built.
# This selects between `head_rev` and `comm_head_rev` and related paramters.
# (See http://firefox-source-docs.mozilla.org/taskcluster/taskcluster/parameters.html#push-information # noqa
# and http://firefox-source-docs.mozilla.org/taskcluster/taskcluster/parameters.html#comm-push-information) # noqa
Required('project-repo-param-prefix'):
basestring,
# This specifies the top level directory of the application being built.
# ie. "browser/" for Firefox, "comm/mail/" for Thunderbird.
Required('product-dir'):
basestring,
Required('treeherder'): {
# Mapping of treeherder group symbols to descriptive names
Required('group-names'): {
basestring: basestring
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run
from taskgraph.util import path
from taskgraph.util.schema import Schema
from voluptuous import Required, Optional
from six import text_type
from pipes import quote as shell_quote
gradlew_schema = Schema({
Required("using"):
"gradlew",
Required("gradlew"): [text_type],
Optional("post-gradlew"): [[text_type]],
# Base work directory used to set up the task.
Required("workdir"):
text_type,
Optional("use-caches"):
bool,
Optional("secrets"): [{
Required("name"): text_type,
Required("path"): text_type,
Required("key"): text_type,
Optional("json"): bool,
}],
})
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.beetmover import craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope,
get_worker_type_for_scope)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Required, Optional
beetmover_checksums_description_schema = schema.extend({
Required('depname', default='build'):
basestring,
Optional('label'):
basestring,
Optional('treeherder'):
task_description_schema['treeherder'],
Optional('locale'):
basestring,
Optional('shipping-phase'):
task_description_schema['shipping-phase'],
Optional('shipping-product'):
task_description_schema['shipping-product'],
})
transforms = TransformSequence()
transforms.add_validate(beetmover_checksums_description_schema)
})
################# VALIDATORS PER REQUEST TYPE ####################
from datetime import datetime
def Date(fmt='%Y-%m-%d'):
return lambda v: datetime.strptime(v, fmt)
schema_date = Schema(Date())
#def CheckFilesList(files_list):
# return lambda files_list: IsFile(f) for f in files_list
# POST /submissions/
study_post_submission_validator = Schema({
Required('name'): All(str, Length(min=1)),
# Required('study_visibility') : All(str, Length(min=1)),
Required('pi_list') : All([str], Length(min=1)),
})
submission_post_validator = Schema({
'files_list' : [str],
#IsDir('dir_path') : All(str, Length(min=1)),
'dir_path' : All(str, Length(min=1)),
Required('sanger_user_id') : All(str, Length(min=1)),
Required('hgi_project') : str, #All(str, Length(min=1)),
Required('study') : study_post_submission_validator,
# Required('reference_genome') : IsFile(str),
'reference_genome' : IsFile(str),
Required('library_metadata') : abstract_library_schema,
"""A helper function for parsing and executing luis.ai skills."""
import logging
import json
import contextlib
import aiohttp
from voluptuous import Required
from opsdroid.const import LUISAI_DEFAULT_URL
_LOGGER = logging.getLogger(__name__)
CONFIG_SCHEMA = {
Required("appid"): str,
Required("appkey"): str,
Required("verbose"): bool,
"min-score": float,
}
async def call_luisai(message, config):
"""Call the luis.ai api and return the response."""
async with aiohttp.ClientSession(trust_env=True) as session:
headers = {"Content-Type": "application/json"}
url = LUISAI_DEFAULT_URL
resp = await session.get(
url + config["appid"] + "?subscription-key=" + config["appkey"] +
"&timezoneOffset=0" + "&verbose=" + str(config["verbose"]) +
"&q=" + message.text,
headers=headers,
resolve_keyed_by,
)
from taskgraph.util.scriptworker import (
add_scope_prefix,
get_beetmover_bucket_scope,
)
from taskgraph.util.taskcluster import get_artifact_prefix
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
from collections import defaultdict
from copy import deepcopy
beetmover_description_schema = schema.extend({
# depname is used in taskref's to identify the taskID of the unsigned things
Required("depname", default="build"):
text_type,
# unique label to describe this beetmover task, defaults to {dep.label}-beetmover
Optional("label"):
text_type,
Required("partner-bucket-scope"):
optionally_keyed_by("release-level", text_type),
Required("partner-public-path"):
Any(None, text_type),
Required("partner-private-path"):
Any(None, text_type),
Optional("extra"):
object,
Required("shipping-phase"):
task_description_schema["shipping-phase"],
Optional("shipping-product"):
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.scriptworker import (generate_beetmover_artifact_map,
generate_beetmover_upstream_artifacts,
get_beetmover_bucket_scope,
get_beetmover_action_scope,
get_worker_type_for_scope,
should_use_artifact_map)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Required, Optional
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
beetmover_checksums_description_schema = schema.extend({
Required('depname', default='build'): basestring,
Optional('label'): basestring,
Optional('treeherder'): task_description_schema['treeherder'],
Optional('locale'): basestring,
Optional('shipping-phase'): task_description_schema['shipping-phase'],
Optional('shipping-product'): task_description_schema['shipping-product'],
Optional('attributes'): task_description_schema['attributes'],
})
transforms = TransformSequence()
transforms.add_validate(beetmover_checksums_description_schema)
@transforms.add
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
"""A helper function for parsing and executing wit.ai skills."""
import logging
import json
import aiohttp
from voluptuous import Required
from opsdroid.const import WITAI_DEFAULT_VERSION
from opsdroid.const import WITAI_API_ENDPOINT
_LOGGER = logging.getLogger(__name__)
CONFIG_SCHEMA = {Required("token"): str, "min-score": float}
async def call_witai(message, config):
"""Call the wit.ai api and return the response."""
async with aiohttp.ClientSession(trust_env=True) as session:
headers = {"Authorization": "Bearer " + config["token"]}
payload = {"v": WITAI_DEFAULT_VERSION, "q": message.text}
resp = await session.get(
WITAI_API_ENDPOINT + "v={}&q={}".format(payload["v"], payload["q"]),
headers=headers,
)
result = await resp.json()
_LOGGER.info(_("wit.ai response - %s."), json.dumps(result))
return result
async def parse_witai(opsdroid, skills, message, config):
]
NECESSITY_OPTIONS = [u'required', u'optional', u'']
VALID_ASSESSMENT_TYPES = [
u'peer-assessment',
u'self-assessment',
u'student-training',
u'staff-assessment',
]
VALID_UPLOAD_FILE_TYPES = [u'image', u'pdf-and-image', u'custom']
# Schema definition for an update from the Studio JavaScript editor.
EDITOR_UPDATE_SCHEMA = Schema({
Required('prompts'): [Schema({
Required('description'): utf8_validator,
})],
Required('prompts_type', default='text'):
Any(All(utf8_validator, In(PROMPTS_TYPES)), None),
Required('title'):
utf8_validator,
Required('feedback_prompt'):
utf8_validator,
Required('feedback_default_text'):
utf8_validator,
Required('submission_start'):
Any(datetime_validator, None),
Required('submission_due'):
Any(datetime_validator, None),
Required('text_response', default='required'):
from __future__ import absolute_import, print_function, unicode_literals
from six import text_type
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by
from taskgraph.util.treeherder import inherit_treeherder_from_dep
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required
transforms = TransformSequence()
langpack_sign_push_description_schema = schema.extend({
Required('label'):
text_type,
Required('description'):
text_type,
Required('worker-type'):
optionally_keyed_by('release-level', text_type),
Required('worker'): {
Required('implementation'):
'push-addons',
Required('channel'):
optionally_keyed_by('project', 'platform', Any('listed', 'unlisted')),
Required('upstream-artifacts'):
None, # Processed here below
},
Required('run-on-projects'): [],
Required('scopes'):
RUN_TASK = os.path.join(
os.path.dirname(os.path.dirname(__file__)), "run-task", "run-task"
)
@memoize
def _run_task_suffix():
"""String to append to cache names under control of run-task."""
return hash_path(RUN_TASK)[0:20]
# A task description is a general description of a TaskCluster task
task_description_schema = Schema(
{
# the label for this task
Required("label"): str,
# description of the task (for metadata)
Required("description"): str,
# attributes for this task
Optional("attributes"): {str: object},
# relative path (from config.path) to the file task was defined in
Optional("job-from"): str,
# dependencies of this task, keyed by name; these are passed through
# verbatim and subject to the interpretation of the Task's get_dependencies
# method.
Optional("dependencies"): {
All(
str,
NotIn(
["self", "decision"],
"Can't use 'self` or 'decision' as depdency names.",
from taskgraph.util.schema import (
optionally_keyed_by,
validate_schema,
Schema,
)
def even_15_minutes(minutes):
if minutes % 15 != 0:
raise ValueError("minutes must be evenly divisible by 15")
cron_yml_schema = Schema({
'jobs': [{
# Name of the crontask (must be unique)
Required('name'):
basestring,
# what to run
# Description of the job to run, keyed by 'type'
Required('job'): {
Required('type'):
'decision-task',
# Treeherder symbol for the cron task
Required('treeherder-symbol'):
basestring,
# --target-tasks-method './mach taskgraph decision' argument
Required('target-tasks-method'):
import bcrypt, re, urllib.parse, urllib.request, flask, json
import api
from api.common import check, validate, safe_fail
from api.common import WebException, InternalException
from api.annotations import log_action
from voluptuous import Required, Length, Schema
_check_email_format = lambda email: re.match(r".+@.+\..{2,}", email
) is not None
user_schema = Schema(
{
Required('email'):
check(("Email must be between 5 and 50 characters.",
[str, Length(min=5, max=50)]),
("Your email does not look like an email address.",
[_check_email_format])),
Required('firstname'):
check(("First Name must be between 1 and 50 characters.",
[str, Length(min=1, max=50)])),
Required('lastname'):
check(("Last Name must be between 1 and 50 characters.",
[str, Length(min=1, max=50)])),
Required('country'):
check(("Please select a country", [str, Length(min=2, max=2)])),
Required('username'):
check(("Usernames must be between 3 and 20 characters.",
[str, Length(min=3, max=20)]),
import json
from dateutil.parser import parse
import singer
from voluptuous import Schema, Required
LOGGER = singer.get_logger()
STATE_SCHEMA = Schema({
Required('bookmarks'): {
str: {
Required('last_record'): str,
Required('field'): str,
}
}
})
def get_last_record_value_for_table(state, table):
to_return = state.get('bookmarks', {}) \
.get(table, {}) \
.get('last_record')
if to_return is not None:
to_return = parse(to_return)
return to_return
def incorporate(state, table, field, value):
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from voluptuous import ALLOW_EXTRA, Any, Required, Schema
option_schema = Schema(
{
Required('description'): Any(basestring, [basestring]),
'required': bool,
'choices': list,
'aliases': list,
'version_added': Any(basestring, float)
},
extra=ALLOW_EXTRA)
doc_schema = Schema(
{
Required('module'): basestring,
'short_description': basestring,
'description': Any(basestring, [basestring]),
'version_added': Any(basestring, float),
'author': Any(None, basestring, [basestring]),
'notes': Any(None, [basestring]),
from taskgraph.util.scriptworker import (get_balrog_server_scope,
get_balrog_channel_scopes)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {
str(k): v
for k, v in task_description_schema.schema.iteritems()
}
transforms = TransformSequence()
# shortcut for a string where task references are allowed
taskref_or_string = Any(basestring, {Required('task-reference'): basestring})
balrog_description_schema = Schema({
# the dependent task (object) for this balrog job, used to inform balrogworker.
Required('dependent-task'):
object,
# unique label to describe this balrog task, defaults to balrog-{dep.label}
Optional('label'):
basestring,
# treeherder is allowed here to override any defaults we use for beetmover. See
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
# below transforms for defaults of various values.
Optional('treeherder'):
task_description_schema['treeherder'],
complexities of worker implementations, scopes, and treeherder annotations.
"""
from __future__ import absolute_import, print_function, unicode_literals
import json
import time
from taskgraph.util.treeherder import split_symbol
from taskgraph.transforms.base import (validate_schema, TransformSequence)
from voluptuous import Schema, Any, Required, Optional, Extra
from .gecko_v2_whitelist import JOB_NAME_WHITELIST, JOB_NAME_WHITELIST_ERROR
# shortcut for a string where task references are allowed
taskref_or_string = Any(basestring, {Required('task-reference'): basestring})
# A task description is a general description of a TaskCluster task
task_description_schema = Schema({
# the label for this task
Required('label'):
basestring,
# description of the task (for metadata)
Required('description'):
basestring,
# attributes for this task
Optional('attributes'): {
basestring: object
},
class RepoDependency(LocalDependency):
PARAM_REPO = "repo"
PARAM_URL = "url"
PARAM_REV = "rev"
PARAM_REV_LOCK = "rev_lock"
REPO_SCHEMA = {
PARAM_REPO: {
Required(PARAM_URL): str,
PARAM_REV: str,
PARAM_REV_LOCK: str,
}
}
def __init__(self, def_repo, stage, *args, **kwargs):
self.def_repo = def_repo
super().__init__(stage, *args, **kwargs)
def _parse_path(self, remote, path):
return None
@property
def is_in_repo(self):
return False
@property
def repo_pair(self):
d = self.def_repo
rev = d.get(self.PARAM_REV_LOCK) or d.get(self.PARAM_REV)
return d[self.PARAM_URL], rev
def __str__(self):
return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])
def _make_repo(self, *, locked=True):
from dvc.external_repo import external_repo
d = self.def_repo
rev = (d.get("rev_lock") if locked else None) or d.get("rev")
return external_repo(d["url"], rev=rev)
def _get_checksum(self, locked=True):
with self._make_repo(locked=locked) as repo:
try:
return repo.find_out_by_relpath(self.def_path).info["md5"]
except OutputNotFoundError:
path = PathInfo(os.path.join(repo.root_dir, self.def_path))
# We are polluting our repo cache with some dir listing here
return self.repo.cache.local.get_checksum(path)
def status(self):
current_checksum = self._get_checksum(locked=True)
updated_checksum = self._get_checksum(locked=False)
if current_checksum != updated_checksum:
return {str(self): "update available"}
return {}
def save(self):
pass
def dumpd(self):
return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
def download(self, to):
with self._make_repo() as repo:
if self.def_repo.get(self.PARAM_REV_LOCK) is None:
self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
if hasattr(repo, "cache"):
repo.cache.local.cache_dir = self.repo.cache.local.cache_dir
repo.pull_to(self.def_path, to.path_info)
def update(self, rev=None):
if rev:
self.def_repo[self.PARAM_REV] = rev
with self._make_repo(locked=False) as repo:
self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
def main():
"""Validate BOTMETA"""
path = '.github/BOTMETA.yml'
try:
with open(path, 'r') as f_path:
botmeta = yaml.safe_load(f_path)
except yaml.error.MarkedYAMLError as ex:
print('%s:%d:%d: YAML load failed: %s' %
(path, ex.context_mark.line + 1, ex.context_mark.column + 1,
re.sub(r'\s+', ' ', str(ex))))
sys.exit()
except Exception as ex: # pylint: disable=broad-except
print('%s:%d:%d: YAML load failed: %s' %
(path, 0, 0, re.sub(r'\s+', ' ', str(ex))))
sys.exit()
list_string_types = list(string_types)
files_schema = Any(
Schema(*string_types),
Schema({
'ignored': Any(list_string_types, *string_types),
'keywords': Any(list_string_types, *string_types),
'labels': Any(list_string_types, *string_types),
'maintainers': Any(list_string_types, *string_types),
'migrated_to': All(
Any(*string_types),
Match(r'^\w+\.\w+$'),
),
'notified': Any(list_string_types, *string_types),
'supershipit': Any(list_string_types, *string_types),
'support': Any("core", "network", "community"),
}))
list_dict_file_schema = [{
str_type: files_schema
} for str_type in string_types]
schema = Schema({
Required('automerge'): bool,
Required('files'): Any(None, *list_dict_file_schema),
Required('macros'): dict, # Any(*list_macros_schema),
})
# Ensure schema is valid
try:
schema(botmeta)
except MultipleInvalid as ex:
for error in ex.errors:
# No way to get line numbers
print('%s:%d:%d: %s' %
(path, 0, 0, humanize_error(botmeta, error)))
# Ensure botmeta is always support:core
botmeta_support = botmeta.get('files', {}).get('.github/BOTMETA.yml',
{}).get('support', '')
if botmeta_support != 'core':
print('%s:%d:%d: .github/BOTMETA.yml MUST be support: core' %
(path, 0, 0))
# Find all path (none-team) macros so we can substitute them
macros = botmeta.get('macros', {})
path_macros = []
for macro in macros:
if macro.startswith('team_'):
continue
path_macros.append(macro)
# Ensure all `files` correspond to a file
for file, file_meta in botmeta['files'].items():
migrated = isinstance(
file_meta, dict) and file_meta.get('migrated_to') is not None
for macro in path_macros:
file = file.replace('$' + macro,
botmeta.get('macros', {}).get(macro, ''))
if not os.path.exists(file) and not migrated:
# Not a file or directory, though maybe the prefix to one?
# https://github.com/ansible/ansibullbot/pull/1023
if not glob.glob('%s*' % file):
print("%s:%d:%d: Can't find '%s.*' in this branch" %
(path, 0, 0, file))
import os
import sys
import yaml
from voluptuous import All, MultipleInvalid, Required, Schema
MQTT_SCHEMA = Schema({
Required("broker_url"): str,
Required("broker_port", default=1883): int,
Required("username", default=""): str,
Required("password", default=""): str,
Required("keepalive", default=5): int,
Required("tls", default=False): bool,
})
CONFIG_SCHEMA = Schema({
Required("domains"): All([str], min=1),
Required("pubkeys_file", default=""): str,
Required("mqtt"): MQTT_SCHEMA,
})
def load_config():
config_file = os.environ.get("WGKEX_CONFIG_FILE", "/etc/wgkex.yaml")
with open(config_file, "r") as stream:
try:
config = CONFIG_SCHEMA(yaml.safe_load(stream))
except MultipleInvalid as ex:
print(f"Config file failed to validate: {ex}", file=sys.stderr)
sys.exit(1)
return config
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Support for running mach python-test tasks (via run-task)
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run
from taskgraph.util.schema import Schema
from voluptuous import Required
python_test_schema = Schema({
Required('using'):
'python-test',
# Python version to use
Required('python-version'):
int,
# The subsuite to run
Required('subsuite'):
basestring,
# Base work directory used to set up the task.
Required('workdir'):
basestring,
})
defaults = {
VALID_ASSESSMENT_TYPES = [
'peer-assessment',
'self-assessment',
'student-training',
'staff-assessment',
]
VALID_UPLOAD_FILE_TYPES = [
'image',
'pdf-and-image',
'custom'
]
# Schema definition for an update from the Studio JavaScript editor.
EDITOR_UPDATE_SCHEMA = Schema({
Required('prompts'): [
Schema({
Required('description'): utf8_validator,
})
],
Required('prompts_type', default='text'): Any(All(utf8_validator, In(PROMPTS_TYPES)), None),
Required('title'): utf8_validator,
Required('feedback_prompt'): utf8_validator,
Required('feedback_default_text'): utf8_validator,
Required('submission_start'): Any(datetime_validator, None),
Required('submission_due'): Any(datetime_validator, None),
Required('text_response', default='required'): Any(All(utf8_validator, In(NECESSITY_OPTIONS)), None),
Required('file_upload_response', default=None): Any(All(utf8_validator, In(NECESSITY_OPTIONS)), None),
'allow_file_upload': bool, # Backwards compatibility.
Required('file_upload_type', default=None): Any(All(utf8_validator, In(VALID_UPLOAD_FILE_TYPES)), None),
'white_listed_file_types': utf8_validator,
from taskgraph.util.taskcluster import get_artifact_prefix
from taskgraph.util.partners import check_if_partners_enabled
from taskgraph.util.platforms import archive_format, executable_extension
from taskgraph.util.workertypes import worker_type_implementation
from taskgraph.transforms.task import task_description_schema
from taskgraph.transforms.repackage import PACKAGE_FORMATS
from voluptuous import Required, Optional
def _by_platform(arg):
return optionally_keyed_by('build-platform', arg)
packaging_description_schema = schema.extend({
# depname is used in taskref's to identify the taskID of the signed things
Required('depname', default='build'):
basestring,
# unique label to describe this repackaging task
Optional('label'):
basestring,
# Routes specific to this task, if defined
Optional('routes'): [basestring],
# passed through directly to the job description
Optional('extra'):
task_description_schema['extra'],
# Shipping product and phase
Optional('shipping-product'):
class RepoDependency(Dependency):
PARAM_REPO = "repo"
PARAM_URL = "url"
PARAM_REV = "rev"
PARAM_REV_LOCK = "rev_lock"
REPO_SCHEMA = {
PARAM_REPO: {
Required(PARAM_URL): str,
PARAM_REV: str,
PARAM_REV_LOCK: str,
}
}
def __init__(self, def_repo, stage, *args, **kwargs):
self.def_repo = def_repo
self._staged_objs: Dict[str, "HashFile"] = {}
super().__init__(stage, *args, **kwargs)
def _parse_path(self, fs, path_info):
return None
@property
def is_in_repo(self):
return False
def __str__(self):
return f"{self.def_path} ({self.def_repo[self.PARAM_URL]})"
def workspace_status(self):
current = self.get_obj(locked=True).hash_info
updated = self.get_obj(locked=False).hash_info
if current != updated:
return {str(self): "update available"}
return {}
def status(self):
return self.workspace_status()
def save(self):
pass
def dumpd(self):
return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
def download(self, to, jobs=None):
from dvc.objects.checkout import checkout
for odb, objs in self.get_used_objs().items():
self.repo.cloud.pull(objs, jobs=jobs, odb=odb)
obj = self.get_obj()
checkout(
to.path_info,
to.fs,
obj,
self.repo.odb.local,
dvcignore=None,
state=self.repo.state,
)
def update(self, rev=None):
if rev:
self.def_repo[self.PARAM_REV] = rev
with self._make_repo(locked=False) as repo:
self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
def changed_checksum(self):
# From current repo point of view what describes RepoDependency is its
# origin project url and rev_lock, and it makes RepoDependency
# immutable, hence its impossible for checksum to change.
return False
def get_used_objs(self,
**kwargs) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
used, _ = self._get_used_and_obj(**kwargs)
return used
def _get_used_and_obj(
self,
obj_only=False,
**kwargs
) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], "HashFile"]:
from dvc.config import NoRemoteError
from dvc.exceptions import NoOutputOrStageError, PathMissingError
from dvc.objects.stage import stage
from dvc.objects.tree import Tree
local_odb = self.repo.odb.local
locked = kwargs.pop("locked", True)
with self._make_repo(locked=locked,
cache_dir=local_odb.cache_dir) as repo:
used_obj_ids = defaultdict(set)
rev = repo.get_rev()
if locked and self.def_repo.get(self.PARAM_REV_LOCK) is None:
self.def_repo[self.PARAM_REV_LOCK] = rev
path_info = PathInfo(repo.root_dir) / str(self.def_path)
if not obj_only:
try:
for odb, obj_ids in repo.used_objs(
[os.fspath(path_info)],
force=True,
jobs=kwargs.get("jobs"),
recursive=True,
).items():
if odb is None:
odb = repo.cloud.get_remote_odb()
odb.read_only = True
self._check_circular_import(odb, obj_ids)
used_obj_ids[odb].update(obj_ids)
except (NoRemoteError, NoOutputOrStageError):
pass
try:
staging, staged_obj = stage(
local_odb,
path_info,
repo.repo_fs,
local_odb.fs.PARAM_CHECKSUM,
)
except FileNotFoundError as exc:
raise PathMissingError(self.def_path,
self.def_repo[self.PARAM_URL]) from exc
staging = copy(staging)
staging.read_only = True
self._staged_objs[rev] = staged_obj
used_obj_ids[staging].add(staged_obj.hash_info)
if isinstance(staged_obj, Tree):
used_obj_ids[staging].update(entry.hash_info
for _, entry in staged_obj)
return used_obj_ids, staged_obj
def _check_circular_import(self, odb, obj_ids):
from dvc.exceptions import CircularImportError
from dvc.fs.repo import RepoFileSystem
from dvc.objects.db.reference import ReferenceObjectDB
from dvc.objects.tree import Tree
if not isinstance(odb, ReferenceObjectDB):
return
def iter_objs():
for hash_info in obj_ids:
if hash_info.isdir:
tree = Tree.load(odb, hash_info)
yield from (odb.get(entry.hash_info) for _, entry in tree)
else:
yield odb.get(hash_info)
checked_urls = set()
for obj in iter_objs():
if not isinstance(obj.fs, RepoFileSystem):
continue
if (obj.fs.repo_url in checked_urls
or obj.fs.root_dir in checked_urls):
continue
self_url = self.repo.url or self.repo.root_dir
if (obj.fs.repo_url is not None and obj.fs.repo_url == self_url
or obj.fs.root_dir == self.repo.root_dir):
raise CircularImportError(self, obj.fs.repo_url, self_url)
checked_urls.update([obj.fs.repo_url, obj.fs.root_dir])
def get_obj(self, filter_info=None, **kwargs):
locked = kwargs.get("locked", True)
rev = self._get_rev(locked=locked)
if rev in self._staged_objs:
return self._staged_objs[rev]
_, obj = self._get_used_and_obj(obj_only=True,
filter_info=filter_info,
**kwargs)
return obj
def _make_repo(self, locked=True, **kwargs):
from dvc.external_repo import external_repo
d = self.def_repo
rev = self._get_rev(locked=locked)
return external_repo(d[self.PARAM_URL], rev=rev, **kwargs)
def _get_rev(self, locked=True):
d = self.def_repo
return (d.get(self.PARAM_REV_LOCK) if locked else None) or d.get(
self.PARAM_REV)
from .gecko_v2_whitelist import JOB_NAME_WHITELIST, JOB_NAME_WHITELIST_ERROR
RUN_TASK = os.path.join(GECKO, 'taskcluster', 'docker', 'recipes', 'run-task')
@memoize
def _run_task_suffix():
"""String to append to cache names under control of run-task."""
return hash_path(RUN_TASK)[0:20]
# shortcut for a string where task references are allowed
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
# A task description is a general description of a TaskCluster task
task_description_schema = Schema({
# the label for this task
Required('label'): basestring,
# description of the task (for metadata)
Required('description'): basestring,
# attributes for this task
Optional('attributes'): {basestring: object},
# dependencies of this task, keyed by name; these are passed through
# verbatim and subject to the interpretation of the Task's get_dependencies
# method.
from voluptuous import Schema, Required, Optional
CONFIG_CONTRACT = Schema([{
Required('table_name'): str,
Required('search_pattern'): str,
Required('key_properties'): [str],
Optional('search_prefix'): str,
Optional('date_overrides'): [str],
Optional('delimiter'): str,
Optional('encoding_module'): str
}])