def _prepare_files_for_upload(paths, names, valid_content_types):
"""
_prepare_files_for_upload takes the paths to the files to upload, the names of them and a list of valid content
types to translate to content types and generates a dictionary which can be uploaded in a post request
:param paths: a list of paths to the files to upload
:param names: the names of files to upload
:param valid_content_types: set of the possible valid content types
:return: a dictionary with as key the name of the file and as value a tuple with: filename, file to upload, and
content type
:exception McmdError when the file on the given path does not exist and when the content type of the file is invalid.
"""
files = {}
for name, path in zip(names, paths):
file_name = get_file_name_from_path(path)
content_type = mimetypes.guess_type(path)[0]
if not os_path.exists(path):
raise McmdError('File [{}] does not exist on path [{}]'.format(
file_name, path.strip(file_name)))
elif content_type in valid_content_types:
try:
files[name] = (file_name, open(path, 'rb'), content_type)
except FileNotFoundError:
raise McmdError('File [{}] does not exist on path [{}]'.format(
file_name, path.strip(file_name)))
else:
raise McmdError(
'File [{}] does not have valid content type [{}], content type should be in {}'
.format(file_name, content_type, valid_content_types))
return files
def _handle_json_error(response_json):
if 'errors' in response_json:
for error in response_json['errors']:
# TODO capture multiple error messages
raise McmdError(error['message'])
elif 'errorMessage' in response_json:
raise McmdError(response_json['errorMessage'])
def _handle_request(request):
response = str()
try:
response = request()
response.raise_for_status()
return response
except requests.HTTPError as e:
if _is_json(response):
_handle_json_error(response.json())
raise McmdError(str(e))
except requests.RequestException as e:
raise McmdError(str(e))
def principal_exists(principal_name, principal_type):
if principal_type == PrincipalType.USER:
return user_exists(principal_name)
elif principal_type == PrincipalType.ROLE:
return role_exists(principal_name)
else:
raise McmdError('Unknown principal type: %s' % principal_type)
def set_host(url):
hosts = _config['host']['auth']
if url in [host['url'] for host in hosts]:
_config['host']['selected'] = url
else:
raise McmdError("There is no host with url {}".format(url))
_persist()
def _get_selected_host_auth():
selected = _config['host']['selected']
hosts = _config['host']['auth']
for host in hosts:
if host['url'] == selected:
return host
raise McmdError("The selected host doesn't exist.")
def _remove_script(script_name):
path = get_scripts_folder().joinpath(script_name)
_check_script_exists(path)
try:
io.start('Removing script %s' % highlight(script_name))
path.unlink()
except OSError as e:
raise McmdError('Error removing script: %s' % str(e))
def _read_script(script_name):
path = get_scripts_folder().joinpath(script_name)
_check_script_exists(path)
try:
with path.open() as f:
for line in f.readlines():
log.info(line.strip())
except OSError as e:
raise McmdError('Error reading script: %s' % str(e))
def get_attachments(issue_num):
validate_issue_number(issue_num)
try:
issue = _molgenis_repo().get_issue(int(issue_num))
except UnknownObjectException:
raise McmdError("Issue #%s doesn't exist" % issue_num)
# GitHub has no API for downloading attachments so we get them from the issue description
urls = _parse_attachment_urls(issue.body)
return [Attachment(url.strip('()')) for url in urls]
def _select_attachment(issue_num, wanted_attachment):
"""Gets attachments from a GitHub issue. If wanted_attachment is specified it will try to select that attachment."""
attachments = github.get_attachments(issue_num)
if len(attachments) == 0:
raise McmdError("Issue #%s doesn't contain any files" % issue_num)
if wanted_attachment:
selected = [a for a in attachments if a.name == wanted_attachment]
if len(selected) == 0:
raise McmdError('There are no attachments named %s.' %
wanted_attachment)
if len(selected) > 1:
raise McmdError('Multiple attachments with name %s found.' %
wanted_attachment)
return selected[0]
else:
if len(attachments) > 1:
return _choose_attachment(attachments)
else:
return attachments[0]
def write(arg_string, success):
try:
history = open(_USER_HISTORY, 'a')
indicator = _INDICATOR_SUCCESS
if not success:
indicator = _INDICATOR_FAILURE
history.write('%s %s\n' % (indicator, arg_string))
except OSError as e:
raise McmdError("Error writing to history: %s" % str(e))
def _find_group(role):
io.debug('Fetching groups')
groups = get(config.api('rest2') + 'sys_sec_Group?attrs=name')
role = lower_kebab(role)
matches = {
len(group['name']): group['name']
for group in groups.json()['items'] if role.startswith(group['name'])
}
if not matches:
raise McmdError('No group found for role %s' % upper_snake(role))
return matches[max(matches, key=int)]
def _create_script(args):
lines = history.read(args.number, args.show_fails)
if len(lines) == 0:
log.info('History is empty.')
return
options = [line[1] for line in lines]
commands = io.checkbox('Pick the lines that will form the script:', options)
file_name = _input_script_name()
try:
script_file = open(get_scripts_folder().joinpath(file_name), 'w')
for command in commands:
script_file.write(command + '\n')
except OSError as e:
raise McmdError("Error writing to script: %s" % str(e))
def add_token(args):
io.start('Adding token %s for user %s' %
(highlight(args.token), highlight(args.user)))
user = get(
config.api('rest2') +
'sys_sec_User?attrs=id&q=username==%s' % args.user)
if user.json()['total'] == 0:
raise McmdError('Unknown user %s' % args.user)
user_id = user.json()['items'][0]['id']
data = {'User': user_id, 'token': args.token}
post(config.api('rest1') + 'sys_sec_Token', data)
def _guess_resource_type(resource_id):
results = dict()
for resource_type in ResourceType:
if resource_exists(resource_id, resource_type):
results[resource_type.get_label()] = resource_id
if len(results) == 0:
raise McmdError('No resources found with id %s' % resource_id)
elif len(results) > 1:
choices = results.keys()
answer = multi_choice(
'Multiple resources found for id %s. Choose one:' % resource_id,
choices)
return ResourceType.of_label(answer)
else:
return ResourceType.of_label(list(results)[0])
def _guess_principal_type(principal_name):
results = dict()
for principal_type in PrincipalType:
if principal_exists(principal_name, principal_type):
results[principal_type.value] = principal_name
if len(results) == 0:
raise McmdError('No principals found with name %s' % principal_name)
elif len(results) > 1:
choices = results.keys()
answer = multi_choice(
'Multiple principals found with name %s. Choose one:' %
principal_name, choices)
return PrincipalType[answer.upper()]
else:
return PrincipalType[list(results)[0].upper()]
def _do_import(file_path, package):
io.start('Importing %s' % (highlight(file_path.name)))
params = {
'action': _get_import_action(file_path.name),
'metadataAction': 'upsert'
}
if package:
params['packageId'] = package
response = post_file(config.api('import'), file_path.resolve(), params)
import_run_url = urljoin(config.get('host', 'selected'), response.text)
status, message = _poll_for_completion(import_run_url)
if status == 'FAILED':
raise McmdError(message)
def _add_host():
url = io.input_("Enter the URL of the host", required=True)
if config.host_exists(url):
raise McmdError("A host with URL {} already exists.".format(url))
username = io.input_(
"Enter the username of the superuser (Default: admin)")
password = io.password(
"Enter the password of the superuser (Default: admin)")
username = '******' if len(username) == 0 else username
password = '******' if len(password) == 0 else password
io.start("Adding host {}".format(highlight(url)))
config.add_host(url, username, password)
io.succeed()
return url
def select_path(file_map, file_name):
"""
select_path selects the path to a file from a dictionary of files with paths
:param file_map: a dictionary with file as key and as value a list of paths l
:param file_name: the name of the file to get the path of
:return: the selected path
:exception McmdError if selected file was not found
"""
if file_name in file_map:
paths = file_map[file_name]
if len(paths) > 1:
path = _choose_file(paths, file_name)
else:
path = paths[0]
else:
raise McmdError('No file found for %s' % file_name)
return path
def enable_theme(args):
"""
enable_theme enables a bootstrap theme
:param args: commandline arguments containing the id of the theme (without .css)
:exception McmdError: when applying the theme fails
:return None
"""
theme = args.theme.replace('.css', '').replace('.min', '')
io.start('Applying theme {}'.format(highlight(theme)))
# Resource can be bootstrap-name.min.css (if molgenis theme), or name.min.css (if uploaded .min.css), or
# name.css (if uploaded as .css).
if one_resource_exists([theme + '.min.css', theme + '.css', 'bootstrap-' + theme + '.min.css'], ResourceType.THEME):
# Molgenis themes start with bootstrap- but this is stripped from the name in the theme manager
try:
post(config.api('set_theme'), theme)
except:
post(config.api('set_theme'), theme.split('bootstrap-')[1])
else:
raise McmdError(
'Applying theme failed. No themes found containing {} in the name'.format(args.theme, 'sys_set_StyleSheet'))
def _import_from_url(args):
file_url = args.resource
file_name = file_url.split("/")[-1]
io.start('Importing from URL %s' % highlight(file_url))
params = {
'action': _get_import_action(file_name),
'metadataAction': 'upsert'
}
if args.to_package:
params['packageId'] = args.to_package
params['url'] = file_url
response = import_by_url(params)
import_run_url = urljoin(config.get('host', 'selected'), response.text)
status, message = _poll_for_completion(import_run_url)
if status == 'FAILED':
raise McmdError(message)
def grant(principal_type, principal_name, resource_type, identifier,
permission):
data = {'radio-' + identifier: permission}
if principal_type == PrincipalType.USER:
data['username'] = principal_name
elif principal_type == PrincipalType.ROLE:
data['rolename'] = principal_name.upper()
else:
raise McmdError('Unknown principal type: %s' % principal_type)
url = config.api('perm') + resource_type.get_resource_name(
) + '/' + principal_type.value
return _handle_request(lambda: requests.post(
url,
headers={
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'x-molgenis-token': token
},
data=data))
def _download_attachment(attachment, issue_num):
issue_folder = get_issues_folder().joinpath(issue_num)
issue_folder.mkdir(parents=True, exist_ok=True)
file_path = issue_folder.joinpath(attachment.name)
if file_path.exists():
overwrite = io.confirm('File %s already exists. Re-download?' %
file_path.name)
if not overwrite:
return file_path
io.start('Downloading %s from GitHub issue %s' %
(highlight(attachment.name), highlight('#' + issue_num)))
try:
r = requests.get(attachment.url)
r.raise_for_status()
with file_path.open('wb') as f:
f.write(r.content)
except (OSError, requests.RequestException, requests.HTTPError) as e:
raise McmdError('Error downloading GitHub attachment: %s' % str(e))
io.succeed()
return file_path
def read(num_lines, include_fails):
lines = deque()
if not path.isfile(get_history_file()):
return lines
try:
with open(_USER_HISTORY, 'r') as history:
for line in history:
line = line.rstrip('\n')
if line.startswith(_INDICATOR_FAILURE):
if include_fails:
lines.append((False, line[2:]))
else:
lines.append((True, line[2:]))
if len(lines) > num_lines:
lines.popleft()
except OSError as e:
raise McmdError("Error reading from history: %s" % str(e))
return lines
def _check_script_exists(path):
if not path.exists():
raise McmdError("Script %s doesn't exist" % path.name)
def clear():
try:
open(_USER_HISTORY, 'w').close()
except OSError as e:
raise McmdError("Error clearing history: %s" % str(e))
def validate_issue_number(issue_num):
try:
int(issue_num)
except ValueError:
raise McmdError('Not a valid issue number: %s' % issue_num)
def ensure_resource_exists(resource_id, resource_type):
if not resource_exists(resource_id, resource_type):
raise McmdError('No %s found with id %s' %
(resource_type.get_label(), resource_id))
def ensure_principal_exists(principal_name, principal_type):
if not principal_exists(principal_name, principal_type):
raise McmdError('No {} found with id {}'.format(
principal_type.value, principal_name))
def _import_from_path(args):
file = Path(args.file)
if not file.is_file():
raise McmdError("File %s doesn't exist" % str(file.resolve()))
_do_import(file, args.to_package)