def import_module_from_github(url, force_reload=False):
url = sanitise_url(url)
reponame = retrieve_project_name(url)
importdir = os.path.join(MODULE_DIRECTORY, 'clones', reponame)
# Delete anything that might left over junk from previous failures (shouldn't happen, but)
if os.path.isdir(importdir):
shutil.rmtree(importdir)
try:
# pull contents from GitHub
try:
git.Repo.clone_from(url, importdir)
except (ValidationError, GitCommandError) as ve:
if type(ve) == GitCommandError:
message = "Received Git error status code {}".format(ve.status)
else:
message = ve.message
raise ValidationError('Unable to clone from GitHub: %s' % (url) +
': %s' % (message))
# load it
version = extract_version(importdir)
ui_info = import_module_from_directory(url, reponame, version,
importdir, force_reload)
except Exception as e:
# Clean up any existing dirs and pass exception up (ValidationErrors will have error message for user)
if os.path.isdir(importdir):
shutil.rmtree(importdir)
raise
log_message('Successfully imported module %s' % url)
return ui_info
def import_module_from_directory(url,
reponame,
version,
importdir,
force_reload=False):
destination_directory = None
ui_info = {}
try:
# check that right files exist
extension_file_mapping = validate_module_structure(importdir)
python_file = extension_file_mapping['py']
json_file = extension_file_mapping['json']
html_file = extension_file_mapping.get('html', None)
# load json file
module_config = get_module_config_from_json(url,
extension_file_mapping,
importdir)
id_name = module_config['id_name']
# Don't allow importing the same version twice, unless forced
if not force_reload:
if ModuleVersion.objects.filter(module__id_name=id_name,
source_version_hash=version):
raise ValidationError(f'Version {version} of module {url}'
' has already been imported')
# Don't allow loading a module with the same id_name from a different
# repo. Prevents replacement attacks.
module_urls = get_already_imported_module_urls()
if module_config["id_name"] in module_urls \
and url != module_urls[module_config["id_name"]]:
source = module_urls[module_config["id_name"]]
if source == '':
source = "Internal"
raise ValidationError(
f"Module {module_config['id_name']} has already been loaded"
f' from a different repo: {source}')
module_config['source_version'] = version
module_config['link'] = url
if 'author' not in module_config:
module_config['author'] = retrieve_author(url)
# Ensure that modules are categorised properly – if a module category
# isn't one of our pre-defined categories, then we just set it to
# other.
if module_config["category"] not in get_categories():
module_config["category"] = "Other"
# The core work of creating a module
destination_directory = create_destination_directory(id_name, version)
move_files_to_final_location(destination_directory, importdir,
[json_file, python_file, html_file])
add_boilerplate_and_check_syntax(destination_directory, python_file)
validate_python_functions(destination_directory, python_file)
# If that succeeds, initialise module in our database
module_version = load_module_from_dict(module_config)
# clean-up
shutil.rmtree(importdir)
if force_reload:
dynamicdispatch.load_module.cache_clear()
# For now, our policy is to update all wfmodules to this just-imported
# version
module = module_version.module
for wfm in WfModule.objects.filter(module_version__module=module):
update_wfm_parameters_to_new_version(wfm, module_version)
except Exception as e:
log_message('Error importing module %s: %s' % (url, str(e)))
if destination_directory is not None:
try:
shutil.rmtree(destination_directory)
except:
pass
try:
shutil.rmtree(destination_directory + '-original')
except:
pass
raise
# return data that we probably want displayed in the UI.
ui_info["category"] = module_config["category"]
ui_info["project"] = reponame
ui_info["author"] = module_config["author"]
ui_info["name"] = module_config["name"]
return ui_info
def import_module_from_directory(url,
reponame,
version,
importdir,
force_reload=False):
destination_directory = None
ui_info = {}
try:
# check that right files exist
extension_file_mapping = validate_module_structure(importdir)
python_file = extension_file_mapping['py']
json_file = extension_file_mapping['json']
html_file = extension_file_mapping.get('html', None)
# load json file
module_config = get_module_config_from_json(url,
extension_file_mapping,
importdir)
id_name = module_config['id_name']
# Don't allow importing the same version twice, unless forced
if not force_reload:
if ModuleVersion.objects.filter(module__id_name=id_name,
source_version_hash=version):
raise ValidationError(
'Version {} of module {} has already been imported'.format(
version, url))
# Don't allow loading a module with the same id_name from a different repo. Prevents replacement attacks.
module_urls = get_already_imported_module_urls()
if module_config["id_name"] in module_urls and url != module_urls[
module_config["id_name"]]:
source = module_urls[module_config["id_name"]]
if source == '':
source = "Internal"
raise ValidationError(
"Module {} has already been loaded, and its source is {}.".
format(module_config["id_name"], source))
module_config["source_version"] = version
module_config["link"] = url
module_config["author"] = module_config[
"author"] if "author" in module_config else retrieve_author(url)
# Ensure that modules are categorised properly – if a module category isn't one of our
# pre-defined categories, then we just set it to other.
if module_config["category"] not in get_categories():
module_config["category"] = "Other"
# The core work of creating a module
destination_directory = create_destination_directory(id_name, version)
move_files_to_final_location(destination_directory, importdir,
[json_file, python_file, html_file])
add_boilerplate_and_check_syntax(destination_directory, python_file)
validate_python_functions(destination_directory, python_file)
# If that succeeds, initialise module in our database
load_module_from_dict(module_config)
# clean-up
shutil.rmtree(importdir)
# data that we probably want displayed in the UI.
ui_info["category"] = module_config["category"]
ui_info["project"] = reponame
ui_info["author"] = module_config["author"]
ui_info["name"] = module_config["name"]
except Exception as e:
log_message('Error importing module %s: %s' % (url, str(e)))
if destination_directory is not None:
try:
shutil.rmtree(destination_directory)
except:
pass
try:
shutil.rmtree(destination_directory + '-original')
except:
pass
raise
return ui_info