def create(self, validated_data):
"""
Create a new `project` with optionally provided `upload_file` and `pipeline`.
The `execute_now` parameter can be provided to execute the Pipeline on creation.
"""
upload_file = validated_data.pop("upload_file", None)
input_urls = validated_data.pop("input_urls", [])
pipeline = validated_data.pop("pipeline", None)
execute_now = validated_data.pop("execute_now", False)
downloads, errors = fetch_urls(input_urls)
if errors:
raise serializers.ValidationError("Could not fetch: " +
"\n".join(errors))
project = super().create(validated_data)
if upload_file:
project.add_uploads([upload_file])
if downloads:
project.add_downloads(downloads)
if pipeline:
project.add_pipeline(pipeline, execute_now)
return project
def clean_input_urls(self):
"""
Fetch the `input_urls` and set the `downloads` objects in the cleaned_data.
A validation error is raised if at least one URL could not be fetched.
"""
input_urls = self.cleaned_data.get("input_urls", [])
self.cleaned_data["downloads"], errors = fetch_urls(input_urls)
if errors:
raise ValidationError("Could not fetch: " + "\n".join(errors))
return input_urls
def test_scanpipe_pipes_fetch_fetch_urls(self, mock_get):
urls = [
"https://example.com/filename.zip",
"https://example.com/archive.tar.gz",
]
mock_get.return_value = mock.Mock(content=b"\x00",
headers={},
status_code=200,
url="mocked_url")
downloads, errors = fetch.fetch_urls(urls)
self.assertEqual(2, len(downloads))
self.assertEqual(urls[0], downloads[0].uri)
self.assertEqual(urls[1], downloads[1].uri)
self.assertEqual(0, len(errors))
mock_get.side_effect = Exception
downloads, errors = fetch.fetch_urls(urls)
self.assertEqual(0, len(downloads))
self.assertEqual(2, len(errors))
self.assertEqual(urls, errors)
def handle_input_urls(self, input_urls):
"""
Fetch provided `input_urls` and store to the project `input` directory.
"""
downloads, errors = fetch_urls(input_urls)
if downloads:
self.project.add_downloads(downloads)
msg = "File(s) downloaded to the project inputs directory:"
self.stdout.write(self.style.SUCCESS(msg))
msg = "\n".join(["- " + downloaded.filename for downloaded in downloads])
self.stdout.write(msg)
if errors:
self.stdout.write(self.style.ERROR("Could not fetch URL(s):"))
msg = "\n".join(["- " + url for url in errors])
self.stdout.write(self.style.ERROR(msg))