def create(self, validated_data): """ Create a new `project` with optionally provided `upload_file` and `pipeline`. The `execute_now` parameter can be provided to execute the Pipeline on creation. """ upload_file = validated_data.pop("upload_file", None) input_urls = validated_data.pop("input_urls", []) pipeline = validated_data.pop("pipeline", None) execute_now = validated_data.pop("execute_now", False) downloads, errors = fetch_urls(input_urls) if errors: raise serializers.ValidationError("Could not fetch: " + "\n".join(errors)) project = super().create(validated_data) if upload_file: project.add_uploads([upload_file]) if downloads: project.add_downloads(downloads) if pipeline: project.add_pipeline(pipeline, execute_now) return project
def clean_input_urls(self): """ Fetch the `input_urls` and set the `downloads` objects in the cleaned_data. A validation error is raised if at least one URL could not be fetched. """ input_urls = self.cleaned_data.get("input_urls", []) self.cleaned_data["downloads"], errors = fetch_urls(input_urls) if errors: raise ValidationError("Could not fetch: " + "\n".join(errors)) return input_urls
def test_scanpipe_pipes_fetch_fetch_urls(self, mock_get): urls = [ "https://example.com/filename.zip", "https://example.com/archive.tar.gz", ] mock_get.return_value = mock.Mock(content=b"\x00", headers={}, status_code=200, url="mocked_url") downloads, errors = fetch.fetch_urls(urls) self.assertEqual(2, len(downloads)) self.assertEqual(urls[0], downloads[0].uri) self.assertEqual(urls[1], downloads[1].uri) self.assertEqual(0, len(errors)) mock_get.side_effect = Exception downloads, errors = fetch.fetch_urls(urls) self.assertEqual(0, len(downloads)) self.assertEqual(2, len(errors)) self.assertEqual(urls, errors)
def handle_input_urls(self, input_urls): """ Fetch provided `input_urls` and store to the project `input` directory. """ downloads, errors = fetch_urls(input_urls) if downloads: self.project.add_downloads(downloads) msg = "File(s) downloaded to the project inputs directory:" self.stdout.write(self.style.SUCCESS(msg)) msg = "\n".join(["- " + downloaded.filename for downloaded in downloads]) self.stdout.write(msg) if errors: self.stdout.write(self.style.ERROR("Could not fetch URL(s):")) msg = "\n".join(["- " + url for url in errors]) self.stdout.write(self.style.ERROR(msg))