Beispiel #1
0
    def test_swift_authenticate_fails_with_missing_params(self, mock_create_context):
        mock_context = mock_create_context.return_value
        mock_swift = mock_context.get_client.return_value

        temp_output = tempfile.NamedTemporaryFile()
        mock_swift.fetch_object.return_value = ["FOOBAR", ]

        # uri with missing auth_endpoint... should fail.
        uri = "swift://%(username)s:%(password)s@%(container)s/%(file)s?" \
              "region=%(region)s&tenant_id=%(tenant_id)s" % self.params
        storage = storagelib.get_storage(uri)
        with self.assertRaises(storagelib.storage.InvalidStorageUri):
            storage.save_to_filename(temp_output.name)

        # uri with missing region... should fail.
        uri = "swift://%(username)s:%(password)s@%(container)s/%(file)s?" \
              "auth_endpoint=%(auth_endpoint)s&tenant_id=%(tenant_id)s" % self.params
        storage = storagelib.get_storage(uri)
        with self.assertRaises(storagelib.storage.InvalidStorageUri):
            storage.save_to_filename(temp_output.name)

        # uri with missing tenant_id... should fail.
        uri = "swift://%(username)s:%(password)s@%(container)s/%(file)s?" \
              "auth_endpoint=%(auth_endpoint)s&region=%(region)s" % self.params
        storage = storagelib.get_storage(uri)
        with self.assertRaises(storagelib.storage.InvalidStorageUri):
            storage.save_to_filename(temp_output.name)
Beispiel #2
0
    def test_swift_save_to_directory(
            self, mock_timeout, mock_create_context, mock_makedirs, mock_path_exists):

        expected_jpg = self.RackspaceObject("file/a/0.jpg", "image/jpg")
        expected_mp4 = self.RackspaceObject("file/a/b/c/1.mp4", "video/mp4")

        expected_files = [
            expected_jpg,
            expected_mp4
        ]
        mock_context = mock_create_context.return_value
        mock_swift = mock_context.get_client.return_value

        mock_swift.list_container_objects.return_value = expected_files

        uri = "swift://{username}:{password}@{container}/{file}?" \
              "auth_endpoint={auth_endpoint}&region={region}" \
              "&tenant_id={tenant_id}".format(**self.params)

        storagelib.get_storage(uri).save_to_directory("/tmp/cat/pants")

        self._assert_default_login_correct(mock_create_context, mock_timeout)
        mock_swift.list_container_objects.assert_called_with(
            self.params["container"], prefix=self.params["file"])

        mock_makedirs.assert_has_calls([
            mock.call("/tmp/cat/pants/a"), mock.call("/tmp/cat/pants/a/b/c")])

        mock_swift.download_object.assert_any_call(
            self.params["container"], expected_jpg, "/tmp/cat/pants/a", structure=False)
        mock_swift.download_object.assert_any_call(
            self.params["container"], expected_mp4, "/tmp/cat/pants/a/b/c", structure=False)
Beispiel #3
0
def fhourstones(mark, storage='console', local_sensor=None, remote_sensor=None):
    res = do_fhourstones(local_sensor=local_sensor,
                         remote_sensor=remote_sensor)
    res['mark'] = mark
    get_storage(storage, 'fhourstones')(res)
    
    return res
Beispiel #4
0
    def test_swift_save_to_directory_fails_after_five_failed_file_download_retries(
            self, mock_timeout, mock_create_context, mock_makedirs, mock_path_exists, mock_uniform,
            mock_sleep):

        expected_jpg = self.RackspaceObject("file/a/0.jpg", "image/jpg")
        expected_mp4 = self.RackspaceObject("file/a/b/c/1.mp4", "video/mp4")

        expected_files = [
            expected_jpg,
            expected_mp4
        ]
        mock_context = mock_create_context.return_value
        mock_swift = mock_context.get_client.return_value

        mock_swift.list_container_objects.return_value = expected_files

        mock_swift.download_object.side_effect = [
            storagelib.storage.TimeoutError,
            IOError,
            IOError,
            IOError,
            RuntimeError,
        ]

        uri = "swift://{username}:{password}@{container}/{file}?" \
              "auth_endpoint={auth_endpoint}&region={region}" \
              "&tenant_id={tenant_id}".format(**self.params)

        with self.assertRaises(RuntimeError):
            storagelib.get_storage(uri).save_to_directory("/tmp/cat/pants")

        self._assert_default_login_correct(mock_create_context, mock_timeout)
        mock_swift.list_container_objects.assert_called_with(
            self.params["container"], prefix=self.params["file"])

        mock_makedirs.assert_called_once_with("/tmp/cat/pants/a")

        self.assertEqual(5, mock_swift.download_object.call_count)
        mock_swift.download_object.assert_has_calls([
            mock.call(self.params["container"], expected_jpg, "/tmp/cat/pants/a", structure=False),
            mock.call(self.params["container"], expected_jpg, "/tmp/cat/pants/a", structure=False),
            mock.call(self.params["container"], expected_jpg, "/tmp/cat/pants/a", structure=False),
            mock.call(self.params["container"], expected_jpg, "/tmp/cat/pants/a", structure=False),
            mock.call(self.params["container"], expected_jpg, "/tmp/cat/pants/a", structure=False)
        ])

        mock_uniform.assert_has_calls([
            mock.call(0, 1),
            mock.call(0, 3),
            mock.call(0, 7),
            mock.call(0, 15)
        ])

        self.assertEqual(4, mock_sleep.call_count)
        mock_sleep.assert_called_with(mock_uniform.return_value)
Beispiel #5
0
    def test_ftp_save_to_filename(self, mock_ftp_class):
        temp_output = tempfile.NamedTemporaryFile()

        mock_results = ["foo", "bar"]

        def mock_retrbinary(command, callback):
            for chunk in mock_results:
                callback(chunk)

            return "226"

        mock_ftp = mock_ftp_class.return_value
        mock_ftp.retrbinary.side_effect = mock_retrbinary

        storage = storagelib.get_storage("ftp://*****:*****@ftp.foo.com/some/dir/file")

        storage.save_to_filename(temp_output.name)

        self.assert_connected(mock_ftp_class, mock_ftp)

        mock_ftp.cwd.assert_called_with("some/dir")
        self.assertEqual(1, mock_ftp.retrbinary.call_count)
        self.assertEqual("RETR file", mock_ftp.retrbinary.call_args[0][0])

        with open(temp_output.name) as output_fp:
            self.assertEqual("foobar", output_fp.read())
Beispiel #6
0
    def __init__(self, uid, session):
        # load storage driver
        self.storage_type = 'awss3'
        self.storage = get_storage(self.storage_type)
        self.uid = uid

        self.session = session
Beispiel #7
0
    def test_local_storage_load_from_directory(self):
        temp_directory = create_temp_nested_directory_with_files()

        temp_output_dir = tempfile.mkdtemp()
        storage = storagelib.get_storage("file://{0}/{1}".format(temp_output_dir, "tmp"))

        storage.load_from_directory(temp_directory["temp_directory"]["path"])

        destination_directory_path = os.path.join(
            temp_output_dir, "tmp")
        destination_input_one_path = os.path.join(
            temp_output_dir, destination_directory_path, temp_directory["temp_input_one"]["name"])
        destination_input_two_path = os.path.join(
            temp_output_dir, destination_directory_path, temp_directory["temp_input_two"]["name"])
        nested_temp_input_path = os.path.join(
            temp_output_dir, destination_directory_path,
            temp_directory["nested_temp_directory"]["path"],
            temp_directory["nested_temp_input"]["name"])

        self.assertTrue(os.path.exists(destination_input_one_path))
        self.assertTrue(os.path.exists(destination_input_two_path))
        self.assertTrue(os.path.exists(nested_temp_input_path))

        with open(destination_input_one_path) as temp_output_fp:
            self.assertEqual("FOO", temp_output_fp.read())

        with open(destination_input_two_path) as temp_output_fp:
            self.assertEqual("BAR", temp_output_fp.read())

        with open(nested_temp_input_path) as temp_output_fp:
            self.assertEqual("FOOBAR", temp_output_fp.read())
Beispiel #8
0
    def test_swift_load_from_directory(self, mock_timeout, mock_create_context):
        mock_swift = mock_create_context.return_value.get_client.return_value

        uri = "swift://{username}:{password}@{container}/{file}?" \
              "auth_endpoint={auth_endpoint}&region={region}" \
              "&tenant_id={tenant_id}".format(**self.params)

        storage = storagelib.get_storage(uri)

        temp_directory = create_temp_nested_directory_with_files()
        storage.load_from_directory(temp_directory["temp_directory"]["path"])

        self._assert_default_login_correct(mock_create_context, mock_timeout)

        mock_swift.upload_file.assert_has_calls([
            mock.call(
                self.params["container"], temp_directory["temp_input_two"]["path"],
                os.path.join(self.params["file"], temp_directory["temp_input_two"]["name"])),
            mock.call(
                self.params["container"], temp_directory["temp_input_one"]["path"],
                os.path.join(self.params["file"], temp_directory["temp_input_one"]["name"])),
            mock.call(
                self.params["container"], temp_directory["nested_temp_input"]["path"],
                os.path.join(
                    self.params["file"], temp_directory["nested_temp_directory"]["name"],
                    temp_directory["nested_temp_input"]["name"]))
        ], any_order=True)
Beispiel #9
0
    def test_save_to_file(self, mock_session_class):
        mock_session = mock_session_class.return_value
        mock_s3 = mock_session.client.return_value

        mock_body = mock.Mock()
        mock_body.read.side_effect = [b"some", b"file", b"contents", None]
        mock_s3.get_object.return_value = {
            "Body": mock_body
        }

        mock_file = mock.Mock()

        storage = storagelib.get_storage(
            "s3://access_key:access_secret@bucket/some/file?region=US_EAST")
        storage.save_to_file(mock_file)

        mock_session_class.assert_called_with(
            aws_access_key_id="access_key",
            aws_secret_access_key="access_secret",
            region_name="US_EAST")

        mock_session.client.assert_called_with("s3")
        mock_s3.get_object.assert_called_with(Bucket="bucket", Key="some/file")
        mock_file.write.assert_has_calls([
            mock.call(b"some"),
            mock.call(b"file"),
            mock.call(b"contents")
        ], any_order=False)
Beispiel #10
0
    def assert_transport_handles_directories(self, transport):
        variable = "TEST_STORAGE_{}_URI".format(transport)
        uri = os.getenv(variable, None)

        if not uri:
            raise unittest.SkipTest("Skipping {} - define {} to test".format(transport, variable))

        uri += "/{}".format(self.dest_prefix)
        parsed_url = urlparse(uri)
        print(
            "Testing using:", parsed_url.scheme, parsed_url.hostname, parsed_url.port,
            parsed_url.path)
        storage = get_storage(uri)

        print("Transport:", transport)
        print("\t* Uploading")
        storage.load_from_directory(self.directory)

        target_directory = tempfile.mkdtemp(prefix="dest-root")

        print("\t* Downloading")
        storage.save_to_directory(target_directory)

        print("\t* Checking")
        try:
            subprocess.check_output(
                ["diff", "-r", self.directory, target_directory], stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as error:
            print("Diff output:\n{}".format(error.output))
            raise
Beispiel #11
0
    def test_ftp_save_to_directory_creates_destination_directory_if_needed(
            self, mock_ftp_class, mock_path_exists, mock_makedirs, mock_chdir):
        mock_ftp = mock_ftp_class.return_value
        mock_ftp.pwd.return_value = "some/dir/file"

        # no files or folders
        mock_ftp.retrlines.side_effect = create_mock_ftp_directory_listing([])

        mock_path_exists.return_value = False

        storage = storagelib.get_storage("ftp://*****:*****@ftp.foo.com/some/dir/file")

        storage.save_to_directory("/cat/pants")

        self.assert_connected(mock_ftp_class, mock_ftp)

        mock_ftp.cwd.assert_has_calls([
            mock.call("/some/dir/file"),
        ])

        mock_makedirs.assert_has_calls([
            mock.call("/cat/pants"),
        ])

        mock_chdir.assert_has_calls([
            mock.call("/cat/pants"),
        ])

        mock_ftp.retrbinary.assert_not_called()
Beispiel #12
0
    def test_save_to_file_with_specific_port(self, mock_ftp_class):
        out_file = StringIO()

        mock_results = ["foo", "bar"]

        def mock_retrbinary(command, callback):
            for chunk in mock_results:
                callback(chunk)

            return "226"

        mock_ftp = mock_ftp_class.return_value
        mock_ftp.retrbinary.side_effect = mock_retrbinary

        storage = storagelib.get_storage("ftp://*****:*****@ftp.foo.com:12345/some/dir/file")

        storage.save_to_file(out_file)

        self.assert_connected(mock_ftp_class, mock_ftp, 12345)

        mock_ftp.cwd.assert_called_with("some/dir")
        self.assertEqual(1, mock_ftp.retrbinary.call_count)
        self.assertEqual("RETR file", mock_ftp.retrbinary.call_args[0][0])

        self.assertEqual("foobar", out_file.getvalue())
Beispiel #13
0
    def test_ftps_scheme_connects_using_ftp_tls_class(self, mock_ftp_tls_class):
        temp_output = tempfile.NamedTemporaryFile()

        mock_results = ["foo", "bar"]

        def mock_retrbinary(command, callback):
            for chunk in mock_results:
                callback(chunk)

            return "226"

        mock_ftp = mock_ftp_tls_class.return_value
        mock_ftp.retrbinary.side_effect = mock_retrbinary

        def assert_tcp_keepalive_already_enabled(username, password):
            # It is important that these already be called before
            # login is called, because FTP_TLS.login replaces the
            # socket instance with an SSL-wrapped socket.
            mock_ftp.sock.setsockopt.assert_any_call(
                socket.SOL_SOCKET, socket.SO_KEEPALIVE,
                storagelib.storage.DEFAULT_FTP_KEEPALIVE_ENABLE)

        mock_ftp.login.side_effect = assert_tcp_keepalive_already_enabled

        storage = storagelib.get_storage("ftps://*****:*****@ftp.foo.com/some/dir/file")

        storage.save_to_filename(temp_output.name)

        mock_ftp_tls_class.assert_called_with(timeout=storagelib.storage.DEFAULT_FTP_TIMEOUT)
        mock_ftp.connect.assert_called_with("ftp.foo.com", port=21)
        mock_ftp.login.assert_called_with("user", "password")
        mock_ftp.prot_p.assert_called_with()
Beispiel #14
0
    def test_connect_sets_tcp_keepalive_options_when_supported(self, mock_ftp_class, mock_socket):
        mock_socket.SOL_SOCKET = socket.SOL_SOCKET
        mock_socket.SOL_TCP = socket.SOL_TCP
        mock_socket.SO_KEEPALIVE = socket.SO_KEEPALIVE

        mock_socket.TCP_KEEPCNT = 1
        mock_socket.TCP_KEEPIDLE = 2
        mock_socket.TCP_KEEPINTVL = 3

        mock_ftp = mock_ftp_class.return_value
        in_file = StringIO("foobar")

        storage = storagelib.get_storage("ftp://*****:*****@ftp.foo.com/some/dir/file")

        storage.load_from_file(in_file)

        mock_ftp.sock.setsockopt.assert_has_calls([
            mock.call(
                socket.SOL_SOCKET, socket.SO_KEEPALIVE,
                storagelib.storage.DEFAULT_FTP_KEEPALIVE_ENABLE),
            mock.call(
                socket.SOL_TCP, mock_socket.TCP_KEEPCNT, storagelib.storage.DEFAULT_FTP_KEEPCNT),
            mock.call(
                socket.SOL_TCP, mock_socket.TCP_KEEPIDLE, storagelib.storage.DEFAULT_FTP_KEEPIDLE),
            mock.call(
                socket.SOL_TCP, mock_socket.TCP_KEEPINTVL, storagelib.storage.DEFAULT_FTP_KEEPINTVL)
        ])
Beispiel #15
0
def main():
    for name, public_site in settings.PUBLIC_SITES.items():
        subject_storage = storage.get_storage(settings, public_site)
        for er_site in public_site.er_sites:
            downloader = er.downloader.SubjectDownloader(
                er_site.token, er_site.host, subject_storage,
                settings.SERVER_URL, public_site.name)
            downloader.download_subjects_and_tracks()
Beispiel #16
0
    def test_ftp_delete_directory(self, mock_ftp_class):
        mock_ftp = mock_ftp_class.return_value

        mock_ftp.pwd.return_value = "some/dir/file"

        mock_ftp.retrlines.side_effect = create_mock_ftp_directory_listing([
            # root
            [
                "drwxrwxr-x 3 test test 4.0K Apr  9 10:54 dir1",
                "drwxrwxr-x 3 test test 4.0K Apr  9 10:54 dir2",
                "-rwxrwxr-x 3 test test 4.0K Apr  9 10:54 file1",
                "-rwxrwxr-x 3 test test 4.0K Apr  9 10:54 file2",
            ],
            # dir1
            [
                "drwxrwxr-x 3 test test 4.0K Apr  9 10:54 dir with spaces",
                "-rwxrwxr-x 3 test test 4.0K Apr  9 10:54 file3",
            ],
            # dir with spaces
            [
                "-rwxrwxr-x 3 test test 4.0K Apr  9 10:54 file with spaces"
            ],
            # dir2
            [
                "drwxrwxr-x 3 test test 4.0K Apr  9 10:54 dir4",
            ],
        ])

        storage = storagelib.get_storage("ftp://*****:*****@ftp.foo.com/some/dir/file")
        storage.delete_directory()

        self.assert_connected(mock_ftp_class, mock_ftp)

        mock_ftp.cwd.assert_has_calls([
            mock.call("/some/dir/file"),
            mock.call("some/dir/file/dir1"),
            mock.call("some/dir/file/dir1/dir with spaces"),
            mock.call("some/dir/file/dir2"),
            mock.call("some/dir/file/dir2/dir4")
        ])
        self.assertEqual(5, mock_ftp.cwd.call_count)

        mock_ftp.delete.assert_has_calls([
            mock.call("/some/dir/file/dir1/dir with spaces/file with spaces"),
            mock.call("/some/dir/file/dir1/file3"),
            mock.call("/some/dir/file/file2"),
            mock.call("/some/dir/file/file1")
        ], any_order=True)
        self.assertEqual(4, mock_ftp.delete.call_count)

        mock_ftp.rmd.assert_has_calls([
            mock.call("/some/dir/file/dir2/dir4"),
            mock.call("/some/dir/file/dir2"),
            mock.call("/some/dir/file/dir1/dir with spaces"),
            mock.call("/some/dir/file/dir1"),
            mock.call("/some/dir/file")
        ])
        self.assertEqual(5, mock_ftp.rmd.call_count)
    def __init__(self, uid):
        # load storage driver
        self.storage = get_storage('mock', uid=uid)
        self.uid = uid

        engine = create_engine('sqlite:///C:\\Users\\xiang\\test.db')

        Session = sessionmaker(bind=engine)
        self.session = Session()
Beispiel #18
0
    def test_load_from_file_creates_intermediate_dirs(self, mock_exists, mock_makedirs, mock_copy):
        mock_exists.return_value = False

        storage = storagelib.get_storage("file:///foo/bar/file")
        storage.load_from_filename("input_file")

        mock_exists.assert_called_with("/foo/bar")
        mock_makedirs.assert_called_with("/foo/bar")
        mock_copy.assert_called_with("input_file", "/foo/bar/file")
Beispiel #19
0
    def test_local_storage_delete_directory(self, mock_remove, mock_rmtree):
        temp_directory = create_temp_nested_directory_with_files()

        storage = storagelib.get_storage(
            "file://{0}".format(temp_directory["temp_directory"]["path"]))
        storage.delete_directory()

        self.assertFalse(mock_remove.called)
        mock_rmtree.assert_called_once_with(temp_directory["temp_directory"]["path"], True)
Beispiel #20
0
    def test_ftp_get_download_url_returns_none_with_empty_base(self, mock_ftp_class):
        ftpuri = "ftp://*****:*****@ftp.foo.com/some/dir/file.txt"

        storage = storagelib.get_storage(ftpuri)

        with self.assertRaises(DownloadUrlBaseUndefinedError):
            storage.get_download_url()

        mock_ftp_class.assert_not_called()
Beispiel #21
0
    def test_local_storage_load_from_file(self):
        in_file = StringIO("foobar")
        temp_output = tempfile.NamedTemporaryFile()

        storage = storagelib.get_storage("file://{0}".format(temp_output.name))
        storage.load_from_file(in_file)

        with open(temp_output.name) as temp_output_fp:
            self.assertEqual("foobar", temp_output_fp.read())
Beispiel #22
0
	def transfer(self,filename,dstSubFolder=None,delete_after=False):
		if 'enable' in self.data.keys() and self.data['enable']:
			self.logger.info(
				'Transfering file from {0} to {1}'.format(
					self.get_uri("source",filename).decode('utf-8'),
					self.get_uri("destination",filename).decode('utf-8')
				)
			)

			source_file = storage.get_storage(self.get_uri("source",filename,showPassword=True))
			dest_file = storage.get_storage(self.get_uri("destination",filename=filename,subFolder=dstSubFolder,showPassword=True))

			if self.data['source']['protocol'] == 'file' and self.data['destination']['protocol'] == 'file':
				destination = "{0}/{1}".format(makePath(self.data['destination']['path'],dstSubFolder),filename)
				if not os.path.exists(os.path.dirname(destination)):
					os.makedirs(os.path.dirname(destination))
				shutil.copyfile("{0}/{1}".format(self.data['source']['path'],filename),destination)
				self.setPerm(destination)
			else:
				if self.data['source']['protocol'] == 'file':
					try:
						dest_file.load_from_filename("{0}/{1}".format(self.data['source']['path'],filename))
					except IOError as e:
						self.logger.error(
							"Connection error to {0}"
							.format(self.data['destination'])
						)
						raise e
				elif self.data['destination']['protocol'] == 'file':
					destination = "{0}/{1}".format(makePath(self.data['destination']['path'],dstSubFolder),filename)
					if not os.path.exists(os.path.dirname(destination)):
						os.makedirs(os.path.dirname(destination))
					source_file.save_to_filename(destination)
					self.setPerm(destination)
				else:
					tmpfile = unicode(tempfile.mkstemp()[1])
					os.remove(tmpfile)
					source_file.save_to_filename(tmpfile)
					dest_file.load_from_filename(tmpfile)
					os.remove(tmpfile)
			if delete_after:
				source_file.delete()
		else:
			self.logger.info('Transferer is disabled. No transfer.')
Beispiel #23
0
    def test_local_storage_get_download_url_returns_none_on_empty_base(self):
        temp_input = tempfile.NamedTemporaryFile()
        temp_input.write("FOOBAR")
        temp_input.flush()

        # blank download_url_base
        storage_uri = "file://{fpath}?download_url_base=".format(fpath=temp_input.name)

        out_storage = storagelib.get_storage(storage_uri)

        with self.assertRaises(DownloadUrlBaseUndefinedError):
            out_storage.get_download_url()

        # no download_url_base
        storage_uri = "file://{fpath}".format(fpath=temp_input.name)
        out_storage = storagelib.get_storage(storage_uri)

        with self.assertRaises(DownloadUrlBaseUndefinedError):
            out_storage.get_download_url()
Beispiel #24
0
    def test_load_from_file_does_not_create_dirs_if_present(
            self, mock_open, mock_exists, mock_makedirs):
        mock_exists.return_value = True
        in_file = StringIO("foobar")

        out_storage = storagelib.get_storage("file:///foobar/is/out")
        out_storage.load_from_file(in_file)

        mock_exists.assert_called_with("/foobar/is")
        self.assertEqual(0, mock_makedirs.call_count)
Beispiel #25
0
    def test_rackspace_authenticate_with_region(self, mock_timeout, mock_create_context):
        uri = "cloudfiles://{username}:{api_key}@{container}/{file}?region=ORD".format(
            username="******", api_key="apikey", container="container", file="file.txt")

        storage = storagelib.get_storage(uri)
        storage.delete()

        self._assert_login_correct(
            mock_create_context, mock_timeout, username="******", password="******", region="ORD",
            public=True)
Beispiel #26
0
    def test_ftp_delete(self, mock_ftp_class):
        mock_ftp = mock_ftp_class.return_value

        storage = storagelib.get_storage("ftp://*****:*****@ftp.foo.com/some/dir/file")
        storage.delete()

        self.assert_connected(mock_ftp_class, mock_ftp)

        mock_ftp.cwd.assert_called_with("some/dir")
        mock_ftp.delete.assert_called_with("file")
Beispiel #27
0
    def test_register_storage_protocol_updates_storage_types(self):

        @storagelib.register_storage_protocol(scheme=self.scheme)
        class MyStorageClass(storagelib.storage.Storage):
            pass

        self.assertIn(self.scheme, storagelib.storage._STORAGE_TYPES)

        uri = "{0}://some/uri/path".format(self.scheme)
        store_obj = storagelib.get_storage(uri)
        self.assertIsInstance(store_obj, MyStorageClass)
Beispiel #28
0
    def test_local_storage_save_to_file(self):
        temp_input = tempfile.NamedTemporaryFile()
        temp_input.write("FOOBAR")
        temp_input.flush()

        out_file = StringIO()

        storage = storagelib.get_storage("file://%s" % (temp_input.name))
        storage.save_to_file(out_file)

        self.assertEqual("FOOBAR", out_file.getvalue())
Beispiel #29
0
    def test_ftp_get_download_url(self, mock_ftp_class):
        download_url_base = urllib.quote_plus("http://hostname/path/to/")

        ftpuri = "ftp://*****:*****@ftp.foo.com/some/dir/file.txt?download_url_base={0}".format(
            download_url_base)

        storage = storagelib.get_storage(ftpuri)
        temp_url = storage.get_download_url()

        mock_ftp_class.assert_not_called()
        self.assertEqual(temp_url, "http://hostname/path/to/file.txt")
Beispiel #30
0
    def test_register_swift_protocol_updates_storage_types(self):

        @storagelib.register_swift_protocol(scheme=self.scheme, auth_endpoint=self.auth_endpoint)
        class MyStorageClass(storagelib.swift_storage.SwiftStorage):
            pass

        self.assertIn(self.scheme, storagelib.storage._STORAGE_TYPES)

        uri = "{0}://username:password@containter/object?region=region-a".format(self.scheme)
        store_obj = storagelib.get_storage(uri)
        self.assertIsInstance(store_obj, MyStorageClass)
Beispiel #31
0
    def test_swift_uses_servicenet_when_requested(self, mock_timeout, mock_create_context):
        mock_swift = mock_create_context.return_value.get_client.return_value

        uri = "swift://%(username)s:%(password)s@%(container)s/%(file)s?" \
              "auth_endpoint=%(auth_endpoint)s&region=%(region)s" \
              "&tenant_id=%(tenant_id)s&public=False" % self.params
        storage = storagelib.get_storage(uri)
        storage.delete()

        self._assert_default_login_correct(mock_create_context, mock_timeout, public=False)
        mock_swift.delete_object.assert_called_with(self.params["container"], self.params["file"])
Beispiel #32
0
 def __init__(self, port=8001, que=5):
     self.port = port
     self.que = que
     self.__parse_config()
     self.storage = storage.get_storage(self.config_options['storage']\
                                       ['type'])
     self.plugins = []
     for plugin in self.config_options['active_plugins']:
         module = __import__(plugin)
         self.plugins.append(getattr(module,
                            self.config_options['active_plugins']\
                            [plugin])(self.config_options['paths']\
                            ['plugins_path']))
Beispiel #33
0
def test_transport(transport, uri, source_directory):
    storage = storagelib.get_storage(uri)

    print "Transport:", transport
    print "\t* Uploading"
    storage.load_from_directory(source_directory)

    target_directory = tempfile.mkdtemp()

    print "\t* Downloading"
    storage.save_to_directory(target_directory)

    with open(os.devnull, 'w') as devnull:
        print "\t* Checking"
        child = subprocess.Popen(
            ["diff", "-r", source_directory, target_directory],
            stderr=devnull,
            stdout=devnull)

        return child.wait()
Beispiel #34
0
 async def fetch_articles(self, url, fetch_min, fetch_max):
     while True:
         print("fetching immonet!")
         page = requests.get(url)
         data = storage.get_storage()
         soup = BeautifulSoup(page.content, 'html.parser')
         articles = soup.find_all("div",
                                  {"id": re.compile("^selObject_\\d*")})
         flats = [build_flat(article) for article in articles]
         flats.reverse()
         for flat in flats:
             if flat['exposeId'] not in data:
                 data[flat['exposeId']] = {
                     "storage":
                     flat,
                     "date_time":
                     datetime.now(tzlocal.get_localzone()).strftime(
                         "%Y-%m-%d %H:%M:%S %z")
                 }
                 storage.set_storage(data)
                 time.sleep(random.randint(1, 5))
                 telegram.send_flat(flat)
         await asyncio.sleep(random.randint(450, 750))
Beispiel #35
0
def root():
    return storage.get_storage()
Beispiel #36
0
def register_routes(app):
    storage = get_storage(app.config['STORAGE'])

    @app.route('/')
    def index():
        return redirect("/docs", code=302)

    @app.route('/set', methods=['GET'])
    def set_pair():
        """
           Create new pair, update if it exists
           ---
           tags:
             - keyValue
           parameters:
             - in: query
               name: k
               required: true
               type: string
               minimum: 1
               maximum: 64
               description: The name of the key
             - in: query
               name: v
               required: true
               type: string
               minimum: 1
               maximum: 256
               description: The value for the given key
           responses:
             200:
               description: Set of key-value pairs
             400:
               description: Key or value are not set correctly
        """
        key = request.args.get('k')
        value = request.args.get('v')

        try:
            validate(key=key, value=value)
        except InvalidRequest as ex:
            return make_response(jsonify({"error": "{0}".format(ex.message)}),
                                 400)

        storage.set(key, value)

        return make_response(jsonify({"message": "OK"}), 200)

    @app.route('/get', methods=['GET'])
    def get_pair():
        """
           Get pair by key
           ---
           tags:
             - keyValue
           parameters:
             - in: query
               name: k
               required: true
               type: string
               minimum: 1
               maximum: 64
               description: The name of the key
           responses:
             200:
               description: Set of key-value pairs
             400:
               description: The key is not set correctly
             404:
               description: No value saved for the key
        """
        key = request.args.get('k')

        try:
            validate(key=key)
            value = storage.get(key)
        except InvalidRequest as ex:
            return make_response(jsonify({"error": "{0}".format(ex.message)}),
                                 400)
        except NoItemFound:
            return make_response(
                jsonify({"error": "No value stored for key: {0}".format(key)}),
                404)

        return make_response(jsonify({key: value}), 200)

    @app.route('/is', methods=['GET'])
    def pair_exists():
        """
           Check if pair exists
           ---
           tags:
             - keyValue
           parameters:
             - in: query
               name: k
               required: true
               type: string
               minimum: 1
               maximum: 64
               description: The name of the key
           responses:
             200:
               description: Return True if pair exists
             400:
               description: The key is not set correctly
             404:
               description: Return False if pair doesn`t exists
        """
        key = request.args.get('k')

        try:
            validate(key=key)
            storage.get(key)
        except InvalidRequest as ex:
            return make_response(jsonify({"error": "{0}".format(ex.message)}),
                                 400)
        except NoItemFound:
            return make_response(jsonify({"message": False}), 404)

        return make_response(jsonify({"message": True}), 200)

    @app.route('/rm', methods=['GET'])
    def remove_pair():
        """
           Remove a pair
           ---
           tags:
             - keyValue
           parameters:
             - in: query
               name: k
               required: true
               type: string
               minimum: 1
               maximum: 64
               description: The name of the key
           responses:
             200:
               description: Set of key-value pairs
             400:
               description: The key is not set correctly
             404:
               description: No value stored for that key
        """
        key = request.args.get('k')

        try:
            validate(key=key)
            storage.remove(key)
        except InvalidRequest as ex:
            return make_response(jsonify({"error": "{0}".format(ex.message)}),
                                 400)
        except NoItemFound:
            return make_response(
                jsonify({"error": "No value stored for key: {0}".format(key)}),
                404)

        return make_response(jsonify({"message": "OK"}), 200)

    @app.route('/clear', methods=['GET'])
    def remove_all_pairs():
        """
           Truncate all values from the storage
           ---
           tags:
             - keyValue
           responses:
             200:
               description: Set of key-value pairs
        """
        storage.remove_all()
        return make_response(jsonify({"message": "OK"}), 200)

    @app.route('/getKeys', methods=['GET'])
    def get_all_keys():
        """
           Get all keys
           ---
           tags:
             - keyValue
           parameters:
             - in: query
               name: page
               required: false
               type: integer
               minimum: 1
               default: 1
               description: Number of the page
             - in: query
               name: per_page
               required: false
               type: integer
               minimum: 1
               default: 10
               description: How many items to be shown on the page
           responses:
             200:
               description: Set of key-value pairs
             404:
               description: No keys stored or wrong paginating
        """
        page, per_page = get_pagination(request.args)
        try:
            data = storage.get_all_keys(page, per_page)
        except NoItemFound:
            return make_response(jsonify({"error": "No keys stored"}), 404)

        return make_response(jsonify(data), 200)

    @app.route('/getValues', methods=['GET'])
    def get_all_values():
        """
           Get all values
           ---
           tags:
             - keyValue
           parameters:
             - in: query
               name: page
               required: false
               type: integer
               minimum: 1
               default: 1
               description: Number of the page
             - in: query
               name: per_page
               required: false
               type: integer
               minimum: 1
               default: 10
               description: How many items to be shown on the page
           responses:
             200:
               description: Set of key-value pairs
             404:
               description: No values stored or wrong paginating
        """

        page, per_page = get_pagination(request.args)
        try:
            data = storage.get_all_values(page, per_page)
        except NoItemFound:
            return make_response(jsonify({"error": "No values stored"}), 404)

        return make_response(jsonify(data), 200)

    @app.route('/getAll', methods=['GET'])
    def get_all():
        """
           Get all key-value pairs
           ---
           tags:
             - keyValue
           parameters:
             - in: query
               name: page
               required: false
               type: integer
               minimum: 1
               default: 1
               description: Number of the page
             - in: query
               name: per_page
               required: false
               type: integer
               minimum: 1
               default: 10
               description: How many items to be shown on the page
           responses:
             200:
               description: Set of key-value pairs
             404:
               description: No pairs stored or wrong paginating
        """

        page, per_page = get_pagination(request.args)

        try:
            data = storage.get_all(page, per_page)
        except NoItemFound:
            return make_response(jsonify({"error": "No pairs stored"}), 404)

        return make_response(jsonify(data), 200)
Beispiel #37
0
#!/usr/bin/env python

from monitor import get_monitor
from storage import get_storage
from auditor import get_auditor
from downloader import get_downloader

if __name__ == "__main__":
	monitor = get_monitor()
	storage = get_storage()
	auditor = get_auditor()
	downloader = get_downloader()

	entries = monitor.poll()
	to_process = storage.filter_entries(entries)
	for entry in to_process:
		print "To process: ", entry

		components = entry.split(".")
		version = components[2].replace("-", ".")
		tag = components[3]

		downloader.download(tag, version)
		hashes = downloader.hash(tag, version)
		result = auditor.audit(entry, hashes)
		storage.store_entry(entry, result)
		downloader.empty_storage(tag, version)
		print "\n\n"
	storage.save()
Beispiel #38
0
def load_subject_storage(public_sites):
    for name, public_site in public_sites.items():
        subject_storages[name] = storage.get_storage(settings, public_site)
Beispiel #39
0
 def __init__(self, plugins_config_path):
     self.get_config_options(plugins_config_path)
     if self.config_options['storage']:
         storage_type = self.config_options['storage']['type']
         self.storage = storage.get_storage(storage_type)
Beispiel #40
0
 def __init__(self, *a, **kw):
     """Init BaseSpider with storage configuration"""
     CrawlSpider.__init__(self, *a, **kw)
     self.source_name = self.get_source_name()
     self.storage = get_storage(self.source_name)
Beispiel #41
0
 def __init__(self, loop, executor):
     self.loop = loop
     self.executor = executor
     self.storage = get_storage()
Beispiel #42
0
 def setUp(self):
     self.app = app.test_client()
     self.client = app.test_client(self)
     self.storage = get_storage(app.config['STORAGE'])
     self.db = db.get_db()