Exemplo n.º 1
0
class KeyfileTest(unittest.TestCase):
    def setUp(self):
        service_connection = MockConnection()
        self.contents = '0123456789'
        bucket = MockBucket(service_connection, 'mybucket')
        key = bucket.new_key('mykey')
        key.set_contents_from_string(self.contents)
        self.keyfile = KeyFile(key)

    def tearDown(self):
        self.keyfile.close()

    def testReadFull(self):
        self.assertEqual(self.keyfile.read(len(self.contents)), self.contents)

    def testReadPartial(self):
        self.assertEqual(self.keyfile.read(5), self.contents[:5])
        self.assertEqual(self.keyfile.read(5), self.contents[5:])

    def testTell(self):
        self.assertEqual(self.keyfile.tell(), 0)
        self.keyfile.read(4)
        self.assertEqual(self.keyfile.tell(), 4)
        self.keyfile.read(6)
        self.assertEqual(self.keyfile.tell(), 10)
        self.keyfile.close()
        try:
            self.keyfile.tell()
        except ValueError, e:
            self.assertEqual(e.message, 'I/O operation on closed file')
Exemplo n.º 2
0
class KeyfileTest(unittest.TestCase):

    def setUp(self):
        service_connection = MockConnection()
        self.contents = '0123456789'
        bucket = MockBucket(service_connection, 'mybucket')
        key = bucket.new_key('mykey')
        key.set_contents_from_string(self.contents)
        self.keyfile = KeyFile(key)

    def tearDown(self):
        self.keyfile.close()

    def testReadFull(self):
        self.assertEqual(self.keyfile.read(len(self.contents)), self.contents)

    def testReadPartial(self):
        self.assertEqual(self.keyfile.read(5), self.contents[:5])
        self.assertEqual(self.keyfile.read(5), self.contents[5:])

    def testTell(self):
        self.assertEqual(self.keyfile.tell(), 0)
        self.keyfile.read(4)
        self.assertEqual(self.keyfile.tell(), 4)
        self.keyfile.read(6)
        self.assertEqual(self.keyfile.tell(), 10)
        self.keyfile.close()
        try:
            self.keyfile.tell()
        except ValueError, e:
            self.assertEqual(str(e), 'I/O operation on closed file')
def fetchxml(awsbucketkey):
    """
    Given an AWS Key() into a bucket, fetch the contents of the XML file stored
    there as a single string.
    If the filename ends in .zip, assume it is a ZIP-encoded single file and return
    that.
    Returns None if the Key's name doesn't end .xml or .xml.zip
    """

    xmlcontent = None
    if awsbucketkey.name.endswith('.xml.zip'):
        try:
            infomsg(awsbucketkey.name + ' ...unzip')
            keyf = KeyFile(awsbucketkey)
            if keyf is None:
                errormsg('ERROR: Failed to open S3 bucket object: ' + awsbucketkey.name)
            else:
                try:
                    zf = ZipFile(keyf)
                    # get just the first file in archive,
                    # as there shouldn't ever be more than one
                    xmlfile = next(iter(zf.infolist()), None)
                    # orig filename and mtime: debugmsg(xmlfile.filename, xmlfile.date_time)
                    xmlcontent = zf.read(xmlfile)

                except BadZipfile as badz:
                    errormsg('ZIP Exception caught while reading zipped S3 file', badz.message)

        except Exception as exc:
            errormsg('Runtime Exception caught while reading zipped S3 file', exc.message)
            raise exc

    elif awsbucketkey.name.endswith('.xml'):
        try:
            infomsg(awsbucketkey.name)
            keyf = KeyFile(awsbucketkey)
            if keyf is None:
                errormsg('ERROR: Failed to open S3 bucket object: ' + awsbucketkey.name)
            else:
                xmlcontent = keyf.read()

        except Exception as exc:
            errormsg('ERROR: Exception caught while reading S3 file', exc.message)
            raise exc

    return xmlcontent 
Exemplo n.º 4
0
 def read(self, length=DEFAULT_READ_SIZE):
   return KeyFile.read(self, length)
Exemplo n.º 5
0
 def read(self, size=-1):
     if self.closed:
         raise ValueError("I/O operation on closed file")
     if size < 0:
         size = self.key.size - self.location
     return KeyFile.read(self, size)
Exemplo n.º 6
0
class KeyfileTest(unittest.TestCase):

    def setUp(self):
        service_connection = MockConnection()
        self.contents = '0123456789'
        bucket = MockBucket(service_connection, 'mybucket')
        key = bucket.new_key('mykey')
        key.set_contents_from_string(self.contents)
        self.keyfile = KeyFile(key)

    def tearDown(self):
        self.keyfile.close()

    def testReadFull(self):
        self.assertEqual(self.keyfile.read(len(self.contents)), self.contents)

    def testReadPartial(self):
        self.assertEqual(self.keyfile.read(5), self.contents[:5])
        self.assertEqual(self.keyfile.read(5), self.contents[5:])

    def testTell(self):
        self.assertEqual(self.keyfile.tell(), 0)
        self.keyfile.read(4)
        self.assertEqual(self.keyfile.tell(), 4)
        self.keyfile.read(6)
        self.assertEqual(self.keyfile.tell(), 10)
        self.keyfile.close()
        try:
            self.keyfile.tell()
        except ValueError as e:
            self.assertEqual(str(e), 'I/O operation on closed file')

    def testSeek(self):
        self.assertEqual(self.keyfile.read(4), self.contents[:4])
        self.keyfile.seek(0)
        self.assertEqual(self.keyfile.read(4), self.contents[:4])
        self.keyfile.seek(5)
        self.assertEqual(self.keyfile.read(5), self.contents[5:])

        # Seeking negative should raise.
        try:
            self.keyfile.seek(-5)
        except IOError as e:
            self.assertEqual(str(e), 'Invalid argument')

        # Reading past end of file is supposed to return empty string.
        self.keyfile.read(10)
        self.assertEqual(self.keyfile.read(20), '')

        # Seeking past end of file is supposed to silently work.
        self.keyfile.seek(50)
        self.assertEqual(self.keyfile.tell(), 50)
        self.assertEqual(self.keyfile.read(1), '')

    def testSeekEnd(self):
        self.assertEqual(self.keyfile.read(4), self.contents[:4])
        self.keyfile.seek(0, os.SEEK_END)
        self.assertEqual(self.keyfile.read(1), '')
        self.keyfile.seek(-1, os.SEEK_END)
        self.assertEqual(self.keyfile.tell(), 9)
        self.assertEqual(self.keyfile.read(1), '9')
        # Test attempt to seek backwards past the start from the end.
        try:
            self.keyfile.seek(-100, os.SEEK_END)
        except IOError as e:
            self.assertEqual(str(e), 'Invalid argument')

    def testSeekCur(self):
        self.assertEqual(self.keyfile.read(1), self.contents[0])
        self.keyfile.seek(1, os.SEEK_CUR)
        self.assertEqual(self.keyfile.tell(), 2)
        self.assertEqual(self.keyfile.read(4), self.contents[2:6])
Exemplo n.º 7
0
class KeyfileTest(unittest.TestCase):
    def setUp(self):
        service_connection = MockConnection()
        self.contents = '0123456789'
        bucket = MockBucket(service_connection, 'mybucket')
        key = bucket.new_key('mykey')
        key.set_contents_from_string(self.contents)
        self.keyfile = KeyFile(key)

    def tearDown(self):
        self.keyfile.close()

    def testReadFull(self):
        self.assertEqual(self.keyfile.read(len(self.contents)), self.contents)

    def testReadPartial(self):
        self.assertEqual(self.keyfile.read(5), self.contents[:5])
        self.assertEqual(self.keyfile.read(5), self.contents[5:])

    def testTell(self):
        self.assertEqual(self.keyfile.tell(), 0)
        self.keyfile.read(4)
        self.assertEqual(self.keyfile.tell(), 4)
        self.keyfile.read(6)
        self.assertEqual(self.keyfile.tell(), 10)
        self.keyfile.close()
        try:
            self.keyfile.tell()
        except ValueError as e:
            self.assertEqual(str(e), 'I/O operation on closed file')

    def testSeek(self):
        self.assertEqual(self.keyfile.read(4), self.contents[:4])
        self.keyfile.seek(0)
        self.assertEqual(self.keyfile.read(4), self.contents[:4])
        self.keyfile.seek(5)
        self.assertEqual(self.keyfile.read(5), self.contents[5:])

        # Seeking negative should raise.
        try:
            self.keyfile.seek(-5)
        except IOError as e:
            self.assertEqual(str(e), 'Invalid argument')

        # Reading past end of file is supposed to return empty string.
        self.keyfile.read(10)
        self.assertEqual(self.keyfile.read(20), '')

        # Seeking past end of file is supposed to silently work.
        self.keyfile.seek(50)
        self.assertEqual(self.keyfile.tell(), 50)
        self.assertEqual(self.keyfile.read(1), '')

    def testSeekEnd(self):
        self.assertEqual(self.keyfile.read(4), self.contents[:4])
        self.keyfile.seek(0, os.SEEK_END)
        self.assertEqual(self.keyfile.read(1), '')
        self.keyfile.seek(-1, os.SEEK_END)
        self.assertEqual(self.keyfile.tell(), 9)
        self.assertEqual(self.keyfile.read(1), '9')
        # Test attempt to seek backwards past the start from the end.
        try:
            self.keyfile.seek(-100, os.SEEK_END)
        except IOError as e:
            self.assertEqual(str(e), 'Invalid argument')

    def testSeekCur(self):
        self.assertEqual(self.keyfile.read(1), self.contents[0])
        self.keyfile.seek(1, os.SEEK_CUR)
        self.assertEqual(self.keyfile.tell(), 2)
        self.assertEqual(self.keyfile.read(4), self.contents[2:6])
Exemplo n.º 8
0
 def read(self, size=-1):
     if self.closed:
         raise ValueError("I/O operation on closed file")
     if size < 0:
         size = self.key.size - self.location
     return KeyFile.read(self, size)
Exemplo n.º 9
0
class KeyfileTest(unittest.TestCase):
    def setUp(self):
        service_connection = MockConnection()
        self.contents = '0123456789'
        bucket = MockBucket(service_connection, 'mybucket')
        key = bucket.new_key('mykey')
        key.set_contents_from_string(self.contents)
        self.keyfile = KeyFile(key)

    def tearDown(self):
        self.keyfile.close()

    def testReadFull(self):
        self.assertEqual(self.keyfile.read(len(self.contents)), self.contents)

    def testReadPartial(self):
        self.assertEqual(self.keyfile.read(5), self.contents[:5])
        self.assertEqual(self.keyfile.read(5), self.contents[5:])

    def testTell(self):
        self.assertEqual(self.keyfile.tell(), 0)
        self.keyfile.read(4)
        self.assertEqual(self.keyfile.tell(), 4)
        self.keyfile.read(6)
        self.assertEqual(self.keyfile.tell(), 10)
        self.keyfile.close()
        try:
            self.keyfile.tell()
        except ValueError as e:
            self.assertEqual(str(e), 'I/O operation on closed file')

    def testSeek(self):
        self.assertEqual(self.keyfile.read(4), self.contents[:4])
        self.keyfile.seek(0)
        self.assertEqual(self.keyfile.read(4), self.contents[:4])
        self.keyfile.seek(5)
        self.assertEqual(self.keyfile.read(5), self.contents[5:])

        # Seeking negative should raise.
        try:
            self.keyfile.seek(-5)
        except IOError as e:
            self.assertEqual(str(e), 'Invalid argument')

        # Reading past end of file is supposed to return empty string.
        self.keyfile.read(10)
        self.assertEqual(self.keyfile.read(20), '')

        # Seeking past end of file is supposed to silently work.
        self.keyfile.seek(50)
        self.assertEqual(self.keyfile.tell(), 50)
        self.assertEqual(self.keyfile.read(1), '')

    def testSeekEnd(self):
        self.assertEqual(self.keyfile.read(4), self.contents[:4])
        self.keyfile.seek(0, os.SEEK_END)
        self.assertEqual(self.keyfile.read(1), '')
        self.keyfile.seek(-1, os.SEEK_END)
        self.assertEqual(self.keyfile.tell(), 9)
        self.assertEqual(self.keyfile.read(1), '9')
        # Test attempt to seek backwards past the start from the end.
        try:
            self.keyfile.seek(-100, os.SEEK_END)
        except IOError as e:
            self.assertEqual(str(e), 'Invalid argument')

    def testSeekCur(self):
        self.assertEqual(self.keyfile.read(1), self.contents[0])
        self.keyfile.seek(1, os.SEEK_CUR)
        self.assertEqual(self.keyfile.tell(), 2)
        self.assertEqual(self.keyfile.read(4), self.contents[2:6])

    def testSetEtag(self):
        # Make sure both bytes and strings work as contents. This is one of the
        # very few places Boto uses the mock key object.
        # https://github.com/GoogleCloudPlatform/gsutil/issues/214#issuecomment-49906044
        self.keyfile.key.data = b'test'
        self.keyfile.key.set_etag()
        self.assertEqual(self.keyfile.key.etag,
                         '098f6bcd4621d373cade4e832627b4f6')

        self.keyfile.key.etag = None
        self.keyfile.key.data = 'test'
        self.keyfile.key.set_etag()
        self.assertEqual(self.keyfile.key.etag,
                         '098f6bcd4621d373cade4e832627b4f6')
Exemplo n.º 10
0
class KeyfileTest(unittest.TestCase):

    def setUp(self):
        service_connection = MockConnection()
        self.contents = '0123456789'
        bucket = MockBucket(service_connection, 'mybucket')
        key = bucket.new_key('mykey')
        key.set_contents_from_string(self.contents)
        self.keyfile = KeyFile(key)

    def tearDown(self):
        self.keyfile.close()

    def testReadFull(self):
        self.assertEqual(self.keyfile.read(len(self.contents)), self.contents)

    def testReadPartial(self):
        self.assertEqual(self.keyfile.read(5), self.contents[:5])
        self.assertEqual(self.keyfile.read(5), self.contents[5:])

    def testTell(self):
        self.assertEqual(self.keyfile.tell(), 0)
        self.keyfile.read(4)
        self.assertEqual(self.keyfile.tell(), 4)
        self.keyfile.read(6)
        self.assertEqual(self.keyfile.tell(), 10)
        self.keyfile.close()
        try:
            self.keyfile.tell()
        except ValueError as e:
            self.assertEqual(str(e), 'I/O operation on closed file')

    def testSeek(self):
        self.assertEqual(self.keyfile.read(4), self.contents[:4])
        self.keyfile.seek(0)
        self.assertEqual(self.keyfile.read(4), self.contents[:4])
        self.keyfile.seek(5)
        self.assertEqual(self.keyfile.read(5), self.contents[5:])

        # Seeking negative should raise.
        try:
            self.keyfile.seek(-5)
        except IOError as e:
            self.assertEqual(str(e), 'Invalid argument')

        # Reading past end of file is supposed to return empty string.
        self.keyfile.read(10)
        self.assertEqual(self.keyfile.read(20), '')

        # Seeking past end of file is supposed to silently work.
        self.keyfile.seek(50)
        self.assertEqual(self.keyfile.tell(), 50)
        self.assertEqual(self.keyfile.read(1), '')

    def testSeekEnd(self):
        self.assertEqual(self.keyfile.read(4), self.contents[:4])
        self.keyfile.seek(0, os.SEEK_END)
        self.assertEqual(self.keyfile.read(1), '')
        self.keyfile.seek(-1, os.SEEK_END)
        self.assertEqual(self.keyfile.tell(), 9)
        self.assertEqual(self.keyfile.read(1), '9')
        # Test attempt to seek backwards past the start from the end.
        try:
            self.keyfile.seek(-100, os.SEEK_END)
        except IOError as e:
            self.assertEqual(str(e), 'Invalid argument')

    def testSeekCur(self):
        self.assertEqual(self.keyfile.read(1), self.contents[0])
        self.keyfile.seek(1, os.SEEK_CUR)
        self.assertEqual(self.keyfile.tell(), 2)
        self.assertEqual(self.keyfile.read(4), self.contents[2:6])

    def testSetEtag(self):
        # Make sure both bytes and strings work as contents. This is one of the
        # very few places Boto uses the mock key object.
        # https://github.com/GoogleCloudPlatform/gsutil/issues/214#issuecomment-49906044
        self.keyfile.key.data = b'test'
        self.keyfile.key.set_etag()
        self.assertEqual(self.keyfile.key.etag, '098f6bcd4621d373cade4e832627b4f6')

        self.keyfile.key.etag = None
        self.keyfile.key.data = 'test'
        self.keyfile.key.set_etag()
        self.assertEqual(self.keyfile.key.etag, '098f6bcd4621d373cade4e832627b4f6')
Exemplo n.º 11
0
 def read(self, length=DEFAULT_READ_SIZE):
   return KeyFile.read(self, length)