def test_copy02(self): """ Test the copy() function some more """ # First we create a 1MB file tmp_file = join(self.tmp_dir, 'NNTPContent_Test.copy', '1MB.rar') # File should not already exist assert(isfile(tmp_file) is False) # Create our file assert(self.touch(tmp_file, size='1MB', random=True) is True) # File should exist now assert(isfile(tmp_file) is True) # Now we want to load it into a NNTPContent object content_a = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir) # We want to make a copy of it content_b = content_a.copy() # Our content should be the same assert(len(content_a) == len(content_b)) assert(content_a.md5() == content_b.md5()) # Because we made a copy, we're working with a different file however # we share the exact same content and filename identifier. assert(content_a == content_b) # Our paths are not the same though; this is what makes us a official # copy assert(content_a.path() != content_b.path())
def test_writes(self): """ More overhead then a normal write() but none the less, using the write() in this class keeps things simple since the file is automatically opened if it was otherwise closed """ # First we create a 1MB file tmp_file = join(self.tmp_dir, 'NNTPContent_Test.write', 'tmp.file') # File should not already exist assert (isfile(tmp_file) is False) # Now we want to create our NNTPContent() object surrouding this # file that does not exist. content = NNTPContent(filepath=tmp_file, work_dir=dirname(tmp_file)) # It's worth noting that this file will 'still' not exist assert (isfile(tmp_file) is False) # we'll write data data = 'hello\r\n' content.write(data) # It's worth noting that this file will ''STILL'' not exist assert (isfile(tmp_file) is False) # Save content assert (content.save() is True) # Now the file 'will' exist assert (isfile(tmp_file) is True) # Open our file and verify it is the data we saved. with open(tmp_file) as f: data_read = f.read() assert (data == data_read)
def test_copy02(self): """ Test the copy() function some more """ # First we create a 1MB file tmp_file = join(self.tmp_dir, 'NNTPContent_Test.copy', '1MB.rar') # File should not already exist assert (isfile(tmp_file) is False) # Create our file assert (self.touch(tmp_file, size='1MB', random=True) is True) # File should exist now assert (isfile(tmp_file) is True) # Now we want to load it into a NNTPContent object content_a = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir) # We want to make a copy of it content_b = content_a.copy() # Our content should be the same assert (len(content_a) == len(content_b)) assert (content_a.md5() == content_b.md5()) # Because we made a copy, we're working with a different file however # we share the exact same content and filename identifier. assert (content_a == content_b) # Our paths are not the same though; this is what makes us a official # copy assert (content_a.path() != content_b.path())
def test_with(self): """ Test the use of the with clause """ # First we create a 1MB file tmp_file = join(self.tmp_dir, 'NNTPContent_Test.with', '1MB.rar') # File should not already exist assert (isfile(tmp_file) is False) # Create our file assert (self.touch(tmp_file, size='1MB', random=True) is True) # File should exist now assert (isfile(tmp_file) is True) # Now we want to load it into a NNTPContent object content_a = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir) content_a.attach() # Create a second object content_b = NNTPContent(work_dir=self.tmp_dir) with content_a as fp: while 1: # Read our data buf = fp.read(1024) if not buf: break # Write our content to disk content_b.write(buf) # At this point we should have a duplicate of our original object assert (len(content_a) == len(content_b)) assert (content_a.md5() == content_b.md5())
def test_writes(self): """ More overhead then a normal write() but none the less, using the write() in this class keeps things simple since the file is automatically opened if it was otherwise closed """ # First we create a 1MB file tmp_file = join(self.tmp_dir, 'NNTPContent_Test.write', 'tmp.file') # File should not already exist assert(isfile(tmp_file) is False) # Now we want to create our NNTPContent() object surrouding this # file that does not exist. content = NNTPContent(filepath=tmp_file, work_dir=dirname(tmp_file)) # It's worth noting that this file will 'still' not exist assert(isfile(tmp_file) is False) # we'll write data data = 'hello\r\n' content.write(data) # It's worth noting that this file will ''STILL'' not exist assert(isfile(tmp_file) is False) # Save content assert(content.save() is True) # Now the file 'will' exist assert(isfile(tmp_file) is True) # Open our file and verify it is the data we saved. with open(tmp_file) as f: data_read = f.read() assert(data == data_read)
def test_from_bestguess(self): """ test from_bestguess() bestguess() does the best of both worlds: from_file() and from_filename(). It never returns None unless you give it bad data. """ # Initialize our mime object m = Mime() # Empty content just gives us an empty response assert(m.from_bestguess(None) is None) assert(m.from_bestguess("") is None) assert(m.from_bestguess(u"") is None) # First we take a binary file image = join(self.var_dir, 'joystick.jpg') c = NNTPContent(image, work_dir=self.tmp_dir) copy = c.copy() # since we have a filename, we can pick it up from that assert(m.from_bestguess(copy.filename).type() == 'image/jpeg') # We can also get it from_file() because even though our temporary # file does not have an extension at all, we can still assert(m.from_bestguess(copy.path()).type() == 'image/jpeg')
def test_with(self): """ Test the use of the with clause """ # First we create a 1MB file tmp_file = join(self.tmp_dir, 'NNTPContent_Test.with', '1MB.rar') # File should not already exist assert(isfile(tmp_file) is False) # Create our file assert(self.touch(tmp_file, size='1MB', random=True) is True) # File should exist now assert(isfile(tmp_file) is True) # Now we want to load it into a NNTPContent object content_a = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir) content_a.attach() # Create a second object content_b = NNTPContent(work_dir=self.tmp_dir) with content_a as fp: while 1: # Read our data buf = fp.read(1024) if not buf: break # Write our content to disk content_b.write(buf) # At this point we should have a duplicate of our original object assert(len(content_a) == len(content_b)) assert(content_a.md5() == content_b.md5())
def test_checksum(self): """Test the assorted checksums supported """ # First we create a 1MB file tmp_file = join( self.tmp_dir, 'NNTPContent_Test.checksum', 'tmpa.tmp') # File should not already exist assert(isfile(tmp_file) is False) # Create a random file assert(self.touch(tmp_file, size='1MB', random=True) is True) # File should exist now assert(isfile(tmp_file) is True) # Now we want to load it into a NNTPContent object content = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir) md5 = content.md5() sha1 = content.sha1() sha256 = content.sha256() assert(md5 is not None) assert(sha1 is not None) assert(sha256 is not None) tmp_file_2 = join( self.tmp_dir, 'NNTPContent_Test.checksum', 'tmp2.rar') # File should not already exist assert(isfile(tmp_file_2) is False) # We'll create a copy of our file assert(content.save(filepath=tmp_file_2, copy=True) is True) # Now it should assert(isfile(tmp_file_2) is True) # Now we'll open the new file we created content_2 = NNTPContent(filepath=tmp_file_2, work_dir=self.tmp_dir) md5_2 = content_2.md5() sha1_2 = content_2.sha1() sha256_2 = content_2.sha256() assert(md5_2 is not None) assert(sha1_2 is not None) assert(sha256_2 is not None) # files should be the same assert(md5 == md5_2) assert(sha1 == sha1_2) assert(sha256 == sha256_2)
def test_password_protection(self): """ Tests the un-raring of content """ # Generate temporary folder to work with work_dir = join(self.tmp_dir, 'CodecRar_Test.rar.pwd', 'tmp') # Now we want to prepare a work folder source_dir = join(self.tmp_dir, 'CodecRar_Test.rar.pwd', 'source') # Initialize Codec cr = CodecRar(work_dir=work_dir, password='******') # create some dummy file entries file_list = [] for i in range(0, 10): # Create some temporary files to work with in our source # directory tmp_file = join(source_dir, 'DSC_IMG%.3d.jpeg' % i) self.touch(tmp_file, size='100K', random=True) # Add our file to the encoding process file_list.append(NNTPContent(tmp_file)) # Now we want to compress this content content = cr.encode(name="mystuff", content=file_list) assert isinstance(content, sortedset) assert len(content) == 1 # Our content should be password protected at this point; so if i # create another CodecRar item (without a password), i should fail to # extract the content. tmp_cr = CodecRar(work_dir=work_dir) decoded = tmp_cr.decode(content) # Bad Password means no results assert decoded is None # We could have saved ourselves time in determining this because # testing doesn't have as much overhead and allows us to check for any # passwords associated with the data assert tmp_cr.test(content) is False # But simply applying a password would have done wonders assert tmp_cr.test(content, password='******') is True # We can extract the contents by passing in the password content = tmp_cr.decode(content, password='******') assert isinstance(content, sortedset) is True assert len(content) == 1 assert isinstance(content[0], NNTPBinaryContent) is True # Content is always attached assert content[0].is_attached() is True
def test_directory_support(self): """ NNTPContent objects can wrap directories too """ my_dir = join(self.tmp_dir, 'NNTPContent', 'my_dir') assert (isdir(my_dir) is False) assert (mkdir(my_dir) is True) assert (isdir(my_dir) is True) # Now create our NNTPContent Object against our directory obj = NNTPContent( filepath=my_dir, work_dir=self.tmp_dir, ) # Directories are never attached by default assert (obj.is_attached() is False) # Deleting the object means the directory remains behind del obj assert (isdir(my_dir) is True) # However obj = NNTPContent( filepath=my_dir, work_dir=self.tmp_dir, ) assert (obj.is_attached() is False) obj.attach() assert (obj.is_attached() is True) # Now we've attached the NNTPContent to the object so deleting the # object destroys the directory del obj assert (exists(my_dir) is False) assert (isdir(my_dir) is False)
def test_directory_support(self): """ NNTPContent objects can wrap directories too """ my_dir = join(self.tmp_dir, 'NNTPContent', 'my_dir') assert(isdir(my_dir) is False) assert(mkdir(my_dir) is True) assert(isdir(my_dir) is True) # Now create our NNTPContent Object against our directory obj = NNTPContent( filepath=my_dir, work_dir=self.tmp_dir, ) # Directories are never attached by default assert(obj.is_attached() is False) # Deleting the object means the directory remains behind del obj assert(isdir(my_dir) is True) # However obj = NNTPContent( filepath=my_dir, work_dir=self.tmp_dir, ) assert(obj.is_attached() is False) obj.attach() assert(obj.is_attached() is True) # Now we've attached the NNTPContent to the object so deleting the # object destroys the directory del obj assert(exists(my_dir) is False) assert(isdir(my_dir) is False)
def encode(self, content, mem_buf=DEFAULT_BUFFER_SIZE): """ Encodes an NNTPContent object passed in """ if isinstance(content, NNTPContent): # Create our ascii instance _encoded = NNTPAsciiContent( filepath=content.filename, part=content.part, total_parts=content.total_parts, sort_no=content.sort_no, work_dir=self.work_dir, # We want to ensure we're working with a unique attached file unique=True, ) else: # If we reach here, we presume our content is a filename # Create our ascii instance _encoded = NNTPAsciiContent( filepath=content, work_dir=self.work_dir, # We want to ensure we're working with a unique attached file unique=True, ) # Convert our content object into an NNTPContent object content = NNTPContent( filepath=content, work_dir=self.work_dir, ) # yEnc (v1.3) begin fmt_ybegin = '=ybegin part=%d total=%d line=%d size=%d name=%s' % ( content.part, content.total_parts, self.linelen, len(content), content.filename, ) # yEnc part fmt_ypart = '=ypart begin=%d end=%d' % ( content.begin() + 1, content.end(), ) if isinstance(content._parent, NNTPContent): # yEnc end fmt_yend = '=yend size=%d part=%d pcrc32=%s crc32=%s' % ( len(content), content.part, content.crc32(), content._parent.crc32(), ) else: # yEnc end fmt_yend = '=yend size=%d part=%d pcrc32=%s' % ( len(content), content.part, content.crc32(), ) # Write =ybegin line _encoded.write(fmt_ybegin + EOL) # Write =ypart line _encoded.write(fmt_ypart + EOL) if not content.open(): return None # Prepare our result set results = "" # Column is used for decoding column = 0 crc = BIN_MASK # We need to parse the content until we either reach # the end of the file or get to an 'end' tag while True: # Read in our data data = content.stream.read(mem_buf) if not data: # We're done break if FAST_YENC_SUPPORT: try: _results, crc, column = encode_string(data, crc, column) # Append our parsed content onto our ongoing buffer results += _results except YencError as e: logger.error("Failed to encode Yenc for %s." % content) logger.debug('Yenc exception: %s' % (str(e))) return None else: # The slow and painful way, the below looks complicated # but it really isn't at the the end of the day; yEnc is # pretty basic; # - first we translate the all of the characters by adding # 42 to their value with the exception of a few special # characters that are explicitly reserved for the yEnc # language (and conflict with the NNTP Server language). # # - next, we need to apply our ENCODE_SPECIAL_MAP to be # sure to handle the characters that are reserved as # special keywords used by both NNTP Servers and the yEnc # protocol itself. # # - finally we want to prevent our string from going on for # to many characters (horizontally). So we need to split # our content up # idx = 0 while idx < len(data): _byte = (ord(data[idx]) + 42) & 0xff if _byte in YENC_ENCODE_ESCAPED_CHARACTERS: _byte = (_byte + 64) & 0xff # Esape Sequence results += '=' # Store our character results += chr(_byte) # Increment Index idx += 1 # Our offset offset = 0 while offset < (len(results) - self.linelen + 1): eol = offset + self.linelen if results[offset:eol][-1] == '=': # Lines can't end with the escape sequence (=). If we get # here then this one did. We just adjust our end-of-line # by 1 and keep moving eol -= 1 _encoded.write(results[offset:eol] + EOL) offset = eol if offset < len(results): results = results[-(len(results) - offset):] else: # reset string results = '' # We're done reading our data content.close() if len(results): # We still have content left in our buffer _encoded.write(results + EOL) # Write footer _encoded.write(fmt_yend + EOL) if _encoded: # close article when complete _encoded.close() # Return our encoded object return _encoded
def test_append(self): """ Test the split() and then append() function """ # First we create a 1MB file tmp_file = join(self.tmp_dir, 'NNTPContent_Test.append', '1MB.rar') # File should not already exist assert (isfile(tmp_file) is False) # Create our file assert (self.touch(tmp_file, size='1MB', random=True) is True) # File should exist now assert (isfile(tmp_file) is True) # Now we want to load it into a NNTPContent object content_a = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir) # Attach our content assert (content_a.is_attached() is False) content_a.attach() # We'll split it in 4 results = content_a.split(strsize_to_bytes('512K')) # Tests that our results are expected assert (isinstance(results, sortedset) is True) assert (len(results) == 2) # Create a new content object content_b = NNTPContent(work_dir=self.tmp_dir) # Our content should be attached assert (content_a.is_attached() is True) assert (content_b.is_attached() is True) # For each of our disassembled items, re-assemble them for content in results: assert (content_b.append(content) is True) # Test that our content is the same again assert (len(content_a) == len(content_b)) assert (content_a.md5() == content_b.md5())
def test_invalid_split_cases(self): """ Test errors that are generated out of the split function """ work_dir = join(self.tmp_dir, 'NNTPContent_Test.chunk') # Now we want to load it into a NNTPContent object content = NNTPContent(work_dir=work_dir) # Nothing to split gives an error assert(content.split() is None) tmp_file = join(self.tmp_dir, 'NNTPContent_Test.chunk', '5K.rar') assert(isfile(tmp_file) is False) assert(self.touch(tmp_file, size='1MB', random=True) is True) assert(isfile(tmp_file) is True) # Now we want to load it into a NNTPContent object content = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir) # No size to split on gives an error assert(content.split(size=0) is None) assert(content.split(size=-1) is None) assert(content.split(size=None) is None) assert(content.split(size='bad_string') is None) # Invalid Memory Limit assert(content.split(mem_buf=0) is None) assert(content.split(mem_buf=-1) is None) assert(content.split(mem_buf=None) is None) assert(content.split(mem_buf='bad_string') is None)
def test_copy01(self): """ Test our copy function The copy function allows us to duplicate an existing NNTPContent object without obstructing the original. Copied content is always attached; so if the object falls out of scope; so does the file. """ my_dir = join(self.tmp_dir, 'NNTPContent', 'test_copy') assert (isdir(my_dir) is False) assert (mkdir(my_dir) is True) assert (isdir(my_dir) is True) # Now create our NNTPContent object witin our directory obj = NNTPContent( filepath='myfile', work_dir=my_dir, ) # Content is attached by default assert (obj.is_attached() is True) obj.detach() assert (obj.is_attached() is False) new_dir = join(my_dir, 'copy') assert (isdir(new_dir) is False) # Create a copy of our object obj_copy = obj.copy() # Successfully loaded files are never attached reguardless # of the original copy assert (obj_copy.is_attached() is True) # Reattach the original so it dies when this test is over obj.attach() assert (obj.is_attached() is True) # Create a copy of our copy obj_copy2 = obj_copy.copy() assert (obj_copy2.is_attached() is True) assert (isfile(obj_copy2.path())) _path = obj_copy2.path() del obj_copy2 assert (isfile(_path) is False) assert (isfile(obj_copy.path()) is True) _path = obj_copy.path() del obj_copy assert (isfile(_path) is False) assert (isfile(obj.path()) is True) _path = obj.path() del obj assert (isfile(_path) is False) # now lets do a few more tests but with actual files this time tmp_file = join(my_dir, '2MB.zip') assert (self.touch(tmp_file, size='2MB', random=True) is True) # Now create our NNTPContent object witin our directory obj = NNTPContent( filepath=tmp_file, work_dir=my_dir, ) assert (isfile(obj.path()) is True) obj_copy = obj.copy() assert (isfile(obj_copy.path()) is True) stats = stat(obj_copy.path()) assert (bytes_to_strsize(stats['size']) == "2.00MB") # Compare that our content is the same assert (compare(obj_copy.path(), obj.path()) is True) # note that the filenames are NOT the same so we are dealing with 2 # distinct copies here assert (isfile(obj_copy.path()) is True) assert (isfile(obj.path()) is True) assert (obj.path() != obj_copy.path())
def test_split(self): """ Test the split() function """ # First we create a 1MB file tmp_file = join(self.tmp_dir, 'NNTPContent_Test.chunk', '1MB.rar') # File should not already exist assert(isfile(tmp_file) is False) # Create our file assert(self.touch(tmp_file, size='1MB', random=True) is True) # File should exist now assert(isfile(tmp_file) is True) # Now we want to load it into a NNTPContent object content = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir) # Loaded files are always detached; The following is loaded # because the path exists. assert(content.is_attached() is False) # We'll split it in 2 results = content.split(strsize_to_bytes('512K')) # Tests that our results are expected assert(isinstance(results, sortedset) is True) assert(len(results) == 2) # We support passing the string format directly in too results = content.split('512K') # Tests that our results are expected assert(isinstance(results, sortedset) is True) assert(len(results) == 2) # Now lets merge them into one again content = NNTPContent(work_dir=self.tmp_dir) assert(content.load(results) is True) # NNTPContent() sets as well as individual objects passed into # load are always attached by default assert(content.is_attached() is True) # Our combined file should be the correct filesize assert(len(content) == strsize_to_bytes('1M')) # Once we save our object, it is no longer attached assert(content.save(filepath=tmp_file) is True) # Now our content is no longer attached assert(content.is_attached() is False) # we'll re-attach it content.attach() # Our file will be gone now if we try to delete it assert(content.is_attached() is True) assert(isfile(tmp_file) is True) del content assert(isfile(tmp_file) is False)
def test_checksum(self): """Test the assorted checksums supported """ # First we create a 1MB file tmp_file = join(self.tmp_dir, 'NNTPContent_Test.checksum', 'tmpa.tmp') # File should not already exist assert (isfile(tmp_file) is False) # Create a random file assert (self.touch(tmp_file, size='1MB', random=True) is True) # File should exist now assert (isfile(tmp_file) is True) # Now we want to load it into a NNTPContent object content = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir) md5 = content.md5() sha1 = content.sha1() sha256 = content.sha256() assert (md5 is not None) assert (sha1 is not None) assert (sha256 is not None) tmp_file_2 = join(self.tmp_dir, 'NNTPContent_Test.checksum', 'tmp2.rar') # File should not already exist assert (isfile(tmp_file_2) is False) # We'll create a copy of our file assert (content.save(filepath=tmp_file_2, copy=True) is True) # Now it should assert (isfile(tmp_file_2) is True) # Now we'll open the new file we created content_2 = NNTPContent(filepath=tmp_file_2, work_dir=self.tmp_dir) md5_2 = content_2.md5() sha1_2 = content_2.sha1() sha256_2 = content_2.sha256() assert (md5_2 is not None) assert (sha1_2 is not None) assert (sha256_2 is not None) # files should be the same assert (md5 == md5_2) assert (sha1 == sha1_2) assert (sha256 == sha256_2)
def test_split(self): """ Test the split() function """ # First we create a 1MB file tmp_file = join(self.tmp_dir, 'NNTPContent_Test.chunk', '1MB.rar') # File should not already exist assert (isfile(tmp_file) is False) # Create our file assert (self.touch(tmp_file, size='1MB', random=True) is True) # File should exist now assert (isfile(tmp_file) is True) # Now we want to load it into a NNTPContent object content = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir) # Loaded files are always detached; The following is loaded # because the path exists. assert (content.is_attached() is False) # We'll split it in 2 results = content.split(strsize_to_bytes('512K')) # Tests that our results are expected assert (isinstance(results, sortedset) is True) assert (len(results) == 2) # We support passing the string format directly in too results = content.split('512K') # Tests that our results are expected assert (isinstance(results, sortedset) is True) assert (len(results) == 2) # Now lets merge them into one again content = NNTPContent(work_dir=self.tmp_dir) assert (content.load(results) is True) # NNTPContent() sets as well as individual objects passed into # load are always attached by default assert (content.is_attached() is True) # Our combined file should be the correct filesize assert (len(content) == strsize_to_bytes('1M')) # Once we save our object, it is no longer attached assert (content.save(filepath=tmp_file) is True) # Now our content is no longer attached assert (content.is_attached() is False) # we'll re-attach it content.attach() # Our file will be gone now if we try to delete it assert (content.is_attached() is True) assert (isfile(tmp_file) is True) del content assert (isfile(tmp_file) is False)
def encode(self, content, mem_buf=DEFAULT_BUFFER_SIZE): """ Encodes an NNTPContent object passed in """ if isinstance(content, NNTPContent): # Create our ascii instance _encoded = NNTPAsciiContent( filepath=content.filename, part=content.part, total_parts=content.total_parts, sort_no=content.sort_no, work_dir=self.work_dir, # We want to ensure we're working with a unique attached file unique=True, ) else: # If we reach here, we presume our content is a filename # Create our ascii instance _encoded = NNTPAsciiContent( filepath=content, work_dir=self.work_dir, # We want to ensure we're working with a unique attached file unique=True, ) # Convert our content object into an NNTPContent object content = NNTPContent( filepath=content, work_dir=self.work_dir, ) # yEnc (v1.3) begin fmt_ybegin = '=ybegin part=%d total=%d line=%d size=%d name=%s' % ( content.part, content.total_parts, self.linelen, len(content), content.filename, ) # yEnc part fmt_ypart = '=ypart begin=%d end=%d' % ( content.begin() + 1, content.end(), ) if isinstance(content._parent, NNTPContent): # yEnc end fmt_yend = '=yend size=%d part=%d pcrc32=%s crc32=%s' % ( len(content), content.part, content.crc32(), content._parent.crc32(), ) else: # yEnc end fmt_yend = '=yend size=%d part=%d pcrc32=%s' % ( len(content), content.part, content.crc32(), ) # Write =ybegin line _encoded.write(fmt_ybegin + EOL) # Write =ypart line _encoded.write(fmt_ypart + EOL) if not content.open(): return None # Prepare our result set results = "" # Column is used for decoding column = 0 crc = BIN_MASK # We need to parse the content until we either reach # the end of the file or get to an 'end' tag while True: # Read in our data data = content.stream.read(mem_buf) if not data: # We're done break if FAST_YENC_SUPPORT: try: _results, crc, column = encode_string(data, crc, column) # Append our parsed content onto our ongoing buffer results += _results except YencError as e: logger.error("Failed to encode Yenc for %s." % content) logger.debug('Yenc exception: %s' % (str(e))) return None else: # The slow and painful way, the below looks complicated # but it really isn't at the the end of the day; yEnc is # pretty basic; # - first we translate the all of the characters by adding # 42 to their value with the exception of a few special # characters that are explicitly reserved for the yEnc # language (and conflict with the NNTP Server language). # # - next, we need to apply our ENCODE_SPECIAL_MAP to be # sure to handle the characters that are reserved as # special keywords used by both NNTP Servers and the yEnc # protocol itself. # # - finally we want to prevent our string from going on for # to many characters (horizontally). So we need to split # our content up # idx = 0 while idx < len(data): _byte = (ord(data[idx]) + 42) & 0xff if _byte in YENC_ENCODE_ESCAPED_CHARACTERS: _byte = (_byte + 64) & 0xff # Esape Sequence results += '=' # Store our character results += chr(_byte) # Increment Index idx += 1 # Our offset offset = 0 while offset < (len(results)-self.linelen+1): eol = offset+self.linelen if results[offset:eol][-1] == '=': # Lines can't end with the escape sequence (=). If we get # here then this one did. We just adjust our end-of-line # by 1 and keep moving eol -= 1 _encoded.write(results[offset:eol] + EOL) offset = eol if offset < len(results): results = results[-(len(results) - offset):] else: # reset string results = '' # We're done reading our data content.close() if len(results): # We still have content left in our buffer _encoded.write(results + EOL) # Write footer _encoded.write(fmt_yend + EOL) if _encoded: # close article when complete _encoded.close() # Return our encoded object return _encoded
def test_rar_errors(self): """ Test that we fail under certain conditions """ # Generate temporary folder to work with work_dir = join(self.tmp_dir, 'CodecRar_Test.rar.fail', 'work') # Now we want to prepare a folder filled with temporary content # Note: this directory is horrible because it's 'within' our work_dir # as a result, adding content should not succeed source_dir = join(work_dir, 'test') # Initialize Codec (without volume_size disables it) cr = CodecRar(work_dir=work_dir) # No files assert len(cr) == 0 tmp_file = join(source_dir, 'temp_file_non-existant') assert isfile(tmp_file) is False # We can't add content that does not exist assert cr.add(tmp_file) is False # Still No files assert len(cr) == 0 # However directories can not cross into our work directory tmp_dir = dirname(work_dir) # We intentionally pick a directory that has the work_dir # as a child within it assert isdir(tmp_dir) # Denied adding the file because it would include the work_dir # if we did assert cr.add(tmp_file) is False # Temporary file (within directory denied in previous check) tmp_file = join(tmp_dir, 'temp_file') assert isfile(tmp_file) is False # Create our temporary file now self.touch(tmp_file, size='120K', random=True) assert isfile(tmp_file) is True # This file is within our work_dir but we're still okay because # it's accessing the file explicitly and the fact it's a file # and not a directory assert cr.add(tmp_file) is True # Now we'll have 1 entry in our list assert len(cr) == 1 # You can't add duplicates assert cr.add(tmp_file) is False # We still have 1 entry assert len(cr) == 1 # Empty NNTPContent() can not be added content = NNTPContent(unique=True, work_dir=self.tmp_dir) # Can't do it assert cr.add(content) is False # Store some data content.write('some data\r\n') # Now we can add it because it has data in it assert cr.add(content) is True # We now have 2 entries assert len(cr) == 2 # We can't add duplicates assert cr.add(content) is False # We still have 2 entries assert len(cr) == 2 # Empty NNTPArticle() can not be added article = NNTPArticle(work_dir=self.tmp_dir) # Can't do it assert cr.add(article) is False # If we add content that's already been added, nothing # new will happen either assert article.add(content) is True # Still can't do (only because it was already added) assert cr.add(article) is False # We still have 2 entries assert len(cr) == 2 # New Empty NNTPContent() can not be added content = NNTPContent(unique=True, work_dir=self.tmp_dir) # We'll add our new content to our article assert article.add(content) is True # Our new content has no data associated with it, so this should # still fail assert cr.add(article) is False # We still have 2 entries assert len(cr) == 2 # Store some new data content.write('some new data\r\n') # Our new content within our article now has data so this will work assert cr.add(article) is True # We now have 3 entries assert len(cr) == 3
def test_append(self): """ Test the split() and then append() function """ # First we create a 1MB file tmp_file = join(self.tmp_dir, 'NNTPContent_Test.append', '1MB.rar') # File should not already exist assert(isfile(tmp_file) is False) # Create our file assert(self.touch(tmp_file, size='1MB', random=True) is True) # File should exist now assert(isfile(tmp_file) is True) # Now we want to load it into a NNTPContent object content_a = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir) # Attach our content assert(content_a.is_attached() is False) content_a.attach() # We'll split it in 4 results = content_a.split(strsize_to_bytes('512K')) # Tests that our results are expected assert(isinstance(results, sortedset) is True) assert(len(results) == 2) # Create a new content object content_b = NNTPContent(work_dir=self.tmp_dir) # Our content should be attached assert(content_a.is_attached() is True) assert(content_b.is_attached() is True) # For each of our disassembled items, re-assemble them for content in results: assert(content_b.append(content) is True) # Test that our content is the same again assert(len(content_a) == len(content_b)) assert(content_a.md5() == content_b.md5())
def test_copy01(self): """ Test our copy function The copy function allows us to duplicate an existing NNTPContent object without obstructing the original. Copied content is always attached; so if the object falls out of scope; so does the file. """ my_dir = join(self.tmp_dir, 'NNTPContent', 'test_copy') assert(isdir(my_dir) is False) assert(mkdir(my_dir) is True) assert(isdir(my_dir) is True) # Now create our NNTPContent object witin our directory obj = NNTPContent( filepath='myfile', work_dir=my_dir, ) # Content is attached by default assert(obj.is_attached() is True) obj.detach() assert(obj.is_attached() is False) new_dir = join(my_dir, 'copy') assert(isdir(new_dir) is False) # Create a copy of our object obj_copy = obj.copy() # Successfully loaded files are never attached reguardless # of the original copy assert(obj_copy.is_attached() is True) # Reattach the original so it dies when this test is over obj.attach() assert(obj.is_attached() is True) # Create a copy of our copy obj_copy2 = obj_copy.copy() assert(obj_copy2.is_attached() is True) assert(isfile(obj_copy2.path())) _path = obj_copy2.path() del obj_copy2 assert(isfile(_path) is False) assert(isfile(obj_copy.path()) is True) _path = obj_copy.path() del obj_copy assert(isfile(_path) is False) assert(isfile(obj.path()) is True) _path = obj.path() del obj assert(isfile(_path) is False) # now lets do a few more tests but with actual files this time tmp_file = join(my_dir, '2MB.zip') assert(self.touch(tmp_file, size='2MB', random=True) is True) # Now create our NNTPContent object witin our directory obj = NNTPContent( filepath=tmp_file, work_dir=my_dir, ) assert(isfile(obj.path()) is True) obj_copy = obj.copy() assert(isfile(obj_copy.path()) is True) stats = stat(obj_copy.path()) assert(bytes_to_strsize(stats['size']) == "2.00MB") # Compare that our content is the same assert(compare(obj_copy.path(), obj.path()) is True) # note that the filenames are NOT the same so we are dealing with 2 # distinct copies here assert(isfile(obj_copy.path()) is True) assert(isfile(obj.path()) is True) assert(obj.path() != obj_copy.path())
def test_7z_errors(self): """ Test that we fail under certain conditions """ # Generate temporary folder to work with work_dir = join(self.tmp_dir, 'Codec7Zip_Test.7z.fail', 'work') # Now we want to prepare a folder filled with temporary content # Note: this directory is horrible because it's 'within' our work_dir # as a result, adding content should not succeed source_dir = join(work_dir, 'test') # Initialize Codec (without volume_size disables it) cr = Codec7Zip(work_dir=work_dir) # No files assert len(cr) == 0 tmp_file = join(source_dir, 'temp_file_non-existant') assert isfile(tmp_file) is False # We can't add content that does not exist assert cr.add(tmp_file) is False # Still No files assert len(cr) == 0 # However directories can not cross into our work directory tmp_dir = dirname(work_dir) # We intentionally pick a directory that has the work_dir # as a child within it assert isdir(tmp_dir) # Denied adding the file because it would include the work_dir # if we did assert cr.add(tmp_file) is False # Temporary file (within directory denied in previous check) tmp_file = join(tmp_dir, 'temp_file') assert isfile(tmp_file) is False # Create our temporary file now self.touch(tmp_file, size='120K', random=True) assert isfile(tmp_file) is True # This file is within our work_dir but we're still okay because # it's accessing the file explicitly and the fact it's a file # and not a directory assert cr.add(tmp_file) is True # Now we'll have 1 entry in our list assert len(cr) == 1 # You can't add duplicates assert cr.add(tmp_file) is False # We still have 1 entry assert len(cr) == 1 # Empty NNTPContent() can not be added content = NNTPContent(unique=True, work_dir=self.tmp_dir) # Can't do it assert cr.add(content) is False # Store some data content.write('some data\r\n') # Now we can add it because it has data in it assert cr.add(content) is True # We now have 2 entries assert len(cr) == 2 # We can't add duplicates assert cr.add(content) is False # We still have 2 entries assert len(cr) == 2 # Empty NNTPArticle() can not be added article = NNTPArticle(work_dir=self.tmp_dir) # Can't do it assert cr.add(article) is False # If we add content that's already been added, nothing # new will happen either assert article.add(content) is True # Still can't do (only because it was already added) assert cr.add(article) is False # We still have 2 entries assert len(cr) == 2 # New Empty NNTPContent() can not be added content = NNTPContent(unique=True, work_dir=self.tmp_dir) # We'll add our new content to our article assert article.add(content) is True # Our new content has no data associated with it, so this should # still fail assert cr.add(article) is False # We still have 2 entries assert len(cr) == 2 # Store some new data content.write('some new data\r\n') # Our new content within our article now has data so this will work assert cr.add(article) is True # We now have 3 entries assert len(cr) == 3
def test_saves(self): """ Saving allows for a variety of inputs, test that they all check out okay """ # First we create a 1MB file tmp_file = join(self.tmp_dir, 'NNTPContent_Test.save', 'testfile.tmp') # File should not already exist assert (isfile(tmp_file) is False) # Create a random file assert (self.touch(tmp_file, size='5MB', random=True) is True) # File should exist now assert (isfile(tmp_file) is True) # Now we want to load it into a NNTPContent object content = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir) # Test our file exists assert (len(content) == strsize_to_bytes('5M')) # By default our load makes it so our file is NOT attached assert (content.is_attached() is False) # as we can't make a copy of our current file on top of the old _filepath = content.path() assert (content.save(copy=True) is True) assert (content.path() == _filepath) assert (content.path() == tmp_file) # File should still be detached assert (content.is_attached() is False) # Filepath shoul still not have changed # Let's attach it content.attach() assert (content.is_attached() is True) # If we call save() a copy parameter, there should be no change assert (content.save(copy=True) is True) # Still no change assert (content.is_attached() is True) # Now lets actually copy it to a new location tmp_file_copy = join(self.tmp_dir, 'NNTPContent_Test.save', 'testfile.copy.tmp') # File should not already exist assert (isfile(tmp_file_copy) is False) # call save using our copy variable and new filename assert (content.save(tmp_file_copy, copy=True) is True) # File should exist now assert (isfile(tmp_file_copy) is True) # Old File should still exist too assert (isfile(tmp_file) is True) # Path should still be the old path and not the new assert (content.path() == tmp_file) # Still no change in attachment assert (content.is_attached() is True) # Create a new file now tmp_file_copy2 = join(self.tmp_dir, 'NNTPContent_Test.save', 'testfile.copy2.tmp') assert (isfile(tmp_file_copy2) is False) # call save with copy set to false; This performs an official # move (adjusting our internal records. assert (content.save(tmp_file_copy2, copy=False) is True) # Old File should no longer exist assert (isfile(tmp_file) is False) # Content should be detached assert (content.is_attached() is False) # New file should exist assert (isfile(tmp_file_copy2) is True) assert (content.path() != _filepath) assert (content.path() == tmp_file_copy2)
def test_saves(self): """ Saving allows for a variety of inputs, test that they all check out okay """ # First we create a 1MB file tmp_file = join( self.tmp_dir, 'NNTPContent_Test.save', 'testfile.tmp') # File should not already exist assert(isfile(tmp_file) is False) # Create a random file assert(self.touch(tmp_file, size='5MB', random=True) is True) # File should exist now assert(isfile(tmp_file) is True) # Now we want to load it into a NNTPContent object content = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir) # Test our file exists assert(len(content) == strsize_to_bytes('5M')) # By default our load makes it so our file is NOT attached assert(content.is_attached() is False) # as we can't make a copy of our current file on top of the old _filepath = content.path() assert(content.save(copy=True) is True) assert(content.path() == _filepath) assert(content.path() == tmp_file) # File should still be detached assert(content.is_attached() is False) # Filepath shoul still not have changed # Let's attach it content.attach() assert(content.is_attached() is True) # If we call save() a copy parameter, there should be no change assert(content.save(copy=True) is True) # Still no change assert(content.is_attached() is True) # Now lets actually copy it to a new location tmp_file_copy = join( self.tmp_dir, 'NNTPContent_Test.save', 'testfile.copy.tmp') # File should not already exist assert(isfile(tmp_file_copy) is False) # call save using our copy variable and new filename assert(content.save(tmp_file_copy, copy=True) is True) # File should exist now assert(isfile(tmp_file_copy) is True) # Old File should still exist too assert(isfile(tmp_file) is True) # Path should still be the old path and not the new assert(content.path() == tmp_file) # Still no change in attachment assert(content.is_attached() is True) # Create a new file now tmp_file_copy2 = join( self.tmp_dir, 'NNTPContent_Test.save', 'testfile.copy2.tmp') assert(isfile(tmp_file_copy2) is False) # call save with copy set to false; This performs an official # move (adjusting our internal records. assert(content.save(tmp_file_copy2, copy=False) is True) # Old File should no longer exist assert(isfile(tmp_file) is False) # Content should be detached assert(content.is_attached() is False) # New file should exist assert(isfile(tmp_file_copy2) is True) assert(content.path() != _filepath) assert(content.path() == tmp_file_copy2)
def test_get_paths(self): """ Test that we fail under certain conditions """ # Generate temporary folder to work with work_dir = join(self.tmp_dir, 'CodecFile_Test', 'work') # Initialize Codec (without volume_size disables it) cr = CodecFile(work_dir=work_dir) # create some dummy file entries tmp_files = set() for i in range(0, 10): # Create some temporary files to work with in our source # directory tmp_file = join(work_dir, 'DSC_IMG%.3d.jpeg' % i) self.touch(tmp_file, size='120K', random=True) # Add a file to our tmp_files list tmp_files.add(tmp_file) # Non-existant file reference invalid_file = join(self.tmp_dir, 'non-existant-file') assert isfile(invalid_file) is False content = NNTPContent( join(work_dir, 'testfile'), work_dir=self.tmp_dir, ) content.write('test data') # Empty NNTPArticle() can not be added article = NNTPArticle(work_dir=self.tmp_dir) # New Empty NNTPContent() can not be added article_content = NNTPContent( join(work_dir, 'testfile2'), work_dir=self.tmp_dir, ) # Store some new data article_content.write('some more test data') # We'll add our new content to our article assert article.add(content) is True # save path sub_dir = join(work_dir, 'subdir') assert mkdir(sub_dir) is True assert isdir(sub_dir) is True # string work assert len(cr.get_paths(self.tmp_dir)) == 1 assert cr.get_paths(self.tmp_dir).pop() == self.tmp_dir # Sub-directories that exist within a root directory also included are # removed assert len(cr.get_paths([self.tmp_dir, sub_dir])) == 1 assert cr.get_paths([self.tmp_dir, sub_dir]).pop() == self.tmp_dir # Invalid files/dirs are not found assert len(cr.get_paths(invalid_file)) == 0 # Create a list of many assorted type of items __set = set([ work_dir, sub_dir, article_content, content, invalid_file, ]) | set(tmp_files) # At the end of the day, the work_dir includes all of the sub-content # and the invalid_file is simply just tossed. However because our # NNTPContent() and NNTPArticle() files are stored outside of our # work_dir, they will also be included in the results results = cr.get_paths(__set) assert len(results) == 3 assert work_dir in results assert content.filepath in results assert article_content.filepath in results # Now if we did the same test but without the work_dir directory, then # we'd have a much larger list; we'll work with lists this time to # show that we support them too __list = [ sub_dir, article_content, content, invalid_file, ] __list.extend(tmp_files) results = cr.get_paths(__list) # +1 for content # +1 for sub_dir assert len(results) == (len(tmp_files) + len(article) + 2) for f in tmp_files: # Each file in our tmp_files will be in our results assert f in results assert content.filepath in results assert article_content.filepath in results assert sub_dir in results
def test_invalid_split_cases(self): """ Test errors that are generated out of the split function """ work_dir = join(self.tmp_dir, 'NNTPContent_Test.chunk') # Now we want to load it into a NNTPContent object content = NNTPContent(work_dir=work_dir) # Nothing to split gives an error assert (content.split() is None) tmp_file = join(self.tmp_dir, 'NNTPContent_Test.chunk', '5K.rar') assert (isfile(tmp_file) is False) assert (self.touch(tmp_file, size='1MB', random=True) is True) assert (isfile(tmp_file) is True) # Now we want to load it into a NNTPContent object content = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir) # No size to split on gives an error assert (content.split(size=0) is None) assert (content.split(size=-1) is None) assert (content.split(size=None) is None) assert (content.split(size='bad_string') is None) # Invalid Memory Limit assert (content.split(mem_buf=0) is None) assert (content.split(mem_buf=-1) is None) assert (content.split(mem_buf=None) is None) assert (content.split(mem_buf='bad_string') is None)