Exemple #1
0
 def test_specified_filenames_yielding_with_dirs(self):
     name, content = self.file_names_and_contents[1]  # .hgignore
     expected = [(self.dir + name, content)]
     real = list(
         iter_unzip_from_bytes(
             self.zipped,
             filenames=['IGNORED-NON-EXISTENT', name, 'another ignored...'],
             yielding_with_dirs=True))
     self.assertCountEqual(expected, real)
Exemple #2
0
 def iter_filenames_and_contents(self, multifile_unpacking=False):
     if self.is_multipart():
         for msg in self.get_payload():
             # recursive calls on sub-messages
             # (NOTE that all of them are instances of this class
             # because all have been created within a call of this
             # class' from_string()/from_file())
             for name, content in msg.iter_filenames_and_contents(
                          multifile_unpacking=multifile_unpacking):
                 yield name, content
     else:
         content_type = self.get_content_type()
         payload = self.get_decoded_payload()
         filename = self.get_filename(None)
         ext = os.path.splitext(filename or '')[1].lower()
         # un-Gzip if necessary
         if (ext in self.GZIP_FILENAME_EXTENSIONS or
               content_type in self.GZIP_CONTENT_TYPES):
             try:
                 payload = gzip_decompress(payload)
             except (IOError, EOFError) as exc:
                 LOGGER.warning('Could not decompress file %r using GZip '
                                'decoder (%s)', filename, exc)
         # un-ZIP if necessary
         elif (ext in self.ZIP_FILENAME_EXTENSIONS or
               content_type in self.ZIP_CONTENT_TYPES):
             try:
                 names_and_contents = list(iter_unzip_from_bytes(payload))
             except (zipfile.BadZipfile, RuntimeError) as exc:
                 LOGGER.warning('Could not unpack file %r using ZIP '
                                'decoder (%s)', filename, exc)
             else:
                 if not names_and_contents:
                     LOGGER.warning('No files in archive %r', filename)
                     # yielding nothing
                     return
                 if multifile_unpacking:
                     # all files from the archive will be yielded (with
                     # their names prefixed with archive file name + '/')
                     name_pattern = (filename or '') + '/{0}'
                     for name, content in names_and_contents:
                         yielded_name = name_pattern.format(name)
                         LOGGER.debug('Yielding file %r...', yielded_name)
                         yield yielded_name, content
                     return
                 # only one file from the archive will be yielded
                 name, payload = names_and_contents[0]
                 if len(names_and_contents) > 1:
                     LOGGER.warning('Archive %r contains more than '
                                    'one file but only one (named '
                                    '%r in the archive) will be '
                                    'yielded as the payload of %r',
                                    filename, name, filename)
         LOGGER.debug('Yielding file %r...', filename)
         yield filename, payload
Exemple #3
0
 def test_all_filenames(self):
     expected = self.file_names_and_contents
     real = list(iter_unzip_from_bytes(self.zipped))
     self.assertCountEqual(expected, real)
Exemple #4
0
 def obtain_orig_data(self):
     data = self.download(self.config['url'])
     [(_, all_rows)] = iter_unzip_from_bytes(data, filenames=[self.CSV_FILENAME])
     return all_rows