def testMakeArchiveSubdir1(self): # test archiving a subdir with a file and extracting relative base = self.tempdir rar = opj(base, "testarchive1.rar") a = archive.Archive(rar, base) drel_top = "A" drel = opj(drel_top, "1", "a") dabs = opj(base, drel) frel = opj(drel, "testfile-1") fabs = opj(base, frel) file_helper.mkdir_p(dabs) file_helper.create_file(fabs, "content-1") a.add_dir(drel_top) self.assertEqual(a.run(), 0) self.assertEqual(file_helper.file_exists(rar), True) file_helper.remove_file(fabs) self.assertEqual(file_helper.file_exists(fabs), False) # test content of archive by extracting: b = archive.Archive(rar, base) syscode = b.extract(target_path=self.tempdir) self.assertEqual(syscode, 0) self.assertEqual(file_helper.file_head(fabs), "content-1")
def testMakeArchive(self): # Part A creates a single file, archives it, then deletes it # Part B extracts the file again from the archive and checks the content base = self.tempdir frel = "testfile" fabs = opj(base, "testfile") rar = opj(base, "testarchive1.rar") # Part A: file_helper.create_file(fabs, "content123test") a = archive.Archive(rar, base) a.add_file(frel) self.assertEqual(a.run(), 0) self.assertEqual(file_helper.file_exists(rar), True) file_helper.remove_file(fabs) self.assertEqual(file_helper.file_exists(fabs), False) # Part B: b = archive.Archive(rar, base) syscode = b.extract(target_path=base) self.assertEqual(syscode, 0) self.assertEqual(file_helper.file_head(fabs), "content123test")
def test_bot(self): arc = archive.Archive(":memory:") bot = archive_bot.ArchiveBot(arc) # Add a new entry bot.handle_message("!add somelink tags: [a, b, c]", no_extras) print(bot.handle_message("!get 1", no_extras)) # Update old entry bot.handle_message("!update 1 tags: [d]", no_extras) print(bot.handle_message("!get 2", no_extras)) # Add a bare entry assert not "error" in bot.handle_message("!add link", no_extras) # Add a complete entry assert not "error" in bot.handle_message( "!add link name: link tags: [a, b, c] read: someone", { "author": ["none"], "file": ("filename.txt", b"some content") }, ) # Find entries assert not "error" in bot.handle_message("!find", no_extras) assert "error" in bot.handle_message('!add "broken link"', no_extras)
def get_path(source): uri = urlparse.urlparse(source) if uri.scheme and uri.path.endswith(".git"): po = remote.GitRepo(source) elif uri.scheme.startswith("svn"): po = remote.SvnRepo(source) elif uri.scheme in ("http", "https", "ftp"): po = remote.Web(source) elif os.path.isfile(source): if any(source.endswith(ext) for ext in ('.zip', '.gz', '.tar')): po = archive.Archive(source) elif source.endswith('.crx'): po = archive.ChromeExtensionCrx(source) elif source.endswith('.xpi'): po = archive.FirefoxExtensionXpi(source) elif source.endswith('.apk'): po = decompilable.Apk(source) elif source.endswith('.jar'): po = decompilable.Jar(source) else: # we do not know what is is logger.debug("unknown: %s" % source) c = Cigma() try: with open(source, 'rb') as f: data = f.read(384) magic = c.cigma(data=data) logger.debug(magic) d_magic = magic["magic"] if not d_magic: po = local.LocalPath(source) else: mimetype = d_magic.get('mimetype', "") if mimetype.startswith( "application/x-executable-32"): po = decompilable.Application32Bits(source) elif mimetype.startswith( "application/x-executable-64"): po = decompilable.Application64Bits(source) elif mimetype.startswith("application/"): petype = PeFileFormat.petype(data) logger.debug(petype) if petype == PeFileFormat.TYPE_PE64: po = decompilable.Application64Bits(source) elif petype == PeFileFormat.TYPE_DOTNET: po = decompilable.ApplicationDotNet(source) else: po = decompilable.Application32Bits(source) else: po = local.LocalPath(source) except Exception as e: logger.exception("exception!") po = local.LocalPath(source) elif os.path.isdir(source): po = local.LocalPath(source) logger.debug("scheme: %s" % uri.scheme) logger.debug(po) return po
def load_prev_image(): global a global position global idx if position <= 1: load_archive(archive_index - 1) load_next_image() else: position -= 1 try: a = iter(archive.Archive(archives[archive_index])) except archive.core.Error: # This is really bad :/ # Skip to next archive. load_archive(archive_index + 1) element = a.__next__() for i in range(1, position): element = a.__next__() buff = element.read() if len(buff) > 0: load_image(buff, element.filename) else: load_prev_image()
def test_archive(self): arc = archive.Archive(":memory:") link = "https://www.google.com/" tags = {"tag1", "tag2"} arc.add(name="name", link=link, tags=tags) self.assertEqual((link, tags), arc.get(1, ["link", "tags"])) new_tag = {"new tag1", "new tag2"} arc.update(1, {"tags": new_tag}) self.assertEqual((new_tag,), arc.get(2, ["tags"])) # Broken links are not allowed self.assertRaises(Exception, arc.add, link="broken link") # Complex update arc.update( 2, { "tags": {"add": ["tag3", "tag4"], "sub": ["new tag1"]}, "name": ["new name"], }, ) result = arc.get(3, ["name", "tags"]) self.assertEqual(result[0], "new name") self.assertEqual(result[1], {"new tag2", "tag4", "tag3"})
def collect(pkg_name): """collect the needed package file from system create a zip file of these files :param pkg_name: the name of needed package """ collect_deb = collector.Collector(pkg_name) collect_deb.extract_installed_package() collect_deb.package_exist() collect_deb.collector_file() # /usr/... # /DEBIAN/... control_info = collect_deb.get_control() if control_info is not None: os.mkdir("donut/DEBIAN") with open("donut/DEBIAN/control", 'w') as file: file.write(control_info) collect_deb.fix_control("donut/DEBIAN/control") collect_deb.extract_md5() time_setter = archive.Archive("{}/donut".format(os.getcwd()), pkg_name, "zip", "1 Jan 18") time_setter.set_default_time() # time-sync time_setter.pack_it( ) # zip them, this file is asserted to be same in any machine collect_deb.clean()
def testMakeArchiveFail1(self): # create an archive of not existing file, this is the only file for the archive! base = self.tempdir a = archive.Archive(opj(base, "testarchive1.rar"), base) a.add_file("this-file-does-not-exist") syscode = a.run() self.assertNotEqual(syscode, 0)
def build_deb(pkg_name): """unzip the packed file and build deb package """ zip_mag = archive.Archive("{}/donut".format(os.getcwd()), pkg_name, "zip", "1 Jan 18") zip_mag.un_zip("{}/Coding/gcc-8-base_donut_decoded.zip".format(os.getcwd()), "{}/donut".format(os.getcwd())) collect_deb = collector.Collector(pkg_name) collect_deb.build("{}/donut".format(os.getcwd()), "{}/{}_donut.deb".format(os.getcwd(), pkg_name))
def load_archive(idx): global a global position global archive_index # By going back the program doesn't exit if it reached the end. if idx < 0: idx = 0 # Don't read futher than you can! if idx >= len(archives): print("Done :)") sys.exit(0) try: a = iter(archive.Archive(archives[idx])) except archive.core.Error: # Shouldn't happen since we already checked the existence of # the file. If this happens, at least don't crash! load_archive(idx + 1) if idx < archive_index: # Here it's a bit hard, since we need to # know how many entries there are. pyarchive # doesn't look like it provised a 'len' field, so... try: while True: a.__next__() position += 1 except StopIteration: position -= 2 # Go one step back... try: a = iter(archive.Archive(archives[idx])) except archive.core.Error: # Shouldn't happen. # Go back to the one that didn't fail. load_archive(idx + 1) for x in range(0, position): a.__next__() else: position = 0 archive_index = idx
def test_update(self): arc = archive.Archive(":memory:") bot = archive_bot.ArchiveBot(arc) bot.handle_message('!add "somelink.com" tags: [a, b]', no_extras) assert not "error" in bot.handle_message( '!update 1 tags: +[c, d] -e link: "otherlink.com" name: name', no_extras)
def archive(self, inputs): v_inputs = inputs self.input_validation(v_inputs) print(v_inputs) archive = a.Archive(source=v_inputs['arc_path'], dest=v_inputs['arc_dest'], format_='zip') archive.publish(key=v_inputs['key'], toc=True) print('Archive complete.')
def testNames(self): ar = archive.Archive('test.tgz') self.assertSetEqual( set(a.filename for a in ar), set([ 'archive/__init__.py', 'archive/core.pyx', 'setup.py', ]))
def testData(self): ar = archive.Archive('test.tgz') self.assertEqual( {a.filename: md5(a.read()).hexdigest() for a in ar}, { 'archive/__init__.py': '29a6a1e050bd42fe24cd17b138d4b08d', 'archive/core.pyx': '1bd9e27890beb0b576a2122e7b57ca8c', 'setup.py': 'de88961c0eca3d7875894eae7d551d18', })
def test_find(self): arc = archive.Archive(":memory:") arc.add(name="link", link="link.com", tags=["atag"]) arc.add(name="other", link="other", tags=["other"]) assert len(arc.find({"name": "link"})) == 1 assert len(arc.find({"link": "link.com"})) == 1 assert len(arc.find({"tags": ["atag"]})) == 1 assert len(arc.find({"keyword": "link"})) == 1
def ArchiveAndUpload(info, archive_latest=False): print '@@@BUILD_STEP dartium_generate_archive@@@' cwd = os.getcwd() dartium_bucket = info.name dartium_bucket = dartium_bucket.replace('multivm', 'multivm-dartium') drt_bucket = dartium_bucket.replace('dartium', 'drt') chromedriver_bucket = dartium_bucket.replace('dartium', 'chromedriver') dartium_archive = dartium_bucket + '-' + info.version drt_archive = drt_bucket + '-' + info.version chromedriver_archive = chromedriver_bucket + '-' + info.version dartium_zip, drt_zip, chromedriver_zip = archive.Archive( SRC_PATH, info.mode, dartium_archive, drt_archive, chromedriver_archive, is_win_ninja=info.is_win_ninja) status = 0 # Upload bleeding-edge builds to old dartium-archive bucket if info.channel == 'be': status = (OldUpload('dartium', dartium_bucket, os.path.abspath(dartium_zip), archive_latest=archive_latest) or OldUpload('drt', drt_bucket, os.path.abspath(drt_zip), archive_latest=archive_latest) or OldUpload('chromedriver', chromedriver_bucket, os.path.abspath(chromedriver_zip), archive_latest=archive_latest)) # Upload to new dart-archive bucket using GCSNamer, but not incremental # or perf builder builds. if not info.is_incremental and not info.is_build: Upload('dartium', os.path.abspath(dartium_zip), info, archive_latest=archive_latest) Upload('drt', os.path.abspath(drt_zip), info, archive_latest=archive_latest) Upload('chromedriver', os.path.abspath(chromedriver_zip), info, archive_latest=archive_latest) os.chdir(cwd) if status != 0: print '@@@STEP_FAILURE@@@' return status
def test_files(self): arc = archive.Archive(":memory:") bot = archive_bot.ArchiveBot(arc) file = ("filename.txt", b"some content here") # Send pseudo-file print(bot.handle_message("!add file", {"file": file})) # Get file print(bot.handle_message("!get 1", {}))
def testMakeArchiveFailSubdirs(self): # create an archive at not existing directory base = self.tempdir aname = opj(self.tempdir, "dir/doesnt/exist", "testarchive1.rar") a = archive.Archive(aname, base) file_helper.create_file(opj(self.tempdir, "testfile")) # empty file is ok a.add_file(opj("testfile")) syscode = a.run() # should fail, because the target dir does not exist: self.assertNotEqual(syscode, 0)
def __call__(self): # TODO: Exception safety # TODO: If a branch name is given, checkout the branch from remote # TODO: Reset back to 'origin/master' afterwards git.reset(self.revision) revhash = git.id() # TODO: Use same content streams for all architectures upload_jobs = [] for arch in arches.arches: date = time.strftime('%Y%m%d') filename = '%s-snapshot-%s-%s-%s' % (self.build_type, date, revhash[:10], arch) try: archive_stream = StringIO.StringIO() archive_obj = archive.Archive(arch, archive_stream) if arch.startswith('darwin'): result, uuid = autobuild.obtain(self.amqp_connection, revhash, arch, [self.build_type]) name, stream = result[0] archive_obj.add(name, stream.read()) else: if self.build_type == 'openclonk': for name, stream in contentiter.ContentIter( groupcontent.snapshot): archive_obj.add(name, stream.read()) arch_iter = architer.ArchIter(self.amqp_connection, arch, revhash, self.build_type) for name, stream in arch_iter: archive_obj.add(name, stream.read()) uuid = arch_iter.uuid archive_filename = archive_obj.get_filename(filename) archive_obj.close() archive_stream.seek(0) upload_jobs.append( (archive_filename, archive_stream, uuid, arch)) except autobuild.AutobuildException as ex: # make an entry for "failed build" archive_filename = archive_obj.get_filename(filename) upload_jobs.append((archive_filename, None, ex.uuid, arch)) uploader = upload.Uploader(self.log, self.dry_release) for archive_filename, archive_stream, uuid, arch in upload_jobs: if archive_stream is not None: # Needed to skip mape osx build(?) uploader.nightly_file(self.build_type, archive_filename, archive_stream, uuid, revhash[:10], arch) return True
def testExclude(self): # test archiving a subdir with a file and extracting relative base = self.tempdir rar = opj(base, "testarchive1.rar") a = archive.Archive(rar, base) drel_top = "A" drel = opj(drel_top, "1", "a") dabs = opj(base, drel) file_helper.mkdir_p(dabs) frel = opj(drel, "testfile-1") fabs = opj(base, frel) file_helper.create_file(fabs, "content-2") frel2 = opj(drel, "testfile-2") fabs2 = opj(base, frel2) file_helper.create_file(fabs2, "content-2") a.add_dir(drel_top) a.exclude(frel) # exclude the first file! self.assertEqual(a.run(), 0) self.assertEqual(file_helper.file_exists(rar), True) file_helper.remove_file(fabs) self.assertEqual(file_helper.file_exists(fabs), False) file_helper.remove_file(fabs2) self.assertEqual(file_helper.file_exists(fabs2), False) # test content of archive by extracting: b = archive.Archive(rar, base) syscode = b.extract(target_path=self.tempdir) self.assertEqual(syscode, 0) # the first file was excluded, may not exist: self.assertEqual(file_helper.file_exists(fabs), False) # the second file must be there: self.assertEqual(file_helper.file_head(fabs2), "content-2")
def acquire_toolchains(self, required, registered, path, debug_calls): return_paths = [] for toolchain in required: if toolchain in registered: # if toolchain is registered as 'local' return empty string if registered[toolchain]["remote"] is False: return_paths.append("") continue toolchain_location = registered[toolchain]["server"] with self.cd(path + "/bin"): # if archive exists but the catalog does not it is probably # a malformed archive - delete it if os.path.isfile(os.path.basename(toolchain_location)): if not os.path.isdir(registered[toolchain]["path"]): self.print_message(self.logtype.INFO, "Toolchain archive seems to ", "be corrupted, redownloading") os.remove(os.path.basename(toolchain_location)) if os.path.isfile( os.path.basename(toolchain_location)) is False: call = "wget " + toolchain_location if self.call_tool(call) != 0: self.print_message(self.logtype.ERROR, "Error while downloading", "toolchain") raise NameError("Required toolchains: " + ", ".join(required)) try: a = archive.Archive( os.path.basename(toolchain_location)) a.extract() except Exception as ext: # the downloaded file is corrupted, delete it os.remove(os.path.basename(toolchain_location)) self.print_message( self.logtype.ERROR, "Error while unpacking", os.path.basename(toolchain_location), "toolchain.", str(ext), "- deleting.") raise NameError("Required toolchains: " + ", ".join(required)) return_paths.append(path + "/bin/" + registered[toolchain]["path"]) else: self.print_message(self.logtype.ERROR, required, "toolchain is not registered") raise NameError("Required toolchains: " + ", ".join(required)) return return_paths
def archive_(aid): nowebp = int(request.args.get('nowebp', 0)) fn = capp.archives[aid]['filename'] ar = archive.Archive(fn) return render_template("archive.html", aid=aid, fn=fn, archive=ar, basename=os.path.basename, enumerate=enumerate, nowebp=nowebp)
def test_upload_two(self, s3_client): e = util.Encrypt(self.encryption_token) s3_client.return_value.download_fileobj.side_effect = ClientError( {}, "download_fileobj") ar = archive.Archive() try: self.make_dirs(self.srcdir, N=2, M=100) ar.upload_many(self.srcdir) self.assertEqual(s3_client.return_value.upload_fileobj.call_count, 2) finally: ar.close()
def testMakeArchiveFileInSubdir(self): # test archiving a file in a subdir and extracting relative base = self.tempdir rar = opj(base, "testarchive1.rar") # Part A: a = archive.Archive(rar, base) drel = opj("A", "1", "a") # relative path to directory dabs = opj(base, drel) # absolute frel = opj("A", "1", "a", "testfile-1") fabs = opj(base, frel) file_helper.mkdir_p(dabs) file_helper.create_file(fabs, "content-1") self.assertEqual(file_helper.file_exists(fabs), True) a.add_file(frel) # must add relative paths # run and check if rar archive exists: self.assertEqual(a.run(), 0) self.assertEqual(file_helper.file_exists(rar), True) # remove original file and check: file_helper.remove_file(fabs) self.assertEqual(file_helper.file_exists(fabs), False) # Part B: b = archive.Archive(rar, base) syscode = b.extract(target_path=base) self.assertEqual(syscode, 0) self.assertEqual(file_helper.file_exists(fabs), True) # TODO: check content: self.assertEqual(file_helper.file_head(fabs), "content-1")
def __init__(self, filename): super().__init__() arch = archive.Archive(filename) all = {} for f in arch: entry = parse_properties(f) pkgname = f.filename.rsplit('/', 1)[0] if pkgname in all: all[pkgname].update(entry) else: all[pkgname] = entry for namever, props in all.items(): p = Package(**props) self.add_package(p)
def extract_to_temp( file: FileStorage, ignore_filter: IgnoreFilterManager, handle_ignore: IgnoreHandling = IgnoreHandling.keep) -> str: """Extracts the contents of file into a temporary directory. :param file: The archive to extract. :param ignore_filter: The files and directories that should be ignored. :param handle_ignore: Determines how ignored files should be handled. :returns: The pathname of the new temporary directory. """ tmpfd, tmparchive = tempfile.mkstemp() try: os.remove(tmparchive) tmparchive += os.path.basename( secure_filename('archive_' + file.filename)) tmpdir = tempfile.mkdtemp() file.save(tmparchive) if handle_ignore == IgnoreHandling.error: arch = archive.Archive(tmparchive) wrong_files = ignore_filter.get_ignored_files_in_archive(arch) if wrong_files: raise IgnoredFilesException(invalid_files=wrong_files) arch.extract(to_path=tmpdir, method='safe') else: archive.extract(tmparchive, to_path=tmpdir, method='safe') if handle_ignore == IgnoreHandling.delete: ignore_filter.delete_from_dir(tmpdir) except (tarfile.ReadError, zipfile.BadZipFile): raise APIException( 'The given archive could not be extracted', "The given archive doesn't seem to be an archive", APICodes.INVALID_ARCHIVE, 400, ) except (InvalidFile, archive.UnsafeArchive) as e: raise APIException( 'The given archive contains invalid files', str(e), APICodes.INVALID_FILE_IN_ARCHIVE, 400, ) finally: os.close(tmpfd) os.remove(tmparchive) return tmpdir
def test_find(self): arc = archive.Archive(":memory:") bot = archive_bot.ArchiveBot(arc) # One single complex entry tags = ", ".join("somerandomtag" + str(e) for e in range(20)) bot.handle_message("!add test name: name tags: [%s]" % tags, no_extras) # Add a bunch of entries to force multiple pages for _ in range(20): bot.handle_message("!add test", no_extras) assert not "error" in bot.handle_message("!find", no_extras) assert not "error" in bot.handle_message("!find page: 1", no_extras) assert not "error" in bot.handle_message("!find page: 2", no_extras)
def testIncremental(self): ar = archive.Archive('test.tgz') for ent in ar: if ent.filename == 'setup.py': break else: raise AssertionError("No setup.py") buf = b"" while True: chunk = ent.read(100) if not chunk: break buf += chunk self.assertEqual( md5(buf).hexdigest(), 'de88961c0eca3d7875894eae7d551d18')
def test_add_get(self): arc = archive.Archive(":memory:") id1 = arc.add(name="link", link="link.com", tags=["a", "b"]) assert not "error" in arc.get(id1) # Adding repeated tags should save just a single tag id2 = arc.add(name="link", link="link.com", tags=["a", "a"]) # Select just the tags self.assertAlmostEqual(arc.get(id2, ["tags"]), ({"a"},)) id3 = arc.add(file=("filename.txt", b"some content")) arc.get(id3)
def __init__(s, path, software): new = False if os.path.isfile(path) else True s.archive = archive.Archive(path) s.metadata = { "createdOn": time.time(), "createdBy": getpass.getuser(), "name": os.path.basename(os.path.splitext(path)[0]), "software": software } with timer.Timer("Loading"): with s.archive: s.metadata = Dict( dict(s.metadata, **decode(s.archive.get("metadata.json", "{}")))) # Metadata s.ref = Dict( reference.Reference( decode(s.archive.get("reference.json", "{}")))) # Reference file s.data = Dict(decode(s.archive.get("data.json", "{}"))) # Storage if new: s.metadata.diff = True s.data.diff = True s.ref.diff = True tree = dict((a, a.split("/")) for a in s.archive.keys()) clipIDs = set(b[1] for a, b in tree.items() if b[0] == "clips") s.clips = Dict({}) if clipIDs: for ID in clipIDs: c = Clip(ID) c.metadata = Dict( dict( c.metadata, **decode( s.archive.get( "clips/%s/metadata.json" % ID, "{}")))) c.data = decode( s.archive.get("clips/%s/data.json" % ID, "{}")) thumbs = sorted([ a for a, b in tree.items() if b[0] == "clips" and b[1] == ID and b[2] == "thumbs" ]) if thumbs: for th in thumbs: c.thumbs.append(s.cache(th)) s.clips[ID] = c s.clips.diff