def translate(source_dir, dest_dir): # Directory with source rpm packages. Conver to abspath source_dir = os.path.abspath(source_dir) # Directory to save translates. Convert to abspath dest_dir = os.path.abspath(dest_dir) #URL for yandex-translate API Url = 'https://translate.yandex.net/api/v1.5/tr.json/translate' # API key for yandex-translate. You can get it from your yandex account ApiKey = 'your API-key' # Translate direction Lang = 'en-ru' # Output format. Can be html or plain Format = 'plain' if os.path.isdir(source_dir) and os.path.isdir(dest_dir): os.chdir(source_dir) for file in os.listdir(source_dir): with rpmfile.open(file) as rpm: orig_text = rpm.headers.get('description', None).decode('utf-8') pkg_name = rpm.headers.get('name', None).decode('utf-8') Data = {'key' : ApiKey, 'text' : orig_text, 'lang' : Lang, 'format' : Format} Request = requests.get(Url, params=Data) Response = Request.json() tr_text = Response.get('text', None) with open(dest_dir + os.path.sep + pkg_name + '.txt', 'w') as f: f.writelines(tr_text) else: print("Check your source and destination directories") sys.exit(1)
def assert_files_mode_and_owner_rpm(project, filename): DIRNAMES_TAG = 1118 DIRINDEXES_TAG = 1116 expected_tags = [ 'basenames', DIRNAMES_TAG, DIRINDEXES_TAG, 'filemodes', 'fileusername', 'filegroupname' ] with rpmfile.open(filename) as rpm: for key in expected_tags: assert key in rpm.headers for i, basename in enumerate(rpm.headers['basenames']): # get filepath basename = basename.decode("utf-8") dirindex = rpm.headers[DIRINDEXES_TAG][i] dirname = rpm.headers[DIRNAMES_TAG][dirindex].decode("utf-8") filepath = os.path.join(dirname, basename) # check fileowner assert rpm.headers['fileusername'][i].decode("utf-8") == 'root' assert rpm.headers['filegroupname'][i].decode("utf-8") == 'root' # check filemodes if filepath.startswith( os.path.join('/usr/share/tarantool/', project['name'], '.rocks')): continue filemode = rpm.headers['filemodes'][i] assert_filemode(project, filepath, filemode)
def test_autoclose(self, rpmpath): """Test that RPMFile.open context manager properly closes rpm file""" rpm_ref = None with rpmfile.open(rpmpath) as rpm: rpm_ref = rpm # Inspect the RPM headers self.assertIn("name", rpm.headers.keys()) self.assertEqual(rpm.headers.get("arch", "noarch"), b"x86_64") members = list(rpm.getmembers()) self.assertEqual(len(members), 13) # Test that subfile does not close parent file by calling close and # then extractfile again fd = rpm.extractfile("rpm-4.13.90-ldflags.patch") calculated = hashlib.md5(fd.read()).hexdigest() self.assertEqual(calculated, "4841553ad9276fffd83df33643839a57") fd.close() fd = rpm.extractfile("rpm.spec") calculated = hashlib.md5(fd.read()).hexdigest() self.assertEqual(calculated, "ed96e23cf7f8dd17c459e0dd4618888c") fd.close() # Test that RPMFile owned file descriptor and that underlying file is really closed self.assertTrue(rpm_ref._fileobj.closed) self.assertTrue(rpm_ref._ownes_fd)
def add_rpm(self, rpm): """Extract a rpm package in the output tar. All files presents in that rpm package will be added to the output tar under the same paths. No user name nor group names will be added to the output. Args: rpm: the rpm file to add """ with rpmfile.open(rpm) as f: # Not sure why this is the header, and not the modes one modes = f.headers[1144] for i, m in enumerate(f.getmembers()): mode = 0o644 name = m.name print(name) if name.startswith('.'): name = name[1:] if name.startswith('/bin') or name.startswith('/sbin'): name = '/usr/local' + name if name.startswith('/usr/bin') or name.startswith('/usr/sbin'): name = '/usr/local' + name.replace('/usr/', '/') if "bin/" in name: mode = 0o755 with self.write_temp_file(data=f.extractfile(m.name).read()) as temp_file: from os.path import basename, dirname self.directory = dirname(name) name = basename(name) self.add_file(temp_file, name, mode=mode)
def assert_pre_and_post_install_scripts_rpm(filename, user_pre_install_script, user_post_install_script): with rpmfile.open(filename) as rpm: user_install_scripts_keys = ['prein', 'postin'] for key in user_install_scripts_keys: assert key in rpm.headers preinst_script = rpm.headers['prein'].decode('ascii') postinst_script = rpm.headers['postin'].decode('ascii') assert_all_lines_in_content(user_pre_install_script, preinst_script) assert_all_lines_in_content(user_post_install_script, postinst_script)
def get_info_from_rpm(path): """Return data pulled from the headers of an RPM file.""" info = {} rpm = rpmfile.open(path) info["name"] = rpm.headers.get("name") ver = rpm.headers.get("version") if re.search('\.s(rc\.)?rpm', path): info["version"] = "{}.{}".format(ver, "src") else: info["version"] = "{}.{}".format(ver, rpm.headers.get("arch")) log.debug("Info from rpm: {}".format(info)) return info
def test_lzma_sudo(self, rpmpath): with rpmfile.open(rpmpath) as rpm: # Inspect the RPM headers self.assertIn('name', rpm.headers.keys()) self.assertEqual(rpm.headers.get('arch', 'noarch'), b'x86_64') members = list(rpm.getmembers()) self.assertEqual(len(members), 1) fd = rpm.extractfile('./usr/bin/sudo') calculated = hashlib.md5(fd.read()).hexdigest() self.assertEqual(calculated, 'a208f3d9170ecfa69a0f4ccc78d2f8f6')
def test_lzma_sudo(self, rpmpath): with rpmfile.open(rpmpath) as rpm: # Inspect the RPM headers self.assertIn("name", rpm.headers.keys()) self.assertEqual(rpm.headers.get("arch", "noarch"), b"x86_64") members = list(rpm.getmembers()) self.assertEqual(len(members), 1) with rpm.extractfile("./usr/bin/sudo") as fd: calculated = hashlib.md5(fd.read()).hexdigest() self.assertEqual(calculated, "a208f3d9170ecfa69a0f4ccc78d2f8f6")
def assert_tarantool_dependency_rpm(filename, tarantool_versions): with rpmfile.open(filename) as rpm: dependency_keys = ['requirename', 'requireversion', 'requireflags'] for key in dependency_keys: assert key in rpm.headers assert len(rpm.headers['requirename']) == 2 assert len(rpm.headers['requireversion']) == 2 assert len(rpm.headers['requireflags']) == 2 assert rpm.headers['requirename'][0].decode('ascii') == 'tarantool' assert rpm.headers['requireversion'][0].decode('ascii') == tarantool_versions["min"]["rpm"] assert rpm.headers['requireflags'][0] == 0x08 | 0x04 # >= assert rpm.headers['requirename'][1].decode('ascii') == 'tarantool' assert rpm.headers['requireversion'][1].decode('ascii') == tarantool_versions["max"]["rpm"] assert rpm.headers['requireflags'][1] == 0x02 # <
def test_autoclose(self, rpmpath): """Test that RPMFile.open context manager properly closes rpm file""" rpm_ref = None with rpmfile.open(rpmpath) as rpm: rpm_ref = rpm # Inspect the RPM headers self.assertIn('name', rpm.headers.keys()) self.assertEqual(rpm.headers.get('arch', 'noarch'), b'x86_64') members = list(rpm.getmembers()) self.assertEqual(len(members), 13) # Test that RPMFile owned file descriptor and that underlying file is really closed self.assertTrue(rpm_ref._fileobj.closed) self.assertTrue(rpm_ref._ownes_fd)
def __init__(self, path, mode='r', ext=None): super(ZFile, self).__init__() if ext is not None: _ext = ext else: _ext = os.path.splitext(path)[-1] if _ext.startswith('.'): _ext = _ext[1:] if zipfile.is_zipfile(path) or _ext == 'zip': self.f = zipfile.ZipFile(path, mode) elif tarfile.is_tarfile(path) or _ext in ('tar', 'tgz', 'gz'): self.f = tarfile.open(path, mode) elif _ext == 'rpm': self.f = rpmfile.open(path, mode + 'b') else: raise ValueError('Unsupported file extension: %s' % path)
def assert_tarantool_dependency_rpm(filename): with rpmfile.open(filename) as rpm: dependency_keys = ['requirename', 'requireversion', 'requireflags'] for key in dependency_keys: assert key in rpm.headers assert len(rpm.headers['requirename']) == 2 assert len(rpm.headers['requireversion']) == 2 assert len(rpm.headers['requireversion']) == 2 min_version = re.findall(r'\d+\.\d+\.\d+', tarantool_version())[0] max_version = str(int(re.findall(r'\d+', tarantool_version())[0]) + 1) assert rpm.headers['requirename'][0].decode('ascii') == 'tarantool' assert rpm.headers['requireversion'][0].decode('ascii') == min_version assert rpm.headers['requireflags'][0] == 0x08 | 0x04 # >= assert rpm.headers['requirename'][1].decode('ascii') == 'tarantool' assert rpm.headers['requireversion'][1].decode('ascii') == max_version assert rpm.headers['requireflags'][1] == 0x02 # <
def assert_dependencies_rpm(filename, deps, tarantool_versions): with rpmfile.open(filename) as rpm: dependency_keys = ['requirename', 'requireversion', 'requireflags'] for key in dependency_keys: assert key in rpm.headers if not tarantool_enterprise_is_used(): deps += ( ("tarantool", 0x08 | 0x04, tarantool_versions["min"]["rpm"]), # >= ("tarantool", 0x02, tarantool_versions["max"]["rpm"]), ) assert len(rpm.headers['requirename']) == len(deps) assert len(rpm.headers['requireversion']) == len(deps) assert len(rpm.headers['requireversion']) == len(deps) for i, dep in enumerate(deps): assert rpm.headers['requirename'][i].decode('ascii') == dep[0] assert rpm.headers['requireflags'][i] == dep[1] assert rpm.headers['requireversion'][i].decode('ascii') == dep[2]
def scan(self, data, file, options, expire_at): tmp_directory = options.get('tmp_directory', '/tmp/') with tempfile.NamedTemporaryFile(dir=tmp_directory) as tmp_data: tmp_data.write(data) tmp_data.flush() try: with rpmfile.open(tmp_data.name) as rpm_obj: extract_name = '' for (key, value) in rpm_obj.headers.items(): if key == 'arch': self.event['architecture'] = value elif key == 'archive_compression': self.event['archive_compression'] = value elif key == 'archive_format': self.event['archive_format'] = value elif key == 'authors': self.event['authors'] = value elif key == 'buildhost': self.event['build_host'] = value elif key == 'buildtime': self.event['build_time'] = value elif key == 'copyright': self.event['copyright'] = value elif key == 'description': if value is not None: self.event['description'] = value.replace(b'\n', b' ') elif key == 'filenames': self.event['filenames'] = value elif key == 'group': self.event['group'] = value elif key == 'name': self.event['name'] = value extract_name = f'{value.decode()}' elif key == 'os': self.event['os'] = value elif key == 'packager': self.event['packager'] = value elif key == 'provides': self.event['provides'] = value elif key == 'release': self.event['release'] = value elif key == 'requirename': self.event['require_name'] = value elif key == 'rpmversion': self.event['rpm_version'] = value elif key == 'serial': self.event['serial'] = value elif key == 'sourcerpm': self.event['source_rpm'] = value elif key == 'summary': self.event['summary'] = value elif key == 'vendor': self.event['vendor'] = value elif key == 'version': self.event['version'] = value elif key == 'url': self.event['url'] = value extract_file = strelka.File( name=extract_name, source=self.name, ) for c in strelka.chunk_string(data[rpm_obj.data_offset:]): self.upload_to_coordinator( extract_file.pointer, c, expire_at, ) self.files.append(extract_file) except ValueError: self.flags.append('value_error')
def upload(*args, context=None, doop=False): valid_archs = set(['x86_64', 'noarch', 'i686', 'i586', 'i386']) major_re = re.compile(""".*(\.|_|-)(rh)?el(?P<version>\d)(u\d)?.*""") filename_re = re.compile( """.*?((\.|_|-)(rh)?el(?P<osmajor>\d)(u\d+)?)?(\.centos)?\.(?P<filearch>[_0-9a-z]+)\.rpm""" ) rpm_info_re = re.compile( """(?P<version>\d+)|debuginfo|(?P<arch>[_0-9a-z]+)""") tasks = set() for rpm_file_info in args: rpm_file_infos = rpm_file_info.split(';') rpm_file_name = rpm_file_infos[0] rpm_good_file_infos = True centos_version = [] filearch = None debuginfo = False for i in rpm_file_infos[1:]: rpm_info_match = rpm_info_re.fullmatch(i) if rpm_info_match is None: print('invalid rpm info "%s"' % rpm_file_info, file=sys.stderr) rpm_good_file_infos = False break else: add_version = rpm_info_match.group('version') new_filearch = rpm_info_match.group('arch') if add_version is not None: centos_version.append(add_version) elif new_filearch is not None: filearch = new_filearch elif i == 'debuginfo': debuginfo = True if not rpm_good_file_infos: continue try: with rpmfile.open(rpm_file_name) as rpm: filename_match = filename_re.fullmatch(basename(rpm_file_name)) # Returned as bytes, ensure it' a str or None for key in ('arch', 'release', 'version', 'release'): if key in rpm.headers: if isinstance(rpm.headers[key], bytes): rpm.headers[key] = rpm.headers[key].decode( 'us-ascii', 'replace') else: rpm.headers[key] = None # the explicit filearch given in the command line wins if filearch is None and filename_match is not None: filearch = filename_match.group('filearch') arch = rpm.headers['arch'] # If the centos_version was not explicit, try to resolve from version or release if len(centos_version) == 0: for try_major in ('version', 'release'): major_match = major_re.fullmatch( rpm.headers[try_major]) if major_match is not None: centos_version = (major_match.group('version'), ) break # The rhel version is not always in the release or the version name # Sometimes, it's in the file name if len( centos_version ) == 0 and filename_match is not None and filename_match.group( 'osmajor') is not None: centos_version = (filename_match.group('osmajor'), ) for i in map(lambda x: x.decode('us-ascii', 'replace'), rpm.headers.get('provides', [])): if i is not None and '-debuginfo' in i: debuginfo = True break for i in map(lambda x: x.decode('us-ascii', 'replace'), rpm.headers.get(1118, [])): if i == '/usr/lib/debug/.build-id/': debuginfo = True break if 'source' in rpm.headers or filearch == 'src': source = True else: source = False except IsADirectoryError as e: continue except AssertionError as e: print('invalid rpm "%s"' % rpm_file_name, file=sys.stderr) continue except Exception as e: print('invalid rpm "%s"' % rpm_file_name, e, file=sys.stderr) continue if source and debuginfo: print('can\'t be both a source and a debuginfo rpm') continue if not source and (arch is None or arch not in valid_archs): print('not a matching rpm: "%s", invalid architecture %s' % (rpm_file_name, arch), file=sys.stderr) continue if source: rpm_type = 'SRPMS' arch = '' elif debuginfo: rpm_type = 'debug' arch = "/%s" % arch else: rpm_type = 'Packages' arch = "/%s" % arch if centos_version == None: print('invalid rpm "%s"' % rpm_file_name, file=sys.stderr) continue if len(centos_version) == 0: centos_version = (5, 6, 7, 8) for v in centos_version: dest = '/%s/%s%s/%s' % (rpm_type, v, arch, basename(rpm_file_name)) if doop: rpm_file = open(rpm_file_name, 'rb') request = context.nexuscnx.perform_request( 'PUT', dest, body=rpm_file, headers={'Content-Type': 'application/x-rpm'}) done_cb = get_done_cb(dest, rpm_file) else: request = context.nexuscnx.perform_request('HEAD', dest) done_cb = get_check_cb(rpm_file_name, dest) op_future = asyncio.ensure_future(request, loop=context.loop) op_future.add_done_callback(done_cb) tasks.add(op_future) if len(tasks) > 10: timeout = None else: timeout = 0 done, tasks = yield from asyncio.wait( tasks, timeout=timeout, loop=context.loop, return_when=asyncio.FIRST_COMPLETED) for i in done: try: i.result() except NexusException: pass if len(tasks) != 0: done, tasks = yield from asyncio.wait( tasks, timeout=None, loop=context.loop, return_when=asyncio.ALL_COMPLETED) for i in done: try: i.result() except NexusException: pass
def scan(self, file_object, options): tempfile_directory = options.get("tempfile_directory", "/tmp/") with tempfile.NamedTemporaryFile( dir=tempfile_directory) as strelka_file: strelka_filename = strelka_file.name strelka_file.write(file_object.data) strelka_file.flush() try: with rpmfile.open(strelka_filename) as rpm_file: child_file = file_object.data[rpm_file.data_offset:] child_filename = f"{self.scanner_name}::size_{len(child_file)}" for (key, value) in rpm_file.headers.items(): if key == "arch": self.metadata["architecture"] = value elif key == "archive_compression": self.metadata["archiveCompression"] = value elif key == "archive_format": self.metadata["archiveFormat"] = value elif key == "authors": self.metadata["authors"] = value elif key == "buildhost": self.metadata["buildHost"] = value elif key == "buildtime": self.metadata[ "buildTime"] = datetime.utcfromtimestamp( value).isoformat(timespec="seconds") elif key == "copyright": self.metadata["copyright"] = value elif key == "description": self.metadata["description"] = value.replace( b"\n", b" ") elif key == "filenames": self.metadata["filenames"] = value elif key == "group": self.metadata["group"] = value elif key == "name": self.metadata["name"] = value child_filename = f"{self.scanner_name}::{value.decode()}" elif key == "os": self.metadata["os"] = value elif key == "packager": self.metadata["packager"] = value elif key == "provides": self.metadata["provides"] = value elif key == "release": self.metadata["release"] = value elif key == "requirename": self.metadata["requireName"] = value elif key == "rpmversion": self.metadata["rpmVersion"] = value elif key == "serial": self.metadata["serial"] = value elif key == "sourcerpm": self.metadata["sourceRpm"] = value elif key == "summary": self.metadata["summary"] = value elif key == "vendor": self.metadata["vendor"] = value elif key == "version": self.metadata["version"] = value elif key == "url": self.metadata["url"] = value child_fo = objects.StrelkaFile( data=child_file, filename=child_filename, depth=file_object.depth + 1, parent_uid=file_object.uid, root_uid=file_object.root_uid, parent_hash=file_object.hash, root_hash=file_object.root_hash, source=self.scanner_name) self.children.append(child_fo) except ValueError: file_object.flags.append(f"{self.scanner_name}::value_error")
def test_issue_19(self, rpmpath): with rpmfile.open(rpmpath) as rpm: self.assertEqual(len(list(rpm.getmembers())), 2)