def translate(source_dir, dest_dir):
# Directory with source rpm packages. Conver to abspath
source_dir = os.path.abspath(source_dir)
# Directory to save translates. Convert to abspath
dest_dir = os.path.abspath(dest_dir)
#URL for yandex-translate API
Url = 'https://translate.yandex.net/api/v1.5/tr.json/translate'
# API key for yandex-translate. You can get it from your yandex account
ApiKey = 'your API-key'
# Translate direction
Lang = 'en-ru'
# Output format. Can be html or plain
Format = 'plain'
if os.path.isdir(source_dir) and os.path.isdir(dest_dir):
os.chdir(source_dir)
for file in os.listdir(source_dir):
with rpmfile.open(file) as rpm:
orig_text = rpm.headers.get('description', None).decode('utf-8')
pkg_name = rpm.headers.get('name', None).decode('utf-8')
Data = {'key' : ApiKey, 'text' : orig_text, 'lang' : Lang, 'format' : Format}
Request = requests.get(Url, params=Data)
Response = Request.json()
tr_text = Response.get('text', None)
with open(dest_dir + os.path.sep + pkg_name + '.txt', 'w') as f:
f.writelines(tr_text)
else:
print("Check your source and destination directories")
sys.exit(1)
def assert_files_mode_and_owner_rpm(project, filename):
DIRNAMES_TAG = 1118
DIRINDEXES_TAG = 1116
expected_tags = [
'basenames', DIRNAMES_TAG, DIRINDEXES_TAG, 'filemodes', 'fileusername',
'filegroupname'
]
with rpmfile.open(filename) as rpm:
for key in expected_tags:
assert key in rpm.headers
for i, basename in enumerate(rpm.headers['basenames']):
# get filepath
basename = basename.decode("utf-8")
dirindex = rpm.headers[DIRINDEXES_TAG][i]
dirname = rpm.headers[DIRNAMES_TAG][dirindex].decode("utf-8")
filepath = os.path.join(dirname, basename)
# check fileowner
assert rpm.headers['fileusername'][i].decode("utf-8") == 'root'
assert rpm.headers['filegroupname'][i].decode("utf-8") == 'root'
# check filemodes
if filepath.startswith(
os.path.join('/usr/share/tarantool/', project['name'],
'.rocks')):
continue
filemode = rpm.headers['filemodes'][i]
assert_filemode(project, filepath, filemode)
def test_autoclose(self, rpmpath):
"""Test that RPMFile.open context manager properly closes rpm file"""
rpm_ref = None
with rpmfile.open(rpmpath) as rpm:
rpm_ref = rpm
# Inspect the RPM headers
self.assertIn("name", rpm.headers.keys())
self.assertEqual(rpm.headers.get("arch", "noarch"), b"x86_64")
members = list(rpm.getmembers())
self.assertEqual(len(members), 13)
# Test that subfile does not close parent file by calling close and
# then extractfile again
fd = rpm.extractfile("rpm-4.13.90-ldflags.patch")
calculated = hashlib.md5(fd.read()).hexdigest()
self.assertEqual(calculated, "4841553ad9276fffd83df33643839a57")
fd.close()
fd = rpm.extractfile("rpm.spec")
calculated = hashlib.md5(fd.read()).hexdigest()
self.assertEqual(calculated, "ed96e23cf7f8dd17c459e0dd4618888c")
fd.close()
# Test that RPMFile owned file descriptor and that underlying file is really closed
self.assertTrue(rpm_ref._fileobj.closed)
self.assertTrue(rpm_ref._ownes_fd)
def add_rpm(self, rpm):
"""Extract a rpm package in the output tar.
All files presents in that rpm package will be added to the
output tar under the same paths. No user name nor group names will
be added to the output.
Args:
rpm: the rpm file to add
"""
with rpmfile.open(rpm) as f:
# Not sure why this is the header, and not the modes one
modes = f.headers[1144]
for i, m in enumerate(f.getmembers()):
mode = 0o644
name = m.name
print(name)
if name.startswith('.'):
name = name[1:]
if name.startswith('/bin') or name.startswith('/sbin'):
name = '/usr/local' + name
if name.startswith('/usr/bin') or name.startswith('/usr/sbin'):
name = '/usr/local' + name.replace('/usr/', '/')
if "bin/" in name:
mode = 0o755
with self.write_temp_file(data=f.extractfile(m.name).read()) as temp_file:
from os.path import basename, dirname
self.directory = dirname(name)
name = basename(name)
self.add_file(temp_file, name, mode=mode)
def assert_pre_and_post_install_scripts_rpm(filename, user_pre_install_script, user_post_install_script):
with rpmfile.open(filename) as rpm:
user_install_scripts_keys = ['prein', 'postin']
for key in user_install_scripts_keys:
assert key in rpm.headers
preinst_script = rpm.headers['prein'].decode('ascii')
postinst_script = rpm.headers['postin'].decode('ascii')
assert_all_lines_in_content(user_pre_install_script, preinst_script)
assert_all_lines_in_content(user_post_install_script, postinst_script)
def get_info_from_rpm(path):
"""Return data pulled from the headers of an RPM file."""
info = {}
rpm = rpmfile.open(path)
info["name"] = rpm.headers.get("name")
ver = rpm.headers.get("version")
if re.search('\.s(rc\.)?rpm', path):
info["version"] = "{}.{}".format(ver, "src")
else:
info["version"] = "{}.{}".format(ver, rpm.headers.get("arch"))
log.debug("Info from rpm: {}".format(info))
return info
def test_lzma_sudo(self, rpmpath):
with rpmfile.open(rpmpath) as rpm:
# Inspect the RPM headers
self.assertIn('name', rpm.headers.keys())
self.assertEqual(rpm.headers.get('arch', 'noarch'), b'x86_64')
members = list(rpm.getmembers())
self.assertEqual(len(members), 1)
fd = rpm.extractfile('./usr/bin/sudo')
calculated = hashlib.md5(fd.read()).hexdigest()
self.assertEqual(calculated, 'a208f3d9170ecfa69a0f4ccc78d2f8f6')
def test_lzma_sudo(self, rpmpath):
with rpmfile.open(rpmpath) as rpm:
# Inspect the RPM headers
self.assertIn("name", rpm.headers.keys())
self.assertEqual(rpm.headers.get("arch", "noarch"), b"x86_64")
members = list(rpm.getmembers())
self.assertEqual(len(members), 1)
with rpm.extractfile("./usr/bin/sudo") as fd:
calculated = hashlib.md5(fd.read()).hexdigest()
self.assertEqual(calculated,
"a208f3d9170ecfa69a0f4ccc78d2f8f6")
def assert_tarantool_dependency_rpm(filename, tarantool_versions):
with rpmfile.open(filename) as rpm:
dependency_keys = ['requirename', 'requireversion', 'requireflags']
for key in dependency_keys:
assert key in rpm.headers
assert len(rpm.headers['requirename']) == 2
assert len(rpm.headers['requireversion']) == 2
assert len(rpm.headers['requireflags']) == 2
assert rpm.headers['requirename'][0].decode('ascii') == 'tarantool'
assert rpm.headers['requireversion'][0].decode('ascii') == tarantool_versions["min"]["rpm"]
assert rpm.headers['requireflags'][0] == 0x08 | 0x04 # >=
assert rpm.headers['requirename'][1].decode('ascii') == 'tarantool'
assert rpm.headers['requireversion'][1].decode('ascii') == tarantool_versions["max"]["rpm"]
assert rpm.headers['requireflags'][1] == 0x02 # <
def test_autoclose(self, rpmpath):
"""Test that RPMFile.open context manager properly closes rpm file"""
rpm_ref = None
with rpmfile.open(rpmpath) as rpm:
rpm_ref = rpm
# Inspect the RPM headers
self.assertIn('name', rpm.headers.keys())
self.assertEqual(rpm.headers.get('arch', 'noarch'), b'x86_64')
members = list(rpm.getmembers())
self.assertEqual(len(members), 13)
# Test that RPMFile owned file descriptor and that underlying file is really closed
self.assertTrue(rpm_ref._fileobj.closed)
self.assertTrue(rpm_ref._ownes_fd)
def __init__(self, path, mode='r', ext=None):
super(ZFile, self).__init__()
if ext is not None:
_ext = ext
else:
_ext = os.path.splitext(path)[-1]
if _ext.startswith('.'):
_ext = _ext[1:]
if zipfile.is_zipfile(path) or _ext == 'zip':
self.f = zipfile.ZipFile(path, mode)
elif tarfile.is_tarfile(path) or _ext in ('tar', 'tgz', 'gz'):
self.f = tarfile.open(path, mode)
elif _ext == 'rpm':
self.f = rpmfile.open(path, mode + 'b')
else:
raise ValueError('Unsupported file extension: %s' % path)
def __init__(self, path, mode='r', ext=None):
super(ZFile, self).__init__()
if ext is not None:
_ext = ext
else:
_ext = os.path.splitext(path)[-1]
if _ext.startswith('.'):
_ext = _ext[1:]
if zipfile.is_zipfile(path) or _ext == 'zip':
self.f = zipfile.ZipFile(path, mode)
elif tarfile.is_tarfile(path) or _ext in ('tar', 'tgz', 'gz'):
self.f = tarfile.open(path, mode)
elif _ext == 'rpm':
self.f = rpmfile.open(path, mode + 'b')
else:
raise ValueError('Unsupported file extension: %s' % path)
def assert_tarantool_dependency_rpm(filename):
with rpmfile.open(filename) as rpm:
dependency_keys = ['requirename', 'requireversion', 'requireflags']
for key in dependency_keys:
assert key in rpm.headers
assert len(rpm.headers['requirename']) == 2
assert len(rpm.headers['requireversion']) == 2
assert len(rpm.headers['requireversion']) == 2
min_version = re.findall(r'\d+\.\d+\.\d+', tarantool_version())[0]
max_version = str(int(re.findall(r'\d+', tarantool_version())[0]) + 1)
assert rpm.headers['requirename'][0].decode('ascii') == 'tarantool'
assert rpm.headers['requireversion'][0].decode('ascii') == min_version
assert rpm.headers['requireflags'][0] == 0x08 | 0x04 # >=
assert rpm.headers['requirename'][1].decode('ascii') == 'tarantool'
assert rpm.headers['requireversion'][1].decode('ascii') == max_version
assert rpm.headers['requireflags'][1] == 0x02 # <
def assert_dependencies_rpm(filename, deps, tarantool_versions):
with rpmfile.open(filename) as rpm:
dependency_keys = ['requirename', 'requireversion', 'requireflags']
for key in dependency_keys:
assert key in rpm.headers
if not tarantool_enterprise_is_used():
deps += (
("tarantool", 0x08 | 0x04, tarantool_versions["min"]["rpm"]), # >=
("tarantool", 0x02, tarantool_versions["max"]["rpm"]),
)
assert len(rpm.headers['requirename']) == len(deps)
assert len(rpm.headers['requireversion']) == len(deps)
assert len(rpm.headers['requireversion']) == len(deps)
for i, dep in enumerate(deps):
assert rpm.headers['requirename'][i].decode('ascii') == dep[0]
assert rpm.headers['requireflags'][i] == dep[1]
assert rpm.headers['requireversion'][i].decode('ascii') == dep[2]
def scan(self, data, file, options, expire_at):
tmp_directory = options.get('tmp_directory', '/tmp/')
with tempfile.NamedTemporaryFile(dir=tmp_directory) as tmp_data:
tmp_data.write(data)
tmp_data.flush()
try:
with rpmfile.open(tmp_data.name) as rpm_obj:
extract_name = ''
for (key, value) in rpm_obj.headers.items():
if key == 'arch':
self.event['architecture'] = value
elif key == 'archive_compression':
self.event['archive_compression'] = value
elif key == 'archive_format':
self.event['archive_format'] = value
elif key == 'authors':
self.event['authors'] = value
elif key == 'buildhost':
self.event['build_host'] = value
elif key == 'buildtime':
self.event['build_time'] = value
elif key == 'copyright':
self.event['copyright'] = value
elif key == 'description':
if value is not None:
self.event['description'] = value.replace(b'\n', b' ')
elif key == 'filenames':
self.event['filenames'] = value
elif key == 'group':
self.event['group'] = value
elif key == 'name':
self.event['name'] = value
extract_name = f'{value.decode()}'
elif key == 'os':
self.event['os'] = value
elif key == 'packager':
self.event['packager'] = value
elif key == 'provides':
self.event['provides'] = value
elif key == 'release':
self.event['release'] = value
elif key == 'requirename':
self.event['require_name'] = value
elif key == 'rpmversion':
self.event['rpm_version'] = value
elif key == 'serial':
self.event['serial'] = value
elif key == 'sourcerpm':
self.event['source_rpm'] = value
elif key == 'summary':
self.event['summary'] = value
elif key == 'vendor':
self.event['vendor'] = value
elif key == 'version':
self.event['version'] = value
elif key == 'url':
self.event['url'] = value
extract_file = strelka.File(
name=extract_name,
source=self.name,
)
for c in strelka.chunk_string(data[rpm_obj.data_offset:]):
self.upload_to_coordinator(
extract_file.pointer,
c,
expire_at,
)
self.files.append(extract_file)
except ValueError:
self.flags.append('value_error')
def upload(*args, context=None, doop=False):
valid_archs = set(['x86_64', 'noarch', 'i686', 'i586', 'i386'])
major_re = re.compile(""".*(\.|_|-)(rh)?el(?P<version>\d)(u\d)?.*""")
filename_re = re.compile(
""".*?((\.|_|-)(rh)?el(?P<osmajor>\d)(u\d+)?)?(\.centos)?\.(?P<filearch>[_0-9a-z]+)\.rpm"""
)
rpm_info_re = re.compile(
"""(?P<version>\d+)|debuginfo|(?P<arch>[_0-9a-z]+)""")
tasks = set()
for rpm_file_info in args:
rpm_file_infos = rpm_file_info.split(';')
rpm_file_name = rpm_file_infos[0]
rpm_good_file_infos = True
centos_version = []
filearch = None
debuginfo = False
for i in rpm_file_infos[1:]:
rpm_info_match = rpm_info_re.fullmatch(i)
if rpm_info_match is None:
print('invalid rpm info "%s"' % rpm_file_info, file=sys.stderr)
rpm_good_file_infos = False
break
else:
add_version = rpm_info_match.group('version')
new_filearch = rpm_info_match.group('arch')
if add_version is not None:
centos_version.append(add_version)
elif new_filearch is not None:
filearch = new_filearch
elif i == 'debuginfo':
debuginfo = True
if not rpm_good_file_infos:
continue
try:
with rpmfile.open(rpm_file_name) as rpm:
filename_match = filename_re.fullmatch(basename(rpm_file_name))
# Returned as bytes, ensure it' a str or None
for key in ('arch', 'release', 'version', 'release'):
if key in rpm.headers:
if isinstance(rpm.headers[key], bytes):
rpm.headers[key] = rpm.headers[key].decode(
'us-ascii', 'replace')
else:
rpm.headers[key] = None
# the explicit filearch given in the command line wins
if filearch is None and filename_match is not None:
filearch = filename_match.group('filearch')
arch = rpm.headers['arch']
# If the centos_version was not explicit, try to resolve from version or release
if len(centos_version) == 0:
for try_major in ('version', 'release'):
major_match = major_re.fullmatch(
rpm.headers[try_major])
if major_match is not None:
centos_version = (major_match.group('version'), )
break
# The rhel version is not always in the release or the version name
# Sometimes, it's in the file name
if len(
centos_version
) == 0 and filename_match is not None and filename_match.group(
'osmajor') is not None:
centos_version = (filename_match.group('osmajor'), )
for i in map(lambda x: x.decode('us-ascii', 'replace'),
rpm.headers.get('provides', [])):
if i is not None and '-debuginfo' in i:
debuginfo = True
break
for i in map(lambda x: x.decode('us-ascii', 'replace'),
rpm.headers.get(1118, [])):
if i == '/usr/lib/debug/.build-id/':
debuginfo = True
break
if 'source' in rpm.headers or filearch == 'src':
source = True
else:
source = False
except IsADirectoryError as e:
continue
except AssertionError as e:
print('invalid rpm "%s"' % rpm_file_name, file=sys.stderr)
continue
except Exception as e:
print('invalid rpm "%s"' % rpm_file_name, e, file=sys.stderr)
continue
if source and debuginfo:
print('can\'t be both a source and a debuginfo rpm')
continue
if not source and (arch is None or arch not in valid_archs):
print('not a matching rpm: "%s", invalid architecture %s' %
(rpm_file_name, arch),
file=sys.stderr)
continue
if source:
rpm_type = 'SRPMS'
arch = ''
elif debuginfo:
rpm_type = 'debug'
arch = "/%s" % arch
else:
rpm_type = 'Packages'
arch = "/%s" % arch
if centos_version == None:
print('invalid rpm "%s"' % rpm_file_name, file=sys.stderr)
continue
if len(centos_version) == 0:
centos_version = (5, 6, 7, 8)
for v in centos_version:
dest = '/%s/%s%s/%s' % (rpm_type, v, arch, basename(rpm_file_name))
if doop:
rpm_file = open(rpm_file_name, 'rb')
request = context.nexuscnx.perform_request(
'PUT',
dest,
body=rpm_file,
headers={'Content-Type': 'application/x-rpm'})
done_cb = get_done_cb(dest, rpm_file)
else:
request = context.nexuscnx.perform_request('HEAD', dest)
done_cb = get_check_cb(rpm_file_name, dest)
op_future = asyncio.ensure_future(request, loop=context.loop)
op_future.add_done_callback(done_cb)
tasks.add(op_future)
if len(tasks) > 10:
timeout = None
else:
timeout = 0
done, tasks = yield from asyncio.wait(
tasks,
timeout=timeout,
loop=context.loop,
return_when=asyncio.FIRST_COMPLETED)
for i in done:
try:
i.result()
except NexusException:
pass
if len(tasks) != 0:
done, tasks = yield from asyncio.wait(
tasks,
timeout=None,
loop=context.loop,
return_when=asyncio.ALL_COMPLETED)
for i in done:
try:
i.result()
except NexusException:
pass
def scan(self, file_object, options):
tempfile_directory = options.get("tempfile_directory", "/tmp/")
with tempfile.NamedTemporaryFile(
dir=tempfile_directory) as strelka_file:
strelka_filename = strelka_file.name
strelka_file.write(file_object.data)
strelka_file.flush()
try:
with rpmfile.open(strelka_filename) as rpm_file:
child_file = file_object.data[rpm_file.data_offset:]
child_filename = f"{self.scanner_name}::size_{len(child_file)}"
for (key, value) in rpm_file.headers.items():
if key == "arch":
self.metadata["architecture"] = value
elif key == "archive_compression":
self.metadata["archiveCompression"] = value
elif key == "archive_format":
self.metadata["archiveFormat"] = value
elif key == "authors":
self.metadata["authors"] = value
elif key == "buildhost":
self.metadata["buildHost"] = value
elif key == "buildtime":
self.metadata[
"buildTime"] = datetime.utcfromtimestamp(
value).isoformat(timespec="seconds")
elif key == "copyright":
self.metadata["copyright"] = value
elif key == "description":
self.metadata["description"] = value.replace(
b"\n", b" ")
elif key == "filenames":
self.metadata["filenames"] = value
elif key == "group":
self.metadata["group"] = value
elif key == "name":
self.metadata["name"] = value
child_filename = f"{self.scanner_name}::{value.decode()}"
elif key == "os":
self.metadata["os"] = value
elif key == "packager":
self.metadata["packager"] = value
elif key == "provides":
self.metadata["provides"] = value
elif key == "release":
self.metadata["release"] = value
elif key == "requirename":
self.metadata["requireName"] = value
elif key == "rpmversion":
self.metadata["rpmVersion"] = value
elif key == "serial":
self.metadata["serial"] = value
elif key == "sourcerpm":
self.metadata["sourceRpm"] = value
elif key == "summary":
self.metadata["summary"] = value
elif key == "vendor":
self.metadata["vendor"] = value
elif key == "version":
self.metadata["version"] = value
elif key == "url":
self.metadata["url"] = value
child_fo = objects.StrelkaFile(
data=child_file,
filename=child_filename,
depth=file_object.depth + 1,
parent_uid=file_object.uid,
root_uid=file_object.root_uid,
parent_hash=file_object.hash,
root_hash=file_object.root_hash,
source=self.scanner_name)
self.children.append(child_fo)
except ValueError:
file_object.flags.append(f"{self.scanner_name}::value_error")
def test_issue_19(self, rpmpath):
with rpmfile.open(rpmpath) as rpm:
self.assertEqual(len(list(rpm.getmembers())), 2)
